brow3c/bro-app-browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve usability for multiple websites crawling

Browse Source
This commit is contained in:
spiduler
2020-10-12 03:30:57 +09:00
parent e3c2dd248f
commit 25dcb68e85
1 changed files with 19 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/component/SeleniumBrowser.js
View File

@@ -7,7 +7,6 @@ const logger = require("logops")
class SeleniumBrowser {
constructor() {
this.website = 'https://www.real.de/'
this.driver = null
this.config = { useHub: true, browser: 'chrome' }
this.capabilities = Capabilities.chrome()
@@ -18,9 +17,6 @@ class SeleniumBrowser {
async setDriver() {
if (this.config.useHub) {
let opts = new Options;
// opts.setAcceptInsecureCerts(true);
// opts.setBrowserVersion('67');
// opts.setPlatform('Windows XP');
opts.addArguments(this.getChromeArgs())
this.driver = await new Builder()
.usingServer('http://spiduler-chrome:4444/wd/hub')
@@ -36,29 +32,25 @@ class SeleniumBrowser {
setLocalDriver() {
let opts = new Options
opts.addArguments(this.getChromeArgs())
// opts.setChromeBinaryPath('/usr/bin/chromedriver')
return new Builder().forBrowser(this.config.browser).setChromeOptions(opts).build();
// let driver = new
// webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
}
async request(uri) {
if (this.isRecaptchaReseponseEmpty) {
return ''
} else {
logger.info('Request URL %s', this.website + uri)
return this.driver.get(this.website + uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML'))
.then(html => this.getHtml(html))
// return await
// driver.findElement(By.css('body')).getAttribute('innerHTML')
// .sendKeys('webdriver', Key.RETURN);
/// await driver.wait(until.titleIs('webdriver - www.real.de'), 1000);
logger.info('URL %s', uri)
if (uri.indexOf('://') > 3) {
return this.driver.get(uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML'))
.then(html => this.getHtml(html))
}else {
logger.info('URL %s is not valid', uri)
return ''
}
}
}
async getHtml(html) {
// return html
// check if it's forbbidden page
if (html.indexOf('grecaptcha') > -1 || html.indexOf('www.google.com/recaptcha/api2/anchor') > -1) {
this.isRecaptchaReseponseEmpty = true
let iframeSrc = await this.driver.findElement(By.css('#grecaptcha iframe')).getAttribute('src')
@@ -83,7 +75,7 @@ class SeleniumBrowser {
do {
maxRetry = maxRetry - 1
res = await this.requestCaptchaResult(res, requestId)
if(maxRetry == 0) break
if (maxRetry == 0) break
} while (res.request == 'CAPCHA_NOT_READY')
this.isRecaptchaReseponseEmpty = false
return res
@@ -92,17 +84,17 @@ class SeleniumBrowser {
// this.driver.findElement(By.css('#g-recaptcha-response')).setAttribute('innerHTML', res.request)
this.driver.executeScript('captchaCallback("' + res.request + '")')
this.driver.wait(10000).then(d => {
this.driver.findElement(By.css('#g-recaptcha-response')).then(function (webElement) {
logger.error(res, 'Failed to overcome recaptcha')
}, function (err) {
// if (err.state && err.state === 'no such element') {
// console.log('Element not found');
// } else {
// webdriver.promise.rejected(err);
// }
logger.info('Success to overcome recaptcha')
})
this.driver.findElement(By.css('#g-recaptcha-response')).then(function (webElement) {
logger.error(res, 'Failed to overcome recaptcha')
}, function (err) {
// if (err.state && err.state === 'no such element') {
// console.log('Element not found');
// } else {
// webdriver.promise.rejected(err);
// }
logger.info('Success to overcome recaptcha')
})
})
}).catch(error => {
logger.error({ message: error.message }, 'Error occured while process with 2captcha')
})