brow3c/bro-app-browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve usability for multiple websites crawling

Browse Source
This commit is contained in:
spiduler
2020-10-12 03:30:57 +09:00
parent e3c2dd248f
commit 25dcb68e85
1 changed files with 19 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/component/SeleniumBrowser.js
View File

@@ -7,7 +7,6 @@ const logger = require("logops")
class SeleniumBrowser { class SeleniumBrowser {
constructor() { constructor() {
this.website = 'https://www.real.de/'
this.driver = null this.driver = null
this.config = { useHub: true, browser: 'chrome' } this.config = { useHub: true, browser: 'chrome' }
this.capabilities = Capabilities.chrome() this.capabilities = Capabilities.chrome()
@@ -18,9 +17,6 @@ class SeleniumBrowser {
async setDriver() { async setDriver() {
if (this.config.useHub) { if (this.config.useHub) {
let opts = new Options; let opts = new Options;
// opts.setAcceptInsecureCerts(true);
// opts.setBrowserVersion('67');
// opts.setPlatform('Windows XP');
opts.addArguments(this.getChromeArgs()) opts.addArguments(this.getChromeArgs())
this.driver = await new Builder() this.driver = await new Builder()
.usingServer('http://spiduler-chrome:4444/wd/hub') .usingServer('http://spiduler-chrome:4444/wd/hub')
@@ -36,29 +32,25 @@ class SeleniumBrowser {
setLocalDriver() { setLocalDriver() {
let opts = new Options let opts = new Options
opts.addArguments(this.getChromeArgs()) opts.addArguments(this.getChromeArgs())
// opts.setChromeBinaryPath('/usr/bin/chromedriver')
return new Builder().forBrowser(this.config.browser).setChromeOptions(opts).build(); return new Builder().forBrowser(this.config.browser).setChromeOptions(opts).build();
// let driver = new
// webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
} }
async request(uri) { async request(uri) {
if (this.isRecaptchaReseponseEmpty) { if (this.isRecaptchaReseponseEmpty) {
return '' return ''
} else { } else {
logger.info('Request URL %s', this.website + uri) logger.info('URL %s', uri)
return this.driver.get(this.website + uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML')) if (uri.indexOf('://') > 3) {
.then(html => this.getHtml(html)) return this.driver.get(uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML'))
// return await .then(html => this.getHtml(html))
// driver.findElement(By.css('body')).getAttribute('innerHTML') }else {
// .sendKeys('webdriver', Key.RETURN); logger.info('URL %s is not valid', uri)
/// await driver.wait(until.titleIs('webdriver - www.real.de'), 1000); return ''
}
} }
} }
async getHtml(html) { async getHtml(html) {
// return html
// check if it's forbbidden page
if (html.indexOf('grecaptcha') > -1 || html.indexOf('www.google.com/recaptcha/api2/anchor') > -1) { if (html.indexOf('grecaptcha') > -1 || html.indexOf('www.google.com/recaptcha/api2/anchor') > -1) {
this.isRecaptchaReseponseEmpty = true this.isRecaptchaReseponseEmpty = true
let iframeSrc = await this.driver.findElement(By.css('#grecaptcha iframe')).getAttribute('src') let iframeSrc = await this.driver.findElement(By.css('#grecaptcha iframe')).getAttribute('src')
@@ -83,7 +75,7 @@ class SeleniumBrowser {
do { do {
maxRetry = maxRetry - 1 maxRetry = maxRetry - 1
res = await this.requestCaptchaResult(res, requestId) res = await this.requestCaptchaResult(res, requestId)
if(maxRetry == 0) break if (maxRetry == 0) break
} while (res.request == 'CAPCHA_NOT_READY') } while (res.request == 'CAPCHA_NOT_READY')
this.isRecaptchaReseponseEmpty = false this.isRecaptchaReseponseEmpty = false
return res return res
@@ -92,17 +84,17 @@ class SeleniumBrowser {
// this.driver.findElement(By.css('#g-recaptcha-response')).setAttribute('innerHTML', res.request) // this.driver.findElement(By.css('#g-recaptcha-response')).setAttribute('innerHTML', res.request)
this.driver.executeScript('captchaCallback("' + res.request + '")') this.driver.executeScript('captchaCallback("' + res.request + '")')
this.driver.wait(10000).then(d => { this.driver.wait(10000).then(d => {
this.driver.findElement(By.css('#g-recaptcha-response')).then(function (webElement) { this.driver.findElement(By.css('#g-recaptcha-response')).then(function (webElement) {
logger.error(res, 'Failed to overcome recaptcha') logger.error(res, 'Failed to overcome recaptcha')
}, function (err) { }, function (err) {
// if (err.state && err.state === 'no such element') { // if (err.state && err.state === 'no such element') {
// console.log('Element not found'); // console.log('Element not found');
// } else { // } else {
// webdriver.promise.rejected(err); // webdriver.promise.rejected(err);
// } // }
logger.info('Success to overcome recaptcha') logger.info('Success to overcome recaptcha')
})
}) })
})
}).catch(error => { }).catch(error => {
logger.error({ message: error.message }, 'Error occured while process with 2captcha') logger.error({ message: error.message }, 'Error occured while process with 2captcha')
}) })