brow3c/bro-app-browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve usability for multiple websites crawling

Browse Source
This commit is contained in:
spiduler
2020-10-12 03:30:57 +09:00
parent e3c2dd248f
commit 25dcb68e85
1 changed files with 19 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/component/SeleniumBrowser.js
View File

@@ -7,7 +7,6 @@ const logger = require("logops")
class SeleniumBrowser {
constructor() {
this.website = 'https://www.real.de/'
this.driver = null
this.config = { useHub: true, browser: 'chrome' }
this.capabilities = Capabilities.chrome()
@@ -18,9 +17,6 @@ class SeleniumBrowser {
async setDriver() {
if (this.config.useHub) {
let opts = new Options;
// opts.setAcceptInsecureCerts(true);
// opts.setBrowserVersion('67');
// opts.setPlatform('Windows XP');
opts.addArguments(this.getChromeArgs())
this.driver = await new Builder()
.usingServer('http://spiduler-chrome:4444/wd/hub')
@@ -36,29 +32,25 @@ class SeleniumBrowser {
setLocalDriver() {
let opts = new Options
opts.addArguments(this.getChromeArgs())
// opts.setChromeBinaryPath('/usr/bin/chromedriver')
return new Builder().forBrowser(this.config.browser).setChromeOptions(opts).build();
// let driver = new
// webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
}
async request(uri) {
if (this.isRecaptchaReseponseEmpty) {
return ''
} else {
logger.info('Request URL %s', this.website + uri)
return this.driver.get(this.website + uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML'))
logger.info('URL %s', uri)
if (uri.indexOf('://') > 3) {
return this.driver.get(uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML'))
.then(html => this.getHtml(html))
// return await
// driver.findElement(By.css('body')).getAttribute('innerHTML')
// .sendKeys('webdriver', Key.RETURN);
/// await driver.wait(until.titleIs('webdriver - www.real.de'), 1000);
}else {
logger.info('URL %s is not valid', uri)
return ''
}
}
}
async getHtml(html) {
// return html
// check if it's forbbidden page
if (html.indexOf('grecaptcha') > -1 || html.indexOf('www.google.com/recaptcha/api2/anchor') > -1) {
this.isRecaptchaReseponseEmpty = true
let iframeSrc = await this.driver.findElement(By.css('#grecaptcha iframe')).getAttribute('src')