spider - crawling headless browser client

src/component/SeleniumBrowser.js

@@ -0,0 +1,75 @@
+const logger = require("logops")
+    , config = require('../../config')
+    , { Builder, Capabilities, By, Key, until } = require('selenium-webdriver')
+    , { Options } = require('selenium-webdriver/chrome');
+
+class SeleniumBrowser {
+    constructor() {
+        this.website = 'https://www.real.de/'
+        this.driver = null
+        this.config = { useHub: true, browser: 'chrome' }
+        this.capabilities = Capabilities.chrome()
+    }
+
+    async setDriver() {
+        if (this.config.useHub) {
+            let opts = new Options;
+            // opts.setAcceptInsecureCerts(true);
+            // opts.setBrowserVersion('67');
+            // opts.setPlatform('Windows XP');
+            opts.addArguments(this.getChromeArgs())
+            this.driver = await new Builder()
+                .usingServer('http://spiduler-chrome:4444/wd/hub')
+                .withCapabilities(this.capabilities)
+                .forBrowser(this.config.browser)
+                .setChromeOptions(opts)
+                .build();
+        } else {
+            this.driver = await this.setLocalDriver()
+        }
+    }
+
+    setLocalDriver() {
+        let opts = new Options
+        opts.addArguments(this.getChromeArgs())
+        // opts.setChromeBinaryPath('/usr/bin/chromedriver')
+        return new Builder().forBrowser(this.config.browser).setChromeOptions(opts).build();
+        // let driver = new webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
+    }
+
+    async request(uri) {
+        logger.info('Request URL %s', this.website + uri)
+        return this.driver.get(this.website + uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML'))
+        // .then(html => {
+        //     // solve the captcha with API
+        //     const token = await solveCaptcha(params, api)
+        // })
+        // return await driver.findElement(By.css('body')).getAttribute('innerHTML')
+        //.sendKeys('webdriver', Key.RETURN);
+        // await driver.wait(until.titleIs('webdriver - www.real.de'), 1000);            
+    }
+
+    async close() {
+
+    }
+
+    getChromeArgs() {
+        let args = ['--no-sandbox', '--headless', '--disable-dev-shm-usage']
+        if (config.proxy) {
+            args.push('--proxy-server=' + config.proxy)
+        }
+        if (config.chrome) {
+            config.chrome.split(' ').map(a => args.push('--' + a))
+        }
+        return args
+    }
+
+    loggingChrome() {
+        let logPath = __dirname + '/storage/chrome-logs.txt'
+        logger.info('Chrome logging into %s', logPath)
+    }
+}
+
+const browser = new SeleniumBrowser
+browser.setDriver()
+module.exports = browser