spider - crawling headless browser client

src/component/PuppeteerBrowser.js

@@ -0,0 +1,168 @@
+const logger = require("logops")
+    , puppeteer = require('puppeteer-core')
+    , config = require('../../config/Config')
+    , { Builder, By, Key, until } = require('selenium-webdriver')
+    // , webdriver = require('selenium-webdriver')
+    , chrome = require('selenium-webdriver/chrome')
+    , { Options } = require('selenium-webdriver/chrome')
+    , { Cluster } = require('puppeteer-cluster');
+
+
+
+
+class PuppeteerBrowser {
+
+    constructor() {
+        this.website = 'https://www.real.de/'
+        this.isCluster = true;
+        this.browser = null;
+        this.page = null;
+        this.cluster = null;
+        this.closing = false
+        this.timeout = 45000
+    }
+
+    getClsuterInfo() {
+        return this.cluster ? {
+            workersCnt: this.cluster.workers.length,
+            workersAvailCnt: this.cluster.workersAvail.length,
+            workersBusyCnt: this.cluster.workersBusy.length,
+            workersStarting: this.cluster.workersStarting,
+            allTargetCount: this.cluster.allTargetCount,
+            errorCount: this.cluster.errorCount,
+            idleResolversCnt: this.cluster.idleResolvers.length,
+            waitForOneResolversCnt: this.cluster.waitForOneResolvers.length,
+            isClosed: this.cluster.isClosed,
+            startTime: this.cluster.startTime,
+            nextWorkerId: this.cluster.nextWorkerId,
+            nextWorkCall: this.cluster.nextWorkCall,
+            lastLaunchedWorkerTime: this.cluster.lastLaunchedWorkerTime
+        } : null
+    }
+
+    async webdriver(uri) {
+        let html = ''
+        let opts = new Options
+        opts.addArguments(this.getChromeArgs())
+        // opts.setChromeBinaryPath('/usr/bin/chromedriver')
+        let driver = await new Builder().forBrowser('chrome').setChromeOptions(opts).build();
+        // let driver = new webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
+        try {
+            logger.info('Request URL %s', this.website + uri)
+            await driver.get(this.website + uri);
+            html = await driver.findElement(By.css('body')).getAttribute('innerHTML')
+            //.sendKeys('webdriver', Key.RETURN);
+            // await driver.wait(until.titleIs('webdriver - www.real.de'), 1000);
+            return html
+        } finally {
+            await driver.quit();
+        }
+    }
+
+    async close() {
+        this.closing = true
+        if (this.isCluster) {
+            if (this.cluster && this.cluster.isClosed === false) {
+                logger.debug({ cluster: this.getClsuterInfo() }, 'Browser closing ...')
+                await this.cluster.idle();
+                await this.cluster.close();
+                this.cluster = null
+                this.closing = false
+                logger.info('Browser cluster closed')
+            } else {
+                logger.info('No browser cluster opened')
+            }
+        } else {
+            if (this.browser != null) {
+                await this.browser.close()
+            }
+            this.browser = null;
+            this.page = null;
+        }
+        return true
+    }
+
+    getChromeArgs() {
+        let args = ['--no-sandbox', '--headless', '--disable-dev-shm-usage']
+        // args.push('--enable-logging=stderr --v=1 > ' + logPath + ' 2>&1')
+        if (config.proxy) {
+            args.push('--proxy-server=' + config.proxy)
+        }
+        if (config.chrome) {
+            config.chrome.split(' ').map(a => args.push('--' + a))
+        }
+        return args
+    }
+
+    loggingChrome() {
+        let logPath = __dirname + '/storage/chrome-logs.txt'
+        logger.info('Chrome logging into %s', logPath)
+    }
+
+    async clustering(url) {
+        if (this.cluster == null) {
+            let logPath = __dirname + '/storage/chrome-logs.txt'
+            logger.info('Chrome logging into %s', logPath)
+            this.cluster = await Cluster.launch({
+                concurrency: Cluster.CONCURRENCY_CONTEXT,
+                maxConcurrency: 10,
+                timeout: this.timeout,
+                workerCreationDelay: 200,
+                puppeteer: puppeteer,
+                puppeteerOptions: {
+                    timeout: this.timeout,
+                    headless: true,
+                    executablePath: '/usr/bin/chromium-browser',
+                    args: this.getChromeArgs()
+                }
+            });
+            await this.cluster.task(async ({ page, data: url }) => {
+                return await this.getHtml(page, url)
+            });
+
+            logger.info('Cluster initialized %j', { cluster: this.getClsuterInfo() })
+        }
+        this.cluster.queue(url);
+        logger.info('Add queue url(%s) cluster updated : %j', url, { cluster: this.getClsuterInfo() })
+        return this.cluster.execute(url);
+
+    }
+
+    async getHtml(page, url) {
+        await page.goto(url, { timeout: this.timeout })
+        await page.waitFor(1500)
+        let bodyHandle = await page.$('body');
+        let html = await page.evaluate(body => body.innerHTML, bodyHandle);
+        logger.debug('Page evaluate with html')
+        await bodyHandle.dispose();
+        return html
+    }
+
+    async request(url) {
+        if (this.page == null) {
+            this.browser = await puppeteer.launch({
+                headless: true,
+                executablePath: '/usr/bin/chromium-browser',
+                args: ['--no-sandbox', '--headless', '--disable-gpu', '--disable-dev-shm-usage']
+            })
+            this.page = await this.browser.newPage()
+        }
+        return await this.getHtml(this.page, url)
+    }
+
+    real(url) {
+        return this.webdriver(url)
+        if (this.closing) {
+            return async () => {
+                return ''
+            }
+        }
+        if (this.isCluster) {
+            return this.clustering(this.website + url)
+        }
+        return this.request(this.website + url)
+    }
+
+}
+
+module.exports = new PuppeteerBrowser