bro-app-browser/src/component/PuppeteerBrowser.js

const logger = require("logops")
    , puppeteer = require('puppeteer-core')
    , config = require('../../config/Config')
    , { Builder, By, Key, until } = require('selenium-webdriver')
    // , webdriver = require('selenium-webdriver')
    , chrome = require('selenium-webdriver/chrome')
    , { Options } = require('selenium-webdriver/chrome')
    , { Cluster } = require('puppeteer-cluster');




class PuppeteerBrowser {

    constructor() {
        this.website = 'https://www.real.de/'
        this.isCluster = true;
        this.browser = null;
        this.page = null;
        this.cluster = null;
        this.closing = false
        this.timeout = 45000
    }

    getClsuterInfo() {
        return this.cluster ? {
            workersCnt: this.cluster.workers.length,
            workersAvailCnt: this.cluster.workersAvail.length,
            workersBusyCnt: this.cluster.workersBusy.length,
            workersStarting: this.cluster.workersStarting,
            allTargetCount: this.cluster.allTargetCount,
            errorCount: this.cluster.errorCount,
            idleResolversCnt: this.cluster.idleResolvers.length,
            waitForOneResolversCnt: this.cluster.waitForOneResolvers.length,
            isClosed: this.cluster.isClosed,
            startTime: this.cluster.startTime,
            nextWorkerId: this.cluster.nextWorkerId,
            nextWorkCall: this.cluster.nextWorkCall,
            lastLaunchedWorkerTime: this.cluster.lastLaunchedWorkerTime
        } : null
    }

    async webdriver(uri) {
        let html = ''
        let opts = new Options
        opts.addArguments(this.getChromeArgs())
        // opts.setChromeBinaryPath('/usr/bin/chromedriver')
        let driver = await new Builder().forBrowser('chrome').setChromeOptions(opts).build();
        // let driver = new webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
        try {
            logger.info('Request URL %s', this.website + uri)
            await driver.get(this.website + uri);
            html = await driver.findElement(By.css('body')).getAttribute('innerHTML')
            //.sendKeys('webdriver', Key.RETURN);
            // await driver.wait(until.titleIs('webdriver - www.real.de'), 1000);
            return html
        } finally {
            await driver.quit();
        }
    }

    async close() {
        this.closing = true
        if (this.isCluster) {
            if (this.cluster && this.cluster.isClosed === false) {
                logger.debug({ cluster: this.getClsuterInfo() }, 'Browser closing ...')
                await this.cluster.idle();
                await this.cluster.close();
                this.cluster = null
                this.closing = false
                logger.info('Browser cluster closed')
            } else {
                logger.info('No browser cluster opened')
            }
        } else {
            if (this.browser != null) {
                await this.browser.close()
            }
            this.browser = null;
            this.page = null;
        }
        return true
    }

    getChromeArgs() {
        let args = ['--no-sandbox', '--headless', '--disable-dev-shm-usage']
        // args.push('--enable-logging=stderr --v=1 > ' + logPath + ' 2>&1')
        if (config.proxy) {
            args.push('--proxy-server=' + config.proxy)
        }
        if (config.chrome) {
            config.chrome.split(' ').map(a => args.push('--' + a))
        }
        return args
    }

    loggingChrome() {
        let logPath = __dirname + '/storage/chrome-logs.txt'
        logger.info('Chrome logging into %s', logPath)
    }

    async clustering(url) {
        if (this.cluster == null) {
            let logPath = __dirname + '/storage/chrome-logs.txt'
            logger.info('Chrome logging into %s', logPath)
            this.cluster = await Cluster.launch({
                concurrency: Cluster.CONCURRENCY_CONTEXT,
                maxConcurrency: 10,
                timeout: this.timeout,
                workerCreationDelay: 200,
                puppeteer: puppeteer,
                puppeteerOptions: {
                    timeout: this.timeout,
                    headless: true,
                    executablePath: '/usr/bin/chromium-browser',
                    args: this.getChromeArgs()
                }
            });
            await this.cluster.task(async ({ page, data: url }) => {
                return await this.getHtml(page, url)
            });

            logger.debug({ cluster: this.getClsuterInfo() }, 'Cluster initialized')
        }
        this.cluster.queue(url);
        logger.debug({ url: url, cluster: this.getClsuterInfo() }, 'Add url to queue cluster updated')
        return this.cluster.execute(url);

    }

    async getHtml(page, url) {
        await page.goto(url, { timeout: this.timeout })
        await page.waitFor(1500)
        let bodyHandle = await page.$('body');
        let html = await page.evaluate(body => body.innerHTML, bodyHandle);
        logger.debug('Page evaluate with html')
        await bodyHandle.dispose();
        return html
    }

    async request(url) {
        if (this.page == null) {
            this.browser = await puppeteer.launch({
                headless: true,
                executablePath: '/usr/bin/chromium-browser',
                args: ['--no-sandbox', '--headless', '--disable-gpu', '--disable-dev-shm-usage']
            })
            this.page = await this.browser.newPage()
        }
        return await this.getHtml(this.page, url)
    }

    real(url) {
        return this.webdriver(url)
        if (this.closing) {
            return async () => {
                return ''
            }
        }
        if (this.isCluster) {
            return this.clustering(this.website + url)
        }
        return this.request(this.website + url)
    }

}

module.exports = new PuppeteerBrowser