bro-app-browser/src/component/SeleniumBrowser.js

const logger = require("logops")
    , config = require('../../config')
    , { Builder, Capabilities, By, Key, until } = require('selenium-webdriver')
    , url = require('url')
    , fetch = require('node-fetch')
    , pool = require('generic-pool')
    , { Options } = require('selenium-webdriver/chrome');

class SeleniumBrowser {
    constructor() {
        this.config = {
            browserFullCount: 0,
            useHub: true, browser: 'chrome',
            browserArgs: ['--no-sandbox', '--headless', '--disable-dev-shm-usage', '--proxy-server=spiduler-tor:8118']
        }
        this.capabilities = Capabilities.chrome()
        this.sleep = m => new Promise(r => setTimeout(r, m))
        this.createPool()
        setInterval(() => this.reCreatePool(), 1000 * 60)
    }

    reCreatePool() {
        this.pool.drain().then(() => {
            return this.pool.clear()
        }).then(() => {
            this.createPool()
        })
    }

    createPool() {
        this.pool = pool.createPool({
            create: () => {
                logger.info('Create new driver instance')
                return this.getDriver();
            },
            destroy: (driver) => {
                logger.info('Quitting driver')
                return driver.quit().then(() => {
                    logger.debug({
                        spare: this.pool.spareResourceCapacity,
                        size: this.pool.size,
                        available: this.pool.available,
                        borrowed: this.pool.borrowed,
                        pending: this.pool.pending
                    }, 'Pool info')
                }).catch(err => {
                    logger.debug({ message: err.message }, 'Quitting error')
                    this.pool.release(driver)
                })
            },
            validate: (driver) => {
                return driver.getSession().then(s => {
                    logger.info('Driver session validation checked (id: %s)', s.getId())
                    return true
                }).catch(err => {
                    this.pool.destroy(driver)
                    logger.debug({ message: err.message }, 'Driver session expired and destroy it')
                    return false
                })
            }
        }, {
            max: 10, min: 1, testOnBorrow: true
        });

        this.pool.on('factoryCreateError', function (err) {
            logger.error({ message: err.message }, 'pool error')
        })

        this.pool.on('factoryDestroyError', function (err) {
            logger.error({ message: err.message }, 'pool error')
        })
    }

    getDriver() {
        let opts = new Options;
        opts.addArguments(this.config.browserArgs)
        return new Builder()
            .usingServer('http://spiduler-chrome:4444/wd/hub')
            .withCapabilities(this.capabilities)
            .forBrowser(this.config.browser)
            .setChromeOptions(opts)
            .build();
    }

    async request(params) {
        let uri = params.uri
        logger.info('URL %s', uri)
        if (uri.indexOf('://') > 3) {
            return this.pool.acquire()
                .then(async driver => {
                    let html = await driver.get(uri)
                        .then(() => driver.findElement(By.css('body')))
                        .then(body => body.getAttribute('innerHTML'))
                    let htmlPromise = this.isCaptchaPage(html) ? this.resolveCaptcha(driver) : this.getHtml(driver, html, params)
                    html = await htmlPromise
                    this.pool.destroy(driver)
                    return html
                })
        } else {
            throw new Error('Invalid URI - ' + uri)
        }
    }

    isCaptchaPage(html) {
        return html.indexOf('grecaptcha') > -1 || html.indexOf('www.google.com/recaptcha/api2/anchor') > -1
    }

    getCaptchaRequestUri(id) {
        return 'https://2captcha.com/res.php?key=62fc5ef52fd4befe266d224796a7ea6f&json=1&action=get&id=' + id
    }

    getHtml(driver, html, params) {
        if (params.waitElements) {
            return driver.get(params.uri)
                .then(() => Promise.all(params.waitElements.map(s => driver.wait(until.elementLocated(By.css(s))))))
                .then(() => driver.findElement(By.css('body')))
                .then(body => body.getAttribute('innerHTML'))
                .then(html => {
                    return html
                })
        } else {
            return html
        }
    }

    resolveCaptcha(driver) {
        return driver.findElement(By.css('#grecaptcha iframe')).getAttribute('src')
            .then(async src => {
                return {
                    "key": "62fc5ef52fd4befe266d224796a7ea6f",
                    "method": "userrecaptcha",
                    "googlekey": url.parse(src, true).query.k,
                    "pageurl": await driver.getCurrentUrl(),
                    "json": 1
                }
            })
            .then(params => {
                let requestUrl = new url.URL('https://2captcha.com/in.php')
                requestUrl.search = new url.URLSearchParams(params).toString();
                return fetch(requestUrl).then(res => res.json()).then(res => {
                    logger.debug(res, '2captcha IN-request response')
                    if (!res.hasOwnProperty('request')) {
                        throw new Error('2captcha IN-request response has no data request id : ' + JSON.stringify(res))
                    }
                    return res.request
                })
            })
            .then(async id => {
                logger.info('Request 2captcha solving result after 20 senconds')
                await this.sleep(1000 * 20)
                let res = await fetch(this.getCaptchaRequestUri(id)).then(res => res.json())
                let maxRetry = 10
                while (res.request == 'CAPCHA_NOT_READY') {
                    await this.sleep(1000 * 5)
                    maxRetry = maxRetry - 1
                    logger.info('2captcha still working so request solving result after 5 senconds')
                    res = await fetch(this.getCaptchaRequestUri(id)).then(res => res.json())
                    if (maxRetry == 0) break
                }
                return res
            })
            .then(res => {
                return driver.executeScript('captchaCallback("' + res.request + '")').then(() => this.sleep(1000 * 5))
            })
    }

    close() {
        return this.pool.drain()
    }
}

const browser = new SeleniumBrowser
module.exports = browser