const logger = require("logops") , config = require('../../config') , { Builder, Capabilities, By, Key, until } = require('selenium-webdriver') , url = require('url') , fetch = require('node-fetch') , pool = require('generic-pool') , { Options } = require('selenium-webdriver/chrome'); class SeleniumBrowser { constructor() { this.config = { browserFullCount: 0, useHub: true, browser: 'chrome', browserArgs: ['--no-sandbox', '--headless', '--disable-dev-shm-usage', '--proxy-server=spiduler-tor:8118'] } this.capabilities = Capabilities.chrome() this.sleep = m => new Promise(r => setTimeout(r, m)) this.createPool() } createPool() { this.pool = pool.createPool({ create: () => { logger.info('Create new driver instance') return this.getDriver(); }, destroy: (driver) => { logger.info('Quitting driver') return driver.quit().catch(err => { logger.debug({ message: err.message }, 'Quitting error') this.pool.release(driver) }) }, validate: (driver) => { return driver.getSession().then(s => { logger.info('Driver session validation checked (id: %s)', s.getId()) return true }).catch(err => { this.pool.destroy(driver) logger.debug({ message: err.message }, 'Driver session expired and destroy it') return false }) } }, { max: 20, min: 1, testOnBorrow: true }); this.pool.on('factoryCreateError', function (err) { logger.error({ message: err.message }, 'pool error') }) this.pool.on('factoryDestroyError', function (err) { logger.error({ message: err.message }, 'pool error') }) setInterval(() => { logger.debug({ spare: this.pool.spareResourceCapacity, size: this.pool.size, available: this.pool.available, borrowed: this.pool.borrowed, pending: this.pool.pending }, 'Pool info') }, 1000 * 60 * 5) } getDriver() { let opts = new Options; opts.addArguments(this.config.browserArgs) return new Builder() .usingServer('http://spiduler-chrome:4444/wd/hub') .withCapabilities(this.capabilities) .forBrowser(this.config.browser) .setChromeOptions(opts) .build(); } async request(params) { let uri = params.uri logger.info('URL %s', uri) if (uri.indexOf('://') > 3) { return this.pool.acquire() .then(async driver => { let html = await driver.get(uri) .then(() => driver.findElement(By.css('body'))) .then(body => body.getAttribute('innerHTML')) let htmlPromise = this.isCaptchaPage(html) ? this.resolveCaptcha(driver) : this.getHtml(driver, html, params) html = await htmlPromise this.pool.destroy(driver) return html }) } else { throw new Error('Invalid URI - ' + uri) } } isCaptchaPage(html) { return html.indexOf('grecaptcha') > -1 || html.indexOf('www.google.com/recaptcha/api2/anchor') > -1 } getCaptchaRequestUri(id) { return 'https://2captcha.com/res.php?key=62fc5ef52fd4befe266d224796a7ea6f&json=1&action=get&id=' + id } getHtml(driver, html, params) { if (params.waitElements) { return driver.get(params.uri) .then(() => Promise.all(params.waitElements.map(s => driver.wait(until.elementLocated(By.css(s)), 10000)))) .then(() => driver.findElement(By.css('body'))) .then(body => body.getAttribute('innerHTML')) .then(html => { return html }) } else { return html } } resolveCaptcha(driver) { return driver.findElement(By.css('#grecaptcha iframe')).getAttribute('src') .then(async src => { return { "key": "62fc5ef52fd4befe266d224796a7ea6f", "method": "userrecaptcha", "googlekey": url.parse(src, true).query.k, "pageurl": await driver.getCurrentUrl(), "json": 1 } }) .then(params => { let requestUrl = new url.URL('https://2captcha.com/in.php') requestUrl.search = new url.URLSearchParams(params).toString(); return fetch(requestUrl).then(res => res.json()).then(res => { logger.debug(res, '2captcha IN-request response') if (!res.hasOwnProperty('request')) { throw new Error('2captcha IN-request response has no data request id : ' + JSON.stringify(res)) } return res.request }) }) .then(async id => { logger.info('Request 2captcha solving result after 20 senconds') await this.sleep(1000 * 20) let res = await fetch(this.getCaptchaRequestUri(id)).then(res => res.json()) let maxRetry = 10 while (res.request == 'CAPCHA_NOT_READY') { await this.sleep(1000 * 5) maxRetry = maxRetry - 1 logger.info('2captcha still working so request solving result after 5 senconds') res = await fetch(this.getCaptchaRequestUri(id)).then(res => res.json()) if (maxRetry == 0) break } return res }) .then(res => { return driver.executeScript('captchaCallback("' + res.request + '")').then(() => this.sleep(1000 * 5)) }) } close() { return this.pool.drain() } } const browser = new SeleniumBrowser module.exports = browser