const logger = require("logops") , puppeteer = require('puppeteer-core') , config = require('../../config/Config') , { Builder, By, Key, until } = require('selenium-webdriver') // , webdriver = require('selenium-webdriver') , chrome = require('selenium-webdriver/chrome') , { Options } = require('selenium-webdriver/chrome') , { Cluster } = require('puppeteer-cluster'); class PuppeteerBrowser { constructor() { this.website = 'https://www.real.de/' this.isCluster = true; this.browser = null; this.page = null; this.cluster = null; this.closing = false this.timeout = 45000 } getClsuterInfo() { return this.cluster ? { workersCnt: this.cluster.workers.length, workersAvailCnt: this.cluster.workersAvail.length, workersBusyCnt: this.cluster.workersBusy.length, workersStarting: this.cluster.workersStarting, allTargetCount: this.cluster.allTargetCount, errorCount: this.cluster.errorCount, idleResolversCnt: this.cluster.idleResolvers.length, waitForOneResolversCnt: this.cluster.waitForOneResolvers.length, isClosed: this.cluster.isClosed, startTime: this.cluster.startTime, nextWorkerId: this.cluster.nextWorkerId, nextWorkCall: this.cluster.nextWorkCall, lastLaunchedWorkerTime: this.cluster.lastLaunchedWorkerTime } : null } async webdriver(uri) { let html = '' let opts = new Options opts.addArguments(this.getChromeArgs()) // opts.setChromeBinaryPath('/usr/bin/chromedriver') let driver = await new Builder().forBrowser('chrome').setChromeOptions(opts).build(); // let driver = new webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build(); try { logger.info('Request URL %s', this.website + uri) await driver.get(this.website + uri); html = await driver.findElement(By.css('body')).getAttribute('innerHTML') //.sendKeys('webdriver', Key.RETURN); // await driver.wait(until.titleIs('webdriver - www.real.de'), 1000); return html } finally { await driver.quit(); } } async close() { this.closing = true if (this.isCluster) { if (this.cluster && this.cluster.isClosed === false) { logger.debug({ cluster: this.getClsuterInfo() }, 'Browser closing ...') await this.cluster.idle(); await this.cluster.close(); this.cluster = null this.closing = false logger.info('Browser cluster closed') } else { logger.info('No browser cluster opened') } } else { if (this.browser != null) { await this.browser.close() } this.browser = null; this.page = null; } return true } getChromeArgs() { let args = ['--no-sandbox', '--headless', '--disable-dev-shm-usage'] // args.push('--enable-logging=stderr --v=1 > ' + logPath + ' 2>&1') if (config.proxy) { args.push('--proxy-server=' + config.proxy) } if (config.chrome) { config.chrome.split(' ').map(a => args.push('--' + a)) } return args } loggingChrome() { let logPath = __dirname + '/storage/chrome-logs.txt' logger.info('Chrome logging into %s', logPath) } async clustering(url) { if (this.cluster == null) { let logPath = __dirname + '/storage/chrome-logs.txt' logger.info('Chrome logging into %s', logPath) this.cluster = await Cluster.launch({ concurrency: Cluster.CONCURRENCY_CONTEXT, maxConcurrency: 10, timeout: this.timeout, workerCreationDelay: 200, puppeteer: puppeteer, puppeteerOptions: { timeout: this.timeout, headless: true, executablePath: '/usr/bin/chromium-browser', args: this.getChromeArgs() } }); await this.cluster.task(async ({ page, data: url }) => { return await this.getHtml(page, url) }); logger.debug({ cluster: this.getClsuterInfo() }, 'Cluster initialized') } this.cluster.queue(url); logger.debug({ url: url, cluster: this.getClsuterInfo() }, 'Add url to queue cluster updated') return this.cluster.execute(url); } async getHtml(page, url) { await page.goto(url, { timeout: this.timeout }) await page.waitFor(1500) let bodyHandle = await page.$('body'); let html = await page.evaluate(body => body.innerHTML, bodyHandle); logger.debug('Page evaluate with html') await bodyHandle.dispose(); return html } async request(url) { if (this.page == null) { this.browser = await puppeteer.launch({ headless: true, executablePath: '/usr/bin/chromium-browser', args: ['--no-sandbox', '--headless', '--disable-gpu', '--disable-dev-shm-usage'] }) this.page = await this.browser.newPage() } return await this.getHtml(this.page, url) } real(url) { return this.webdriver(url) if (this.closing) { return async () => { return '' } } if (this.isCluster) { return this.clustering(this.website + url) } return this.request(this.website + url) } } module.exports = new PuppeteerBrowser