const logger = require("logops") , { Builder, Capabilities, By, Key, until } = require('selenium-webdriver') , url = require('url') , fetch = require('node-fetch') , pool = require('generic-pool') , { Options } = require('selenium-webdriver/chrome'); class Test { constructor() { this.config = { stopping: false, browserFullCount: 0, useHub: true, browser: 'chrome', browserArgs: ['--no-sandbox', '--headless', '--disable-dev-shm-usage', '--proxy-server=spiduler-tor:8118'] } this.capabilities = Capabilities.chrome() this.sleep = m => new Promise(r => setTimeout(r, m)) this.pool = pool.createPool({ create: () => { logger.info('Create new driver instance') return this.getDriver(); }, destroy: (driver) => { logger.info('Quitting driver') return driver.quit() }, validate: (driver) => { logger.debug({ spare: this.pool.spareResourceCapacity, size: this.pool.size, available: this.pool.available, borrowed: this.pool.borrowed, pending: this.pool.pending }, 'Pool info') if(this.pool.borrowed > 8) { this.pool.clear() } return driver.getSession().then(s => { logger.info('Driver session validation checked (id: %s)', s.getId()) return true }).catch(err => { this.pool.destroy(driver) logger.debug({ message: err.message }, 'Driver session expired and destroy it') return false }) } }, { max: 2, min: 1, testOnBorrow: true, acquireTimeoutMillis: 1000 }); this.pool.on('factoryCreateError', function (err) { logger.error({ message: err.message }, 'pool error') }) this.pool.on('factoryDestroyError', function (err) { logger.error({ message: err.message }, 'pool error') }) // this.pool.use(driver => { // return new Promise((resolve, reject) => { // if (this.pool.available == 0) this.config.browserFullCount++ // if (this.config.browserFullCount > 100) { // this.pool.clear() // this.config.browserFullCount = 0 // reject(new Error('browser full count more than 100 times')) // } else { // resolve(driver) // } // }) // }) } getDriver() { let opts = new Options; opts.addArguments(this.config.browserArgs) return new Builder() .usingServer('http://spiduler-chrome:4444/wd/hub') .withCapabilities(this.capabilities) .forBrowser(this.config.browser) .setChromeOptions(opts) .build(); } async request(params) { let uri = params.uri logger.info('URL %s', uri) if (uri.indexOf('://') > 3) { return this.pool.acquire() .then(async driver => { let html = await driver.get(uri) .then(() => driver.findElement(By.css('body'))) .then(body => body.getAttribute('innerHTML')) let htmlPromise = this.isCaptchaPage(html) ? this.resolveCaptcha(driver) : this.getHtml(driver, html, params) html = await htmlPromise this.pool.release(driver) return html }) } else { throw new Error('Invalid URI - ' + uri) } } isCaptchaPage(html) { return html.indexOf('grecaptcha') > -1 || html.indexOf('www.google.com/recaptcha/api2/anchor') > -1 } getCaptchaRequestUri(id) { return 'https://2captcha.com/res.php?key=62fc5ef52fd4befe266d224796a7ea6f&json=1&action=get&id=' + id } getHtml(driver, html, params) { if (params.waitElements) { return driver.get(params.uri) .then(() => Promise.all(params.waitElements.map(s => driver.wait(until.elementLocated(By.css(s)))))) .then(() => driver.findElement(By.css('body'))) .then(body => body.getAttribute('innerHTML')) .then(html => { return html }) } else { return html } } close() { clearTimeout(intervals) return this.pool.drain().then(function () { this.pool.clear(); }) } } const test = new Test const intervals = setInterval(() => { test.request({uri : 'https://www.naver.com'}).then(html => console.log(html.length)) }, 2000) exitHandler = (options, exitCode) => { if (options.cleanup) logger.info('cleanup') if (exitCode || exitCode === 0) logger.info('exit code :', exitCode) test.close().then(() => process.exit()); } /** * Process shutdown events handling */ //do something when app is closing process.on('exit', exitHandler.bind(null, { cleanup: true })); //catches ctrl+c event process.on('SIGINT', exitHandler.bind(null, { exit: true })); process.on('SIGTERM', exitHandler.bind(null, { cleanup: true }));