spider - crawling headless browser client
This commit is contained in:
spiduler
72
src/component/CaptchaComponent.js
Normal file
72
src/component/CaptchaComponent.js
Normal file
@@ -0,0 +1,72 @@
|
||||
const logger = require('logops')
|
||||
, fetch = require('node-fetch')
|
||||
|
||||
|
||||
|
||||
class CaptchaComponent {
|
||||
constructor() {
|
||||
this.api = {
|
||||
in: 'https://' + service + '/in.php',
|
||||
res: 'https://' + service + '/res.php',
|
||||
key: process.env.APIKEY || 'YOUR_API_KEY',
|
||||
pollingInterval: 5000
|
||||
}
|
||||
this.params = {
|
||||
key: this.api.key,
|
||||
method: 'userrecaptcha',
|
||||
googlekey: recapParams.sitekey,
|
||||
pageurl: url,
|
||||
json: 1
|
||||
}
|
||||
}
|
||||
|
||||
submitCaptcha(captchaParams) {
|
||||
captchaParams.soft_id = '2496'
|
||||
return fetch(this.api.in, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(captchaParams)
|
||||
}).then(res => res.json()).then(res => {
|
||||
if (res.status === 1) {
|
||||
return res.request
|
||||
} else {
|
||||
logger.error(res.request, 'Captcha submit get response with error')
|
||||
}
|
||||
}).catch(err => logger.error(err))
|
||||
}
|
||||
|
||||
getAnswer(id) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const polling = setInterval(() => {
|
||||
fetch(`${this.api.res}?key=${this.api.key}&json=1&action=get&id=${id}`, {
|
||||
method: 'GET',
|
||||
headers: { "Content-Type": "application/json" }
|
||||
}).then(res => res.json()).then(res => {
|
||||
if (res.status === 1) {
|
||||
clearInterval(polling)
|
||||
return res.request
|
||||
} else if (res.request !== 'CAPCHA_NOT_READY') {
|
||||
clearInterval(polling)
|
||||
logger.error(res.request, 'Captcha answer get response with error')
|
||||
}
|
||||
}).catch(err => {
|
||||
clearInterval(polling)
|
||||
logger.error(err, 'Captcha answer get response with error')
|
||||
})
|
||||
}, this.api.pollingInterval)
|
||||
})
|
||||
}
|
||||
|
||||
async solveCaptcha(params) {
|
||||
try {
|
||||
let id = await this.submitCaptcha(params)
|
||||
let answer = await this.getAnswer(id)
|
||||
return answer
|
||||
} catch (e) {
|
||||
return e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
module.exports = new CaptchaComponent
|
||||
|
||||
168
src/component/PuppeteerBrowser.js
Normal file
168
src/component/PuppeteerBrowser.js
Normal file
@@ -0,0 +1,168 @@
|
||||
const logger = require("logops")
|
||||
, puppeteer = require('puppeteer-core')
|
||||
, config = require('../../config/Config')
|
||||
, { Builder, By, Key, until } = require('selenium-webdriver')
|
||||
// , webdriver = require('selenium-webdriver')
|
||||
, chrome = require('selenium-webdriver/chrome')
|
||||
, { Options } = require('selenium-webdriver/chrome')
|
||||
, { Cluster } = require('puppeteer-cluster');
|
||||
|
||||
|
||||
|
||||
|
||||
class PuppeteerBrowser {
|
||||
|
||||
constructor() {
|
||||
this.website = 'https://www.real.de/'
|
||||
this.isCluster = true;
|
||||
this.browser = null;
|
||||
this.page = null;
|
||||
this.cluster = null;
|
||||
this.closing = false
|
||||
this.timeout = 45000
|
||||
}
|
||||
|
||||
getClsuterInfo() {
|
||||
return this.cluster ? {
|
||||
workersCnt: this.cluster.workers.length,
|
||||
workersAvailCnt: this.cluster.workersAvail.length,
|
||||
workersBusyCnt: this.cluster.workersBusy.length,
|
||||
workersStarting: this.cluster.workersStarting,
|
||||
allTargetCount: this.cluster.allTargetCount,
|
||||
errorCount: this.cluster.errorCount,
|
||||
idleResolversCnt: this.cluster.idleResolvers.length,
|
||||
waitForOneResolversCnt: this.cluster.waitForOneResolvers.length,
|
||||
isClosed: this.cluster.isClosed,
|
||||
startTime: this.cluster.startTime,
|
||||
nextWorkerId: this.cluster.nextWorkerId,
|
||||
nextWorkCall: this.cluster.nextWorkCall,
|
||||
lastLaunchedWorkerTime: this.cluster.lastLaunchedWorkerTime
|
||||
} : null
|
||||
}
|
||||
|
||||
async webdriver(uri) {
|
||||
let html = ''
|
||||
let opts = new Options
|
||||
opts.addArguments(this.getChromeArgs())
|
||||
// opts.setChromeBinaryPath('/usr/bin/chromedriver')
|
||||
let driver = await new Builder().forBrowser('chrome').setChromeOptions(opts).build();
|
||||
// let driver = new webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
|
||||
try {
|
||||
logger.info('Request URL %s', this.website + uri)
|
||||
await driver.get(this.website + uri);
|
||||
html = await driver.findElement(By.css('body')).getAttribute('innerHTML')
|
||||
//.sendKeys('webdriver', Key.RETURN);
|
||||
// await driver.wait(until.titleIs('webdriver - www.real.de'), 1000);
|
||||
return html
|
||||
} finally {
|
||||
await driver.quit();
|
||||
}
|
||||
}
|
||||
|
||||
async close() {
|
||||
this.closing = true
|
||||
if (this.isCluster) {
|
||||
if (this.cluster && this.cluster.isClosed === false) {
|
||||
logger.debug({ cluster: this.getClsuterInfo() }, 'Browser closing ...')
|
||||
await this.cluster.idle();
|
||||
await this.cluster.close();
|
||||
this.cluster = null
|
||||
this.closing = false
|
||||
logger.info('Browser cluster closed')
|
||||
} else {
|
||||
logger.info('No browser cluster opened')
|
||||
}
|
||||
} else {
|
||||
if (this.browser != null) {
|
||||
await this.browser.close()
|
||||
}
|
||||
this.browser = null;
|
||||
this.page = null;
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
getChromeArgs() {
|
||||
let args = ['--no-sandbox', '--headless', '--disable-dev-shm-usage']
|
||||
// args.push('--enable-logging=stderr --v=1 > ' + logPath + ' 2>&1')
|
||||
if (config.proxy) {
|
||||
args.push('--proxy-server=' + config.proxy)
|
||||
}
|
||||
if (config.chrome) {
|
||||
config.chrome.split(' ').map(a => args.push('--' + a))
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
loggingChrome() {
|
||||
let logPath = __dirname + '/storage/chrome-logs.txt'
|
||||
logger.info('Chrome logging into %s', logPath)
|
||||
}
|
||||
|
||||
async clustering(url) {
|
||||
if (this.cluster == null) {
|
||||
let logPath = __dirname + '/storage/chrome-logs.txt'
|
||||
logger.info('Chrome logging into %s', logPath)
|
||||
this.cluster = await Cluster.launch({
|
||||
concurrency: Cluster.CONCURRENCY_CONTEXT,
|
||||
maxConcurrency: 10,
|
||||
timeout: this.timeout,
|
||||
workerCreationDelay: 200,
|
||||
puppeteer: puppeteer,
|
||||
puppeteerOptions: {
|
||||
timeout: this.timeout,
|
||||
headless: true,
|
||||
executablePath: '/usr/bin/chromium-browser',
|
||||
args: this.getChromeArgs()
|
||||
}
|
||||
});
|
||||
await this.cluster.task(async ({ page, data: url }) => {
|
||||
return await this.getHtml(page, url)
|
||||
});
|
||||
|
||||
logger.info('Cluster initialized %j', { cluster: this.getClsuterInfo() })
|
||||
}
|
||||
this.cluster.queue(url);
|
||||
logger.info('Add queue url(%s) cluster updated : %j', url, { cluster: this.getClsuterInfo() })
|
||||
return this.cluster.execute(url);
|
||||
|
||||
}
|
||||
|
||||
async getHtml(page, url) {
|
||||
await page.goto(url, { timeout: this.timeout })
|
||||
await page.waitFor(1500)
|
||||
let bodyHandle = await page.$('body');
|
||||
let html = await page.evaluate(body => body.innerHTML, bodyHandle);
|
||||
logger.debug('Page evaluate with html')
|
||||
await bodyHandle.dispose();
|
||||
return html
|
||||
}
|
||||
|
||||
async request(url) {
|
||||
if (this.page == null) {
|
||||
this.browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
executablePath: '/usr/bin/chromium-browser',
|
||||
args: ['--no-sandbox', '--headless', '--disable-gpu', '--disable-dev-shm-usage']
|
||||
})
|
||||
this.page = await this.browser.newPage()
|
||||
}
|
||||
return await this.getHtml(this.page, url)
|
||||
}
|
||||
|
||||
real(url) {
|
||||
return this.webdriver(url)
|
||||
if (this.closing) {
|
||||
return async () => {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
if (this.isCluster) {
|
||||
return this.clustering(this.website + url)
|
||||
}
|
||||
return this.request(this.website + url)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
module.exports = new PuppeteerBrowser
|
||||
75
src/component/SeleniumBrowser.js
Normal file
75
src/component/SeleniumBrowser.js
Normal file
@@ -0,0 +1,75 @@
|
||||
const logger = require("logops")
|
||||
, config = require('../../config')
|
||||
, { Builder, Capabilities, By, Key, until } = require('selenium-webdriver')
|
||||
, { Options } = require('selenium-webdriver/chrome');
|
||||
|
||||
class SeleniumBrowser {
|
||||
constructor() {
|
||||
this.website = 'https://www.real.de/'
|
||||
this.driver = null
|
||||
this.config = { useHub: true, browser: 'chrome' }
|
||||
this.capabilities = Capabilities.chrome()
|
||||
}
|
||||
|
||||
async setDriver() {
|
||||
if (this.config.useHub) {
|
||||
let opts = new Options;
|
||||
// opts.setAcceptInsecureCerts(true);
|
||||
// opts.setBrowserVersion('67');
|
||||
// opts.setPlatform('Windows XP');
|
||||
opts.addArguments(this.getChromeArgs())
|
||||
this.driver = await new Builder()
|
||||
.usingServer('http://spiduler-chrome:4444/wd/hub')
|
||||
.withCapabilities(this.capabilities)
|
||||
.forBrowser(this.config.browser)
|
||||
.setChromeOptions(opts)
|
||||
.build();
|
||||
} else {
|
||||
this.driver = await this.setLocalDriver()
|
||||
}
|
||||
}
|
||||
|
||||
setLocalDriver() {
|
||||
let opts = new Options
|
||||
opts.addArguments(this.getChromeArgs())
|
||||
// opts.setChromeBinaryPath('/usr/bin/chromedriver')
|
||||
return new Builder().forBrowser(this.config.browser).setChromeOptions(opts).build();
|
||||
// let driver = new webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
|
||||
}
|
||||
|
||||
async request(uri) {
|
||||
logger.info('Request URL %s', this.website + uri)
|
||||
return this.driver.get(this.website + uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML'))
|
||||
// .then(html => {
|
||||
// // solve the captcha with API
|
||||
// const token = await solveCaptcha(params, api)
|
||||
// })
|
||||
// return await driver.findElement(By.css('body')).getAttribute('innerHTML')
|
||||
//.sendKeys('webdriver', Key.RETURN);
|
||||
// await driver.wait(until.titleIs('webdriver - www.real.de'), 1000);
|
||||
}
|
||||
|
||||
async close() {
|
||||
|
||||
}
|
||||
|
||||
getChromeArgs() {
|
||||
let args = ['--no-sandbox', '--headless', '--disable-dev-shm-usage']
|
||||
if (config.proxy) {
|
||||
args.push('--proxy-server=' + config.proxy)
|
||||
}
|
||||
if (config.chrome) {
|
||||
config.chrome.split(' ').map(a => args.push('--' + a))
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
loggingChrome() {
|
||||
let logPath = __dirname + '/storage/chrome-logs.txt'
|
||||
logger.info('Chrome logging into %s', logPath)
|
||||
}
|
||||
}
|
||||
|
||||
const browser = new SeleniumBrowser
|
||||
browser.setDriver()
|
||||
module.exports = browser
|
||||
Reference in New Issue
Block a user