spider - crawling headless browser client
This commit is contained in:
spiduler
@@ -1,33 +0,0 @@
|
||||
const logger = require("logops")
|
||||
, puppeteer = require('puppeteer');
|
||||
|
||||
|
||||
|
||||
|
||||
class HeadlessBrowser {
|
||||
|
||||
constructor() {
|
||||
this.browser = null;
|
||||
this.page = null;
|
||||
}
|
||||
|
||||
async request(url) {
|
||||
if (this.page == null) {
|
||||
this.browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
executablePath: '/usr/bin/chromium-browser',
|
||||
args: ['--no-sandbox', '--headless', '--disable-gpu', '--disable-dev-shm-usage']
|
||||
})
|
||||
this.page = await this.browser.newPage()
|
||||
}
|
||||
await this.page.goto(url)
|
||||
return this.page.evaluate(() => document.documentElement.outerHTML)
|
||||
}
|
||||
|
||||
real(url) {
|
||||
return this.request('https://www.real.de/' + url)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
module.exports = new HeadlessBrowser
|
||||
65
config/index.js
Normal file
65
config/index.js
Normal file
@@ -0,0 +1,65 @@
|
||||
const commandLineArgs = require('command-line-args')
|
||||
, logger = require("logops");
|
||||
|
||||
class Config {
|
||||
|
||||
constructor() {
|
||||
this.options = {}
|
||||
this.requires = ['env', 'gateway', 'quix24']
|
||||
this.init()
|
||||
this.validate()
|
||||
this.finalize()
|
||||
}
|
||||
|
||||
init() {
|
||||
try {
|
||||
this.options = commandLineArgs([
|
||||
{ name: 'env', alias: 'e', type: String, defaultValue: ['production'] },
|
||||
{ name: 'host', type: String },
|
||||
{ name: 'port', type: Number },
|
||||
{ name: 'database', alias: 'd', type: String },
|
||||
{ name: 'username', alias: 'u', type: String },
|
||||
{ name: 'password', alias: 'p', type: String },
|
||||
{ name: 'mongo', alias: 'm', type: String },
|
||||
{ name: 'redis', alias: 'r', type: String },
|
||||
{ name: 'level', alias: 'l', type: String },
|
||||
{ name: 'gateway', alias: 'g', type: String },
|
||||
{ name: 'quix24', alias: 'q', type: String }
|
||||
]);
|
||||
} catch (e) {
|
||||
logger.debug('Command line arguments interpret failed :', e.message)
|
||||
logger.debug('expected arguments : ', JSON.stringify(this.requires))
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
finalize() {
|
||||
if (this.options.env == 'development') {
|
||||
this.options.level = this.options.level ? this.options.level : "DEBUG"
|
||||
logger.format = logger.formatters.dev;
|
||||
} else {
|
||||
this.options.level = this.options.level ? this.options.level : "WARN"
|
||||
}
|
||||
logger.setLevel(this.options.level)
|
||||
logger.info('Environment setting options', this.options)
|
||||
}
|
||||
|
||||
validate() {
|
||||
for (let key in this.options) {
|
||||
if (this.options[key]) {
|
||||
delete this.requires[this.requires.indexOf(key)]
|
||||
}
|
||||
}
|
||||
this.requires = this.requires.filter(function (el) {
|
||||
return el != null;
|
||||
})
|
||||
if (this.requires.length) {
|
||||
logger.debug('Process terminated invalid required arguments: ', JSON.stringify(requires))
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
let config = new Config;
|
||||
module.exports = config.options
|
||||
37
forbidden.html
Normal file
37
forbidden.html
Normal file
@@ -0,0 +1,37 @@
|
||||
|
||||
|
||||
<div class="main-container">
|
||||
<div class="container">
|
||||
<div class="row">
|
||||
<div class="col-sm-push-1 col-md-7 col-md-push-2 col-lg-5 col-lg-push-3">
|
||||
<div class="top-holder">
|
||||
<h2 class="headline text-bold text-primary">Forbidden</h2>
|
||||
<span class="img-holder">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 126 55"><title>real Logo</title><g id="real-logo-noclaim"><path d="M126,.9s.17-.89-.72-.89H113.76A1.09,1.09,0,0,0,112.7.9l-9.51,51.35s-.17.89.72.89h11.47a1.08,1.08,0,0,0,1.05-.89Z" style="fill:#003a70"></path><path d="M56.67,28.13H40.87c1.32-4.05,4.73-8.33,10.3-8.33,4.72,0,5.93,3.8,5.5,8.33m10.69.32C68.58,15.67,61.75,10.09,52,10.09c-16.58,0-23.74,15-24.67,24.83-1,10.59,4.87,19.25,17.31,19.25a44.44,44.44,0,0,0,12.51-2.12,1.22,1.22,0,0,0,.87-.83h0l2.08-8.16h0a.72.72,0,0,0-.73-.88,1,1,0,0,0-.34.06h0a28.45,28.45,0,0,1-10.47,2.22c-7.86,0-9.52-3.55-9.23-8.08H64.89a1.11,1.11,0,0,0,1-.79h0a55.16,55.16,0,0,0,1.46-7.13m24.75,6c-1,5.67-4.29,9.95-9.9,9.95a4.37,4.37,0,0,1-4.44-4.2c0-3.72,4.59-6.23,10.85-6a18.54,18.54,0,0,1,3.49.24m9.06,17.88c1.49-12.29,4.69-21.5,4.69-28C105.86,14,98.43,10,89.41,10A37.45,37.45,0,0,0,76,12.26h0a.9.9,0,0,0-.58.72h0l-1.06,8.55h0a.72.72,0,0,0,.73.89.8.8,0,0,0,.34-.07h0a21.55,21.55,0,0,1,11-2.63c4.3,0,7,2.43,7,5.42a6.49,6.49,0,0,1-.15,1.7c-1.82-.16-4.44-.4-7-.4-11.58,0-20.89,5.09-20.89,15.45,0,8.16,4.52,12.21,12.67,12.21,5.09,0,10-2,12.73-6.88v-.16l.15.16-.49,5h0a.89.89,0,0,0,.86.89h9a.88.88,0,0,0,.88-.81Z" style="fill:#003a70"></path><path d="M32.86,10v0a.42.42,0,0,0-.39-.52h0C30.23,9.47,22,10.26,19,20.1l1.7-8.5h0a1.09,1.09,0,0,0,0-.18.89.89,0,0,0-.89-.89H8.6a.88.88,0,0,0-.87.73h0L.32,52.06v.06h0c0,.05,0,.09,0,.13a.89.89,0,0,0,.89.89H12.88a.9.9,0,0,0,.86-.68h0l4.67-22s3.19-10,11.14-8.52h.08a1.12,1.12,0,0,0,1-.84h0L32.87,10Z" style="fill:#003a70"></path></g>
|
||||
</svg>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="_content">
|
||||
<p class="text-md">
|
||||
You were blocked due to suspicious requests to our site.
|
||||
</p>
|
||||
<p class="text-md">Please contact us via mail to robocop<span class="specialsymbol"></span>real-digital.de, if you have questions.</p>
|
||||
<div class="mt-lg">
|
||||
<p id="recaptcha-hint">Please resolve the captcha below for unblock.</p>
|
||||
<p id="recaptcha-success" class="text-success"></p>
|
||||
<p id="recaptcha-error" class="text-error"></p>
|
||||
<div id="grecaptcha"><div style="width: 304px; height: 78px;"><div><iframe src="https://www.google.com/recaptcha/api2/anchor?ar=1&k=6LcZsQgTAAAAAH3ulW8VyA-vm-GtnL0NabxlBLVd&co=aHR0cHM6Ly93d3cucmVhbC5kZTo0NDM.&hl=en&v=yXSLJBpiFoTYkexaPhFknpU7&theme=dark&size=normal&cb=rk3en1qdbpnt" width="304" height="78" role="presentation" name="a-1jsf1yhfxwqo" frameborder="0" scrolling="no" sandbox="allow-forms allow-popups allow-same-origin allow-scripts allow-top-navigation allow-modals allow-popups-to-escape-sandbox"></iframe></div><textarea id="g-recaptcha-response" name="g-recaptcha-response" class="g-recaptcha-response" style="width: 250px; height: 40px; border: 1px solid rgb(193, 193, 193); margin: 10px 25px; padding: 0px; resize: none; display: none;"></textarea></div><iframe style="display: none;"></iframe></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="footer-container">
|
||||
<a href="https://www.real.de/rechtliches/impressum/">Impressum</a> <a href="https://www.real.de/rechtliches/datenschutz/">Datenschutz</a>
|
||||
</div>
|
||||
</div>
|
||||
<script src="//www.google.com/recaptcha/api.js?onload=onloadCallback&render=explicit" async="" defer=""></script>
|
||||
|
||||
|
||||
<div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(204, 204, 204); box-shadow: rgba(0, 0, 0, 0.2) 2px 2px 3px; position: absolute; transition: visibility 0s linear 0.3s, opacity 0.3s linear; opacity: 0; visibility: hidden; z-index: 2000000000; left: 0px; top: -10000px;"><div style="width: 100%; height: 100%; position: fixed; top: 0px; left: 0px; z-index: 2000000000; background-color: rgb(255, 255, 255); opacity: 0.05;"></div><div class="g-recaptcha-bubble-arrow" style="border: 11px solid transparent; width: 0px; height: 0px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 2000000000;"></div><div class="g-recaptcha-bubble-arrow" style="border: 10px solid transparent; width: 0px; height: 0px; position: absolute; pointer-events: none; margin-top: -10px; z-index: 2000000000;"></div><div style="z-index: 2000000000; position: relative;"><iframe title="recaptcha challenge" src="https://www.google.com/recaptcha/api2/bframe?hl=en&v=yXSLJBpiFoTYkexaPhFknpU7&k=6LcZsQgTAAAAAH3ulW8VyA-vm-GtnL0NabxlBLVd&cb=6efox323yu0m" name="c-1jsf1yhfxwqo" frameborder="0" scrolling="no" sandbox="allow-forms allow-popups allow-same-origin allow-scripts allow-top-navigation allow-modals allow-popups-to-escape-sandbox" style="width: 100%; height: 100%;"></iframe></div></div>
|
||||
60
index.js
60
index.js
@@ -2,8 +2,10 @@ const express = require("express")
|
||||
, expressLogging = require("express-logging")
|
||||
, logger = require("logops")
|
||||
, hostname = require('os').hostname()
|
||||
, browser = require('./HeadlessBrowser')
|
||||
, port = 32023;
|
||||
, browser = require('./src/service/HeadlessService')
|
||||
, port = 32101
|
||||
, { NoSuchSessionError } = require("selenium-webdriver/lib/error")
|
||||
// , { puppeteerErrors } = require("puppeteer-core/lib/cjs/puppeteer/common/Errors");
|
||||
|
||||
|
||||
logger.getContext = function getContext() {
|
||||
@@ -21,6 +23,58 @@ app.listen(port, () => {
|
||||
logger.info({ app: 'Headless browser' }, 'Startup');
|
||||
});
|
||||
|
||||
|
||||
// process.stdin.resume();//so the program will not close instantly
|
||||
|
||||
exitHandler = (options, exitCode) => {
|
||||
if (options.cleanup) logger.info('cleanup')
|
||||
if (exitCode || exitCode === 0) logger.info('exit code :', exitCode)
|
||||
browser.close().then(() => process.exit());
|
||||
}
|
||||
|
||||
/**
|
||||
* Process shutdown events handling
|
||||
*/
|
||||
//do something when app is closing
|
||||
process.on('exit', exitHandler.bind(null, { cleanup: true }));
|
||||
|
||||
//catches ctrl+c event
|
||||
process.on('SIGINT', exitHandler.bind(null, { exit: true }));
|
||||
|
||||
process.on('SIGTERM', exitHandler.bind(null, { cleanup: true }));
|
||||
|
||||
// setTimeout(() => {
|
||||
// process.on('SIGKILL', exitHandler.bind(null, {exit:true}));
|
||||
// }, 2000)
|
||||
|
||||
|
||||
// catches "kill pid" (for example: nodemon restart)
|
||||
// process.on('SIGUSR1', exitHandler.bind(null, {exit:true}));
|
||||
// process.on('SIGUSR2', exitHandler.bind(null, {exit:true}));
|
||||
|
||||
//catches uncaught exceptions
|
||||
// process.on('uncaughtException', exitHandler.bind(null, {exit:true}));
|
||||
|
||||
/**
|
||||
* request handler
|
||||
*/
|
||||
app.post('/', async (req, res, next) => {
|
||||
browser.real(req.body.url).then(html => res.send(html))
|
||||
const requestPage = () => browser.real(req.body.url).then(html => res.send(html)).catch(e => {
|
||||
if(e instanceof NoSuchSessionError) {
|
||||
browser.reset()
|
||||
logger.error({ reason: e.message, url: req.url }, 'Browser session expired')
|
||||
}else {
|
||||
logger.error({ reason: e.message, url: req.url }, 'Browser error occured')
|
||||
}
|
||||
res.status(500).send({ code: 500, message: e.message })
|
||||
// browser.close().then(() => e instanceof puppeteerErrors.TimeoutError ? setTimeout(() => requestPage(), 3000) : res.status(500).send(e.message))
|
||||
})
|
||||
requestPage()
|
||||
})
|
||||
|
||||
app.get('/restart', (req, res, next) => {
|
||||
// browser.close().then(() => {
|
||||
// res.json({message: 'Browser closed and lanuch with new request'})
|
||||
// })
|
||||
process.exit()
|
||||
})
|
||||
|
||||
856
package-lock.json
generated
856
package-lock.json
generated
@@ -1,856 +0,0 @@
|
||||
{
|
||||
"name": "cx-amazon-crawler",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"@types/node": {
|
||||
"version": "14.6.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.6.2.tgz",
|
||||
"integrity": "sha512-onlIwbaeqvZyniGPfdw/TEhKIh79pz66L1q06WUQqJLnAb6wbjvOtepLYTGHTqzdXgBYIE3ZdmqHDGsRsbBz7A==",
|
||||
"optional": true
|
||||
},
|
||||
"@types/yauzl": {
|
||||
"version": "2.9.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz",
|
||||
"integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"accepts": {
|
||||
"version": "1.3.7",
|
||||
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz",
|
||||
"integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==",
|
||||
"requires": {
|
||||
"mime-types": "~2.1.24",
|
||||
"negotiator": "0.6.2"
|
||||
}
|
||||
},
|
||||
"agent-base": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz",
|
||||
"integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g=="
|
||||
},
|
||||
"array-flatten": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
|
||||
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
|
||||
},
|
||||
"balanced-match": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
|
||||
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
|
||||
},
|
||||
"base64-js": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
|
||||
"integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
|
||||
},
|
||||
"bl": {
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
|
||||
"integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
|
||||
"requires": {
|
||||
"buffer": "^5.5.0",
|
||||
"inherits": "^2.0.4",
|
||||
"readable-stream": "^3.4.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"inherits": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"body-parser": {
|
||||
"version": "1.19.0",
|
||||
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
|
||||
"integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
|
||||
"requires": {
|
||||
"bytes": "3.1.0",
|
||||
"content-type": "~1.0.4",
|
||||
"debug": "2.6.9",
|
||||
"depd": "~1.1.2",
|
||||
"http-errors": "1.7.2",
|
||||
"iconv-lite": "0.4.24",
|
||||
"on-finished": "~2.3.0",
|
||||
"qs": "6.7.0",
|
||||
"raw-body": "2.4.0",
|
||||
"type-is": "~1.6.17"
|
||||
}
|
||||
},
|
||||
"brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
|
||||
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
}
|
||||
},
|
||||
"buffer": {
|
||||
"version": "5.6.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
|
||||
"integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
|
||||
"requires": {
|
||||
"base64-js": "^1.0.2",
|
||||
"ieee754": "^1.1.4"
|
||||
}
|
||||
},
|
||||
"buffer-crc32": {
|
||||
"version": "0.2.13",
|
||||
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
|
||||
"integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
|
||||
},
|
||||
"bytes": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
|
||||
"integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
|
||||
},
|
||||
"chownr": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
|
||||
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
|
||||
},
|
||||
"colors": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
|
||||
"integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA=="
|
||||
},
|
||||
"concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
|
||||
},
|
||||
"content-disposition": {
|
||||
"version": "0.5.3",
|
||||
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
|
||||
"integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==",
|
||||
"requires": {
|
||||
"safe-buffer": "5.1.2"
|
||||
}
|
||||
},
|
||||
"content-type": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
|
||||
"integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
|
||||
},
|
||||
"cookie": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz",
|
||||
"integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg=="
|
||||
},
|
||||
"cookie-signature": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
|
||||
"integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
|
||||
},
|
||||
"debug": {
|
||||
"version": "2.6.9",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
|
||||
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
|
||||
"requires": {
|
||||
"ms": "2.0.0"
|
||||
}
|
||||
},
|
||||
"depd": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
|
||||
"integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak="
|
||||
},
|
||||
"destroy": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
|
||||
"integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
|
||||
},
|
||||
"devtools-protocol": {
|
||||
"version": "0.0.781568",
|
||||
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.781568.tgz",
|
||||
"integrity": "sha512-9Uqnzy6m6zEStluH9iyJ3iHyaQziFnMnLeC8vK0eN6smiJmIx7+yB64d67C2lH/LZra+5cGscJAJsNXO+MdPMg=="
|
||||
},
|
||||
"ee-first": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
|
||||
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
|
||||
},
|
||||
"encodeurl": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
|
||||
"integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
|
||||
},
|
||||
"end-of-stream": {
|
||||
"version": "1.4.4",
|
||||
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
|
||||
"integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
|
||||
"requires": {
|
||||
"once": "^1.4.0"
|
||||
}
|
||||
},
|
||||
"escape-html": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
|
||||
"integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
|
||||
},
|
||||
"etag": {
|
||||
"version": "1.8.1",
|
||||
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
|
||||
"integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
|
||||
},
|
||||
"express": {
|
||||
"version": "4.17.1",
|
||||
"resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
|
||||
"integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
|
||||
"requires": {
|
||||
"accepts": "~1.3.7",
|
||||
"array-flatten": "1.1.1",
|
||||
"body-parser": "1.19.0",
|
||||
"content-disposition": "0.5.3",
|
||||
"content-type": "~1.0.4",
|
||||
"cookie": "0.4.0",
|
||||
"cookie-signature": "1.0.6",
|
||||
"debug": "2.6.9",
|
||||
"depd": "~1.1.2",
|
||||
"encodeurl": "~1.0.2",
|
||||
"escape-html": "~1.0.3",
|
||||
"etag": "~1.8.1",
|
||||
"finalhandler": "~1.1.2",
|
||||
"fresh": "0.5.2",
|
||||
"merge-descriptors": "1.0.1",
|
||||
"methods": "~1.1.2",
|
||||
"on-finished": "~2.3.0",
|
||||
"parseurl": "~1.3.3",
|
||||
"path-to-regexp": "0.1.7",
|
||||
"proxy-addr": "~2.0.5",
|
||||
"qs": "6.7.0",
|
||||
"range-parser": "~1.2.1",
|
||||
"safe-buffer": "5.1.2",
|
||||
"send": "0.17.1",
|
||||
"serve-static": "1.14.1",
|
||||
"setprototypeof": "1.1.1",
|
||||
"statuses": "~1.5.0",
|
||||
"type-is": "~1.6.18",
|
||||
"utils-merge": "1.0.1",
|
||||
"vary": "~1.1.2"
|
||||
}
|
||||
},
|
||||
"express-logging": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/express-logging/-/express-logging-1.1.1.tgz",
|
||||
"integrity": "sha1-YoOWGMurW7NhDxocFIU1L+nSbCo=",
|
||||
"requires": {
|
||||
"on-headers": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"extract-zip": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
|
||||
"integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==",
|
||||
"requires": {
|
||||
"@types/yauzl": "^2.9.1",
|
||||
"debug": "^4.1.1",
|
||||
"get-stream": "^5.1.0",
|
||||
"yauzl": "^2.10.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
|
||||
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
|
||||
"requires": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"fd-slicer": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
|
||||
"integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=",
|
||||
"requires": {
|
||||
"pend": "~1.2.0"
|
||||
}
|
||||
},
|
||||
"finalhandler": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz",
|
||||
"integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==",
|
||||
"requires": {
|
||||
"debug": "2.6.9",
|
||||
"encodeurl": "~1.0.2",
|
||||
"escape-html": "~1.0.3",
|
||||
"on-finished": "~2.3.0",
|
||||
"parseurl": "~1.3.3",
|
||||
"statuses": "~1.5.0",
|
||||
"unpipe": "~1.0.0"
|
||||
}
|
||||
},
|
||||
"find-up": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
|
||||
"integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
|
||||
"requires": {
|
||||
"locate-path": "^5.0.0",
|
||||
"path-exists": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"forwarded": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
|
||||
"integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
|
||||
},
|
||||
"fresh": {
|
||||
"version": "0.5.2",
|
||||
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
|
||||
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
|
||||
},
|
||||
"fs-constants": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
|
||||
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
|
||||
},
|
||||
"fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
|
||||
},
|
||||
"get-stream": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
|
||||
"integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
|
||||
"requires": {
|
||||
"pump": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"glob": {
|
||||
"version": "7.1.6",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
|
||||
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
|
||||
"requires": {
|
||||
"fs.realpath": "^1.0.0",
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "^3.0.4",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"http-errors": {
|
||||
"version": "1.7.2",
|
||||
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
|
||||
"integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
|
||||
"requires": {
|
||||
"depd": "~1.1.2",
|
||||
"inherits": "2.0.3",
|
||||
"setprototypeof": "1.1.1",
|
||||
"statuses": ">= 1.5.0 < 2",
|
||||
"toidentifier": "1.0.0"
|
||||
}
|
||||
},
|
||||
"http-status-codes": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/http-status-codes/-/http-status-codes-1.4.0.tgz",
|
||||
"integrity": "sha512-JrT3ua+WgH8zBD3HEJYbeEgnuQaAnUeRRko/YojPAJjGmIfGD3KPU/asLdsLwKjfxOmQe5nXMQ0pt/7MyapVbQ=="
|
||||
},
|
||||
"https-proxy-agent": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz",
|
||||
"integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==",
|
||||
"requires": {
|
||||
"agent-base": "5",
|
||||
"debug": "4"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
|
||||
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
|
||||
"requires": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"iconv-lite": {
|
||||
"version": "0.4.24",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
|
||||
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
|
||||
"requires": {
|
||||
"safer-buffer": ">= 2.1.2 < 3"
|
||||
}
|
||||
},
|
||||
"ieee754": {
|
||||
"version": "1.1.13",
|
||||
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
|
||||
"integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
|
||||
},
|
||||
"inflight": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
|
||||
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
|
||||
"requires": {
|
||||
"once": "^1.3.0",
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"inherits": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
|
||||
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
|
||||
},
|
||||
"ipaddr.js": {
|
||||
"version": "1.9.1",
|
||||
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
|
||||
"integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
|
||||
},
|
||||
"locate-path": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
|
||||
"integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
|
||||
"requires": {
|
||||
"p-locate": "^4.1.0"
|
||||
}
|
||||
},
|
||||
"lodash": {
|
||||
"version": "4.17.20",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
|
||||
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
|
||||
},
|
||||
"logops": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/logops/-/logops-2.1.1.tgz",
|
||||
"integrity": "sha512-4VKuufobS4yzCsU3iyKJSudYhZfzg1aHO6j2aTdYf1K1kWz6+Mhfi91BqD+RXtokLpSfocc9vOkzbsun5U4OiQ==",
|
||||
"requires": {
|
||||
"colors": "^1.1.2",
|
||||
"lodash": "^4.17.15",
|
||||
"safe-json-stringify": "^1.2.0",
|
||||
"serr": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"media-typer": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
|
||||
"integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
|
||||
},
|
||||
"merge-descriptors": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
|
||||
"integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
|
||||
},
|
||||
"methods": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
|
||||
"integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
|
||||
},
|
||||
"mime": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
|
||||
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
|
||||
},
|
||||
"mime-db": {
|
||||
"version": "1.44.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz",
|
||||
"integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg=="
|
||||
},
|
||||
"mime-types": {
|
||||
"version": "2.1.27",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz",
|
||||
"integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==",
|
||||
"requires": {
|
||||
"mime-db": "1.44.0"
|
||||
}
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
|
||||
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
},
|
||||
"mkdirp-classic": {
|
||||
"version": "0.5.3",
|
||||
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
|
||||
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
|
||||
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
|
||||
},
|
||||
"negotiator": {
|
||||
"version": "0.6.2",
|
||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
|
||||
"integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
|
||||
},
|
||||
"on-finished": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
|
||||
"integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
|
||||
"requires": {
|
||||
"ee-first": "1.1.1"
|
||||
}
|
||||
},
|
||||
"on-headers": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz",
|
||||
"integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA=="
|
||||
},
|
||||
"once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
|
||||
"requires": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"p-limit": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
|
||||
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
|
||||
"requires": {
|
||||
"p-try": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"p-locate": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
|
||||
"integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
|
||||
"requires": {
|
||||
"p-limit": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"p-try": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
|
||||
"integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="
|
||||
},
|
||||
"parseurl": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
|
||||
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
|
||||
},
|
||||
"path-exists": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
|
||||
"integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="
|
||||
},
|
||||
"path-is-absolute": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
|
||||
},
|
||||
"path-to-regexp": {
|
||||
"version": "0.1.7",
|
||||
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
|
||||
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
|
||||
},
|
||||
"pend": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
|
||||
"integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
|
||||
},
|
||||
"pkg-dir": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
|
||||
"integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==",
|
||||
"requires": {
|
||||
"find-up": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"progress": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
|
||||
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
|
||||
},
|
||||
"proxy-addr": {
|
||||
"version": "2.0.6",
|
||||
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
|
||||
"integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==",
|
||||
"requires": {
|
||||
"forwarded": "~0.1.2",
|
||||
"ipaddr.js": "1.9.1"
|
||||
}
|
||||
},
|
||||
"proxy-from-env": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
|
||||
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
|
||||
},
|
||||
"pump": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
|
||||
"integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
|
||||
"requires": {
|
||||
"end-of-stream": "^1.1.0",
|
||||
"once": "^1.3.1"
|
||||
}
|
||||
},
|
||||
"puppeteer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.2.1.tgz",
|
||||
"integrity": "sha512-PZoZG7u+T6N1GFWBQmGVG162Ak5MAy8nYSVpeeQrwJK2oYUlDWpHEJPcd/zopyuEMTv7DiztS1blgny1txR2qw==",
|
||||
"requires": {
|
||||
"debug": "^4.1.0",
|
||||
"devtools-protocol": "0.0.781568",
|
||||
"extract-zip": "^2.0.0",
|
||||
"https-proxy-agent": "^4.0.0",
|
||||
"mime": "^2.0.3",
|
||||
"pkg-dir": "^4.2.0",
|
||||
"progress": "^2.0.1",
|
||||
"proxy-from-env": "^1.0.0",
|
||||
"rimraf": "^3.0.2",
|
||||
"tar-fs": "^2.0.0",
|
||||
"unbzip2-stream": "^1.3.3",
|
||||
"ws": "^7.2.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
|
||||
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
|
||||
"requires": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"mime": {
|
||||
"version": "2.4.6",
|
||||
"resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz",
|
||||
"integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA=="
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"qs": {
|
||||
"version": "6.7.0",
|
||||
"resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
|
||||
"integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ=="
|
||||
},
|
||||
"range-parser": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
|
||||
"integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
|
||||
},
|
||||
"raw-body": {
|
||||
"version": "2.4.0",
|
||||
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
|
||||
"integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
|
||||
"requires": {
|
||||
"bytes": "3.1.0",
|
||||
"http-errors": "1.7.2",
|
||||
"iconv-lite": "0.4.24",
|
||||
"unpipe": "1.0.0"
|
||||
}
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "3.6.0",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
|
||||
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
|
||||
"requires": {
|
||||
"inherits": "^2.0.3",
|
||||
"string_decoder": "^1.1.1",
|
||||
"util-deprecate": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
||||
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
|
||||
"requires": {
|
||||
"glob": "^7.1.3"
|
||||
}
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"safe-json-stringify": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz",
|
||||
"integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg=="
|
||||
},
|
||||
"safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
|
||||
},
|
||||
"send": {
|
||||
"version": "0.17.1",
|
||||
"resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
|
||||
"integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
|
||||
"requires": {
|
||||
"debug": "2.6.9",
|
||||
"depd": "~1.1.2",
|
||||
"destroy": "~1.0.4",
|
||||
"encodeurl": "~1.0.2",
|
||||
"escape-html": "~1.0.3",
|
||||
"etag": "~1.8.1",
|
||||
"fresh": "0.5.2",
|
||||
"http-errors": "~1.7.2",
|
||||
"mime": "1.6.0",
|
||||
"ms": "2.1.1",
|
||||
"on-finished": "~2.3.0",
|
||||
"range-parser": "~1.2.1",
|
||||
"statuses": "~1.5.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"ms": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
|
||||
"integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"serr": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/serr/-/serr-1.0.1.tgz",
|
||||
"integrity": "sha1-dKW55/rdW1X4qF5+crwApBm25II=",
|
||||
"requires": {
|
||||
"lodash": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"serve-static": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
|
||||
"integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
|
||||
"requires": {
|
||||
"encodeurl": "~1.0.2",
|
||||
"escape-html": "~1.0.3",
|
||||
"parseurl": "~1.3.3",
|
||||
"send": "0.17.1"
|
||||
}
|
||||
},
|
||||
"setprototypeof": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
|
||||
"integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw=="
|
||||
},
|
||||
"statuses": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
|
||||
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
|
||||
},
|
||||
"string_decoder": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
|
||||
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
|
||||
"requires": {
|
||||
"safe-buffer": "~5.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"tar-fs": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz",
|
||||
"integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==",
|
||||
"requires": {
|
||||
"chownr": "^1.1.1",
|
||||
"mkdirp-classic": "^0.5.2",
|
||||
"pump": "^3.0.0",
|
||||
"tar-stream": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"tar-stream": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.3.tgz",
|
||||
"integrity": "sha512-Z9yri56Dih8IaK8gncVPx4Wqt86NDmQTSh49XLZgjWpGZL9GK9HKParS2scqHCC4w6X9Gh2jwaU45V47XTKwVA==",
|
||||
"requires": {
|
||||
"bl": "^4.0.1",
|
||||
"end-of-stream": "^1.4.1",
|
||||
"fs-constants": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^3.1.1"
|
||||
}
|
||||
},
|
||||
"through": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
|
||||
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
|
||||
},
|
||||
"toidentifier": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
|
||||
"integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
|
||||
},
|
||||
"type-is": {
|
||||
"version": "1.6.18",
|
||||
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
|
||||
"integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
|
||||
"requires": {
|
||||
"media-typer": "0.3.0",
|
||||
"mime-types": "~2.1.24"
|
||||
}
|
||||
},
|
||||
"unbzip2-stream": {
|
||||
"version": "1.4.3",
|
||||
"resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
|
||||
"integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==",
|
||||
"requires": {
|
||||
"buffer": "^5.2.1",
|
||||
"through": "^2.3.8"
|
||||
}
|
||||
},
|
||||
"unpipe": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
|
||||
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
|
||||
},
|
||||
"util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
|
||||
},
|
||||
"utils-merge": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
|
||||
"integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
|
||||
},
|
||||
"vary": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
|
||||
"integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
|
||||
},
|
||||
"wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
|
||||
},
|
||||
"ws": {
|
||||
"version": "7.3.1",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz",
|
||||
"integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA=="
|
||||
},
|
||||
"yauzl": {
|
||||
"version": "2.10.0",
|
||||
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
|
||||
"integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=",
|
||||
"requires": {
|
||||
"buffer-crc32": "~0.2.3",
|
||||
"fd-slicer": "~1.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "cx-amazon-crawler",
|
||||
"name": "spd-app-headless",
|
||||
"version": "1.0.0",
|
||||
"description": "C-Xtra headless browser for crawling service",
|
||||
"main": "index.js",
|
||||
@@ -10,10 +10,13 @@
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"command-line-args": "^5.1.1",
|
||||
"express": "^4.17.1",
|
||||
"express-logging": "^1.1.1",
|
||||
"http-status-codes": "^1.4.0",
|
||||
"logops": "^2.1.1",
|
||||
"puppeteer": "^5.2.1"
|
||||
"puppeteer-cluster": "^0.22.0",
|
||||
"puppeteer-core": "^1.9.0",
|
||||
"selenium-webdriver": "^4.0.0-alpha.7"
|
||||
}
|
||||
}
|
||||
|
||||
72
src/component/CaptchaComponent.js
Normal file
72
src/component/CaptchaComponent.js
Normal file
@@ -0,0 +1,72 @@
|
||||
const logger = require('logops')
|
||||
, fetch = require('node-fetch')
|
||||
|
||||
|
||||
|
||||
class CaptchaComponent {
|
||||
constructor() {
|
||||
this.api = {
|
||||
in: 'https://' + service + '/in.php',
|
||||
res: 'https://' + service + '/res.php',
|
||||
key: process.env.APIKEY || 'YOUR_API_KEY',
|
||||
pollingInterval: 5000
|
||||
}
|
||||
this.params = {
|
||||
key: this.api.key,
|
||||
method: 'userrecaptcha',
|
||||
googlekey: recapParams.sitekey,
|
||||
pageurl: url,
|
||||
json: 1
|
||||
}
|
||||
}
|
||||
|
||||
submitCaptcha(captchaParams) {
|
||||
captchaParams.soft_id = '2496'
|
||||
return fetch(this.api.in, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(captchaParams)
|
||||
}).then(res => res.json()).then(res => {
|
||||
if (res.status === 1) {
|
||||
return res.request
|
||||
} else {
|
||||
logger.error(res.request, 'Captcha submit get response with error')
|
||||
}
|
||||
}).catch(err => logger.error(err))
|
||||
}
|
||||
|
||||
getAnswer(id) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const polling = setInterval(() => {
|
||||
fetch(`${this.api.res}?key=${this.api.key}&json=1&action=get&id=${id}`, {
|
||||
method: 'GET',
|
||||
headers: { "Content-Type": "application/json" }
|
||||
}).then(res => res.json()).then(res => {
|
||||
if (res.status === 1) {
|
||||
clearInterval(polling)
|
||||
return res.request
|
||||
} else if (res.request !== 'CAPCHA_NOT_READY') {
|
||||
clearInterval(polling)
|
||||
logger.error(res.request, 'Captcha answer get response with error')
|
||||
}
|
||||
}).catch(err => {
|
||||
clearInterval(polling)
|
||||
logger.error(err, 'Captcha answer get response with error')
|
||||
})
|
||||
}, this.api.pollingInterval)
|
||||
})
|
||||
}
|
||||
|
||||
async solveCaptcha(params) {
|
||||
try {
|
||||
let id = await this.submitCaptcha(params)
|
||||
let answer = await this.getAnswer(id)
|
||||
return answer
|
||||
} catch (e) {
|
||||
return e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
module.exports = new CaptchaComponent
|
||||
|
||||
168
src/component/PuppeteerBrowser.js
Normal file
168
src/component/PuppeteerBrowser.js
Normal file
@@ -0,0 +1,168 @@
|
||||
const logger = require("logops")
|
||||
, puppeteer = require('puppeteer-core')
|
||||
, config = require('../../config/Config')
|
||||
, { Builder, By, Key, until } = require('selenium-webdriver')
|
||||
// , webdriver = require('selenium-webdriver')
|
||||
, chrome = require('selenium-webdriver/chrome')
|
||||
, { Options } = require('selenium-webdriver/chrome')
|
||||
, { Cluster } = require('puppeteer-cluster');
|
||||
|
||||
|
||||
|
||||
|
||||
class PuppeteerBrowser {
|
||||
|
||||
constructor() {
|
||||
this.website = 'https://www.real.de/'
|
||||
this.isCluster = true;
|
||||
this.browser = null;
|
||||
this.page = null;
|
||||
this.cluster = null;
|
||||
this.closing = false
|
||||
this.timeout = 45000
|
||||
}
|
||||
|
||||
getClsuterInfo() {
|
||||
return this.cluster ? {
|
||||
workersCnt: this.cluster.workers.length,
|
||||
workersAvailCnt: this.cluster.workersAvail.length,
|
||||
workersBusyCnt: this.cluster.workersBusy.length,
|
||||
workersStarting: this.cluster.workersStarting,
|
||||
allTargetCount: this.cluster.allTargetCount,
|
||||
errorCount: this.cluster.errorCount,
|
||||
idleResolversCnt: this.cluster.idleResolvers.length,
|
||||
waitForOneResolversCnt: this.cluster.waitForOneResolvers.length,
|
||||
isClosed: this.cluster.isClosed,
|
||||
startTime: this.cluster.startTime,
|
||||
nextWorkerId: this.cluster.nextWorkerId,
|
||||
nextWorkCall: this.cluster.nextWorkCall,
|
||||
lastLaunchedWorkerTime: this.cluster.lastLaunchedWorkerTime
|
||||
} : null
|
||||
}
|
||||
|
||||
async webdriver(uri) {
|
||||
let html = ''
|
||||
let opts = new Options
|
||||
opts.addArguments(this.getChromeArgs())
|
||||
// opts.setChromeBinaryPath('/usr/bin/chromedriver')
|
||||
let driver = await new Builder().forBrowser('chrome').setChromeOptions(opts).build();
|
||||
// let driver = new webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
|
||||
try {
|
||||
logger.info('Request URL %s', this.website + uri)
|
||||
await driver.get(this.website + uri);
|
||||
html = await driver.findElement(By.css('body')).getAttribute('innerHTML')
|
||||
//.sendKeys('webdriver', Key.RETURN);
|
||||
// await driver.wait(until.titleIs('webdriver - www.real.de'), 1000);
|
||||
return html
|
||||
} finally {
|
||||
await driver.quit();
|
||||
}
|
||||
}
|
||||
|
||||
async close() {
|
||||
this.closing = true
|
||||
if (this.isCluster) {
|
||||
if (this.cluster && this.cluster.isClosed === false) {
|
||||
logger.debug({ cluster: this.getClsuterInfo() }, 'Browser closing ...')
|
||||
await this.cluster.idle();
|
||||
await this.cluster.close();
|
||||
this.cluster = null
|
||||
this.closing = false
|
||||
logger.info('Browser cluster closed')
|
||||
} else {
|
||||
logger.info('No browser cluster opened')
|
||||
}
|
||||
} else {
|
||||
if (this.browser != null) {
|
||||
await this.browser.close()
|
||||
}
|
||||
this.browser = null;
|
||||
this.page = null;
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
getChromeArgs() {
|
||||
let args = ['--no-sandbox', '--headless', '--disable-dev-shm-usage']
|
||||
// args.push('--enable-logging=stderr --v=1 > ' + logPath + ' 2>&1')
|
||||
if (config.proxy) {
|
||||
args.push('--proxy-server=' + config.proxy)
|
||||
}
|
||||
if (config.chrome) {
|
||||
config.chrome.split(' ').map(a => args.push('--' + a))
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
loggingChrome() {
|
||||
let logPath = __dirname + '/storage/chrome-logs.txt'
|
||||
logger.info('Chrome logging into %s', logPath)
|
||||
}
|
||||
|
||||
async clustering(url) {
|
||||
if (this.cluster == null) {
|
||||
let logPath = __dirname + '/storage/chrome-logs.txt'
|
||||
logger.info('Chrome logging into %s', logPath)
|
||||
this.cluster = await Cluster.launch({
|
||||
concurrency: Cluster.CONCURRENCY_CONTEXT,
|
||||
maxConcurrency: 10,
|
||||
timeout: this.timeout,
|
||||
workerCreationDelay: 200,
|
||||
puppeteer: puppeteer,
|
||||
puppeteerOptions: {
|
||||
timeout: this.timeout,
|
||||
headless: true,
|
||||
executablePath: '/usr/bin/chromium-browser',
|
||||
args: this.getChromeArgs()
|
||||
}
|
||||
});
|
||||
await this.cluster.task(async ({ page, data: url }) => {
|
||||
return await this.getHtml(page, url)
|
||||
});
|
||||
|
||||
logger.info('Cluster initialized %j', { cluster: this.getClsuterInfo() })
|
||||
}
|
||||
this.cluster.queue(url);
|
||||
logger.info('Add queue url(%s) cluster updated : %j', url, { cluster: this.getClsuterInfo() })
|
||||
return this.cluster.execute(url);
|
||||
|
||||
}
|
||||
|
||||
async getHtml(page, url) {
|
||||
await page.goto(url, { timeout: this.timeout })
|
||||
await page.waitFor(1500)
|
||||
let bodyHandle = await page.$('body');
|
||||
let html = await page.evaluate(body => body.innerHTML, bodyHandle);
|
||||
logger.debug('Page evaluate with html')
|
||||
await bodyHandle.dispose();
|
||||
return html
|
||||
}
|
||||
|
||||
async request(url) {
|
||||
if (this.page == null) {
|
||||
this.browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
executablePath: '/usr/bin/chromium-browser',
|
||||
args: ['--no-sandbox', '--headless', '--disable-gpu', '--disable-dev-shm-usage']
|
||||
})
|
||||
this.page = await this.browser.newPage()
|
||||
}
|
||||
return await this.getHtml(this.page, url)
|
||||
}
|
||||
|
||||
real(url) {
|
||||
return this.webdriver(url)
|
||||
if (this.closing) {
|
||||
return async () => {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
if (this.isCluster) {
|
||||
return this.clustering(this.website + url)
|
||||
}
|
||||
return this.request(this.website + url)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
module.exports = new PuppeteerBrowser
|
||||
75
src/component/SeleniumBrowser.js
Normal file
75
src/component/SeleniumBrowser.js
Normal file
@@ -0,0 +1,75 @@
|
||||
const logger = require("logops")
|
||||
, config = require('../../config')
|
||||
, { Builder, Capabilities, By, Key, until } = require('selenium-webdriver')
|
||||
, { Options } = require('selenium-webdriver/chrome');
|
||||
|
||||
class SeleniumBrowser {
|
||||
constructor() {
|
||||
this.website = 'https://www.real.de/'
|
||||
this.driver = null
|
||||
this.config = { useHub: true, browser: 'chrome' }
|
||||
this.capabilities = Capabilities.chrome()
|
||||
}
|
||||
|
||||
async setDriver() {
|
||||
if (this.config.useHub) {
|
||||
let opts = new Options;
|
||||
// opts.setAcceptInsecureCerts(true);
|
||||
// opts.setBrowserVersion('67');
|
||||
// opts.setPlatform('Windows XP');
|
||||
opts.addArguments(this.getChromeArgs())
|
||||
this.driver = await new Builder()
|
||||
.usingServer('http://spiduler-chrome:4444/wd/hub')
|
||||
.withCapabilities(this.capabilities)
|
||||
.forBrowser(this.config.browser)
|
||||
.setChromeOptions(opts)
|
||||
.build();
|
||||
} else {
|
||||
this.driver = await this.setLocalDriver()
|
||||
}
|
||||
}
|
||||
|
||||
setLocalDriver() {
|
||||
let opts = new Options
|
||||
opts.addArguments(this.getChromeArgs())
|
||||
// opts.setChromeBinaryPath('/usr/bin/chromedriver')
|
||||
return new Builder().forBrowser(this.config.browser).setChromeOptions(opts).build();
|
||||
// let driver = new webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
|
||||
}
|
||||
|
||||
async request(uri) {
|
||||
logger.info('Request URL %s', this.website + uri)
|
||||
return this.driver.get(this.website + uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML'))
|
||||
// .then(html => {
|
||||
// // solve the captcha with API
|
||||
// const token = await solveCaptcha(params, api)
|
||||
// })
|
||||
// return await driver.findElement(By.css('body')).getAttribute('innerHTML')
|
||||
//.sendKeys('webdriver', Key.RETURN);
|
||||
// await driver.wait(until.titleIs('webdriver - www.real.de'), 1000);
|
||||
}
|
||||
|
||||
async close() {
|
||||
|
||||
}
|
||||
|
||||
getChromeArgs() {
|
||||
let args = ['--no-sandbox', '--headless', '--disable-dev-shm-usage']
|
||||
if (config.proxy) {
|
||||
args.push('--proxy-server=' + config.proxy)
|
||||
}
|
||||
if (config.chrome) {
|
||||
config.chrome.split(' ').map(a => args.push('--' + a))
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
loggingChrome() {
|
||||
let logPath = __dirname + '/storage/chrome-logs.txt'
|
||||
logger.info('Chrome logging into %s', logPath)
|
||||
}
|
||||
}
|
||||
|
||||
const browser = new SeleniumBrowser
|
||||
browser.setDriver()
|
||||
module.exports = browser
|
||||
24
src/service/HeadlessService.js
Normal file
24
src/service/HeadlessService.js
Normal file
@@ -0,0 +1,24 @@
|
||||
const logger = require('logops')
|
||||
, browser = require('../component/SeleniumBrowser')
|
||||
|
||||
|
||||
|
||||
class HeadlessService {
|
||||
constructor() {
|
||||
|
||||
}
|
||||
|
||||
real(url) {
|
||||
return browser.request(url)
|
||||
}
|
||||
|
||||
close() {
|
||||
return browser.close()
|
||||
}
|
||||
|
||||
reset() {
|
||||
browser.setDriver()
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new HeadlessService
|
||||
Reference in New Issue
Block a user