improve usability for multiple websites crawling

src/component/SeleniumBrowser.js

@@ -7,7 +7,6 @@ const logger = require("logops")
 
 class SeleniumBrowser {
     constructor() {
-        this.website = 'https://www.real.de/'
         this.driver = null
         this.config = { useHub: true, browser: 'chrome' }
         this.capabilities = Capabilities.chrome()
@@ -18,9 +17,6 @@ class SeleniumBrowser {
     async setDriver() {
         if (this.config.useHub) {
             let opts = new Options;
-            // opts.setAcceptInsecureCerts(true);
-            // opts.setBrowserVersion('67');
-            // opts.setPlatform('Windows XP');
             opts.addArguments(this.getChromeArgs())
             this.driver = await new Builder()
                 .usingServer('http://spiduler-chrome:4444/wd/hub')
@@ -36,29 +32,25 @@ class SeleniumBrowser {
     setLocalDriver() {
         let opts = new Options
         opts.addArguments(this.getChromeArgs())
-        // opts.setChromeBinaryPath('/usr/bin/chromedriver')
         return new Builder().forBrowser(this.config.browser).setChromeOptions(opts).build();
-        // let driver = new
-        // webdriver.Builder().forBrowser('chrome').setChromeOptions(this.getChromeArgs()).build();
     }
 
     async request(uri) {
         if (this.isRecaptchaReseponseEmpty) {
             return ''
         } else {
-            logger.info('Request URL %s', this.website + uri)
+            logger.info('URL %s', uri)
-            return this.driver.get(this.website + uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML'))
+            if (uri.indexOf('://') > 3) {
+                return this.driver.get(uri).then(d => this.driver.findElement(By.css('body')).getAttribute('innerHTML'))
                     .then(html => this.getHtml(html))
-            // return await
+            }else {
-            // driver.findElement(By.css('body')).getAttribute('innerHTML')
+                logger.info('URL %s is not valid', uri)
-            // .sendKeys('webdriver', Key.RETURN);
+                return ''
-            /// await driver.wait(until.titleIs('webdriver - www.real.de'), 1000);
+            }
         }
     }
 
     async getHtml(html) {
-        // return html
-        // check if it's forbbidden page
         if (html.indexOf('grecaptcha') > -1 || html.indexOf('www.google.com/recaptcha/api2/anchor') > -1) {
             this.isRecaptchaReseponseEmpty = true
             let iframeSrc = await this.driver.findElement(By.css('#grecaptcha iframe')).getAttribute('src')
@@ -83,7 +75,7 @@ class SeleniumBrowser {
                 do {
                     maxRetry = maxRetry - 1
                     res = await this.requestCaptchaResult(res, requestId)
-                    if(maxRetry == 0) break
+                    if (maxRetry == 0) break
                 } while (res.request == 'CAPCHA_NOT_READY')
                 this.isRecaptchaReseponseEmpty = false
                 return res