first commit

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

index.js

@@ -0,0 +1,12 @@
+const express = require("express")
+    , router = require('./router')    
+    , { logger, expressLogger } = require('./src/component/Logger')
+    , port = 33103
+
+const app = express();
+app.use(express.json());
+app.use(expressLogger);
+app.listen(port, () => {
+    logger.info('Feature Service Startup');
+});
+app.use('/', router);

package.json

@@ -0,0 +1,20 @@
+{
+  "name": "spd-app-andygrace",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@elastic/ecs-winston-format": "^1.1.0",
+    "amqplib": "^0.8.0",
+    "express": "^4.17.1",
+    "express-winston": "^4.1.0",
+    "http-status-codes": "^2.1.4",
+    "node-fetch": "^2.6.1",
+    "winston": "^3.2.1"
+  }
+}

router.js

@@ -0,0 +1,25 @@
+const express = require("express")
+    , router = express.Router()
+    , { StatusCodes, getReasonPhrase } = require('http-status-codes')
+    , { logger } = require('./src/component/Logger')
+    // , migrateController = require('./src/controller/MigrateController')
+
+// a middleware function with no mount path. This code is executed for every request to the router
+router.use(function (req, res, next) {
+    // logger.info("Request body", {err: req.body});
+    // logger.error('oops there is a problem', { foo: 'bar' })
+    // logger.warn('responsebbbbb ', req.body)
+    next();
+});
+
+router.get('/', function (req, res) {
+    res.json(req.headers)
+});
+
+router.post('/', function (req, res) {
+    crawlService.start(req).then(result => {
+        res.json(result)
+    })
+});
+
+module.exports = router

src/component/AmqpTransporter.js

@@ -0,0 +1,184 @@
+const { URL } = require('url');
+const TransportStream = require('winston-transport');
+const hostname = require('os').hostname;
+const amqplib = require('amqplib');
+const amqp = require('amqplib/callback_api');
+const config = {
+    name: 'rabbitmq-logger',
+    level: 'info',
+    logToConsole: false,
+    url: process.env.WINSTON_RABBITMQ_URL || 'amqp://ibpcorp:KjUuC754D5xTSpqf@localhost:49072/ibpcorp',
+    socketOpts: {},
+    exchange: 'logs',
+    exchangeOptions: {
+        durable: true,
+        autoDelete: false
+    },
+    routingKey: 'ibp_logs',
+    timestamp: function () {
+        return new Date().toISOString();
+    }
+}
+
+class AmqpTransport extends TransportStream {
+
+    constructor(options) {
+        super(options);
+        TransportStream.call(this, options);
+        this.config = { ...config, ...options }
+        this.validate()
+        this.initialize();        
+        if (!this.config.lazyInit && !this.config.logToConsole) { setImmediate(async () => await this.initializeRabbitmq()) }
+    }
+
+    /**
+     *
+     *Validates critical options passed as params in constructor
+     * @memberof RabbitmqTransport
+     */
+    validate() {
+        if (typeof this.config.url !== 'string') {
+            console.log('[AmqpTransport]: RabbitMQ url must be of type string');
+            throw new TypeError('[AmqpTransport]: RabbitMQ url must be of type string');
+        }
+
+
+        this.url = new URL(this.config.url);
+        this.config.host = (this.url).host;
+
+        if (!this.url.protocol.match(/amqp/)) {
+            console.log('[AmqpTransport]: Incorrect protocol, must be amqp');
+            throw new Error('[AmqpTransport]: Incorrect protocol, must be amqp');
+        }
+    }
+
+    /**
+     *
+     *Initializes properties and debug functions
+     * @returns
+     * @memberof RabbitmqTransport
+     */
+    initialize() {
+        this.name = this.config.name;
+        this.level = this.config.level;
+
+        this.debug = this.level === 'debug' && this.config.debug && typeof this.config.debug === 'function' ? this.config.debug : console.debug;
+
+        if (this.config.logToConsole) {
+            this.log = this.logToConsole;
+            return;
+        }
+    }
+
+
+    /**
+     *Initializes RabbitMQ connection
+     *
+     * @memberof RabbitmqTransport
+     */
+    async initializeRabbitmq() {
+        // try {
+        this.connection = await this.createConnection(this.config.url);
+        this.loggingChannel = await this.createChannel(this.connection);
+        // } catch (err) {
+        //     this.close().catch((e) => this.debug(e.message));
+        //     console.log('[RabbitmqTransport]: ' + err);
+        //     throw new Error('[RabbitmqTransport]: ' + err);
+        // }
+    }
+
+    /**
+     *Closes open connections if any
+     *
+     * @memberof RabbitmqTransport
+     */
+    async close() {
+        if (this.loggingChannel) { await this.loggingChannel.close(); }
+        delete this.loggingChannel;
+
+        if (this.connection) { await this.connection.close(); }
+        delete this.connection;
+    }
+
+    /**
+     *Creates a new RabbitMQ connection
+     *
+     * @param {*} url
+     * @param {*} socketOpts
+     * @returns {*} connection
+     * @memberof RabbitmqTransport
+     */
+    async createConnection(url, socketOpts) {
+        const connection = await amqplib.connect(url, socketOpts);
+        this.debug('[RabbitmqTransport]: Connection established');
+        connection.on('error', (err) => {
+            if (err.message !== 'Connection closing') {
+                throw err;
+            }
+        });
+        connection.on('close', () => {
+            this.debug('[RabbitmqTransport]: Connection Closed');
+        });
+        return connection;
+    }
+
+    /**
+     *Creates a new RabbitMQ channel 
+     *
+     * @param {*} connection
+     * @returns {*} channel
+     * @memberof RabbitmqTransport
+     */
+    async createChannel(connection) {
+        const channel = await connection.createConfirmChannel();
+        channel.assertExchange(this.config.exchange, 'topic', this.config.exchangeOptions);
+        this.debug('[RabbitmqTransport]: Channel Created');
+
+        channel.on('error', (err) => {
+            throw new Error('[RabbitmqTransport]: ' + err);
+        });
+        channel.on('close', () => {
+            this.debug('[RabbitmqTransport]: Channel Closed');
+        });
+        return channel;
+    }
+
+    /**
+     *Logs to console if mq logging is disabled
+     *
+     * @param {*} level
+     * @param {*} msg
+     * @param {*} meta
+     * @param {*} callback
+     * @memberof RabbitmqTransport
+     */
+    logToConsole(message, callback) {
+        console.log(message);
+        callback();
+    }
+
+    /**
+     *Main logging function that sends logs to RabbitMQ
+     *
+     * @param {*} level
+     * @param {*} msg
+     * @param {*} meta
+     * @param {*} callback
+     * @memberof RabbitmqTransport
+     */
+    log(message, callback) {
+
+
+        try {
+            this.loggingChannel.publish(this.config.exchange, this.config.routingKey, Buffer.from(JSON.stringify(message)),);
+            this.debug('[RabbitmqTransport]: Logged to ' + this.config.exchange + '/' + this.config.routingKey);
+            callback();
+        } catch (ex) {
+            // throw new Error('[RabbitmqTransport]: ' + ex.message);
+            console.log('[RabbitmqTransport]: ' + ex.message)
+        }
+    }
+
+}
+
+module.exports = AmqpTransport

src/component/Logger.js

@@ -0,0 +1,51 @@
+const express = require('express')
+    , winston = require('winston')
+    , { StatusCodes, getReasonPhrase } = require('http-status-codes')
+    , expressWinston = require('express-winston')
+    , ecsFormat = require('@elastic/ecs-winston-format')
+    , AmqpTransport = require('./AmqpTransporter')
+
+
+// const amqpTransporter = new AmqpTransport({})
+
+const logger = winston.createLogger({
+    level: 'info',
+    format: ecsFormat({ convertReqRes: true }),
+    defaultMeta: { service: { name: 'feature' } },
+    transports: [
+        //
+        // - Write all logs with level `error` and below to `error.log`
+        // - Write all logs with level `info` and below to `combined.log`
+        //
+        // new winston.transports.File({ filename: 'error.log', level: 'error' }),
+        // new winston.transports.File({ filename: 'combined.log' }),
+        new winston.transports.Console(),
+        // amqpTransporter
+    ]
+});
+
+// if (process.env.NODE_ENV !== 'production') {
+//     logger.add(new winston.transports.Console({
+//         format: winston.format.simple(),
+//     }));
+// }
+
+const expressLogger = expressWinston.logger({
+    transports: [
+        new winston.transports.Console(),
+        // amqpTransporter
+    ],
+    format: ecsFormat(),
+    baseMeta: { service: { name: 'feature' } },
+    meta: false, // optional: control whether you want to log the meta data about the request (default to true)
+    metaField: 'service',
+    msg: "HTTP {{req.method}} {{req.url}}", // optional: customize the default logging message. E.g. "{{res.statusCode}} {{req.method}} {{res.responseTime}}ms {{req.url}}"
+    expressFormat: false, // Use the default Express/morgan request formatting. Enabling this will override any msg if true. Will only output colors with colorize set to true
+    colorize: false, // Color the text and status code, using the Express/morgan color palette (text: gray, status: default green, 3XX cyan, 4XX yellow, 5XX red).
+    ignoreRoute: function (req, res) { return false; } // optional: allows to skip some log messages based on request and/or response
+})
+
+module.exports = {
+    logger: logger,
+    expressLogger: expressLogger
+}

src/component/RestClient.js

@@ -0,0 +1,53 @@
+const fetch = require("node-fetch")
+    , logger = require("./Logger").logger
+    , TokenExpiredException = require('../exception/TokenExpiredException')
+    , config = require('../config');
+
+
+class RestClient {
+
+    constructor() {
+        this.token = null
+        this.gateway = 'http://' + config.gateway + '/api/v1/'
+        this.clientId = '7tmK4rF8As8CtRw5'
+        this.clientSecret = 'cgzLcVFku5zEvXkCKNZsAbxXQENFqPDx2kfjnMGd3m9BczehmFt2pw9r9MdASSFt'
+    }
+
+    auth() {
+        let url = this.gateway + 'oauth/token?client_id=' + this.clientId + '&grant_type=client_credentials&client_secret=' + this.clientSecret
+        return fetch(url, { method: 'GET', headers: { 'content-type': 'application/json' } }).then(res => res.json()).then(res => {
+            logger.debug(res, 'Token from %s', this.gateway)
+            this.token = res.access_token
+            return res.access_token
+        })
+    }
+
+    headers(additionalheaders) {
+        let headers = {
+            "Authorization": "Bearer " + this.token,
+            "Content-Type": "application/json"
+        }
+        return additionalheaders ? Object.assign(headers, additionalheaders) : headers
+    }
+
+    response(res) {
+        if (res.code == 406) {
+            throw new TokenExpiredException("Token expired " + this.gateway)
+        }
+        return res;
+    }
+
+    async request(fetchable) {
+        if (!this.token) await this.auth()
+        return fetchable().catch(error => {
+            logger.error({ message: error.message }, 'Fetchable failed with error')
+            if (error.status == 406) {
+                logger.info('Requesting token %s', this.gateway)
+                return this.auth().then(() => fetchable())
+            }
+        })
+    }
+}
+
+
+module.exports = RestClient;

src/config/index.js

@@ -0,0 +1,70 @@
+const commandLineArgs = require('command-line-args')
+  , sites = require('./sites.json')
+  , logger = require("logops");
+
+class Config {
+
+  constructor() {
+    this.options = {}
+    // this.requires = ['env', 'gateway', 'quix24', 'mongo']
+    this.requires = ['env']
+    this.init()
+    this.validate()
+    this.finalize()
+  }
+
+  init() {
+    try {
+      this.options = Object.assign(this.options, commandLineArgs([
+        { name: 'env', alias: 'e', type: String, defaultValue: ['production'] },
+        { name: 'host', type: String },
+        { name: 'port', type: Number },
+        { name: 'database', alias: 'd', type: String },
+        { name: 'username', alias: 'u', type: String },
+        { name: 'password', alias: 'p', type: String },
+        { name: 'mongo', alias: 'm', type: String },
+        { name: 'redis', alias: 'r', type: String },
+        { name: 'level', alias: 'l', type: String },
+        { name: 'gateway', alias: 'g', type: String },
+        { name: 'quix24', alias: 'q', type: String }
+      ]));
+    } catch (e) {
+      logger.debug('Command line arguments interpret failed :', e.message)
+      logger.debug('expected arguments : ', JSON.stringify(this.requires))
+      process.exit(1)
+    }
+  }
+
+  finalize() {
+    if (this.options.env == 'development') {
+      this.options.level = this.options.level ? this.options.level : "DEBUG"
+      logger.formatters.dev.omit = ['pid', 'port', 'hostname', 'app'];
+      // logger.format = logger.formatters.dev;
+    } else {
+      this.options.level = this.options.level ? this.options.level : "WARN"
+    }
+    logger.formatters.json.omit = ['pid', 'port', 'hostname', 'app'];
+    logger.setLevel(this.options.level)
+    logger.debug(this.options, 'Environment setting options')
+    this.options.sites = sites
+  }
+
+  validate() {
+    for (let key in this.options) {
+      if (this.options[key]) {
+        delete this.requires[this.requires.indexOf(key)]
+      }
+    }
+    this.requires = this.requires.filter(function (el) {
+      return el != null;
+    })
+    if (this.requires.length) {
+      logger.debug('Process terminated invalid required arguments: ', JSON.stringify(this.requires))
+      process.exit(1)
+    }
+  }
+
+}
+
+let config = new Config;
+module.exports = config.options

src/config/sites.json

@@ -0,0 +1,113 @@
+{
+    "melon": {
+        "hostname": "https://www.melon.com",
+        "startUrl": "/genre/song_list.htm?gnrCode=GN2100",
+        "title": "멜론"
+    },
+    "hosanna": {
+        "hostname": "https://www.hosanna.net",
+        "startUrl": "/bbs/list.asp?bbsid=yard_sale",
+        "ajaxUrl": "/bbs/popup.asp?BBSID=yard_sale&No=[postId]",
+        "title": "호산나넷"
+    },
+    "cjob": {
+        "hostname": "https://www.cjob.co.kr",
+        "startUrl": "",
+        "title": "기독정보넷"
+    },
+    "godpeople": {
+        "hostname": "https://www.godpeople.com",
+        "title": "god피플"
+    },
+    "ccs33": {
+        "hostname": "http://www.ccs33.com/home/agree.php",
+        "title": ""
+    },
+    "koreacit": {
+        "hostname": "http://www.koreacit.net/",
+        "title": ""
+    },
+    "ch8949": {
+        "hostname": "http://www.ch8949.com/",
+        "title": ""
+    },
+    "daum": {
+        "hostname": "http://cafe.daum.net/alohochristian",
+        "title": ""
+    },
+    "seng1": {
+        "hostname": "http://www.seng1.net/",
+        "title": ""
+    },
+    "amen365": {
+        "hostname": "http://www.amen365.com/",
+        "title": ""
+    },
+    "kcdc": {
+        "hostname": "https://www.kcdc.net/index.php",
+        "title": ""
+    },
+    "andygrace": {
+        "hostname": "http://www.기독교구인구직.com",
+        "startUrl": "/bbs/board.php?bo_table=guin&page=[pageNo]",
+        "title": "기독구인넷",
+        "login": {
+            "anchor": "login",
+            "css": {
+                "username": "form > div > table > tbody > tr > td:nth-child(1) > table > tbody > tr:nth-child(1) > td:nth-child(2) > input",
+                "password": "#pw2 > input",
+                "submit": "form > div > table > tbody > tr > td:nth-child(2) > input[type=image]"
+            },
+            "key": {
+                "username": "alexkoo",
+                "password": "9zWqDanCSsfSx7C9"
+            },
+            "uri": "http://www.기독교구인구직.com/index.php"
+        }
+    },
+    "kidokmarket": {
+        "hostname": "https://cafe.naver.com/kidokmarket",
+        "title": ""
+    },
+    "npca": {
+        "hostname": "http://cafe.daum.net/npca",
+        "title": ""
+    },
+    "ezipsa": {
+        "hostname": "http://www.ezipsa.net/@sermon",
+        "title": "이집사"
+    },
+    "kidokjungbo": {
+        "hostname": "http://www.kidokjungbo.com/bbs/board.php?bo_table=rental",
+        "title": "기독정보닷컴"
+    },
+    "ryghlaoao": {
+        "hostname": "http://cafe.daum.net/ryghlaoao",
+        "title": "교회매매 교회임대"
+    },
+    "church1122": {
+        "hostname": "http://church1122.com",
+        "title": "하나공인중개사"
+    },
+    "sousu7708": {
+        "hostname": "http://cafe.daum.net/sousu7708",
+        "title": "교회후임자 임대 매매정보"
+    },
+    "solomonchurch": {
+        "hostname": "http://cafe.daum.net/solomonchurch",
+        "title": "교회부동산"
+    },
+    "kidokjob": {
+        "hostname": "http://cafe.daum.net/kidokjob",
+        "title": "기독목회종합"
+    },
+    "diakonia": {
+        "hostname": "http://www.diakonia.net",
+        "title": "디아코니아넷"
+    },
+    "yes24": {
+        "hostname": "http://www.yes24.com",
+        "startUrl": "/24/Category/Display/001001021003006?PageNumber=[pageNo]",
+        "title": "yes24"
+    }
+}

src/controller/BaseController.js

@@ -0,0 +1,30 @@
+const logger = require('logops')
+    , { StatusCodes, getReasonPhrase } = require('http-status-codes');
+
+class BaseController {
+    constructor() {
+
+    }
+
+    getMethodName(req) {        
+        let method = req.params.what.split('_').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join('')
+        logger.info({method: method}, 'Translated method')
+        return req.method.toLowerCase() + method
+    }
+
+    getRequestParams(req) {
+        return Object.assign(req.body, req.query, req.params)
+    }
+
+    errorHandler(err, callback) {
+        logger.error(err.message)
+        process.send({ cmd: 'outKey', key: keyService.get() });
+        callback({
+            code: StatusCodes.INTERNAL_SERVER_ERROR,
+            message: getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR),
+            data: { reason: err.message }
+        })
+    }
+}
+
+module.exports = BaseController

src/controller/BookController.js

@@ -0,0 +1,16 @@
+const logger = require("logops")
+    , bookService = require('../service/BookService');
+
+class BookController {
+    getMethodName(req) {
+        return req.method.toLowerCase()
+            + req.params.what.split('_').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join('')
+    }
+
+    book(req, callback) {        
+        let params = Object.assign(req.body, req.query, req.params)        
+        bookService.get(req.params.what, params).then(data => callback(data))
+    }
+}
+
+module.exports = new BookController;

src/controller/JobController.js

@@ -0,0 +1,24 @@
+const logger = require("logops")
+    , jobService = require('../service/JobService')
+    , BaseController = require("./BaseController");
+
+class JobController extends BaseController {
+
+    constructor() {
+        super()
+    }
+
+    job(req, callback) {                
+        this[this.getMethodName(req)](req.params.what.split('_')[0], this.getRequestParams(req), callback)
+    }
+
+    getAndygrace(site, params, callback) {
+        jobService.get(site, params).then(data => callback(data))        
+    }
+
+    getAndygraceDetail(site, params, callback) {
+        jobService.getDetail(site, params).then(data => callback(data))        
+    }
+}
+
+module.exports = new JobController;

src/controller/RealestateController.js

@@ -0,0 +1,16 @@
+const logger = require("logops")
+    , realestateService = require('../service/RealestateService');
+
+class RealestateController {
+    getMethodName(req) {
+        return req.method.toLowerCase()
+            + req.params.what.split('_').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join('')
+    }
+
+    realestate(req, callback) {        
+        let params = Object.assign(req.body, req.query, req.params)        
+        realestateService.get(req.params.what, params).then(data => callback(data))
+    }
+}
+
+module.exports = new RealestateController;

src/controller/index.js

@@ -0,0 +1,15 @@
+'use strict';
+
+const fs = require('fs')
+  , path = require('path')
+  , basename = path.basename(__filename)
+  , controllers = {};
+
+fs.readdirSync(__dirname).filter(file => {
+  return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js');
+}).forEach(file => {
+  const controller = require('./' + file)
+  controllers[file.slice(0, -3).toLowerCase().replace('controller', '')] = controller;
+});
+
+module.exports = controllers;

src/crawler/book/Yes24.js

@@ -0,0 +1,33 @@
+const cheerio = require('cheerio')
+
+
+class Yes24 {
+
+    cleanup(txt) {
+        return typeof txt == 'string' ? txt.replace(/[\-\t\n\r]+/g, '').replace(/[\s]{2}/g, '').trim() : txt
+    }
+
+    parse(html) {
+        let $ = cheerio.load(html)
+        let items = []
+        $('#category_layout .cCont_goodsSet').map((i, item) => {
+            item = $(item)
+            items.push({
+                title: this.cleanup(item.find('.goods_name a').eq(0).text().trim()),
+                subTitle: this.cleanup(item.find('.goods_name a').eq(0).next().text()),
+                author: item.find('.goods_auth a').text(),
+                publisher: item.find('.goods_pub').text(),
+                publishedAt: item.find('.goods_date').text(),
+                price: item.find('.goods_price em').eq(0).text(),
+                summary: this.cleanup(item.find('.goods_read').text()),
+                coverImg: item.find('.goods_imgSet img').attr('src')
+                // image: item.find('img').attr('src'),
+                // location: item.find('.sigungu').text().trim(),
+                // category: item.find('.rents').prop('title')
+            });
+        })
+        return items
+    }
+}
+
+module.exports = new Yes24

src/crawler/ccm/Melon.js

@@ -0,0 +1,21 @@
+const cheerio = require('cheerio')
+
+
+class Melon {
+    parse(html) {
+        let $ = cheerio.load(html)
+        let items = []
+        $('.service_list_song table tbody tr').map((i, item) => {
+            item = $(item)
+            items.push({
+                title: item.find('.wrap_song_info a').text().trim(),
+                image: item.find('img').attr('src'),
+                url: 'https://www.melon.com/album/detail.htm?albumId='+ item.find('.wrap a').attr('href').match(/\d+/)[0],
+                album: item.find('.wrap_song_info .rank02 a').text().trim()
+            });
+        })
+        return items
+    }
+}
+
+module.exports = new Melon

src/crawler/job/Andygrace.js

@@ -0,0 +1,59 @@
+const cheerio = require('cheerio')
+
+
+class Andygrace {
+
+    parse(html) {
+        let $ = cheerio.load(html)
+        let items = []
+        let table = $('body > table > tbody > tr:nth-child(1) > td:nth-child(2) > table > tbody > tr:nth-child(2) > td > table > tbody > tr:nth-child(2) > td > table > tbody > tr > td > form:nth-child(2) > table:nth-child(8)')        
+        table.find('tr').map((i, item) => {
+            item = $(item)
+            let url = item.find('td a').attr('href')
+            if(!url) return
+            items.push({
+                url: item.find('td a').attr('href').replace('..',''),                
+                title: item.find('td:eq(2)').text().trim()
+            });
+        })
+        return items
+    }
+
+    parseDetail(html) {
+        let $ = cheerio.load(html)
+        let items = []
+        let table = $('body > table > tbody > tr:nth-child(1) > td:nth-child(2) > table > tbody > tr:nth-child(2) > td > table > tbody > tr:nth-child(2) > td > table:nth-child(6) > tbody > tr > td > table:nth-child(4) > tbody > tr:nth-child(1) > td:nth-child(2) > table')        
+        table.find('tr').map((i, item) => {
+            item = $(item)            
+            let name = item.find('td').eq(0).text()
+            if(!name) return
+            items.push({
+                name: this.cleanup(name),                
+                info: this.cleanup(item.find('td').eq(1).text())
+            });
+        })
+
+        table = $('body > table > tbody > tr:nth-child(1) > td:nth-child(2) > table > tbody > tr:nth-child(2) > td > table > tbody > tr:nth-child(2) > td > table:nth-child(6) > tbody > tr > td > table:nth-child(7)')        
+        table.find('tr').map((i, item) => {
+            item = $(item)            
+            let name = item.find('td').eq(0).text()
+            if(!name) return
+            items.push({
+                name: this.cleanup(name),                
+                info: this.cleanup(item.find('td').eq(1).text())
+            });
+        })
+
+        items.push({
+            name: 'detail',
+            info: $('#ContentsLayer').html()
+        })
+        return items
+    }
+
+    cleanup(txt) {
+        return txt.trim().replace(/[\s][2]/, ' ')
+    }
+}
+
+module.exports = new Andygrace

src/crawler/realestate/Cjob.js

@@ -0,0 +1,22 @@
+const cheerio = require('cheerio')
+
+
+class Cjob {
+
+    parse(html) {
+        let $ = cheerio.load(html)
+        let items = []
+        $('.special_list li').map((i, item) => {
+            item = $(item)
+            items.push({
+                title: item.find('.txt_g').text().trim(),
+                image: item.find('img').attr('src'),
+                location: item.find('.sigungu').text().trim(),
+                category: item.find('.rents').prop('title')
+            });
+        })
+        return items
+    }
+}
+
+module.exports = new Cjob

src/crawler/realestate/Hosanna.js

@@ -0,0 +1,47 @@
+const config = require('../../config')
+    , cheerio = require('cheerio')
+    , url = require('url')
+    , fetch = require('node-fetch')
+    , querystring = require('querystring')
+
+
+class Hosanna {
+
+    constructor() {
+        this.site = config.sites.hosanna
+    }
+
+    parse(html) {
+        let $ = cheerio.load(html)
+        let posts = []
+        $('#board tbody tr').map((i, row) => {
+            row = $(row).find('td')
+            posts.push({
+                postId: querystring.parse(url.parse(row.eq(0).find('a').attr('href')).query).No,
+                title: row.eq(0).find('a').text().trim(),
+                writer: row.eq(1).text().trim(),
+                createdAt: row.eq(2).text().trim()
+            });
+        })
+        return Promise.all(posts.map(item => fetch(this.site.hostname + this.site.ajaxUrl.replace('[postId]', item.postId)).then(res => res.text()).then(html => {
+            let post = this.parseAjax(html)
+            return Object.assign(item, post);
+        })))
+    }
+
+    parseAjax(html) {
+        let $ = cheerio.load(html)
+        let info = $('.modal-header div').text().replace(/[\n\r\s]+/g, '').split('|');
+        let infoKey = { user: 'writer', phone: 'mobile', envelope: 'email', 'thumbs-up': 'likes' }
+
+        let post = {
+            title: $('.modal-title').text().trim()
+        }
+        $('.modal-header .glyphicon').map((k, el) => {
+            post[infoKey[$(el).removeClass('glyphicon').attr('class').replace('glyphicon-', '')]] = info[k]
+        })
+        return post
+    }
+}
+
+module.exports = new Hosanna

src/exception/TokenExpiredException.js

@@ -0,0 +1,16 @@
+class TokenExpiredException extends Error {  
+    constructor (message) {
+      super(message)
+  
+      // assign the error class name in your custom error (as a shortcut)
+      this.name = this.constructor.name
+  
+      // capturing the stack trace keeps the reference to your error class
+      Error.captureStackTrace(this, this.constructor);
+  
+      // you may also assign additional properties to your error
+      this.status = 406
+    }
+  }
+  
+  module.exports = TokenExpiredException  

src/service/BrowserService.js

@@ -0,0 +1,31 @@
+const RestClient = require("../component/RestClient")
+, loginService = require('./LoginService')
+
+class BrowserService extends RestClient {
+
+    constructor(){
+        super()
+        this.gateway = 'http://ibpcorp.c-xtra.com:32100/api/v1/'
+        this.loginParams = {
+            "anchor": "login",
+            "uri": "https://cplace.christiandaily.co.kr/rankup_module/rankup_member/login.html",
+            "sessionId": "",
+            "css": {
+                "username": "input[name=id]",
+                "password": "input[name=passwd]",
+                "submit": "table input[type=image]"
+            },
+            "key": {
+                "username": "alexkoo",
+                "password": "9zWqDanCSsfSx7C9"
+            }
+        }
+    }
+
+    login() {
+        return loginService.login(this)
+    }
+
+}
+
+module.exports = new BrowserService

src/service/CrawlService.js

@@ -0,0 +1,5 @@
+class LinkService {
+
+}
+
+module.exports = new LinkService

src/service/JobService.js

@@ -0,0 +1,66 @@
+const config = require('../config')
+    , fetch = require('node-fetch')
+    , iconv = require("iconv-lite")
+    , crawlerFactory = require("./CrawlerFactory")
+    , navigateService = require('./NavigateService')
+    , boardService = require('./BoardService');
+
+class JobService {
+
+    constructor() {
+        this.page = 0
+        this.sessionId = null
+    }
+
+    login(params) {        
+        return navigateService.login(params).then(body => {
+            this.sessionId = body.sessionId
+            return body
+        })
+    }
+
+    get(site, params) {
+        if(!this.sessionId) {
+            return this.login(config.sites[site].login)
+        }
+        if (params.hasOwnProperty('page')) {
+            this.page = params.page
+        } else {
+            ++this.page
+        }
+        return fetch(config.sites[site].hostname + config.sites[site].startUrl.replace('[pageNo]', this.page), { compress: false }).then(res => res.buffer()).then(buffer => {
+            return iconv.decode(buffer, "EUC-KR").toString()
+        }).then(html => crawlerFactory.crawlers[site].parse(html)).then(result => {
+            if (result.length) {
+                Promise.all(result.map(job => {
+                    return boardService.createPost(Object.assign(job, {
+                        category: "job",
+                        username: "cx",
+                        contentUrl: job.url
+                    }))
+                }))
+                return result
+            } else {
+                this.page = 0
+                return []
+            }
+        })
+    }
+
+    getDetail(site, params) {
+        return fetch(params.url, { compress: false }).then(res => res.buffer()).then(buffer => {
+            return iconv.decode(buffer, "EUC-KR").toString()
+        }).then(html => crawlerFactory.crawlers[site].parseDetail(html)).then(result => {
+            if (result.length) {
+                return result
+            } else {
+                this.page = 0
+                return []
+            }
+        })
+    }
+
+}
+
+
+module.exports = new JobService

src/service/LinkService.js

@@ -0,0 +1,5 @@
+class LinkService {
+
+}
+
+module.exports = new LinkService

src/service/LoginService.js

@@ -0,0 +1,44 @@
+const fetch = require('node-fetch')
+
+class LoginService {
+
+    constructor() {
+        this.sessionId = ''
+    }
+
+    login(browser) {
+        let params = {
+            "anchor": "login",
+            "uri": "https://cplace.christiandaily.co.kr/rankup_module/rankup_member/login.html",
+            "sessionId": "",
+            "css": {
+                "username": "input[name=id]",
+                "password": "input[name=passwd]",
+                "submit": "table input[type=image]"
+            },
+            "key": {
+                "username": "alexkoo",
+                "password": "9zWqDanCSsfSx7C9"
+            },
+            "name": {
+                "username": "id",
+                "password": "passwd"
+            },
+            "xpath": {
+                "username": "\/\/*[@id=\"container_sub\"]/div[2]/table/tbody/tr/td[1]/table/tbody/tr[2]/td/table/tbody/tr[1]/td[2]/input",
+                "password": "\/\/*[@id=\"container_sub\"]/div[2]/table/tbody/tr/td[1]/table/tbody/tr[2]/td/table/tbody/tr[2]/td[2]/input",
+                "submit": "\/\/*[@id=\"container_sub\"]/div[2]/table/tbody/tr/td[1]/table/tbody/tr[2]/td/table/tbody/tr[1]/td[3]/input"
+            }
+        }
+
+        return browser.request(() => fetch(browser.gateway + 'browse', {
+            method: 'POST',
+            headers: browser.headers(),
+            body: JSON.stringify(params)
+        }).then(res => res.json()).then(res => browser.response(res))).catch(err => {
+            logger.error(err.message, 'Browser login failed')
+            return {}
+        })
+    }
+
+}

src/service/NavigateService.js

@@ -0,0 +1,40 @@
+const logger = require('logops')
+    , fetch = require('node-fetch')
+    , RestClient = require("../component/RestClient");
+
+class NavigateService extends RestClient {
+
+    constructor() {
+        super()
+        this.gateway = 'http://ibpcorp.c-xtra.com:32100/api/v1/'
+        this.loginParams = {
+            "anchor": "login",
+            "uri": "https://cplace.christiandaily.co.kr/rankup_module/rankup_member/login.html",
+            "sessionId": "",
+            "css": {
+                "username": "input[name=id]",
+                "password": "input[name=passwd]",
+                "submit": "table input[type=image]"
+            },
+            "key": {
+                "username": "alexkoo",
+                "password": "9zWqDanCSsfSx7C9"
+            }
+        }
+    }
+
+    login(params) {
+        this.loginParams.css = Object.assign(this.loginParams.css, params.css)
+        this.loginParams.uri = params.uri
+        return this.request(() => fetch(this.gateway + 'browse/', {
+            method: 'POST',
+            headers: this.headers(),
+            body: JSON.stringify(params)
+        }).then(res => res.json()).then(res => this.response(res))).catch(err => {
+            logger.error(err.message, 'Browser login failed')
+            return {}
+        })
+    }
+}
+
+module.exports = new NavigateService

src/service/Service.js

@@ -0,0 +1,5 @@
+module.exports = class Service {
+    async async(any) {
+        return any
+    }
+}