diff --git a/config/config.js.example b/config/config.js.example new file mode 100644 index 0000000..eb1a8f8 --- /dev/null +++ b/config/config.js.example @@ -0,0 +1,14 @@ +let path = require('path'); + +let rootPath = path.normalize(__dirname + '/..') + +let config = { + root: rootPath, + app: { + name: 'spider' + }, + db: 'mongodb://localhost/ghost-development', + logs_path: '/Users/zhl/Documents/logs/spider' +}; + +module.exports = config; diff --git a/package.json b/package.json index ced9900..498ef84 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,12 @@ "license": "ISC", "dependencies": { "bluebird": "^3.5.3", + "bunyan": "^1.8.12", + "file-stream-rotator": "^0.4.1", + "fs-extra": "^7.0.1", + "glob": "^7.1.3", "mongoose": "^5.2.15", + "request": "^2.88.0", "request-promise": "^4.2.4", "ws": "^6.1.2" }, diff --git a/src/app.js b/src/app.js index e69de29..d41840b 100644 --- a/src/app.js +++ b/src/app.js @@ -0,0 +1,27 @@ +import mongoose from 'mongoose'; +import Promise from 'bluebird'; +import logger from './utils/logger'; +import glob from 'glob'; +import config from '../config/config'; +import hoh8 from './sites/hoh8'; + +mongoose.Promise = Promise; + + +const db = mongoose.connection; + +db.on('error', function (err) { + logger.error(err); + process.exit(1); +}); +db.once('open', function () { + logger.info('Connected to db.'); + hoh8.run(); +}); +mongoose.connect(config.db, {promiseLibrary: Promise, useNewUrlParser: true}); + +let models = glob.sync(config.root + './models/*.js'); +models.forEach(function (model) { + require(model); +}); + diff --git a/src/models/SpiderData.js b/src/models/SpiderData.js new file mode 100644 index 0000000..a2a081d --- /dev/null +++ b/src/models/SpiderData.js @@ -0,0 +1,53 @@ +import mongoose from 'mongoose'; + +let Schema = mongoose.Schema; + + +let SpiderDataSchema = new Schema({ + id: {type: Number}, + data: {type: Schema.Types.Mixed}, + type: {type: String}, + status: {type: Number, default: 0} +}, { + collection: 'spider_data', + timestamps: true +}); + +class SpiderDataClass { + static async updateData(data) { + try { + let record = await SpiderDataModel.findOne({'data.id': data.id}); + record.data = data; + record.status = 1; + await record.save(); + } catch (err) { + console.log(err); + } + } + static async saveList(list, type) { + for(let record of list) { + if (record.id > 0) { + let sdata = new SpiderDataModel({ + type: type, + data: record, + status: 0 + }); + try { + await sdata.save(); + } catch (err) { + console.log(err); + } + } + } + } +} + +SpiderDataSchema.loadClass(SpiderDataClass); + +SpiderDataSchema.query.byType = function(type) { + return this.where({ type: type, status: 0}); +}; + +let SpiderDataModel = mongoose.model('SpiderData', SpiderDataSchema); + +export default SpiderDataModel; diff --git a/src/sites/hoh8.js b/src/sites/hoh8.js new file mode 100644 index 0000000..65c25c3 --- /dev/null +++ b/src/sites/hoh8.js @@ -0,0 +1,129 @@ +import socketClient from '../socket/client'; +import SpiderData from '../models/SpiderData'; +import netUtil from '../utils/net.util'; + +let client = socketClient; + +const getMovieList = (id) => { + let msg = `{"path":"/video/searchVideoList","body":"{\\"pageNum\\":${id},\\"kw\\":null,\\"type\\":\\"所有类型\\",\\"region\\":\\"所有地区\\",\\"date\\":\\"所有年代\\"}","reqId":"movielist000${id}"}`; + client.send(msg); +} +const getMovieInfo = () => { + let record = list[current]; + let msg = `{"path":"/video/getVideoDtl.page","body":"{\\"id\\":\\"${record.data.id}\\"}","reqId":"movieinfo${record.id}"}`; + client.send(msg); +} +const getGameList = (id) => { + let msg = `{"path":"/hg/getGameList","body":"{\\"pageNum\\":${id},\\"kw\\":\\"\\"}","reqId":"gamelist000${id}"}`; + client.send(msg); +} + +const getGameInfo = () => { + let record = list[current]; + let msg = `{"path":"/hg/getGameDtl.page","body":"{\\"gameId\\":\\"${record.id}\\"}","reqId":"gameinfo${record.id}"}`; + client.send(msg); +} + +const getVideoToken = (id) => { + let msg = `{"path":"/deal/getVideoADToken","body":"{}","reqId":"videotoken${id}"}`; + client.send(msg); +} + +const updateMoney = async (appId, token, appName) => { + let url = 'https://game.hoh8.cn/hh/deal/getVideoADReward.anon'; + let jsonData = {"appId": appId,"appName":appName,"token":token,"userId":uid}; + let data = { + deviceType: 5, + appVersion: 1, + jsonData: JSON.stringify(jsonData) + } + return netUtil.postForm(url, data); +} + +const parseAllRecord = async (type) => { + try { + list = await SpiderData.find({type: type, status: 0}); + console.log(list.length); + if (type === 'movie') { + getMovieInfo(); + } else if (type === 'game') { + getGameInfo(); + } + } catch (err) { + console.log(err); + } + +} + +let current = 0; +let list = []; +let uid = '2407044'; +export default { + run: function () { + let url = `wss://game.hoh8.cn/hh/websocket?userId=${uid}&token=c402cdea067bb6d1ff51090edbe0d645&roomId=0&groupId=global`; + client.open(url, ''); + client.onopen = (e) => { + console.log('socket on open'); + // getMovieList(0); + // getGameList(0); + // parseAllRecord('movie'); + getVideoToken(0); + } + client.onmessage = async (data, flags, number) => { + console.log(data); + data = JSON.parse(data); + let id = parseInt(data.reqId.replace('movielist000', '')); + if (data.reqId.indexOf('movielist') === 0 && data.data.length > 0) { + try { + await SpiderData.saveList(data.data, 'movie'); + getMovieList(id + 1); + } catch (err) { + console.log(err); + } + } else if (data.reqId.indexOf('movieinfo') === 0) { + await SpiderData.updateData(data.data); + console.log('begin next movie'); + current++; + if (current < list.length) { + getMovieInfo(); + } else { + console.log('finish parse movie info') + process.exit(1); + } + } else if (data.reqId.indexOf('gamelist') === 0 && data.data.length > 0) { + try { + await SpiderData.saveList(data.data, 'game'); + getGameList(id + 1); + } catch (err) { + console.log(err); + } + } else if (data.reqId.indexOf('gameinfo') === 0) { + await SpiderData.updateData(data.data); + console.log('begin next game'); + current++; + if (current < list.length) { + getGameInfo(); + } else { + console.log('finish parse movie info') + process.exit(1); + } + } else if (data.reqId.indexOf('videotoken') === 0) { + const appId = data.data.appId; + const videoToken = data.data.token; + const appName = data.data.appName; + console.log(`appId: ${appId}, token: ${videoToken}`); + try { + const result = await updateMoney(appId, videoToken, appName); + console.log(result); + setTimeout(function () { + if (current ++ < 10) { + getVideoToken(current); + } + }, 15000) + } catch (err) { + console.log(err); + } + } + } + } +} diff --git a/src/socket/client.js b/src/socket/client.js new file mode 100644 index 0000000..a2795a9 --- /dev/null +++ b/src/socket/client.js @@ -0,0 +1,98 @@ +import WebSocket from 'ws'; +import logger from '../utils/logger'; + +function WebSocketClient() { + this.number = 0; // Message number + this.autoReconnectInterval = 5 * 1000; // ms +} + +WebSocketClient.prototype.open = function (url, id) { + this.url = url; + this.id = id; + this.instance = new WebSocket(this.url); + this.instance.on('open', () => { + this.onopen(); + }); + this.instance.on('message', (data, flags) => { + this.number++; + this.onmessage(data, flags, this.number); + }); + this.instance.on('close', (e) => { + switch (e) { + case 1000: // CLOSE_NORMAL + logger.info("WebSocket: closed"); + break; + default: // Abnormal closure + this.reconnect(e); + break; + } + this.onclose(e); + }); + this.instance.on('error', (e) => { + switch (e.code) { + case 'ECONNREFUSED': + this.reconnect(e); + break; + default: + this.onerror(e); + break; + } + }); +}; +WebSocketClient.prototype.send = function (data, option) { + try { + this.instance.send(data, option); + } catch (e) { + this.instance.emit('error', e); + } +}; +WebSocketClient.prototype.sendTestText = function (data, option) { + try { + let msg = { + msg_type: 'text', + content: data + }; + let msgObj = { + type: 'm', + bot_id: '591bf8c21d2860389057fafa', + group: '6334778342@chatroom', + messages: [msg] + }; + this.instance.send(JSON.stringify(msgObj), option); + } catch (e) { + this.instance.emit('error', e); + } +}; +WebSocketClient.prototype.reconnect = function (e) { + logger.info(`WebSocketClient: retry in ${this.autoReconnectInterval}ms`, e); + this.instance.removeAllListeners(); + let that = this; + setTimeout(function () { + logger.info("WebSocketClient: reconnecting..."); + that.open(that.url, that.id); + }, this.autoReconnectInterval); +}; +WebSocketClient.prototype.onopen = function (e) { + logger.info(arguments, "WebSocketClient: open"); +}; +WebSocketClient.prototype.onmessage = function (data, flags, number) { + logger.info(arguments, "WebSocketClient: message"); +}; +WebSocketClient.prototype.onerror = function (e) { + logger.info(arguments, "WebSocketClient: error"); +}; +WebSocketClient.prototype.onclose = function (e) { + logger.info(arguments, "WebSocketClient: closed"); +}; + + +WebSocketClient.client = null; + +WebSocketClient.getClient = function () { + if (this.client === null) { + this.client = new WebSocketClient(); + } + return this.client; +}; + +export default WebSocketClient.getClient(); diff --git a/src/utils/logger.js b/src/utils/logger.js new file mode 100644 index 0000000..ed62540 --- /dev/null +++ b/src/utils/logger.js @@ -0,0 +1,99 @@ +import fs from 'fs-extra'; +import FileStreamRotator from 'file-stream-rotator'; +import bunyan from 'bunyan'; +import config from '../../config/config'; + + +let env = process.env.NODE_ENV || 'development'; +let isDev = env === 'development'; + +const logDir = config.logs_path; +fs.existsSync(logDir) || fs.mkdirSync(logDir); + +let logger = null; +let createLogger = function(appName) { + appName = !appName ? config.app.name : appName; + let streams = [{ + level: 'info', + stream: FileStreamRotator.getStream({ + date_format: 'YYYYMMDD', + filename: `${logDir}/${appName}-%DATE%.log`, + frequency: 'daily', + verbose: false + }) + }]; + if (isDev) { + streams.push({ + level: 'debug', + stream: process.stdout + }) + } + return bunyan.createLogger({ + name: appName, + serializers: bunyan.stdSerializers, + streams: streams, + src: false + }) +}; +export default { + info(obj, msg) { + if (!logger) { + logger = createLogger(global.app_name); + } + if (msg) { + logger.info(obj, msg); + } else { + logger.info(obj); + } + }, + error(obj, msg) { + if (!logger) { + logger = createLogger(global.app_name); + } + if (msg) { + logger.error(obj, msg); + } else { + logger.error(obj); + } + }, + warn(obj, msg) { + if (!logger) { + logger = createLogger(global.app_name); + } + if (msg) { + logger.warn(obj, msg); + } else { + logger.warn(obj); + } + }, + debug(obj, msg) { + if (!logger) { + logger = createLogger(global.app_name); + } + if (msg) { + logger.debug(obj, msg); + } else { + logger.debug(obj); + } + }, + trace(obj, msg) { + if (!logger) { + logger = createLogger(global.app_name); + } + if (msg) { + logger.trace(obj, msg); + } else { + logger.trace(obj); + } + }, + fatal(obj, msg) { + if (!logger) { + logger = createLogger(global.app_name); + } + if (msg) { + logger.fatal(obj, msg); + } else { + logger.fatal(obj); + } + } +} diff --git a/src/utils/net.util.js b/src/utils/net.util.js new file mode 100644 index 0000000..0c82a76 --- /dev/null +++ b/src/utils/net.util.js @@ -0,0 +1,27 @@ +import request from 'request'; +import Promise from 'bluebird'; + +const requestData = (options) => { + return new Promise((resolve, reject) => { + request(options, (err, response, body) => { + if (err) { + return reject(err); + } + resolve(body); + }); + }); +} +export default { + postForm(url, data) { + const options = { + method: 'POST', + url: url, + headers: { + 'Cache-Control': 'no-cache', + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', + }, + form: data, + }; + return requestData(options); + } +}