From f4d92336c717dac9e32ada5a6ebebd6b0af36877 Mon Sep 17 00:00:00 2001 From: zhl Date: Tue, 2 Apr 2019 19:47:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=96=87=E4=BB=B6=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/config.js.example | 3 +- package.json | 4 + src/sites/hoh8.js | 147 ++++++++++++++++++++----------------- src/utils/download.util.js | 53 +++++++++++++ src/utils/string.util.js | 11 +++ 5 files changed, 151 insertions(+), 67 deletions(-) create mode 100644 src/utils/download.util.js create mode 100644 src/utils/string.util.js diff --git a/config/config.js.example b/config/config.js.example index eb1a8f8..9cf41d9 100644 --- a/config/config.js.example +++ b/config/config.js.example @@ -8,7 +8,8 @@ let config = { name: 'spider' }, db: 'mongodb://localhost/ghost-development', - logs_path: '/Users/zhl/Documents/logs/spider' + logs_path: '/Users/zhl/Documents/logs/spider', + download_path: '/Users/zhl/Documents/spider', }; module.exports = config; diff --git a/package.json b/package.json index 498ef84..17bbe27 100644 --- a/package.json +++ b/package.json @@ -15,11 +15,15 @@ "author": "zhl", "license": "ISC", "dependencies": { + "async": "^2.6.2", "bluebird": "^3.5.3", "bunyan": "^1.8.12", + "dateformat": "^3.0.3", "file-stream-rotator": "^0.4.1", + "fs": "0.0.1-security", "fs-extra": "^7.0.1", "glob": "^7.1.3", + "mkdirp": "^0.5.1", "mongoose": "^5.2.15", "request": "^2.88.0", "request-promise": "^4.2.4", diff --git a/src/sites/hoh8.js b/src/sites/hoh8.js index ba533ee..a6c16a0 100644 --- a/src/sites/hoh8.js +++ b/src/sites/hoh8.js @@ -1,6 +1,8 @@ import socketClient from '../socket/client'; import SpiderData from '../models/SpiderData'; import netUtil from '../utils/net.util'; +import downloadQueue from '../utils/download.util'; +import stringUtil from '../utils/string.util'; let client = socketClient; @@ -78,7 +80,17 @@ const parseAllRecord = async (type) => { } catch (err) { console.log(err); } - +} +const downloadAllMovieImg = async () => { + try { + list = await SpiderData.find({type: 'movie', status: 1}).sort({'data.id': 1}); + for (let record of list) { + let data = record.data; + downloadQueue.add_file_queue({type: 'movie', name: stringUtil.getFileNameFromUrl(data.img), url: data.img}); + } + } catch (err) { + console.log(err); + } } let current = 0; @@ -86,70 +98,73 @@ let list = []; export default { run: function () { - let url = `wss://game.hoh8.cn/hh/websocket?userId=${uid}&token=${token}&roomId=0&groupId=global`; - client.open(url, ''); - client.onopen = (e) => { - console.log('socket on open'); - // getMovieList(0); - // getGameList(0); - // parseAllRecord('movie'); - getVideoToken(0); - } - client.onmessage = async (data, flags, number) => { - console.log(data); - data = JSON.parse(data); - let id = parseInt(data.reqId.replace('movielist000', '')); - if (data.reqId.indexOf('movielist') === 0 && data.data.length > 0) { - try { - await SpiderData.saveList(data.data, 'movie'); - getMovieList(id + 1); - } catch (err) { - console.log(err); - } - } else if (data.reqId.indexOf('movieinfo') === 0) { - await SpiderData.updateData(data.data); - console.log('begin next movie'); - current++; - if (current < list.length) { - getMovieInfo(); - } else { - console.log('finish parse movie info') - process.exit(1); - } - } else if (data.reqId.indexOf('gamelist') === 0 && data.data.length > 0) { - try { - await SpiderData.saveList(data.data, 'game'); - getGameList(id + 1); - } catch (err) { - console.log(err); - } - } else if (data.reqId.indexOf('gameinfo') === 0) { - await SpiderData.updateData(data.data); - console.log('begin next game'); - current++; - if (current < list.length) { - getGameInfo(); - } else { - console.log('finish parse movie info') - process.exit(1); - } - } else if (data.reqId.indexOf('videotoken') === 0) { - const appId = data.data.appId; - const videoToken = data.data.token; - const appName = data.data.appName; - console.log(`appId: ${appId}, token: ${videoToken}`); - try { - const result = await updateMoney(appId, videoToken, appName); - console.log(result); - setTimeout(function () { - if (current ++ < 20) { - getVideoToken(current); - } - }, 15000) - } catch (err) { - console.log(err); - } - } - } + downloadAllMovieImg(); } + // run: function () { + // let url = `wss://game.hoh8.cn/hh/websocket?userId=${uid}&token=${token}&roomId=0&groupId=global`; + // client.open(url, ''); + // client.onopen = (e) => { + // console.log('socket on open'); + // // getMovieList(0); + // // getGameList(0); + // // parseAllRecord('movie'); + // getVideoToken(0); + // } + // client.onmessage = async (data, flags, number) => { + // console.log(data); + // data = JSON.parse(data); + // let id = parseInt(data.reqId.replace('movielist000', '')); + // if (data.reqId.indexOf('movielist') === 0 && data.data.length > 0) { + // try { + // await SpiderData.saveList(data.data, 'movie'); + // getMovieList(id + 1); + // } catch (err) { + // console.log(err); + // } + // } else if (data.reqId.indexOf('movieinfo') === 0) { + // await SpiderData.updateData(data.data); + // console.log('begin next movie'); + // current++; + // if (current < list.length) { + // getMovieInfo(); + // } else { + // console.log('finish parse movie info') + // process.exit(1); + // } + // } else if (data.reqId.indexOf('gamelist') === 0 && data.data.length > 0) { + // try { + // await SpiderData.saveList(data.data, 'game'); + // getGameList(id + 1); + // } catch (err) { + // console.log(err); + // } + // } else if (data.reqId.indexOf('gameinfo') === 0) { + // await SpiderData.updateData(data.data); + // console.log('begin next game'); + // current++; + // if (current < list.length) { + // getGameInfo(); + // } else { + // console.log('finish parse movie info') + // process.exit(1); + // } + // } else if (data.reqId.indexOf('videotoken') === 0) { + // const appId = data.data.appId; + // const videoToken = data.data.token; + // const appName = data.data.appName; + // console.log(`appId: ${appId}, token: ${videoToken}`); + // try { + // const result = await updateMoney(appId, videoToken, appName); + // console.log(result); + // setTimeout(function () { + // if (current ++ < 20) { + // getVideoToken(current); + // } + // }, 15000) + // } catch (err) { + // console.log(err); + // } + // } + // } + // } } diff --git a/src/utils/download.util.js b/src/utils/download.util.js new file mode 100644 index 0000000..4abf717 --- /dev/null +++ b/src/utils/download.util.js @@ -0,0 +1,53 @@ +import config from '../../config/config'; +import fs from 'fs'; +import request from 'request'; +import async from 'async'; +import dateformat from 'dateformat'; +import mkdirp from 'mkdirp'; +import logger from '../utils/logger'; + + +/** + * 文件下载队列 + * */ +let q = async.queue(function (file, cb) { + let local_path = `${config.download_path}/${dateformat('yyyymmdd')}/${file.type}`; + if (!fs.existsSync(local_path)){ + mkdirp.sync(local_path); + } + if (file.name) { + local_path = local_path + '/' + file.name; + } + if (fs.existsSync(local_path)) { + return cb() + } + let stream = request + .get({ + url: file.url, + timeout: 1000 + }) + .on('response', function(response) { + + }) + .on('error', function(err) { + cb(err); + }) + .pipe(fs.createWriteStream(local_path)); + stream.on('finish',() => { + console.info(file, 'file download success'); + cb() + }) +}, 2); +q.drain = function(){ + console.info('all queue done'); +}; +module.exports = { + add_file_queue(file) { + console.log('add file to queue', file); + q.push(file, function(err){ + if (err) { + console.log('error download file: ', file.url, err); + } + }); + } +} diff --git a/src/utils/string.util.js b/src/utils/string.util.js new file mode 100644 index 0000000..92d75c8 --- /dev/null +++ b/src/utils/string.util.js @@ -0,0 +1,11 @@ + +export default { + /** + * 从url中截取文件名 + * @param {string} url + * @return {string} filename + * */ + getFileNameFromUrl(url) { + return url.substring(url.lastIndexOf('/')+1) + } +}