diff --git a/package.json b/package.json index 2bfeedb..9bd4ada 100644 --- a/package.json +++ b/package.json @@ -16,9 +16,12 @@ "author": "", "license": "ISC", "dependencies": { + "bunyan": "^1.8.12", "cheerio": "^1.0.0-rc.3", "cross-env": "^6.0.0", "express": "^4.17.1", + "file-stream-rotator": "^0.5.5", + "fs-extra": "^8.1.0", "moment": "^2.24.0", "mongoose": "^5.7.1", "node-schedule": "^1.3.2", diff --git a/src/app.js b/src/app.js index 7152b04..82b6e12 100644 --- a/src/app.js +++ b/src/app.js @@ -4,10 +4,12 @@ import schedule from 'node-schedule' import config from '../config/config' import getCate from './spider/cate' +import logger from '../src/utils/logger' + const db = mongoose.connection db.on('error', function(err) { - console.log(err) + logger.error(err) process.exit(1) }) db.once('open', function() { @@ -20,7 +22,8 @@ mongoose.connect(config.db_taptap, { const scheduleCronstyle = () => { console.log('TapTap爬虫正在运行！') - const timer = schedule.scheduleJob('0 0 10 * * *', () => { + logger.info({}, new Date() + '开始收集数据！') + const timer = schedule.scheduleJob('0 0 9 * * *', () => { console.log(new Date() + '开始收集数据！') getAllData() }) diff --git a/src/model/Logs.js b/src/model/Logs.js new file mode 100644 index 0000000..3d16516 --- /dev/null +++ b/src/model/Logs.js @@ -0,0 +1,17 @@ +'use strict' +import mongoose from 'mongoose' + +/** + * 游戏信息 + */ +const Logs = new mongoose.Schema( + { + errmsg: {type: Object} + }, + { + collection: 'logs', + timestamps: true, + } +) + +export default mongoose.model('Logs', Logs) diff --git a/src/spider/cate.js b/src/spider/cate.js index 06e0952..cdd493a 100644 --- a/src/spider/cate.js +++ b/src/spider/cate.js @@ -7,16 +7,17 @@ import config from '../../config/config' import {clearInterval} from 'timers' import parseHtml from '../utils/parseHtml' - import getDetails from './details' +import logger from '../utils/logger' + let next = '' let timer = '' let data = [] function getCate(cateName) { next = config.taptap[cateName] - timer = setInterval(getData, 3000, cateName) + timer = setInterval(getData, 2000, cateName) } async function getData(cateName) { @@ -31,7 +32,7 @@ async function getData(cateName) { data = [] } } catch (err) { - console.log(err) + logger.error(err) } } @@ -103,6 +104,7 @@ function analyze(str, cateName) { data.push(info) }) } catch (err) { + logger.error(err) console.log(err) } } diff --git a/src/spider/details.js b/src/spider/details.js index fa12859..b114d83 100644 --- a/src/spider/details.js +++ b/src/spider/details.js @@ -4,6 +4,7 @@ require('superagent-proxy')(request) import formWatch from '../utils/formWatch' import saveData from '../utils/saveData' +import logger from '../utils/logger' import {clearInterval} from 'timers' @@ -14,7 +15,7 @@ let allData = [] function getDetails(cateName, data) { allData = [] curIdx = 0 - timer = setInterval(getData, 3000, {cateName, data}) + timer = setInterval(getData, 2000, {cateName, data}) } async function getData({cateName, data}) { @@ -31,7 +32,7 @@ async function getData({cateName, data}) { allData = [] } } catch (err) { - console.log(err) + logger.error(err) } } @@ -48,7 +49,7 @@ function analyze(str, cateName, info) { info.topic = parseInt($('a[data-taptap-tab="topic"] small').text()) || 0 allData.push(info) } catch (err) { - console.log(err) + logger.error(err) } } diff --git a/src/utils/logger.js b/src/utils/logger.js new file mode 100644 index 0000000..4d98a75 --- /dev/null +++ b/src/utils/logger.js @@ -0,0 +1,110 @@ +'use strict'; +import fs from 'fs-extra'; +import FileStreamRotator from 'file-stream-rotator'; +import bunyan from 'bunyan'; +import config from '../../config/config' +import Logs from '../model/Logs' + +const env = process.env.NODE_ENV || 'development'; +const isDev = env === 'development'; + + +const logDir = config.logs_path; +fs.existsSync(logDir) || fs.mkdirSync(logDir); + + + +let logger = null; +const createLogger = function(appName) { + const streams = [{ + level: 'info', + stream: FileStreamRotator.getStream({ + date_format: 'YYYYMMDD', + filename: `${logDir}/${'taptap'}-%DATE%.log`, + frequency: 'daily', + verbose: false, + }), + }]; + if (isDev) { + streams.push({ + level: 'debug', + stream: process.stdout, + }); + } + return bunyan.createLogger({ + name: appName, + serializers: bunyan.stdSerializers, + streams: streams, + src: false, + }); +}; +export default { + info(obj, msg) { + if (!logger) { + logger = createLogger('taptap'); + } + if (msg) { + logger.info(obj, msg); + } else { + logger.info(obj); + } + }, + error(obj, msg) { + if (!logger) { + logger = createLogger('taptap'); + } + if (msg) { + logger.error(obj, msg); + } else { + logger.error(obj); + } + }, + warn(obj, msg) { + if (!logger) { + logger = createLogger('taptap'); + } + if (msg) { + logger.warn(obj, msg); + } else { + logger.warn(obj); + } + }, + debug(obj, msg) { + if (!logger) { + logger = createLogger('taptap'); + } + if (msg) { + logger.debug(obj, msg); + } else { + logger.debug(obj); + } + }, + trace(obj, msg) { + if (!logger) { + logger = createLogger('taptap'); + } + if (msg) { + logger.trace(obj, msg); + } else { + logger.trace(obj); + } + }, + fatal(obj, msg) { + if (!logger) { + logger = createLogger('taptap'); + } + if (msg) { + logger.fatal(obj, msg); + } else { + logger.fatal(obj); + } + }, + db(err) { + const obj = new Logs({ + errmsg: err + }); + obj.save().then(()=>{}).catch((err)=> { + logger.error(err); + }); + }, +}; diff --git a/src/utils/saveData.js b/src/utils/saveData.js index 1920cba..37c9284 100644 --- a/src/utils/saveData.js +++ b/src/utils/saveData.js @@ -1,10 +1,12 @@ import Game from '../model/Game' +import logger from '../utils/logger' export default function saveData(data) { try { Game.insertMany(data) + logger.info({}, `${data.length}条数据已保存`) console.log(`${data.length}条数据已保存`) } catch (err) { - console.log(err) + logger.error(err) } }