增加日志

package.json

@@ -16,9 +16,12 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "bunyan": "^1.8.12",
     "cheerio": "^1.0.0-rc.3",
     "cross-env": "^6.0.0",
     "express": "^4.17.1",
+    "file-stream-rotator": "^0.5.5",
+    "fs-extra": "^8.1.0",
     "moment": "^2.24.0",
     "mongoose": "^5.7.1",
     "node-schedule": "^1.3.2",

src/app.js

@@ -4,10 +4,12 @@ import schedule from 'node-schedule'
 import config from '../config/config'
 import getCate from './spider/cate'
 
+import logger from '../src/utils/logger'
+
 const db = mongoose.connection
 
 db.on('error', function(err) {
-  console.log(err)
+  logger.error(err)
   process.exit(1)
 })
 db.once('open', function() {
@@ -20,7 +22,8 @@ mongoose.connect(config.db_taptap, {
 
 const scheduleCronstyle = () => {
   console.log('TapTap爬虫正在运行！')
-  const timer = schedule.scheduleJob('0 0 10 * * *', () => {
+  logger.info({}, new Date() + '开始收集数据！')
+  const timer = schedule.scheduleJob('0 0 9 * * *', () => {
     console.log(new Date() + '开始收集数据！')
     getAllData()
   })

src/model/Logs.js

@@ -0,0 +1,17 @@
+'use strict'
+import mongoose from 'mongoose'
+
+/**
+ * 游戏信息
+ */
+const Logs = new mongoose.Schema(
+  {
+    errmsg: {type: Object}
+  },
+  {
+    collection: 'logs',
+    timestamps: true,
+  }
+)
+
+export default mongoose.model('Logs', Logs)

src/spider/cate.js

@@ -7,16 +7,17 @@ import config from '../../config/config'
 import {clearInterval} from 'timers'
 
 import parseHtml from '../utils/parseHtml'
-
 import getDetails from './details'
 
+import logger from '../utils/logger'
+
 let next = ''
 let timer = ''
 let data = []
 
 function getCate(cateName) {
   next = config.taptap[cateName]
-  timer = setInterval(getData, 3000, cateName)
+  timer = setInterval(getData, 2000, cateName)
 }
 
 async function getData(cateName) {
@@ -31,7 +32,7 @@ async function getData(cateName) {
       data = []
     }
   } catch (err) {
-    console.log(err)
+    logger.error(err)
   }
 }
 
@@ -103,6 +104,7 @@ function analyze(str, cateName) {
       data.push(info)
     })
   } catch (err) {
+    logger.error(err)
     console.log(err)
   }
 }

src/spider/details.js

@@ -4,6 +4,7 @@ require('superagent-proxy')(request)
 
 import formWatch from '../utils/formWatch'
 import saveData from '../utils/saveData'
+import logger from '../utils/logger'
 
 import {clearInterval} from 'timers'
 
@@ -14,7 +15,7 @@ let allData = []
 function getDetails(cateName, data) {
   allData = []
   curIdx = 0
-  timer = setInterval(getData, 3000, {cateName, data})
+  timer = setInterval(getData, 2000, {cateName, data})
 }
 
 async function getData({cateName, data}) {
@@ -31,7 +32,7 @@ async function getData({cateName, data}) {
       allData = []
     }
   } catch (err) {
-    console.log(err)
+    logger.error(err)
   }
 }
 
@@ -48,7 +49,7 @@ function analyze(str, cateName, info) {
     info.topic = parseInt($('a[data-taptap-tab="topic"] small').text()) || 0
     allData.push(info)
   } catch (err) {
-    console.log(err)
+    logger.error(err)
   }
 }
 

src/utils/logger.js

@@ -0,0 +1,110 @@
+'use strict';
+import fs from 'fs-extra';
+import FileStreamRotator from 'file-stream-rotator';
+import bunyan from 'bunyan';
+import config from '../../config/config'
+import Logs from '../model/Logs'
+
+const env = process.env.NODE_ENV || 'development';
+const isDev = env === 'development';
+
+
+const logDir = config.logs_path;
+fs.existsSync(logDir) || fs.mkdirSync(logDir);
+
+
+
+let logger = null;
+const createLogger = function(appName) {
+  const streams = [{
+    level: 'info',
+    stream: FileStreamRotator.getStream({
+      date_format: 'YYYYMMDD',
+      filename: `${logDir}/${'taptap'}-%DATE%.log`,
+      frequency: 'daily',
+      verbose: false,
+    }),
+  }];
+  if (isDev) {
+    streams.push({
+      level: 'debug',
+      stream: process.stdout,
+    });
+  }
+  return bunyan.createLogger({
+    name: appName,
+    serializers: bunyan.stdSerializers,
+    streams: streams,
+    src: false,
+  });
+};
+export default {
+  info(obj, msg) {
+    if (!logger) {
+      logger = createLogger('taptap');
+    }
+    if (msg) {
+      logger.info(obj, msg);
+    } else {
+      logger.info(obj);
+    }
+  },
+  error(obj, msg) {
+    if (!logger) {
+      logger = createLogger('taptap');
+    }
+    if (msg) {
+      logger.error(obj, msg);
+    } else {
+      logger.error(obj);
+    }
+  },
+  warn(obj, msg) {
+    if (!logger) {
+      logger = createLogger('taptap');
+    }
+    if (msg) {
+      logger.warn(obj, msg);
+    } else {
+      logger.warn(obj);
+    }
+  },
+  debug(obj, msg) {
+    if (!logger) {
+      logger = createLogger('taptap');
+    }
+    if (msg) {
+      logger.debug(obj, msg);
+    } else {
+      logger.debug(obj);
+    }
+  },
+  trace(obj, msg) {
+    if (!logger) {
+      logger = createLogger('taptap');
+    }
+    if (msg) {
+      logger.trace(obj, msg);
+    } else {
+      logger.trace(obj);
+    }
+  },
+  fatal(obj, msg) {
+    if (!logger) {
+      logger = createLogger('taptap');
+    }
+    if (msg) {
+      logger.fatal(obj, msg);
+    } else {
+      logger.fatal(obj);
+    }
+  },
+  db(err) {
+    const obj = new Logs({
+        errmsg: err
+    });
+    obj.save().then(()=>{}).catch((err)=> {
+      logger.error(err);
+    });
+  },
+};

src/utils/saveData.js

@@ -1,10 +1,12 @@
 import Game from '../model/Game'
+import logger from '../utils/logger'
 
 export default function saveData(data) {
   try {
     Game.insertMany(data)
+    logger.info({}, `${data.length}条数据已保存`)
     console.log(`${data.length}条数据已保存`)
   } catch (err) {
-    console.log(err)
+    logger.error(err)
   }
 }