添加全民小说下载的方法

docs/qmxs.md

@@ -0,0 +1,78 @@
+# 全民小说
+
+## 分类列表
+
+https://bc.dzjgmp.com/core/book/type.api?sex=1
+
+|  字段  | 类型   | 说明    |
+|----|----|----|
+|sex| Number   |  性别 1：男， 2：女  |
+
+Response:
+
+```
+{ code: 1,
+  data: 
+   [ { ltype_id: 0,
+       ltype_image: 'https://res.jnmzsd.com/create/15457022763611.png',
+       ltype_name: '玄幻',
+       ltype_list: [Array] },
+   ]
+}
+```
+
+## 列表查询接口
+
+https://bc.dzjgmp.com/core/book/type_all.api?sex=1&ltype=0&stype=-1&status=-1&page=1&limit=20
+
+|  字段  | 类型   | 说明    |
+|----|----|----|
+|sex| Number   |  性别 0:未指定，1：男， 2：女  |
+|ltype|  Number  | 分类的id，从0开始，具体含义从分类列表获取    |
+|stype|  Number  | 子分类的id，-1：所有，其他：从分类列表的ltype_list获取    |
+|status| Number   |连载状态 -1：所有，1： 完结，2：连载    |
+|page|  Number  |  分页页码，从1开始  |
+|limit|  Number  |  每页数量  |
+
+Response:
+
+```
+{ code: 1,
+  data: 
+   { total: 34248,
+     current_page: 1,
+     limit: 20,
+     from: 1,
+     to: 20,
+     last_page: 1713,
+     list: []
+   }
+}
+```
+
+单本书的结构
+```
+{ book_id: 668,
+  name: '逆天邪神',
+  author: '火星引力',
+  image: 'https://res.jnmzsd.com/zssq/308/542a5838a5ae10f815039a7f.jpeg',
+  sex: 1,
+  ltype: '玄幻',
+  stype: '东方玄幻',
+  remark: '掌天毒之珠，承邪神之血，修逆天之力，一代邪神，君临天下！【添加微信公众号：火星引力】【我们的yy频道：49554】......各位书友要是觉得《逆天邪神》还不错的话请不要忘记向您QQ群和微博里的朋友推荐哦！',
+  status: 2,
+  last_chapter: '第1385章 极怒凤炎',
+  updated_at: '2019-04-02 12:20:00' }
+```
+
+## 书籍详情
+
+https://cache.dzjgmp.com/book.api?book_id=10887
+
+## 数据章节列表
+
+https://cache.dzjgmp.com/book_chapter.api?book_id=10887&site_id=wsg
+
+## 章节详情
+
+https://cache.dzjgmp.com/chapter.api?chapter_id=19873&crawl_book_id=839413&id=418&site_id=wsg

src/app.js

@@ -4,6 +4,7 @@ import logger from './utils/logger';
 import glob from 'glob';
 import config from '../config/config';
 import hoh8 from './sites/hoh8';
+import book from './sites/book';
 
 mongoose.Promise = Promise;
 
@@ -16,7 +17,8 @@ db.on('error', function (err) {
 });
 db.once('open', function () {
   logger.info('Connected to db.');
-  hoh8.run();
+  // hoh8.run();
+  book.run();
 });
 mongoose.connect(config.db, {promiseLibrary: Promise, useNewUrlParser: true});
 

src/models/SpiderCategory.js

@@ -0,0 +1,23 @@
+import mongoose from 'mongoose';
+
+let Schema = mongoose.Schema;
+
+
+let SpiderCategorySchema = new Schema({
+  data: {type: Schema.Types.Mixed},
+  type: {type: String},
+  status: {type: Number, default: 0}
+}, {
+  collection: 'spider_category',
+  timestamps: true
+});
+
+class SpiderCategoryClass {
+}
+
+SpiderCategorySchema.loadClass(SpiderCategoryClass);
+
+
+let SpiderCategoryModel = mongoose.model('SpiderCategory', SpiderCategorySchema);
+
+export default SpiderCategoryModel;

src/models/SpiderData.js

@@ -4,7 +4,6 @@ let Schema = mongoose.Schema;
 
 
 let SpiderDataSchema = new Schema({
-  id: {type: Number},
   data: {type: Schema.Types.Mixed},
   type: {type: String},
   status: {type: Number, default: 0}

src/sites/book.js

@@ -0,0 +1,112 @@
+import netUtil from '../utils/net.util';
+import SpiderCategory from '../models/SpiderCategory';
+import SpiderData from '../models/SpiderData';
+
+const generateHeader = () => {
+  return {
+    'User-Agent': 'quanminxiaoshuo_apple/1.0.9 CFNetwork/976 Darwin/18.2.0',
+    'ver': '5.0',
+    'pt': 'ios',
+    'package': 'quanminxiaoshuo',
+    'token': 'tc3ahpxco5ads8utnsbhxfuypkstc2oi',
+    'user': '2300097'
+  }
+}
+/**
+ * 获取分类列表
+ * @param {number} sex 性别 1: 男，2：女
+ * */
+const categoryList = async (sex) => {
+  let url = `https://bc.dzjgmp.com/core/book/type.api?sex=${0}`
+  let data = await netUtil.getData(url, generateHeader());
+  data = JSON.parse(data);
+  for (let obj of data.data) {
+    obj.sex = sex;
+    let record = new SpiderCategory({
+      data: obj,
+      type: 'book'
+    });
+    await record.save();
+  }
+}
+/**
+ * 获取某分类下所有书籍列表
+ * @param {number} sex sex 性别 1: 男，2：女
+ * @param {number} ltype 大类
+ * @param {number} page 分页 1开始
+ * */
+const bookList = async (sex, ltype, page) => {
+  console.log(`get book list, sex: ${sex}, ltype: ${ltype}, page: ${page}`);
+  let url = `https://bc.dzjgmp.com/core/book/type_all.api?sex=${sex}&ltype=${ltype}&stype=-1&status=-1&page=${page}&limit=20`;
+  let data = await netUtil.getData(url, generateHeader());
+  data = JSON.parse(data);
+  return data;
+}
+
+const bookInfo = async (bookId) => {
+  console.log(`parse book: ${bookId}`);
+  let url = `https://cache.dzjgmp.com/book.api?book_id=${bookId}`;
+  let data = await netUtil.getData(url, generateHeader());
+  data = JSON.parse(data);
+  return data.data;
+}
+// 处理单页的列表数据
+const parsePageObj = async (dataArr) => {
+  for(let data of dataArr) {
+    let record = new SpiderData({
+      data: data,
+      type: 'book'
+    })
+    await record.save();
+  }
+}
+// 获取所有分类下的数据列表
+const parseAllCategory = async () => {
+    try {
+      let cateList = await SpiderCategory.find({type: 'book'});
+      for(const cate of cateList) {
+        let firstPage = await bookList(cate.data.sex, cate.data.ltype_id, 1);
+        let totalPage = firstPage.data.last_page;
+        cate.data.total = firstPage.data.total;
+        cate.markModified('data');
+        await cate.save();
+        console.log(`parse ltype: ${cate.data.ltype_id}, totalPage: ${totalPage}`);
+        await parsePageObj(firstPage.data.list);
+        for(let i = 2; i <= totalPage; i ++) {
+          let pageObj = await bookList(cate.data.sex, cate.data.ltype_id, i);
+          await parsePageObj(pageObj.data.list);
+        }
+      }
+    } catch (err) {
+      console.log(err);
+    }
+}
+
+const parseAllBookList = async () => {
+  let bookList = await SpiderData.find({status: 0}).sort({'data.book_id': 1}).limit(1);
+  for(let book of bookList) {
+    console.time('oneBook');
+    let record = await bookInfo(book.data.book_id);
+    for (const key in record) {
+      if ({}.hasOwnProperty.call(record, key)) {
+        book.data[key] = record[key];
+      }
+    }
+    book.markModified('data');
+    console.timeEnd('oneBook')
+    await book.save();
+  }
+}
+
+
+
+export default {
+  run: async () => {
+    // await categoryList(1);
+    // await categoryList(2);
+    // await bookList(1, 0, 1);
+    // 获取所有分类下的数据列表
+    // await parseAllCategory();
+    await parseAllBookList();
+  }
+}

src/utils/net.util.js

@@ -12,16 +12,28 @@ const requestData = (options) => {
   });
 }
 export default {
-  postForm(url, data) {
+  postForm(url, data, header) {
+    header = header || {
+      'Cache-Control': 'no-cache',
+      'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
+    }
     const options = {
       method: 'POST',
       url: url,
-      headers: {
+      headers: header,
-        'Cache-Control': 'no-cache',
-        'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
-      },
       form: data,
     };
     return requestData(options);
+  },
+  getData(url, header) {
+    header = header || {
+      'Cache-Control': 'no-cache',
+    }
+    const options = {
+      method: 'GET',
+      url: url,
+      headers: header,
+    };
+    return requestData(options);
   }
 }