diff --git a/README.md b/README.md index 6cee8bf..8025631 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,4 @@ sites/hoh8.js ## 20190403 全民小说抓取 sites/book.js +sites/bookChapter.js diff --git a/src/sites/book.js b/src/sites/book.js index 47f3a0c..89f2452 100644 --- a/src/sites/book.js +++ b/src/sites/book.js @@ -158,22 +158,33 @@ const parseOneBook = async (book) => { // 有些源无法获取到章节列表, 所以挨个获取,直到获取到数据 if (book.data.sites && book.data.sites.length > 0) { for(let site of book.data.sites) { - let chapterListData = await bookChapterList(bookId, site.site); - if (chapterListData.code === 1) { - hasChapter = true; - chapterListData.book_id = bookId; - chapterListData.site = site.site; - let chapterRecord = new BookChapter({ - data: chapterListData, - type: 'book_chapter' - }); - await chapterRecord.save(); + try { + let chapterListData = await bookChapterList(bookId, site.site); + if (chapterListData.code === 1) { + hasChapter = true; + chapterListData.book_id = bookId; + chapterListData.site = site.site; + const chapterExist = await BookChapter.findOne({'data.book_id': bookId, 'data.site': site.site}); + if (!chapterExist) { + let chapterRecord = new BookChapter({ + data: chapterListData, + type: 'book_chapter' + }); + await chapterRecord.save(); + } + } + } catch (err) { + console.log(err); } } } if (hasChapter) { - book.status = 2; - await book.save(); + try { + book.status = 2; + await book.save(); + } catch (err) { + console.log(err); + } } // let bar = multi.newBar(' '+book.data.name+' [:bar] :percent :etas', { // complete: '=', @@ -216,6 +227,11 @@ export default { // 获取所有分类下的数据列表 // await parseAllCategory(2, 0); // 获取所有数据的详情 - await parseAllBookList(1, 0); + for(let i = 0; i < 14; i++){ + await parseAllBookList(1, i); + } + for(let i = 0; i < 9; i++){ + await parseAllBookList(2, i); + } } }