node/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改处理失败记录的逻辑

Browse Source
This commit is contained in:
zhl 2019-05-09 20:19:13 +08:00
parent a833d7dd52
commit 7e3e83f25b
3 changed files with 16 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/app.js
View File

@ -28,8 +28,8 @@ db.once('open', async function () {
// movie.run(); // movie.run();
// bookChapter.run(); // bookChapter.run();
// await dandanzan.run(); // await dandanzan.run();
// await commonTask.run(); await commonTask.run();
websiteSchedule.scheduleUpdateAll(); // websiteSchedule.scheduleUpdateAll();
}); });
mongoose.connect(config.db, {promiseLibrary: Promise, useNewUrlParser: true}); mongoose.connect(config.db, {promiseLibrary: Promise, useNewUrlParser: true});

21
src/sites/common.js
View File

@ -7,13 +7,18 @@ import generalQueue from '../utils/general.queue';
const parseOneRecord = async function (record) { const parseOneRecord = async function (record) {
let classTask = require(`./${record.className}`).default; let classTask = require(`./${record.className}`).default;
try { if(typeof classTask[record.methodName] === 'function'){
if(typeof classTask[record.methodName] === 'function'){ try {
await classTask[record.methodName](record.params); generalQueue.addQueue({
run: async function () {
await classTask[record.methodName](record.params);
}
})
} catch (err) {
console.log('error parse false record', record.id);
} }
} catch (err) {
console.log('error parse false record', record.id);
} }
} }
const parseAllRecord = async function () { const parseAllRecord = async function () {
@ -32,11 +37,7 @@ const parseAllRecord = async function () {
if (records) { if (records) {
for(const record of records) { for(const record of records) {
try { try {
generalQueue.addQueue({ await parseOneRecord(record);
run: async function () {
await parseOneRecord(record);
}
})
} catch (err) { } catch (err) {
console.log('error parse record:', record); console.log('error parse record:', record);
} }

5
src/utils/general.queue.js
View File

@ -7,9 +7,10 @@ let finishCb;
let q = async.queue( async (reqObj, cb) => { let q = async.queue( async (reqObj, cb) => {
try { try {
await reqObj.run(); await reqObj.run();
cb(); (cb) && (cb());
} catch (err) { } catch (err) {
cb(err); console.log('error parse queue:', reqObj.name, err);
(cb) && (cb(err));
} }
}, 30); }, 30);
q.drain = function(){ q.drain = function(){