node/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

电影增加字段is_modify

Browse Source
This commit is contained in:
zhl 2019-05-16 09:50:24 +08:00
parent 539ff78e70
commit c23a86e102
2 changed files with 43 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/models/Movies.js
View File

@ -44,6 +44,8 @@ const Movies = new Schema({
price: {type: Number, default: 100},
// 是否是新记录
is_new: {type: Boolean, default: true},
// 是否已修改
is_modify: {type: Boolean, default: false},
// 视频分类，movie: 电影；tv: 电视剧； show: 综艺节目
category: {type: String},
// 是否已删除
@ -55,15 +57,32 @@ const Movies = new Schema({
collection: 'movies',
timestamps: true,
});
const MovieModel = mongoose.model('Movies', Movies);
MovieModel.findByGid = function(gid) {
return MovieModel.findOne({gid: gid, deleted: false});
};
MovieModel.updateOne = async function (gid, record) {
const query = {gid: gid};
const options = {upsert: true, setDefaultsOnInsert:true};
await MovieModel.update(query, record, options);
class MovieClass {
static findByGid = function(gid) {
return MovieModel.findOne({gid: gid, deleted: false});
}
static updateOne = async function (gid, record) {
const query = {gid: gid};
const options = {upsert: true, setDefaultsOnInsert:true};
await MovieModel.update(query, record, options);
}
updateResources(arr) {
let isModify = false;
if (this.resources.length !== arr.length) {
isModify = true;
} else {
for (let i = 0; i < this.resources.length; i ++) {
if (this.resources[i].title !== arr[i].title
|| this.resources[i].link !== arr[i].link) {
isModify = true;
break;
}
}
}
this.resources = arr;
this.is_modify = isModify;
}
}
Movies.loadClass(MovieClass);
const MovieModel = mongoose.model('Movies', Movies);
export default MovieModel;

26
src/sites/dandanzan.js
View File

@ -77,9 +77,10 @@ const parseOnePage = async ({subLink, category, sortIdx}) => {
category: category,
sortIdx: sortIdx,
is_new: true,
is_modify: true,
})
} else {
record.resources = resourceArr;
record.updateResources(resourceArr);
record.sortIdx = sortIdx;
}
await record.save();
@ -170,17 +171,18 @@ const parseAllMovie = async (category, beginNo = 1) => {
export default {
run: async () => {
return new Promise(async (resolve, reject) => {
generalQueue.setCb(function () {
console.log('====crawl movie finished', dateformat('yyyy-mm-dd HH:MM:ss'));
resolve()
})
console.log('====crawl movie begin', dateformat('yyyy-mm-dd HH:MM:ss'));
await parseAllMovie('movie');
await parseAllMovie('tv');
await parseAllMovie('show');
await parseAllMovie('cartoon');
})
// return new Promise(async (resolve, reject) => {
// generalQueue.setCb(function () {
// console.log('====crawl movie finished', dateformat('yyyy-mm-dd HH:MM:ss'));
// resolve()
// })
// console.log('====crawl movie begin', dateformat('yyyy-mm-dd HH:MM:ss'));
// await parseAllMovie('movie');
// await parseAllMovie('tv');
// await parseAllMovie('show');
// await parseAllMovie('cartoon');
// })
parseOnePage({subLink:'/dianshiju/31701.html', category:'tv', sortIdx:0});
},
parseListPage: parseListPage,