node/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改蛋蛋赞电影抓取规则，如果当前数据已存在，则只更新播放链接

Browse Source
This commit is contained in:
zhl 2019-05-07 21:43:24 +08:00
parent 45d6c70775
commit 593bd3e758
2 changed files with 31 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
src/sites/dandanzan.js
View File

@ -19,7 +19,8 @@ const parseOnePage = async (subLink, category) => {
console.log(`>>>>>>>>>>>> ${url} not found`); console.log(`>>>>>>>>>>>> ${url} not found`);
} else { } else {
let resourceStr = stringUtil.getContentByReg(html,/links='(.+?)\|'/); let resourceStr = stringUtil.getContentByReg(html,/links='(.+?)\|'/);
let idx = parseInt(subLink.replace('/dianying/', '').replace('.html', '') let idx = parseInt(subLink.replace('/dianying/', '')
.replace('.html', '')
.replace('/dongman/', '') .replace('/dongman/', '')
.replace('/dianshiju/', '') .replace('/dianshiju/', '')
.replace('/zongyi/', '')); .replace('/zongyi/', ''));
@ -36,22 +37,27 @@ const parseOnePage = async (subLink, category) => {
let daoYan = $('meta[property="og:video:director"]').attr('content'); let daoYan = $('meta[property="og:video:director"]').attr('content');
let zhuYan = $('meta[property="og:video:actor"]').attr('content').replace(/ \/ /g, ','); let zhuYan = $('meta[property="og:video:actor"]').attr('content').replace(/ \/ /g, ',');
let region = $('meta[property="og:video:area"]').attr('content'); let region = $('meta[property="og:video:area"]').attr('content');
let record = { let record = await Movie.findByGid(idx);
gid: idx, if (!record) {
name: $('meta[property="og:title"]').attr('content'), record = {
type: typeArr, gid: idx,
resources: resourceArr, name: $('meta[property="og:title"]').attr('content'),
daoYan: daoYan ? daoYan.split(',') : [], type: typeArr,
zhuYan: zhuYan ? zhuYan.split(',') : [], resources: resourceArr,
score: Number($('meta[property="og:video:score"]').attr('content')), daoYan: daoYan ? daoYan.split(',') : [],
img: $('meta[property="og:image"]').attr('content'), zhuYan: zhuYan ? zhuYan.split(',') : [],
introduce: $('meta[property="og:description"]').attr('content'), score: Number($('meta[property="og:video:score"]').attr('content')),
nameAlias: $('meta[property="og:video:alias"]').attr('content'), img: $('meta[property="og:image"]').attr('content'),
region: region ? region.split(',') : [], introduce: $('meta[property="og:description"]').attr('content'),
year: Number($('meta[property="og:video:release_date"]').attr('content')), nameAlias: $('meta[property="og:video:alias"]').attr('content'),
category: category, region: region ? region.split(',') : [],
year: Number($('meta[property="og:video:release_date"]').attr('content')),
category: category,
}
} else {
record.resources = resourceArr;
} }
await Movie.updateOne(idx, record); await record.save();
console.log(`@@@@@ ${subLink} @ ${record.name} saved`); console.log(`@@@@@ ${subLink} @ ${record.name} saved`);
} }
} catch (err) { } catch (err) {
@ -124,34 +130,12 @@ const parseAllMovie = async (category) => {
export default { export default {
run: async () => { run: async () => {
// await parseAllMovie('movie'); await parseAllMovie('movie');
// await parseAllMovie('tv'); // await parseAllMovie('tv');
// await parseAllMovie('show'); // await parseAllMovie('show');
// await parseAllMovie('cartoon'); // await parseAllMovie('cartoon');
// console.log('all done'); // console.log('all done');
let html = await netUtil.getData('https://wechat-test.kingsome.cn/', {}) // let html = await netUtil.getData('https://wechat-test.kingsome.cn/', {})
console.log(html); // console.log(html);
// var ProxyLists = require('proxy-lists');
//
// var options = {
// countries: ['cn'],
// protocols: ['https'],
// };
//
// var gettingProxies = ProxyLists.getProxies(options);
// gettingProxies.on('data', function(proxies) {
// // Received some proxies.
// console.log(proxies);
// });
//
// gettingProxies.on('error', function(error) {
// // Some error has occurred.
// // console.error(error);
// });
//
// gettingProxies.once('end', function() {
// // Done getting proxies.
// console.log('finish get proxy');
// });
} }
} }

8
src/utils/string.util.js
View File

@ -11,8 +11,12 @@ export default {
removeHtml(content, replceEnter) { removeHtml(content, replceEnter) {
if (replceEnter) { if (replceEnter) {
return content.replace(/<.+?>/g, '').replace(/\r\n/g, '<br/>').replace(/\s/g, '') return content.replace(/<.+?>/g, '')
.replace(/(<br\/>)+/g, '<br/>').replace(/^<br\/>/, '').replace(/<br\/>$/, ''); .replace(/\r\n/g, '<br/>')
.replace(/\s/g, '')
.replace(/(<br\/>)+/g, '<br/>')
.replace(/^<br\/>/, '')
.replace(/<br\/>$/, '');
} else { } else {
return content.replace(/<.+?>/g, '').replace(/\s/g, ''); return content.replace(/<.+?>/g, '').replace(/\s/g, '');
} }