From 593bd3e7587ea88abacc0cc566327e6e5ff673e2 Mon Sep 17 00:00:00 2001 From: zhl Date: Tue, 7 May 2019 21:43:24 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=9B=8B=E8=9B=8B=E8=B5=9E?= =?UTF-8?q?=E7=94=B5=E5=BD=B1=E6=8A=93=E5=8F=96=E8=A7=84=E5=88=99=EF=BC=8C?= =?UTF-8?q?=E5=A6=82=E6=9E=9C=E5=BD=93=E5=89=8D=E6=95=B0=E6=8D=AE=E5=B7=B2?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=EF=BC=8C=E5=88=99=E5=8F=AA=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E6=92=AD=E6=94=BE=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sites/dandanzan.js | 66 +++++++++++++++------------------------- src/utils/string.util.js | 8 +++-- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/src/sites/dandanzan.js b/src/sites/dandanzan.js index d7a2875..4485b91 100644 --- a/src/sites/dandanzan.js +++ b/src/sites/dandanzan.js @@ -19,7 +19,8 @@ const parseOnePage = async (subLink, category) => { console.log(`>>>>>>>>>>>> ${url} not found`); } else { let resourceStr = stringUtil.getContentByReg(html,/links='(.+?)\|'/); - let idx = parseInt(subLink.replace('/dianying/', '').replace('.html', '') + let idx = parseInt(subLink.replace('/dianying/', '') + .replace('.html', '') .replace('/dongman/', '') .replace('/dianshiju/', '') .replace('/zongyi/', '')); @@ -36,22 +37,27 @@ const parseOnePage = async (subLink, category) => { let daoYan = $('meta[property="og:video:director"]').attr('content'); let zhuYan = $('meta[property="og:video:actor"]').attr('content').replace(/ \/ /g, ','); let region = $('meta[property="og:video:area"]').attr('content'); - let record = { - gid: idx, - name: $('meta[property="og:title"]').attr('content'), - type: typeArr, - resources: resourceArr, - daoYan: daoYan ? daoYan.split(',') : [], - zhuYan: zhuYan ? zhuYan.split(',') : [], - score: Number($('meta[property="og:video:score"]').attr('content')), - img: $('meta[property="og:image"]').attr('content'), - introduce: $('meta[property="og:description"]').attr('content'), - nameAlias: $('meta[property="og:video:alias"]').attr('content'), - region: region ? region.split(',') : [], - year: Number($('meta[property="og:video:release_date"]').attr('content')), - category: category, + let record = await Movie.findByGid(idx); + if (!record) { + record = { + gid: idx, + name: $('meta[property="og:title"]').attr('content'), + type: typeArr, + resources: resourceArr, + daoYan: daoYan ? daoYan.split(',') : [], + zhuYan: zhuYan ? zhuYan.split(',') : [], + score: Number($('meta[property="og:video:score"]').attr('content')), + img: $('meta[property="og:image"]').attr('content'), + introduce: $('meta[property="og:description"]').attr('content'), + nameAlias: $('meta[property="og:video:alias"]').attr('content'), + region: region ? region.split(',') : [], + year: Number($('meta[property="og:video:release_date"]').attr('content')), + category: category, + } + } else { + record.resources = resourceArr; } - await Movie.updateOne(idx, record); + await record.save(); console.log(`@@@@@ ${subLink} @ ${record.name} saved`); } } catch (err) { @@ -124,34 +130,12 @@ const parseAllMovie = async (category) => { export default { run: async () => { - // await parseAllMovie('movie'); + await parseAllMovie('movie'); // await parseAllMovie('tv'); // await parseAllMovie('show'); // await parseAllMovie('cartoon'); // console.log('all done'); - let html = await netUtil.getData('https://wechat-test.kingsome.cn/', {}) - console.log(html); - // var ProxyLists = require('proxy-lists'); - // - // var options = { - // countries: ['cn'], - // protocols: ['https'], - // }; - // - // var gettingProxies = ProxyLists.getProxies(options); - // gettingProxies.on('data', function(proxies) { - // // Received some proxies. - // console.log(proxies); - // }); - // - // gettingProxies.on('error', function(error) { - // // Some error has occurred. - // // console.error(error); - // }); - // - // gettingProxies.once('end', function() { - // // Done getting proxies. - // console.log('finish get proxy'); - // }); + // let html = await netUtil.getData('https://wechat-test.kingsome.cn/', {}) + // console.log(html); } } diff --git a/src/utils/string.util.js b/src/utils/string.util.js index a2a3a39..1129549 100644 --- a/src/utils/string.util.js +++ b/src/utils/string.util.js @@ -11,8 +11,12 @@ export default { removeHtml(content, replceEnter) { if (replceEnter) { - return content.replace(/<.+?>/g, '').replace(/\r\n/g, '
').replace(/\s/g, '') - .replace(/()+/g, '
').replace(/^/, '').replace(/$/, ''); + return content.replace(/<.+?>/g, '') + .replace(/\r\n/g, '
') + .replace(/\s/g, '') + .replace(/()+/g, '
') + .replace(/^/, '') + .replace(/$/, ''); } else { return content.replace(/<.+?>/g, '').replace(/\s/g, ''); }