diff --git a/src/sites/dandanzan.js b/src/sites/dandanzan.js index d7a2875..4485b91 100644 --- a/src/sites/dandanzan.js +++ b/src/sites/dandanzan.js @@ -19,7 +19,8 @@ const parseOnePage = async (subLink, category) => { console.log(`>>>>>>>>>>>> ${url} not found`); } else { let resourceStr = stringUtil.getContentByReg(html,/links='(.+?)\|'/); - let idx = parseInt(subLink.replace('/dianying/', '').replace('.html', '') + let idx = parseInt(subLink.replace('/dianying/', '') + .replace('.html', '') .replace('/dongman/', '') .replace('/dianshiju/', '') .replace('/zongyi/', '')); @@ -36,22 +37,27 @@ const parseOnePage = async (subLink, category) => { let daoYan = $('meta[property="og:video:director"]').attr('content'); let zhuYan = $('meta[property="og:video:actor"]').attr('content').replace(/ \/ /g, ','); let region = $('meta[property="og:video:area"]').attr('content'); - let record = { - gid: idx, - name: $('meta[property="og:title"]').attr('content'), - type: typeArr, - resources: resourceArr, - daoYan: daoYan ? daoYan.split(',') : [], - zhuYan: zhuYan ? zhuYan.split(',') : [], - score: Number($('meta[property="og:video:score"]').attr('content')), - img: $('meta[property="og:image"]').attr('content'), - introduce: $('meta[property="og:description"]').attr('content'), - nameAlias: $('meta[property="og:video:alias"]').attr('content'), - region: region ? region.split(',') : [], - year: Number($('meta[property="og:video:release_date"]').attr('content')), - category: category, + let record = await Movie.findByGid(idx); + if (!record) { + record = { + gid: idx, + name: $('meta[property="og:title"]').attr('content'), + type: typeArr, + resources: resourceArr, + daoYan: daoYan ? daoYan.split(',') : [], + zhuYan: zhuYan ? zhuYan.split(',') : [], + score: Number($('meta[property="og:video:score"]').attr('content')), + img: $('meta[property="og:image"]').attr('content'), + introduce: $('meta[property="og:description"]').attr('content'), + nameAlias: $('meta[property="og:video:alias"]').attr('content'), + region: region ? region.split(',') : [], + year: Number($('meta[property="og:video:release_date"]').attr('content')), + category: category, + } + } else { + record.resources = resourceArr; } - await Movie.updateOne(idx, record); + await record.save(); console.log(`@@@@@ ${subLink} @ ${record.name} saved`); } } catch (err) { @@ -124,34 +130,12 @@ const parseAllMovie = async (category) => { export default { run: async () => { - // await parseAllMovie('movie'); + await parseAllMovie('movie'); // await parseAllMovie('tv'); // await parseAllMovie('show'); // await parseAllMovie('cartoon'); // console.log('all done'); - let html = await netUtil.getData('https://wechat-test.kingsome.cn/', {}) - console.log(html); - // var ProxyLists = require('proxy-lists'); - // - // var options = { - // countries: ['cn'], - // protocols: ['https'], - // }; - // - // var gettingProxies = ProxyLists.getProxies(options); - // gettingProxies.on('data', function(proxies) { - // // Received some proxies. - // console.log(proxies); - // }); - // - // gettingProxies.on('error', function(error) { - // // Some error has occurred. - // // console.error(error); - // }); - // - // gettingProxies.once('end', function() { - // // Done getting proxies. - // console.log('finish get proxy'); - // }); + // let html = await netUtil.getData('https://wechat-test.kingsome.cn/', {}) + // console.log(html); } } diff --git a/src/utils/string.util.js b/src/utils/string.util.js index a2a3a39..1129549 100644 --- a/src/utils/string.util.js +++ b/src/utils/string.util.js @@ -11,8 +11,12 @@ export default { removeHtml(content, replceEnter) { if (replceEnter) { - return content.replace(/<.+?>/g, '').replace(/\r\n/g, '
').replace(/\s/g, '') - .replace(/()+/g, '
').replace(/^/, '').replace(/$/, ''); + return content.replace(/<.+?>/g, '') + .replace(/\r\n/g, '
') + .replace(/\s/g, '') + .replace(/()+/g, '
') + .replace(/^/, '') + .replace(/$/, ''); } else { return content.replace(/<.+?>/g, '').replace(/\s/g, ''); }