node/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

抓取蛋蛋赞网站前更新代理

Browse Source
This commit is contained in:
zhl 2019-05-08 21:04:44 +08:00
parent c2add8ff00
commit c25eb6d213
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/app.js
View File

@ -24,8 +24,8 @@ db.once('open', function () {
// book.run(); // book.run();
// movie.run(); // movie.run();
// bookChapter.run(); // bookChapter.run();
// dandanzan.run(); dandanzan.run();
proxy.run(); // proxy.run();
}); });
mongoose.connect(config.db, {promiseLibrary: Promise, useNewUrlParser: true}); mongoose.connect(config.db, {promiseLibrary: Promise, useNewUrlParser: true});

2
src/sites/dandanzan.js
View File

@ -3,6 +3,7 @@ import cheerio from "cheerio";
import stringUtil from '../utils/string.util'; import stringUtil from '../utils/string.util';
import Movie from '../models/Movies'; import Movie from '../models/Movies';
import generalQueue from '../utils/general.queue'; import generalQueue from '../utils/general.queue';
import proxy from './proxy';
const URL_BASE = 'https://www.dandanzan.com' const URL_BASE = 'https://www.dandanzan.com'
const maxIdx = 100000; const maxIdx = 100000;
@ -145,6 +146,7 @@ const parseAllMovie = async (category) => {
export default { export default {
run: async () => { run: async () => {
await proxy.run();
await parseAllMovie('movie'); await parseAllMovie('movie');
await parseAllMovie('tv'); await parseAllMovie('tv');
await parseAllMovie('show'); await parseAllMovie('show');