node/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改抓取重试逻辑

Browse Source
This commit is contained in:
zhl 2019-05-09 11:57:23 +08:00
parent d9f106d9ef
commit 6e4ef6793a
2 changed files with 45 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/sites/dandanzan.js
View File

@ -25,16 +25,7 @@ const parseOnePage = async ({subLink, category, sortIdx}) => {
lastStatus: true,
})
} catch(err) {
console.log('parse page with network error, try again :', url);
try {
html = await proxyUtil.getDataProxy(url)
await CrawlRecord.updateRecord({url: url,
className: 'dandanzan',
methodName: 'parseOnePage',
params: {subLink, category, sortIdx},
lastStatus: true,
})
} catch (err2) {
console.log('parse page with network error:', url);
await CrawlRecord.updateRecord({url: url,
className: 'dandanzan',
methodName: 'parseOnePage',
@ -42,7 +33,6 @@ const parseOnePage = async ({subLink, category, sortIdx}) => {
lastStatus: false,
})
}
}
if (!html) {
return false;
}
@ -127,16 +117,7 @@ const parseListPage = async ({idx, category}) => {
lastStatus: true,
})
} catch (err) {
console.log('parse page with network error, try again :', url);
try {
html = await proxyUtil.getDataProxy(url)
await CrawlRecord.updateRecord({url: url,
className: 'dandanzan',
methodName: 'parseListPage',
params: {idx, category},
lastStatus: true,
})
} catch (err2) {
console.log('parse page with network error:', url);
await CrawlRecord.updateRecord({url: url,
className: 'dandanzan',
methodName: 'parseListPage',
@ -144,7 +125,6 @@ const parseListPage = async ({idx, category}) => {
lastStatus: false,
})
}
}
if (html) {
const $ = cheerio.load(html);
let hrefs = $('.thumbnail');

32
src/utils/proxy.util.js
View File

@ -36,8 +36,31 @@ export default {
}
const proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
return new Promise(async (resolve, reject) => {
let response
try {
let response = await request.get(url)
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err) {
console.log('parse page with network error, try again :', url);
try {
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err2) {
reject(err2)
}
}
if(response.statusCode === 200 ){
resolve(response.text);
} else {
console.log('parse page with statusCode: ', statusCode, url);
try {
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
@ -45,10 +68,11 @@ export default {
if(response.statusCode === 200 ){
resolve(response.text);
} else {
reject(new Error(' server response code: ' + response.statusCode));
reject(new Error('parse page with error statusCode: ' + response.statusCode))
}
} catch (err2) {
reject(err2)
}
} catch (err) {
reject(err);
}
})
},