node/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改抓取重试逻辑

Browse Source
This commit is contained in:
zhl 2019-05-09 11:57:23 +08:00
parent d9f106d9ef
commit 6e4ef6793a
2 changed files with 45 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/sites/dandanzan.js
View File

@ -25,23 +25,13 @@ const parseOnePage = async ({subLink, category, sortIdx}) => {
lastStatus: true, lastStatus: true,
}) })
} catch(err) { } catch(err) {
console.log('parse page with network error, try again :', url); console.log('parse page with network error:', url);
try { await CrawlRecord.updateRecord({url: url,
html = await proxyUtil.getDataProxy(url) className: 'dandanzan',
await CrawlRecord.updateRecord({url: url, methodName: 'parseOnePage',
className: 'dandanzan', params: {subLink, category, sortIdx},
methodName: 'parseOnePage', lastStatus: false,
params: {subLink, category, sortIdx}, })
lastStatus: true,
})
} catch (err2) {
await CrawlRecord.updateRecord({url: url,
className: 'dandanzan',
methodName: 'parseOnePage',
params: {subLink, category, sortIdx},
lastStatus: false,
})
}
} }
if (!html) { if (!html) {
return false; return false;
@ -127,23 +117,13 @@ const parseListPage = async ({idx, category}) => {
lastStatus: true, lastStatus: true,
}) })
} catch (err) { } catch (err) {
console.log('parse page with network error, try again :', url); console.log('parse page with network error:', url);
try { await CrawlRecord.updateRecord({url: url,
html = await proxyUtil.getDataProxy(url) className: 'dandanzan',
await CrawlRecord.updateRecord({url: url, methodName: 'parseListPage',
className: 'dandanzan', params: {idx, category},
methodName: 'parseListPage', lastStatus: false,
params: {idx, category}, })
lastStatus: true,
})
} catch (err2) {
await CrawlRecord.updateRecord({url: url,
className: 'dandanzan',
methodName: 'parseListPage',
params: {idx, category},
lastStatus: false,
})
}
} }
if (html) { if (html) {
const $ = cheerio.load(html); const $ = cheerio.load(html);

38
src/utils/proxy.util.js
View File

@ -36,19 +36,43 @@ export default {
} }
const proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link; const proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
let response
try { try {
let response = await request.get(url) response = await request.get(url)
.set('User-Agent', random_useragent.getRandom()) .set('User-Agent', random_useragent.getRandom())
.proxy(proxy) .proxy(proxy)
.retry(2) .retry(2)
.timeout(15000); .timeout(15000);
if(response.statusCode === 200 ){
resolve(response.text);
} else {
reject(new Error(' server response code: ' + response.statusCode));
}
} catch (err) { } catch (err) {
reject(err); console.log('parse page with network error, try again :', url);
try {
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err2) {
reject(err2)
}
}
if(response.statusCode === 200 ){
resolve(response.text);
} else {
console.log('parse page with statusCode: ', statusCode, url);
try {
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
if(response.statusCode === 200 ){
resolve(response.text);
} else {
reject(new Error('parse page with error statusCode: ' + response.statusCode))
}
} catch (err2) {
reject(err2)
}
} }
}) })
}, },