node/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

重试时更换proxy

Browse Source
This commit is contained in:
zhl 2019-05-09 11:59:54 +08:00
parent 6e4ef6793a
commit 4d7280079f
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/utils/proxy.util.js
View File

@ -34,7 +34,7 @@ export default {
if (proxys.length === 0) { if (proxys.length === 0) {
proxys = await ProxyInfo.availableList(); proxys = await ProxyInfo.availableList();
} }
const proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link; let proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
let response let response
try { try {
@ -46,6 +46,7 @@ export default {
} catch (err) { } catch (err) {
console.log('parse page with network error, try again :', url); console.log('parse page with network error, try again :', url);
try { try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url) response = await request.get(url)
.set('User-Agent', random_useragent.getRandom()) .set('User-Agent', random_useragent.getRandom())
.proxy(proxy) .proxy(proxy)
@ -60,6 +61,7 @@ export default {
} else { } else {
console.log('parse page with statusCode: ', statusCode, url); console.log('parse page with statusCode: ', statusCode, url);
try { try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url) response = await request.get(url)
.set('User-Agent', random_useragent.getRandom()) .set('User-Agent', random_useragent.getRandom())
.proxy(proxy) .proxy(proxy)