node/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of git.kingsome.cn:node/spider

Browse Source
This commit is contained in:
zhl 2019-05-09 12:00:05 +08:00
commit eec83ed4f0
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/utils/proxy.util.js
View File

@ -34,7 +34,7 @@ export default {
if (proxys.length === 0) {
proxys = await ProxyInfo.availableList();
}
const proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
let proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
return new Promise(async (resolve, reject) => {
let response
try {
@ -46,6 +46,7 @@ export default {
} catch (err) {
console.log('parse page with network error, try again :', url);
try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
@ -60,6 +61,7 @@ export default {
} else {
console.log('parse page with statusCode: ', statusCode, url);
try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)