import stringUtil from './string.util'; import ProxyInfo from '../models/spider/ProxyInfo'; import Promise from 'bluebird'; import random_useragent from 'random-useragent'; let request = require('superagent'); require('superagent-proxy')(request); let proxys = []; export default { async randomProxy() { if (proxys.length === 0) { proxys = await ProxyInfo.availableList(); } return proxys[stringUtil.randomNum(0, proxys.length - 1)].link; }, async checkProxy(proxy){ return new Promise(async (resolve, reject) => { try { let response = await request.get('https://wechat-test.kingsome.cn/api/stat').proxy(proxy).timeout(5000); if(response.statusCode === 200 && response.text === '1' ){ resolve(); } else { reject(new Error(' server response code: ' + response.statusCode)); } } catch (err) { reject(err); } }) }, // 通过代理来get数据 async getDataProxy(url) { if (proxys.length === 0) { proxys = await ProxyInfo.availableList(); } const proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link; return new Promise(async (resolve, reject) => { let response try { response = await request.get(url) .set('User-Agent', random_useragent.getRandom()) .proxy(proxy) .retry(2) .timeout(15000); } catch (err) { console.log('parse page with network error, try again :', url); try { response = await request.get(url) .set('User-Agent', random_useragent.getRandom()) .proxy(proxy) .retry(2) .timeout(15000); } catch (err2) { reject(err2) } } if(response.statusCode === 200 ){ resolve(response.text); } else { console.log('parse page with statusCode: ', statusCode, url); try { response = await request.get(url) .set('User-Agent', random_useragent.getRandom()) .proxy(proxy) .retry(2) .timeout(15000); if(response.statusCode === 200 ){ resolve(response.text); } else { reject(new Error('parse page with error statusCode: ' + response.statusCode)) } } catch (err2) { reject(err2) } } }) }, }