node/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update sth

Browse Source
This commit is contained in:
cebgcontract 2022-12-21 17:27:59 +08:00
parent 50d9bd1d46
commit 11b5584b29
3 changed files with 76 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/sites/proxy.js
View File

@ -179,7 +179,7 @@ export default {
run: async () => {
try {
console.log('====begin update proxy:', dateformat('yyyy-mm-dd HH:MM:ss'));
await parseXiladaili(50);
// await parseXiladaili(50);
// await parseXicidaili(20);
await parseNimadailidaili(50);
await checkAllProxy(true);

21
src/utils/net.util.js
View File

@ -2,6 +2,7 @@ import request from 'request';
import Promise from 'bluebird';
import random_useragent from 'random-useragent';
import proxy from './proxy.util';
import child_process from 'child_process'
const iconv = require('iconv-lite');
@ -22,6 +23,26 @@ const requestData = (options, encoding) => {
});
});
}
const curlData = (url) => {
let urlExec = `curl ${url}`
return new Promise((resolve, reject) => {
let child = child_process.exec(urlExec, function(err, stdout, stderr) {
console.log(stdout);
if (err) {
reject && reject(err)
return
}
if (stderr) {
reject && reject(stderr)
return
}
resolve && resolve(stdout)
});
})
}
export default {
postForm(url, data, header, gzip) {
header = header || {

87
src/utils/proxy.util.js
View File

@ -34,48 +34,69 @@ export default {
if (proxys.length === 0) {
proxys = await ProxyInfo.availableList();
}
let proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
return new Promise(async (resolve, reject) => {
let response
try {
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err) {
console.log('parse page with network error, try again :', url);
if (proxys.length === 0) {
return new Promise(async (resolve, reject) => {
let response
try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err2) {
reject(err2)
}
}
if(response && response.statusCode === 200 ){
resolve(response.text);
} else {
console.log('parse page with statusCode: ', (response) && (response.statusCode), url);
try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
if(response && response.statusCode === 200 ){
resolve(response.text);
} else {
reject(new Error('parse page with error statusCode: ' + (response) && (response.statusCode)))
reject && reject(response.statusCode)
}
} catch (err2) {
reject(err2)
resolve(response.text);
} catch(err) {
reject && reject(err)
}
}
})
})
} else {
let proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
return new Promise(async (resolve, reject) => {
let response
try {
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err) {
console.log('parse page with network error, try again :', url);
try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err2) {
reject(err2)
}
}
if(response && response.statusCode === 200 ){
resolve(response.text);
} else {
console.log('parse page with statusCode: ', (response) && (response.statusCode), url);
try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
if(response && response.statusCode === 200 ){
resolve(response.text);
} else {
reject(new Error('parse page with error statusCode: ' + (response) && (response.statusCode)))
}
} catch (err2) {
reject(err2)
}
}
})
}
},
}