node/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update sth

Browse Source
This commit is contained in:
cebgcontract 2022-12-21 17:27:59 +08:00
parent 50d9bd1d46
commit 11b5584b29
3 changed files with 76 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/sites/proxy.js
View File

@ -179,7 +179,7 @@ export default {
run: async () => { run: async () => {
try { try {
console.log('====begin update proxy:', dateformat('yyyy-mm-dd HH:MM:ss')); console.log('====begin update proxy:', dateformat('yyyy-mm-dd HH:MM:ss'));
await parseXiladaili(50); // await parseXiladaili(50);
// await parseXicidaili(20); // await parseXicidaili(20);
await parseNimadailidaili(50); await parseNimadailidaili(50);
await checkAllProxy(true); await checkAllProxy(true);

21
src/utils/net.util.js
View File

@ -2,6 +2,7 @@ import request from 'request';
import Promise from 'bluebird'; import Promise from 'bluebird';
import random_useragent from 'random-useragent'; import random_useragent from 'random-useragent';
import proxy from './proxy.util'; import proxy from './proxy.util';
import child_process from 'child_process'
const iconv = require('iconv-lite'); const iconv = require('iconv-lite');
@ -22,6 +23,26 @@ const requestData = (options, encoding) => {
}); });
}); });
} }
const curlData = (url) => {
let urlExec = `curl ${url}`
return new Promise((resolve, reject) => {
let child = child_process.exec(urlExec, function(err, stdout, stderr) {
console.log(stdout);
if (err) {
reject && reject(err)
return
}
if (stderr) {
reject && reject(stderr)
return
}
resolve && resolve(stdout)
});
})
}
export default { export default {
postForm(url, data, header, gzip) { postForm(url, data, header, gzip) {
header = header || { header = header || {

87
src/utils/proxy.util.js
View File

@ -34,48 +34,69 @@ export default {
if (proxys.length === 0) { if (proxys.length === 0) {
proxys = await ProxyInfo.availableList(); proxys = await ProxyInfo.availableList();
} }
let proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link; if (proxys.length === 0) {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
let response let response
try {
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err) {
console.log('parse page with network error, try again :', url);
try { try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url) response = await request.get(url)
.set('User-Agent', random_useragent.getRandom()) .set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err2) {
reject(err2)
}
}
if(response && response.statusCode === 200 ){
resolve(response.text);
} else {
console.log('parse page with statusCode: ', (response) && (response.statusCode), url);
try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2) .retry(2)
.timeout(15000); .timeout(15000);
if(response && response.statusCode === 200 ){ if(response && response.statusCode === 200 ){
resolve(response.text); resolve(response.text);
} else { } else {
reject(new Error('parse page with error statusCode: ' + (response) && (response.statusCode))) reject && reject(response.statusCode)
} }
} catch (err2) { resolve(response.text);
reject(err2) } catch(err) {
reject && reject(err)
} }
} })
}) } else {
let proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
return new Promise(async (resolve, reject) => {
let response
try {
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err) {
console.log('parse page with network error, try again :', url);
try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
} catch (err2) {
reject(err2)
}
}
if(response && response.statusCode === 200 ){
resolve(response.text);
} else {
console.log('parse page with statusCode: ', (response) && (response.statusCode), url);
try {
proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
response = await request.get(url)
.set('User-Agent', random_useragent.getRandom())
.proxy(proxy)
.retry(2)
.timeout(15000);
if(response && response.statusCode === 200 ){
resolve(response.text);
} else {
reject(new Error('parse page with error statusCode: ' + (response) && (response.statusCode)))
}
} catch (err2) {
reject(err2)
}
}
})
}
}, },
} }