From 11b5584b29b2eb55afe138bd2906a3f64e14fd62 Mon Sep 17 00:00:00 2001
From: cebgcontract <99630598+cebgcontract@users.noreply.github.com>
Date: Wed, 21 Dec 2022 17:27:59 +0800
Subject: [PATCH] update sth

---
 src/sites/proxy.js      |  2 +-
 src/utils/net.util.js   | 21 ++++++++++
 src/utils/proxy.util.js | 87 +++++++++++++++++++++++++----------------
 3 files changed, 76 insertions(+), 34 deletions(-)

diff --git a/src/sites/proxy.js b/src/sites/proxy.js
index 63add0f..36ce6b4 100644
--- a/src/sites/proxy.js
+++ b/src/sites/proxy.js
@@ -179,7 +179,7 @@ export default {
   run: async () => {
     try {
       console.log('====begin update proxy:', dateformat('yyyy-mm-dd HH:MM:ss'));
-      await parseXiladaili(50);
+      // await parseXiladaili(50);
       // await parseXicidaili(20);
       await parseNimadailidaili(50);
       await checkAllProxy(true);
diff --git a/src/utils/net.util.js b/src/utils/net.util.js
index 3d91662..f0ceacd 100644
--- a/src/utils/net.util.js
+++ b/src/utils/net.util.js
@@ -2,6 +2,7 @@ import request from 'request';
 import Promise from 'bluebird';
 import random_useragent from 'random-useragent';
 import proxy from './proxy.util';
+import child_process from 'child_process'
 
 
 const iconv = require('iconv-lite');
@@ -22,6 +23,26 @@ const requestData = (options, encoding) => {
     });
   });
 }
+
+const curlData = (url) => {
+  let urlExec = `curl ${url}`
+  return new Promise((resolve, reject) => {
+    let child = child_process.exec(urlExec, function(err, stdout, stderr) {
+      console.log(stdout);
+      if (err) {
+        reject && reject(err)
+        return
+      }
+      if (stderr) {
+        reject && reject(stderr)
+        return
+      }
+      resolve && resolve(stdout)
+    });
+  })
+
+}
+
 export default {
   postForm(url, data, header, gzip) {
     header = header || {
diff --git a/src/utils/proxy.util.js b/src/utils/proxy.util.js
index ae210cd..45654ca 100644
--- a/src/utils/proxy.util.js
+++ b/src/utils/proxy.util.js
@@ -34,48 +34,69 @@ export default {
     if (proxys.length === 0) {
       proxys = await ProxyInfo.availableList();
     }
-    let proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
-    return new Promise(async (resolve, reject) => {
-      let response
-      try {
-        response = await request.get(url)
-          .set('User-Agent', random_useragent.getRandom())
-          .proxy(proxy)
-          .retry(2)
-          .timeout(15000);
-      } catch (err) {
-        console.log('parse page with network error, try again :', url);
+    if (proxys.length === 0) {
+      return new Promise(async (resolve, reject) => {
+        let response
         try {
-          proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
           response = await request.get(url)
             .set('User-Agent', random_useragent.getRandom())
-            .proxy(proxy)
-            .retry(2)
-            .timeout(15000);
-        } catch (err2) {
-          reject(err2)
-        }
-      }
-      if(response && response.statusCode === 200 ){
-        resolve(response.text);
-      } else {
-        console.log('parse page with statusCode: ', (response) && (response.statusCode), url);
-        try {
-          proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
-          response = await request.get(url)
-            .set('User-Agent', random_useragent.getRandom())
-            .proxy(proxy)
             .retry(2)
             .timeout(15000);
           if(response && response.statusCode === 200 ){
             resolve(response.text);
           } else {
-            reject(new Error('parse page with error statusCode: ' + (response) && (response.statusCode)))
+            reject && reject(response.statusCode)
           }
-        } catch (err2) {
-          reject(err2)
+          resolve(response.text);
+        } catch(err) {
+          reject && reject(err)
         }
-      }
-    })
+      })
+    } else {
+      let proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
+      return new Promise(async (resolve, reject) => {
+        let response
+        try {
+          response = await request.get(url)
+            .set('User-Agent', random_useragent.getRandom())
+            .proxy(proxy)
+            .retry(2)
+            .timeout(15000);
+        } catch (err) {
+          console.log('parse page with network error, try again :', url);
+          try {
+            proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
+            response = await request.get(url)
+              .set('User-Agent', random_useragent.getRandom())
+              .proxy(proxy)
+              .retry(2)
+              .timeout(15000);
+          } catch (err2) {
+            reject(err2)
+          }
+        }
+        if(response && response.statusCode === 200 ){
+          resolve(response.text);
+        } else {
+          console.log('parse page with statusCode: ', (response) && (response.statusCode), url);
+          try {
+            proxy = 'http://' + proxys[stringUtil.randomNum(0, proxys.length - 1)].link;
+            response = await request.get(url)
+              .set('User-Agent', random_useragent.getRandom())
+              .proxy(proxy)
+              .retry(2)
+              .timeout(15000);
+            if(response && response.statusCode === 200 ){
+              resolve(response.text);
+            } else {
+              reject(new Error('parse page with error statusCode: ' + (response) && (response.statusCode)))
+            }
+          } catch (err2) {
+            reject(err2)
+          }
+        }
+      })
+    }
+
   },
 }