1

config/config.py

@@ -10,7 +10,13 @@ class Settings(BaseSettings):
     admin_email: str = "pengtao@kingsome.cn"
     items_per_user: int = 50
     is_debug: bool = True
-
+    mongo_info = {
+        "host": 'localhost',
+        "user": "admin",
+        "pswd": "kingsome",
+        "port": 27017,
+        "db": "jump"
+    }
     MONGODB_URL = "mongodb://admin:kingsome@localhost"
     platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3"
     all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset={offset}&platForm={platformid}&systemList=&type=4&version=3"

scripts/get_jump_data.py

@@ -0,0 +1,132 @@
+from sys import int_info
+import pymongo
+import datetime
+from bson.objectid import ObjectId
+from bson import json_util
+import copy
+import re
+import pdb
+import json
+import requests
+import time
+
+mongo_info = {
+    "host": 'localhost',
+    "user": "admin",
+    "pswd": "kingsome",
+    "port": 27017,
+    "db": "jump"
+}
+platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3"
+all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=30&offset={offset}&platForm={platformid}&systemList=&type=4&version=3"
+# all_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3"
+
+game_info_url = "https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id={gameid}&path=find_discount_gamedetail_switch&platform={platformid}&version=3"
+game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gameid}&platform={platformid}"
+game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all"
+
+mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
+mongo_auth = mongo_client['admin']
+mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd'])
+mongo_db = mongo_client.jump
+
+
+def set_jump_data() -> bool:
+    if not set_platform():
+        print("get platform info failed!")
+    else:
+        if not set_all_game():
+            print("get game list failed!")
+
+    return True
+
+
+def set_platform() -> bool:
+    info = get_url_data(platform_url)
+    for platform in info.get('data'):
+        platform['uptime'] = datetime.datetime.strftime(
+            datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
+        platformAlias = platform.get('platformAlias')
+        existing_platform = mongo_db["platform"].find_one(
+            {"platformAlias": platformAlias})
+        if not existing_platform:
+            new_platform = mongo_db["platform"].insert_one(platform)
+            created_platform = mongo_db["platform"].find_one(
+                {"_id": new_platform.inserted_id})
+            print(f"insert platform {created_platform}")
+        else:
+            print(f"key found with {existing_platform}")
+    return True
+
+
+def set_all_game() -> bool:
+    ids = mongo_db['platform'].find({}, {
+        "_id": 0,
+        "moduleId": 1,
+        "gameNum": 1
+    })
+
+    for id in ids:
+        platformid = id.get("moduleId")
+        nums = int(id.get("gameNum"))
+        ii = 0
+        i = 0
+        while 1:
+            url = all_game_url.format(offset=ii, platformid=platformid)
+            info = get_url_data(url)
+            print(f"url={url}")
+            if info.get("data"):
+                for item in info.get("data"):
+                    i += 1
+                    item['uptime'] = datetime.datetime.strftime(
+                        datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
+                    gameid = item.get("oldGameId")
+                    mongo_db['history_price'].update_one({'oldGameId': gameid},
+                                                         {'$set': item},
+                                                         upsert=True)
+            else:
+                print(info)
+            if ii > nums:
+                break
+            ii += 30
+            print(f"total={i}")
+    return True
+
+
+def set_history_price() -> bool:
+    gameid_info = mongo_db['gameinfo'].find({}, {
+        "_id": 0,
+        "oldGameId": 1,
+        "platform": 1,
+    })
+    for line in gameid_info:
+        try:
+            oldGameId = line.get("oldGameId", 0)
+            platform = line.get("platform", 0)
+            url = game_history_price_url.format(gameid=oldGameId,
+                                                platformid=platform)
+
+            data = get_url_data(url).get('data')
+            data['oldGameId'] = oldGameId
+            data['uptime'] = datetime.datetime.strftime(
+                datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
+            mongo_db['history_price'].update_one({'oldGameId': oldGameId},
+                                                 {'$set': data},
+                                                 upsert=True)
+        except:
+            print(f"get history price error with {line} {url}")
+
+
+def get_url_data(url: str, data=None) -> json:
+    requests.adapters.DEFAULT_RETRIES = 5
+    s = requests.session()
+    s.keep_alive = False
+    headers = {'Connection': 'close'}
+    time.sleep(1)
+    res = s.get(url=url, params=data, timeout=10, headers=headers).json()
+    return res
+
+
+if __name__ == "__main__":
+    #set_jump_data()
+    set_history_price()

scripts/sync_data.py

@@ -2,45 +2,73 @@ import json
 from pydantic.fields import T
 from config.config import settings
 import requests
-from starlette.requests import Request
-from starlette.responses import JSONResponse
-from fastapi.encoders import jsonable_encoder
 from scripts.logger import logger
+import time
 
 
 async def get_jump_data(db) -> bool:
-    platform_info = get_platform()
-    platforms = platform_info.get('data')
-    for platform in platforms:
+    if not await get_platform(db):
+        logger.error("get platform info failed!")
+    else:
+        if not await get_all_game(db):
+            logger.error("get game list failed!")
+
+    return True
+
+
+async def get_platform(db) -> json:
+    info = get_url_data(settings.platform_url)
+    for platform in info.get('data'):
         platformAlias = platform.get('platformAlias')
         existing_platform = await db["platform"].find_one(
             {"platformAlias": platformAlias})
         if not existing_platform:
-            platform = jsonable_encoder(a)
+            #platform = jsonable_encoder(platform)
             new_platform = await db["platform"].insert_one(platform)
             created_platform = await db["platform"].find_one(
                 {"_id": new_platform.inserted_id})
             logger.info(f"insert platform {created_platform}")
         else:
             logger.info(f"key found with {existing_platform}")
-
-    # all_game = get_all_game()
-    # #gameids = all_game['data']
-    # print(all_game)
     return True
-    # for gameid in gameids:
-    #     game_info = get_game_info(gameid)
 
 
-def get_platform() -> json:
-    info = get_url_data(settings.platform_url)
-    return info
+async def get_all_game(db) -> json:
+    ids = await db['platform'].find({}, {
+        "_id": 0,
+        "moduleId": 1,
+        "gameNum": 1
+    }).to_list(10)
+    for id in ids:
+        platformid = id.get("moduleId")
+        nums = int(id.get("gameNum")) // 10 + 1
+        for i in range(0, nums):
 
+            url = settings.all_game_url.format(offset=i, platformid=platformid)
+            info = get_url_data(url)
+            #logger.debug(info.get("data"))
+            ll = len(info.get("data"))
+            logger.debug(f"collect {nums}\t, {i},{url},{ll}")
+            if info.get("data"):
+                for item in info.get("data"):
+                    gameid = item.get("oldGameId", 0)
+                    existing_gameid = await db["gameinfo"].find_one(
+                        {"oldGameId": gameid})
+                    if not existing_gameid:
+                        #logger.debug(item)
+                        new_gameinfo = await db["gameinfo"].insert_one(item)
+                        created_gameinfo = await db["gameinfo"].find_one(
+                            {"_id": new_gameinfo.inserted_id})
+                        # logger.info(f"insert gameinfo {created_gameinfo}")
+                    else:
+                        # logger.info(
+                        #     f"key found with {existing_gameid} in gameinfo")
+                        await db["gameinfo"].update_one({"oldGameId": gameid},
+                                                        {"$set": item})
+            else:
+                break
 
-def get_all_game() -> json:
-    info = get_url_data(settings.all_game_url)
-    print(info)
-    return None
+    return True
 
 
 def get_game_info(gameid: int) -> dict:
@@ -60,5 +88,6 @@ def get_url_data(url: str, data=None) -> json:
     s = requests.session()
     s.keep_alive = False
     headers = {'Connection': 'close'}
+    time.sleep(1)
     res = s.get(url=url, params=data, timeout=10, headers=headers).json()
     return res