from sys import int_info import pymongo import datetime from bson.objectid import ObjectId from bson import json_util import copy import re import pdb import json import requests import time mongo_info = { "host": 'localhost', "user": "admin", "pswd": "kingsome", "port": 27017, "db": "jump" } platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3" all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=30&offset={offset}&platForm={platformid}&systemList=&type=4&version=3" # all_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3" game_info_url = "https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id={gameid}&path=find_discount_gamedetail_switch&platform={platformid}&version=3" game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gameid}&platform={platformid}" game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all" mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port']) mongo_auth = mongo_client['admin'] mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd']) mongo_db = mongo_client.jump def set_jump_data() -> bool: if not set_platform(): print("get platform info failed!") else: if not set_all_game(): print("get game list failed!") return True def set_platform() -> bool: info = get_url_data(platform_url) for platform in info.get('data'): platform['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") platformAlias = platform.get('platformAlias') existing_platform = mongo_db["platform"].find_one( {"platformAlias": platformAlias}) if not existing_platform: new_platform = mongo_db["platform"].insert_one(platform) created_platform = mongo_db["platform"].find_one( {"_id": new_platform.inserted_id}) print(f"insert platform {created_platform}") else: print(f"key found with {existing_platform}") return True def set_all_game() -> bool: ids = mongo_db['platform'].find({}, { "_id": 0, "moduleId": 1, "gameNum": 1 }) for id in ids: platformid = id.get("moduleId") nums = int(id.get("gameNum")) ii = 0 i = 0 while 1: url = all_game_url.format(offset=ii, platformid=platformid) info = get_url_data(url) print(f"url={url}") if info.get("data"): for item in info.get("data"): i += 1 item['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") gameid = item.get("oldGameId") mongo_db['history_price'].update_one({'oldGameId': gameid}, {'$set': item}, upsert=True) else: print(info) if ii > nums: break ii += 30 print(f"total={i}") return True def set_history_price() -> bool: gameid_info = mongo_db['gameinfo'].find({}, { "_id": 0, "oldGameId": 1, "platform": 1, }) for line in gameid_info: try: oldGameId = line.get("oldGameId", 0) platform = line.get("platform", 0) url = game_history_price_url.format(gameid=oldGameId, platformid=platform) data = get_url_data(url).get('data') data['oldGameId'] = oldGameId data['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") mongo_db['history_price'].update_one({'oldGameId': oldGameId}, {'$set': data}, upsert=True) except: print(f"get history price error with {line} {url}") def get_url_data(url: str, data=None) -> json: requests.adapters.DEFAULT_RETRIES = 5 s = requests.session() s.keep_alive = False headers = {'Connection': 'close'} time.sleep(1) res = s.get(url=url, params=data, timeout=10, headers=headers).json() return res if __name__ == "__main__": #set_jump_data() set_history_price()