import pymongo import datetime import json import requests import time import os def get_log(log_path_files="/data/logs/ops/ops.log"): from loguru import logger logger.add(log_path_files, rotation="500 MB", enqueue=True) return logger if os.getenv('env') == 'dev': logger = get_log("get_jump.log") mongo_info = { "host": 'localhost', "user": "admin", "pswd": "kingsome", "port": 27017, "db": "jump" } mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port']) mongo_auth = mongo_client['admin'] mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd']) mongo_db = mongo_client.jump else: logger = get_log("/data/logs/ops/get_jump.log") mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"} mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port']) mongo_db = mongo_client.jump # 即将推出:https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=1&version=16 # 热门新游:https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=2&version=16 # 正在流行:https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=3&version=16 platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3" discount_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3" all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=30&offset={offset}&platForm={platformid}&systemList=&type=4&version=3" game_info_url = "https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id={gameid}&path=find_discount_gamedetail_switch&platform={platformid}&version=3" game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gameid}&platform={platformid}" game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all" def set_jump_data() -> bool: if not set_platform(): logger.error("get platform info failed!") else: if not set_all_game(): logger.error("get game list failed!") return True def set_platform() -> bool: info = get_url_data(platform_url) for platform in info.get('data'): try: platform['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") platformAlias = platform.get('platformAlias') mongo_db['platform'].update_one({'platformAlias': platformAlias}, {'$set': platform}, upsert=True) except: logger.error(f"some error with set platform with {platform}") return True def set_all_game() -> bool: ids = mongo_db['platform'].find({}, { "_id": 0, "moduleId": 1, "gameNum": 1 }) for id in ids: platformid = id.get("moduleId") ii = 0 while 1: url = all_game_url.format(offset=ii, platformid=platformid) info = get_url_data(url) ii += 30 try: if info.get("data"): for item in info.get("data"): item['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") gameid = item.get("oldGameId") cutOff = item.get("cutOff") m_cutOff = mongo_db['gameinfo'].find_one( {"oldGameId": gameid}, { "_id": 0, "cutOff": 1, "cuttime": 1 }) if not m_cutOff: mm_cutOff = 0 item['cuttime'] = 0 else: mm_cutOff = m_cutOff.get('cutOff') # 判断是否打折,修改cuttime数值 if not cutOff: item['cuttime'] = 0 elif mm_cutOff == 0 and cutOff > 0: item['cuttime'] = item['uptime'] else: pass mongo_db['gameinfo'].update_one({'oldGameId': gameid}, {'$set': item}, upsert=True) else: logger.debug(f"get {url} {info}") break except: logger.error(f"get gameinfo failed with {info}") logger.info(f"platformid = {id} \ttotal={ii}") return True def set_gameinfo_ext() -> bool: gameid_info = mongo_db['gameinfo'].find({}, { "_id": 0, "oldGameId": 1, "platform": 1, }) i = 0 for line in gameid_info: try: oldGameId = line.get("oldGameId", 0) platform = line.get("platform", 0) url = game_info_url.format(gameid=oldGameId, platformid=platform) data = get_url_data(url=url, timesleep=1).get('data') logger.info(f"start get {oldGameId} {platform} game info ext!") data['oldGameId'] = oldGameId data['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") mongo_db['gameinfoext'].update_one({'oldGameId': oldGameId}, {'$set': data}, upsert=True) i += 1 logger.info(f"collect gameinfoext {i}") except Exception as e: logger.error(f"get game info error with {line} {url} {e}") logger.info(f"get game ext info total= {i}!") def set_game_price() -> bool: gameid_info = mongo_db['gameinfo'].find({}, { "_id": 0, "oldGameId": 1, "platform": 1, }) i = 0 for line in gameid_info: try: oldGameId = line.get("oldGameId", 0) platform = line.get("platform", 0) logger.info(f"start get {oldGameId} {platform} game price!") url = game_price_url.format(gameid=oldGameId, platformid=platform) data = get_url_data(url).get('data') data['oldGameId'] = oldGameId data['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") mongo_db['gameprice'].update_one({'oldGameId': oldGameId}, {'$set': data}, upsert=True) i += 1 except: logger.error(f"get game price error with {line} {url}") logger.info(f"get game price total={i} ") def set_history_price() -> bool: gameid_info = mongo_db['gameinfo'].find({}, { "_id": 0, "oldGameId": 1, "platform": 1, }) i = 0 for line in gameid_info: try: oldGameId = line.get("oldGameId", 0) platform = line.get("platform", 0) logger.info(f"start get {oldGameId} {platform} history price!") url = game_history_price_url.format(gameid=oldGameId, platformid=platform) data = get_url_data(url).get('data') data['oldGameId'] = oldGameId data['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") mongo_db['history_price'].update_one({'oldGameId': oldGameId}, {'$set': data}, upsert=True) i += 1 except: logger.error(f"get history price error with {line} {url}") logger.info(f"get history price total={i}") def set_discount() -> bool: collections = mongo_db.list_collection_names() if "discount_price" in collections: logger.debug(f"find discount_price remove it \n {collections}") mongo_db.drop_collection("discount_price") ids = mongo_db['platform'].find({}, {"_id": 0, "moduleId": 1}) #ids = [{"moduleId": 1}] for id in ids: platformid = id.get("moduleId") i = 0 while 1: url = discount_game_url.format(offset=i, platformid=platformid) info = get_url_data(url) i += 10 try: if info.get("data"): for item in info.get("data"): item['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") gameid = item.get("oldGameId") if item.get("platform") == 53: item["platform"] = 52 mongo_db['discount_price'].update_one( {'oldGameId': gameid}, {'$set': item}, upsert=True) else: logger.debug(f"get {url} {info}") break except: logger.error(f"get gameinfo failed with {info}") logger.info(f"platformid = {id} \ttotal={i}") return True def get_url_data(url: str, timesleep=0.5, data=None) -> json: requests.adapters.DEFAULT_RETRIES = 5 s = requests.session() s.keep_alive = False headers = {'Connection': 'close'} time.sleep(timesleep) res = s.get(url=url, params=data, timeout=10, headers=headers).json() return res def main_handler(event, context): set_jump_data() time.sleep(5) set_history_price() time.sleep(1) #set_discount() #time.sleep(1) set_game_price() time.sleep(1) set_gameinfo_ext() if __name__ == '__main__': main_handler("", "")