diff --git a/main.py b/main.py index 3dec4b0..9662203 100644 --- a/main.py +++ b/main.py @@ -193,7 +193,11 @@ async def getgameinfoext(request: Request, oldGameId: int): db = request.app.state.mongo gameinfoext = db["gameinfoext"].find_one({"oldGameId": oldGameId}, {"_id": 0}) - + if not gameinfoext: + return JSONResponse( + status_code=starlette.status.HTTP_200_OK, + content="", + ) dlc_data = gameinfoext.get('dlcList') new_data = list() for one_line in dlc_data: diff --git a/scripts/get_inc_data.py b/scripts/get_inc_data.py index c9ed8b1..928d12e 100644 --- a/scripts/get_inc_data.py +++ b/scripts/get_inc_data.py @@ -13,7 +13,7 @@ def get_log(log_path_files="/data/logs/ops/ops.log"): if os.getenv('env') == 'dev': - logger = get_log("get_jump.log") + logger = get_log("get_inc_jump.log") mongo_info = { "host": 'localhost', "user": "admin", @@ -26,7 +26,7 @@ if os.getenv('env') == 'dev': mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd']) mongo_db = mongo_client.jump else: - logger = get_log("/data/logs/ops/get_jump.log") + logger = get_log("/data/logs/ops/get_inc_jump.log") mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"} mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port']) mongo_db = mongo_client.jump @@ -41,92 +41,11 @@ game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gam game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all" -def set_jump_data() -> bool: - if not set_platform(): - logger.error("get platform info failed!") - else: - if not set_all_game(): - logger.error("get game list failed!") - return True - - -def set_platform() -> bool: - info = get_url_data(platform_url) - for platform in info.get('data'): - try: - platform['uptime'] = datetime.datetime.strftime( - datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") - platformAlias = platform.get('platformAlias') - mongo_db['platform'].update_one({'platformAlias': platformAlias}, - {'$set': platform}, - upsert=True) - except: - logger.error(f"some error with set platform with {platform}") - return True - - -def set_all_game() -> bool: - ids = mongo_db['platform'].find({}, { - "_id": 0, - "moduleId": 1, - "gameNum": 1 - }) - - for id in ids: - platformid = id.get("moduleId") - ii = 0 - while 1: - url = all_game_url.format(offset=ii, platformid=platformid) - info = get_url_data(url) - ii += 30 - try: - if info.get("data"): - for item in info.get("data"): - item['uptime'] = datetime.datetime.strftime( - datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") - gameid = item.get("oldGameId") - cutOff = item.get("cutOff") - m_cutOff = mongo_db['gameinfo'].find_one( - {"oldGameId": gameid}, { - "_id": 0, - "cutOff": 1, - "cuttime": 1 - }) - if not m_cutOff: - mm_cutOff = 0 - else: - mm_cutOff = m_cutOff.get('cutOff') - # 判断是否打折，修改cuttime数值 - if not cutOff: - item['cuttime'] = 0 - elif mm_cutOff == 0 and cutOff > 0: - item['cuttime'] = item['uptime'] - else: - pass - - mongo_db['gameinfo'].update_one({'oldGameId': gameid}, - {'$set': item}, - upsert=True) - else: - logger.debug(f"get {url} {info}") - break - except: - logger.error(f"get gameinfo failed with {info}") - - logger.info(f"platformid = {id} \ttotal={ii}") - return True - - -def set_gameinfo_ext() -> bool: - gameid_info = mongo_db['gameinfo'].find({}, { - "_id": 0, - "oldGameId": 1, - "platform": 1, - }) +def set_gameinfo_ext(gameids: list) -> bool: i = 0 - for line in gameid_info: + for line in gameids: try: - oldGameId = line.get("oldGameId", 0) + oldGameId = line.get("gameid", 0) platform = line.get("platform", 0) url = game_info_url.format(gameid=oldGameId, platformid=platform) @@ -139,22 +58,17 @@ def set_gameinfo_ext() -> bool: {'$set': data}, upsert=True) i += 1 - logger.info(f"collect gameinfoext {i}") + logger.info(f"collect gameinfoext with {oldGameId}") except Exception as e: logger.error(f"get game info error with {line} {url} {e}") logger.info(f"get game ext info total= {i}!") -def set_game_price() -> bool: - gameid_info = mongo_db['gameinfo'].find({}, { - "_id": 0, - "oldGameId": 1, - "platform": 1, - }) +def set_game_price(gameids: list) -> bool: i = 0 - for line in gameid_info: + for line in gameids: try: - oldGameId = line.get("oldGameId", 0) + oldGameId = line.get("gameid", 0) platform = line.get("platform", 0) logger.info(f"start get {oldGameId} {platform} game price!") url = game_price_url.format(gameid=oldGameId, platformid=platform) @@ -167,21 +81,17 @@ def set_game_price() -> bool: {'$set': data}, upsert=True) i += 1 + logger.info(f"get game price with {oldGameId}") except: logger.error(f"get game price error with {line} {url}") logger.info(f"get game price total={i} ") -def set_history_price() -> bool: - gameid_info = mongo_db['gameinfo'].find({}, { - "_id": 0, - "oldGameId": 1, - "platform": 1, - }) +def set_history_price(gameids: list) -> bool: i = 0 - for line in gameid_info: + for line in gameids: try: - oldGameId = line.get("oldGameId", 0) + oldGameId = line.get("gameid", 0) platform = line.get("platform", 0) logger.info(f"start get {oldGameId} {platform} history price!") url = game_history_price_url.format(gameid=oldGameId, @@ -194,46 +104,13 @@ def set_history_price() -> bool: mongo_db['history_price'].update_one({'oldGameId': oldGameId}, {'$set': data}, upsert=True) + logger.info(f"get history price with {oldGameId}") i += 1 except: logger.error(f"get history price error with {line} {url}") logger.info(f"get history price total={i}") -def set_discount() -> bool: - collections = mongo_db.list_collection_names() - if "discount_price" in collections: - logger.debug(f"find discount_price remove it \n {collections}") - mongo_db.drop_collection("discount_price") - ids = mongo_db['platform'].find({}, {"_id": 0, "moduleId": 1}) - #ids = [{"moduleId": 1}] - for id in ids: - platformid = id.get("moduleId") - i = 0 - while 1: - url = discount_game_url.format(offset=i, platformid=platformid) - info = get_url_data(url) - i += 10 - try: - if info.get("data"): - for item in info.get("data"): - item['uptime'] = datetime.datetime.strftime( - datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") - gameid = item.get("oldGameId") - if item.get("platform") == 53: - item["platform"] = 52 - mongo_db['discount_price'].update_one( - {'oldGameId': gameid}, {'$set': item}, upsert=True) - else: - logger.debug(f"get {url} {info}") - break - except: - logger.error(f"get gameinfo failed with {info}") - - logger.info(f"platformid = {id} \ttotal={i}") - return True - - def get_url_data(url: str, timesleep=0.5, data=None) -> json: requests.adapters.DEFAULT_RETRIES = 5 s = requests.session() @@ -244,13 +121,14 @@ def get_url_data(url: str, timesleep=0.5, data=None) -> json: return res -def get_inc_gamelist() -> bool: +def get_inc_gamelist() -> list: ids = mongo_db['platform'].find({}, { "_id": 0, "moduleId": 1, "gameNum": 1 }) diff_gameid = [] + logger.info("start get game list") for id in ids: platformid = id.get("moduleId") ii = 0 @@ -261,51 +139,69 @@ def get_inc_gamelist() -> bool: try: if info.get("data"): for item in info.get("data"): + item['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") gameid = item.get("oldGameId") + platform = item.get("platform") cutOff = item.get("cutOff") + price = item.get("price") + #logger.debug(f"begin check {gameid} {platform}") m_cutOff = mongo_db['gameinfo'].find_one( {"oldGameId": gameid}, { "_id": 0, "cutOff": 1, - "cuttime": 1 + "cuttime": 1, + "price": 1 }) if not m_cutOff: mm_cutOff = 0 + item['cuttime'] = 0 else: + # 判断是否打折，修改cuttime数值 mm_cutOff = m_cutOff.get('cutOff') - # 判断是否打折，修改cuttime数值 + mprice = m_cutOff.get('price') + if price != mprice: + temp = {} + temp["gameid"] = gameid + temp["platform"] = platform + diff_gameid.append(temp) + logger.info( + f"find {gameid} {platform} price was changed {price} {mprice}!" + ) if not cutOff: item['cuttime'] = 0 elif mm_cutOff == 0 and cutOff > 0: item['cuttime'] = item['uptime'] else: pass - + # logger.debug( + # f"change {gameid} {platform} gameinfo find={diff_gameid}!" + # ) mongo_db['gameinfo'].update_one({'oldGameId': gameid}, {'$set': item}, upsert=True) + if len(diff_gameid) >= 2: + return diff_gameid else: - logger.debug(f"get {url} {info}") + logger.debug(f"get {url} {info} ") break except: logger.error(f"get gameinfo failed with {info}") logger.info(f"platformid = {id} \ttotal={ii}") - return True + return diff_gameid def main_handler(event, context): - set_jump_data() - time.sleep(5) - set_history_price() - time.sleep(1) - #set_discount() - #time.sleep(1) - set_game_price() - time.sleep(1) - set_gameinfo_ext() + change_gameids = get_inc_gamelist() + if change_gameids: + print(change_gameids) + set_game_price(change_gameids) + time.sleep(1) + set_history_price(change_gameids) + time.sleep(1) + set_gameinfo_ext(change_gameids) if __name__ == '__main__': diff --git a/scripts/get_jump_data.py b/scripts/get_jump_data.py index 48db678..9b2e9fc 100644 --- a/scripts/get_jump_data.py +++ b/scripts/get_jump_data.py @@ -94,6 +94,7 @@ def set_all_game() -> bool: }) if not m_cutOff: mm_cutOff = 0 + item['cuttime'] = 0 else: mm_cutOff = m_cutOff.get('cutOff') # 判断是否打折，修改cuttime数值