import pymongo import datetime import json import requests import time import os import pdb # todo: getgamelist->is_check:0 get_gameext->is_check:1 # todo: finish_check: find(gamelist:{is_check:0}) get_gameext again def get_log(log_path_files="/data/logs/ops/ops.log"): from loguru import logger logger.add(log_path_files, rotation="500 MB", enqueue=True) return logger if os.getenv('env') == 'dev': logger = get_log("get_inc_jump.log") mongo_info = { "host": 'localhost', "user": "admin", "pswd": "kingsome", "port": 27017, "db": "jump" } mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port']) mongo_auth = mongo_client['admin'] mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd']) mongo_db = mongo_client.jump else: logger = get_log("/data/logs/ops/get_inc_jump.log") mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"} mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port']) mongo_db = mongo_client.jump # 即将推出:https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=1&version=16 # 热门新游:https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=2&version=16 # 正在流行:https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=3&version=16 platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3" discount_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3" all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=30&offset={offset}&platForm={platformid}&systemList=&type=4&version=3" game_info_url = "https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id={gameid}&path=find_discount_gamedetail_switch&platform={platformid}&version=3" game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gameid}&platform={platformid}" game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all" def set_gamelist_ischeck(gameid: int) -> bool: newvlues = {"is_check": 1} mongo_db['gameinfo'].update_one()({ 'oldGameId': gameid }, { '$set': newvlues }, upsert=True) return True def set_gameinfo_ext(gameids: list) -> bool: i = 0 for line in gameids: try: oldGameId = line.get("gameid", 0) platform = line.get("platform", 0) url = game_info_url.format(gameid=oldGameId, platformid=platform) data = get_url_data(url=url, timesleep=1).get('data') logger.info(f"start get {oldGameId} {platform} game info ext!") data['oldGameId'] = oldGameId data['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") mongo_db['gameinfoext'].update_one({'oldGameId': oldGameId}, {'$set': data}, upsert=True) set_gamelist_ischeck(oldGameId) i += 1 logger.info(f"collect gameinfoext with {oldGameId}") except Exception as e: logger.error(f"get game info error with {line} {url} {e}") logger.info(f"get game ext info total= {i}!") def set_game_price(gameids: list) -> bool: i = 0 for line in gameids: try: oldGameId = line.get("gameid", 0) platform = line.get("platform", 0) logger.info(f"start get {oldGameId} {platform} game price!") url = game_price_url.format(gameid=oldGameId, platformid=platform) data = get_url_data(url).get('data') data['oldGameId'] = oldGameId data['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") mongo_db['gameprice'].update_one({'oldGameId': oldGameId}, {'$set': data}, upsert=True) i += 1 logger.info(f"get game price with {oldGameId}") except: logger.error(f"get game price error with {line} {url}") logger.info(f"get game price total={i} ") def set_history_price(gameids: list) -> bool: i = 0 for line in gameids: try: oldGameId = line.get("gameid", 0) platform = line.get("platform", 0) logger.info(f"start get {oldGameId} {platform} history price!") url = game_history_price_url.format(gameid=oldGameId, platformid=platform) data = get_url_data(url).get('data') data['oldGameId'] = oldGameId data['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") mongo_db['history_price'].update_one({'oldGameId': oldGameId}, {'$set': data}, upsert=True) logger.info(f"get history price with {oldGameId}") i += 1 except: logger.error(f"get history price error with {line} {url}") logger.info(f"get history price total={i}") def get_url_data(url: str, timesleep=0.5, data=None) -> json: requests.adapters.DEFAULT_RETRIES = 5 s = requests.session() s.keep_alive = False headers = {'Connection': 'close'} time.sleep(timesleep) res = s.get(url=url, params=data, timeout=10, headers=headers).json() return res def get_inc_gamelist() -> list: ids = mongo_db['platform'].find({}, { "_id": 0, "moduleId": 1, "gameNum": 1 }) diff_gameid = [] logger.info("start get game list") for id in ids: platformid = id.get("moduleId") ii = 0 while 1: url = all_game_url.format(offset=ii, platformid=platformid) info = get_url_data(url) ii += 30 try: if info.get("data"): for item in info.get("data"): item['uptime'] = datetime.datetime.strftime( datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") gameid = item.get("oldGameId") platform = item.get("platform") cutOff = item.get("cutOff") price = item.get("price") #logger.debug(f"begin check {gameid} {platform}") m_cutOff = mongo_db['gameinfo'].find_one( {"oldGameId": gameid}, { "_id": 0, "cutOff": 1, "cuttime": 1, "price": 1 }) if not m_cutOff: mm_cutOff = 0 item['cuttime'] = 0 item["is_check"] = 0 else: # 判断是否打折,修改cuttime数值 mm_cutOff = m_cutOff.get('cutOff') mprice = m_cutOff.get('price') if abs(round(price) - round(mprice)) > 1: temp = {} temp["gameid"] = gameid temp["platform"] = platform diff_gameid.append(temp) logger.info( f"find {gameid} {platform} price was changed {round(price)} {round(mprice)}!" ) if not cutOff: item['cuttime'] = 0 elif mm_cutOff == 0 and cutOff > 0: item['cuttime'] = item['uptime'] else: pass # logger.debug( # f"change {gameid} {platform} gameinfo find={diff_gameid}!" # ) mongo_db['gameinfo'].update_one({'oldGameId': gameid}, {'$set': item}, upsert=True) # if len(diff_gameid) >= 2: # return diff_gameid else: logger.debug(f"get {url} {info} ") break except: logger.error(f"get gameinfo failed with {info}") logger.info(f"platformid = {id} \ttotal={ii}") return diff_gameid # check gamelist is_check=0,get ext again def check_gameinfoext() -> bool: check_failed = mongo_db['gameinfo'].find({"is_check": 0}, { "_id": 0, "oldGameId": 1 }) set_gameinfo_ext(list(check_failed)) def update_is_check() -> bool: ids = mongo_db['gameinfo'].find({}, {"_id": 0, "oldGameId": 1}) for id in list(ids): check = mongo_db['gameinfoext'].find({"oldGameId": id}) if check: logger.info(f"update {id} set is_check =1!") set_gamelist_ischeck(id) else: set_gameinfo_ext([id]) return True def main_handler(event, context): change_gameids = get_inc_gamelist() logger.info(f"find diff {len(change_gameids)}") # if change_gameids: # print(change_gameids) # set_game_price(change_gameids) # time.sleep(1) # set_history_price(change_gameids) # time.sleep(1) # set_gameinfo_ext(change_gameids) if __name__ == '__main__': main_handler("", "")