249 lines
9.9 KiB
Python
249 lines
9.9 KiB
Python
import pymongo
|
||
import datetime
|
||
import json
|
||
import requests
|
||
import time
|
||
import os
|
||
import pdb
|
||
# todo: getgamelist->is_check:0 get_gameext->is_check:1
|
||
# todo: finish_check: find(gamelist:{is_check:0}) get_gameext again
|
||
|
||
|
||
def get_log(log_path_files="/data/logs/ops/ops.log"):
|
||
from loguru import logger
|
||
logger.add(log_path_files, rotation="500 MB", enqueue=True)
|
||
return logger
|
||
|
||
|
||
if os.getenv('env') == 'dev':
|
||
logger = get_log("get_inc_jump.log")
|
||
mongo_info = {
|
||
"host": 'localhost',
|
||
"user": "admin",
|
||
"pswd": "kingsome",
|
||
"port": 27017,
|
||
"db": "jump"
|
||
}
|
||
mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
|
||
mongo_auth = mongo_client['admin']
|
||
mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd'])
|
||
mongo_db = mongo_client.jump
|
||
else:
|
||
logger = get_log("/data/logs/ops/get_inc_jump.log")
|
||
mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"}
|
||
mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
|
||
mongo_db = mongo_client.jump
|
||
# 即将推出：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=1&version=16
|
||
# 热门新游：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=2&version=16
|
||
# 正在流行：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=3&version=16
|
||
platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3"
|
||
discount_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3"
|
||
all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=30&offset={offset}&platForm={platformid}&systemList=&type=4&version=3"
|
||
game_info_url = "https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id={gameid}&path=find_discount_gamedetail_switch&platform={platformid}&version=3"
|
||
game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gameid}&platform={platformid}"
|
||
game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all"
|
||
|
||
|
||
def set_gamelist_ischeck(gameid: int) -> bool:
|
||
newvlues = {"is_check": 1}
|
||
mongo_db['gameinfo'].update_one()({
|
||
'oldGameId': gameid
|
||
}, {
|
||
'$set': newvlues
|
||
},
|
||
upsert=True)
|
||
return True
|
||
|
||
|
||
def set_gameinfo_ext(gameids: list) -> bool:
|
||
i = 0
|
||
for line in gameids:
|
||
try:
|
||
oldGameId = line.get("gameid", 0)
|
||
platform = line.get("platform", 0)
|
||
url = game_info_url.format(gameid=oldGameId, platformid=platform)
|
||
|
||
data = get_url_data(url=url, timesleep=1).get('data')
|
||
logger.info(f"start get {oldGameId} {platform} game info ext!")
|
||
data['oldGameId'] = oldGameId
|
||
data['uptime'] = datetime.datetime.strftime(
|
||
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||
mongo_db['gameinfoext'].update_one({'oldGameId': oldGameId},
|
||
{'$set': data},
|
||
upsert=True)
|
||
set_gamelist_ischeck(oldGameId)
|
||
|
||
i += 1
|
||
logger.info(f"collect gameinfoext with {oldGameId}")
|
||
except Exception as e:
|
||
logger.error(f"get game info error with {line} {url} {e}")
|
||
logger.info(f"get game ext info total= {i}!")
|
||
|
||
|
||
def set_game_price(gameids: list) -> bool:
|
||
i = 0
|
||
for line in gameids:
|
||
try:
|
||
oldGameId = line.get("gameid", 0)
|
||
platform = line.get("platform", 0)
|
||
logger.info(f"start get {oldGameId} {platform} game price!")
|
||
url = game_price_url.format(gameid=oldGameId, platformid=platform)
|
||
|
||
data = get_url_data(url).get('data')
|
||
data['oldGameId'] = oldGameId
|
||
data['uptime'] = datetime.datetime.strftime(
|
||
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||
mongo_db['gameprice'].update_one({'oldGameId': oldGameId},
|
||
{'$set': data},
|
||
upsert=True)
|
||
i += 1
|
||
logger.info(f"get game price with {oldGameId}")
|
||
except:
|
||
logger.error(f"get game price error with {line} {url}")
|
||
logger.info(f"get game price total={i} ")
|
||
|
||
|
||
def set_history_price(gameids: list) -> bool:
|
||
i = 0
|
||
for line in gameids:
|
||
try:
|
||
oldGameId = line.get("gameid", 0)
|
||
platform = line.get("platform", 0)
|
||
logger.info(f"start get {oldGameId} {platform} history price!")
|
||
url = game_history_price_url.format(gameid=oldGameId,
|
||
platformid=platform)
|
||
|
||
data = get_url_data(url).get('data')
|
||
data['oldGameId'] = oldGameId
|
||
data['uptime'] = datetime.datetime.strftime(
|
||
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||
mongo_db['history_price'].update_one({'oldGameId': oldGameId},
|
||
{'$set': data},
|
||
upsert=True)
|
||
logger.info(f"get history price with {oldGameId}")
|
||
i += 1
|
||
except:
|
||
logger.error(f"get history price error with {line} {url}")
|
||
logger.info(f"get history price total={i}")
|
||
|
||
|
||
def get_url_data(url: str, timesleep=0.5, data=None) -> json:
|
||
requests.adapters.DEFAULT_RETRIES = 5
|
||
s = requests.session()
|
||
s.keep_alive = False
|
||
headers = {'Connection': 'close'}
|
||
time.sleep(timesleep)
|
||
res = s.get(url=url, params=data, timeout=10, headers=headers).json()
|
||
return res
|
||
|
||
|
||
def get_inc_gamelist() -> list:
|
||
ids = mongo_db['platform'].find({}, {
|
||
"_id": 0,
|
||
"moduleId": 1,
|
||
"gameNum": 1
|
||
})
|
||
diff_gameid = []
|
||
logger.info("start get game list")
|
||
for id in ids:
|
||
platformid = id.get("moduleId")
|
||
ii = 0
|
||
while 1:
|
||
url = all_game_url.format(offset=ii, platformid=platformid)
|
||
info = get_url_data(url)
|
||
ii += 30
|
||
try:
|
||
if info.get("data"):
|
||
for item in info.get("data"):
|
||
|
||
item['uptime'] = datetime.datetime.strftime(
|
||
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||
gameid = item.get("oldGameId")
|
||
platform = item.get("platform")
|
||
cutOff = item.get("cutOff")
|
||
price = item.get("price")
|
||
#logger.debug(f"begin check {gameid} {platform}")
|
||
m_cutOff = mongo_db['gameinfo'].find_one(
|
||
{"oldGameId": gameid}, {
|
||
"_id": 0,
|
||
"cutOff": 1,
|
||
"cuttime": 1,
|
||
"price": 1
|
||
})
|
||
if not m_cutOff:
|
||
mm_cutOff = 0
|
||
item['cuttime'] = 0
|
||
item["is_check"] = 0
|
||
else:
|
||
# 判断是否打折，修改cuttime数值
|
||
mm_cutOff = m_cutOff.get('cutOff')
|
||
mprice = m_cutOff.get('price')
|
||
if abs(round(price) - round(mprice)) > 1:
|
||
|
||
temp = {}
|
||
temp["gameid"] = gameid
|
||
temp["platform"] = platform
|
||
diff_gameid.append(temp)
|
||
logger.info(
|
||
f"find {gameid} {platform} price was changed {round(price)} {round(mprice)}!"
|
||
)
|
||
if not cutOff:
|
||
item['cuttime'] = 0
|
||
elif mm_cutOff == 0 and cutOff > 0:
|
||
item['cuttime'] = item['uptime']
|
||
else:
|
||
pass
|
||
# logger.debug(
|
||
# f"change {gameid} {platform} gameinfo find={diff_gameid}!"
|
||
# )
|
||
mongo_db['gameinfo'].update_one({'oldGameId': gameid},
|
||
{'$set': item},
|
||
upsert=True)
|
||
# if len(diff_gameid) >= 2:
|
||
# return diff_gameid
|
||
else:
|
||
logger.debug(f"get {url} {info} ")
|
||
break
|
||
except:
|
||
logger.error(f"get gameinfo failed with {info}")
|
||
|
||
logger.info(f"platformid = {id} \ttotal={ii}")
|
||
return diff_gameid
|
||
|
||
|
||
# check gamelist is_check=0,get ext again
|
||
def check_gameinfoext() -> bool:
|
||
check_failed = mongo_db['gameinfo'].find({"is_check": 0}, {
|
||
"_id": 0,
|
||
"oldGameId": 1
|
||
})
|
||
set_gameinfo_ext(list(check_failed))
|
||
|
||
|
||
def update_is_check() -> bool:
|
||
ids = mongo_db['gameinfo'].find({}, {"_id": 0, "oldGameId": 1})
|
||
for id in list(ids):
|
||
check = mongo_db['gameinfoext'].find({"oldGameId": id})
|
||
if check:
|
||
logger.info(f"update {id} set is_check =1!")
|
||
set_gamelist_ischeck(id)
|
||
else:
|
||
set_gameinfo_ext([id])
|
||
return True
|
||
|
||
|
||
def main_handler(event, context):
|
||
change_gameids = get_inc_gamelist()
|
||
logger.info(f"find diff {len(change_gameids)}")
|
||
# if change_gameids:
|
||
# print(change_gameids)
|
||
# set_game_price(change_gameids)
|
||
# time.sleep(1)
|
||
# set_history_price(change_gameids)
|
||
# time.sleep(1)
|
||
# set_gameinfo_ext(change_gameids)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main_handler("", "")
|