259 lines
9.7 KiB
Python
259 lines
9.7 KiB
Python
import pymongo
|
||
import datetime
|
||
import json
|
||
import requests
|
||
import time
|
||
import os
|
||
|
||
|
||
def get_log(log_path_files="/data/logs/ops/ops.log"):
|
||
from loguru import logger
|
||
logger.add(log_path_files, rotation="500 MB", enqueue=True)
|
||
return logger
|
||
|
||
|
||
if os.getenv('env') == 'dev':
|
||
logger = get_log("get_jump.log")
|
||
mongo_info = {
|
||
"host": 'localhost',
|
||
"user": "admin",
|
||
"pswd": "kingsome",
|
||
"port": 27017,
|
||
"db": "jump"
|
||
}
|
||
mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
|
||
mongo_auth = mongo_client['admin']
|
||
mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd'])
|
||
mongo_db = mongo_client.jump
|
||
else:
|
||
logger = get_log("/data/logs/ops/get_jump.log")
|
||
mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"}
|
||
mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
|
||
mongo_db = mongo_client.jump
|
||
# 即将推出：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=1&version=16
|
||
# 热门新游：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=2&version=16
|
||
# 正在流行：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=3&version=16
|
||
platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3"
|
||
discount_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3"
|
||
all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=30&offset={offset}&platForm={platformid}&systemList=&type=4&version=3"
|
||
game_info_url = "https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id={gameid}&path=find_discount_gamedetail_switch&platform={platformid}&version=3"
|
||
game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gameid}&platform={platformid}"
|
||
game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all"
|
||
|
||
|
||
def set_jump_data() -> bool:
|
||
if not set_platform():
|
||
logger.error("get platform info failed!")
|
||
else:
|
||
if not set_all_game():
|
||
logger.error("get game list failed!")
|
||
return True
|
||
|
||
|
||
def set_platform() -> bool:
|
||
info = get_url_data(platform_url)
|
||
for platform in info.get('data'):
|
||
try:
|
||
platform['uptime'] = datetime.datetime.strftime(
|
||
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||
platformAlias = platform.get('platformAlias')
|
||
mongo_db['platform'].update_one({'platformAlias': platformAlias},
|
||
{'$set': platform},
|
||
upsert=True)
|
||
except:
|
||
logger.error(f"some error with set platform with {platform}")
|
||
return True
|
||
|
||
|
||
def set_all_game() -> bool:
|
||
ids = mongo_db['platform'].find({}, {
|
||
"_id": 0,
|
||
"moduleId": 1,
|
||
"gameNum": 1
|
||
})
|
||
|
||
for id in ids:
|
||
platformid = id.get("moduleId")
|
||
ii = 0
|
||
while 1:
|
||
url = all_game_url.format(offset=ii, platformid=platformid)
|
||
info = get_url_data(url)
|
||
ii += 30
|
||
try:
|
||
if info.get("data"):
|
||
for item in info.get("data"):
|
||
item['uptime'] = datetime.datetime.strftime(
|
||
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||
gameid = item.get("oldGameId")
|
||
cutOff = item.get("cutOff")
|
||
m_cutOff = mongo_db['gameinfo'].find_one(
|
||
{"oldGameId": gameid}, {
|
||
"_id": 0,
|
||
"cutOff": 1,
|
||
"cuttime": 1
|
||
})
|
||
if not m_cutOff:
|
||
mm_cutOff = 0
|
||
else:
|
||
mm_cutOff = m_cutOff.get('cutOff')
|
||
# 判断是否打折，修改cuttime数值
|
||
if not cutOff:
|
||
item['cuttime'] = 0
|
||
elif mm_cutOff == 0 and cutOff > 0:
|
||
item['cuttime'] = item['uptime']
|
||
else:
|
||
pass
|
||
|
||
mongo_db['gameinfo'].update_one({'oldGameId': gameid},
|
||
{'$set': item},
|
||
upsert=True)
|
||
else:
|
||
logger.debug(f"get {url} {info}")
|
||
break
|
||
except:
|
||
logger.error(f"get gameinfo failed with {info}")
|
||
|
||
logger.info(f"platformid = {id} \ttotal={ii}")
|
||
return True
|
||
|
||
|
||
def set_gameinfo_ext() -> bool:
|
||
gameid_info = mongo_db['gameinfo'].find({}, {
|
||
"_id": 0,
|
||
"oldGameId": 1,
|
||
"platform": 1,
|
||
})
|
||
i = 0
|
||
for line in gameid_info:
|
||
try:
|
||
oldGameId = line.get("oldGameId", 0)
|
||
platform = line.get("platform", 0)
|
||
url = game_info_url.format(gameid=oldGameId, platformid=platform)
|
||
|
||
data = get_url_data(url=url, timesleep=1).get('data')
|
||
#print(data)
|
||
data['oldGameId'] = oldGameId
|
||
data['uptime'] = datetime.datetime.strftime(
|
||
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||
mongo_db['gameinfoext'].update_one({'oldGameId': oldGameId},
|
||
{'$set': data},
|
||
upsert=True)
|
||
i += 1
|
||
logger.info(f"collect gameinfoext {i}")
|
||
except Exception as e:
|
||
logger.error(f"get game info error with {line} {url} {e}")
|
||
logger.info(f"get game ext info total= {i}!")
|
||
|
||
|
||
def set_game_price() -> bool:
|
||
gameid_info = mongo_db['gameinfo'].find({}, {
|
||
"_id": 0,
|
||
"oldGameId": 1,
|
||
"platform": 1,
|
||
})
|
||
i = 0
|
||
for line in gameid_info:
|
||
try:
|
||
oldGameId = line.get("oldGameId", 0)
|
||
platform = line.get("platform", 0)
|
||
url = game_price_url.format(gameid=oldGameId, platformid=platform)
|
||
|
||
data = get_url_data(url).get('data')
|
||
data['oldGameId'] = oldGameId
|
||
data['uptime'] = datetime.datetime.strftime(
|
||
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||
mongo_db['gameprice'].update_one({'oldGameId': oldGameId},
|
||
{'$set': data},
|
||
upsert=True)
|
||
i += 1
|
||
except:
|
||
logger.error(f"get game price error with {line} {url}")
|
||
logger.info(f"get game price total={i} ")
|
||
|
||
|
||
def set_history_price() -> bool:
|
||
gameid_info = mongo_db['gameinfo'].find({}, {
|
||
"_id": 0,
|
||
"oldGameId": 1,
|
||
"platform": 1,
|
||
})
|
||
i = 0
|
||
for line in gameid_info:
|
||
try:
|
||
oldGameId = line.get("oldGameId", 0)
|
||
platform = line.get("platform", 0)
|
||
url = game_history_price_url.format(gameid=oldGameId,
|
||
platformid=platform)
|
||
|
||
data = get_url_data(url).get('data')
|
||
data['oldGameId'] = oldGameId
|
||
data['uptime'] = datetime.datetime.strftime(
|
||
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||
mongo_db['history_price'].update_one({'oldGameId': oldGameId},
|
||
{'$set': data},
|
||
upsert=True)
|
||
i += 1
|
||
except:
|
||
logger.error(f"get history price error with {line} {url}")
|
||
logger.info(f"get history price total={i}")
|
||
|
||
|
||
def set_discount() -> bool:
|
||
collections = mongo_db.list_collection_names()
|
||
if "discount_price" in collections:
|
||
logger.debug(f"find discount_price remove it \n {collections}")
|
||
mongo_db.drop_collection("discount_price")
|
||
ids = mongo_db['platform'].find({}, {"_id": 0, "moduleId": 1})
|
||
#ids = [{"moduleId": 1}]
|
||
for id in ids:
|
||
platformid = id.get("moduleId")
|
||
i = 0
|
||
while 1:
|
||
url = discount_game_url.format(offset=i, platformid=platformid)
|
||
info = get_url_data(url)
|
||
i += 10
|
||
try:
|
||
if info.get("data"):
|
||
for item in info.get("data"):
|
||
item['uptime'] = datetime.datetime.strftime(
|
||
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||
gameid = item.get("oldGameId")
|
||
if item.get("platform") == 53:
|
||
item["platform"] = 52
|
||
mongo_db['discount_price'].update_one(
|
||
{'oldGameId': gameid}, {'$set': item}, upsert=True)
|
||
else:
|
||
logger.debug(f"get {url} {info}")
|
||
break
|
||
except:
|
||
logger.error(f"get gameinfo failed with {info}")
|
||
|
||
logger.info(f"platformid = {id} \ttotal={i}")
|
||
return True
|
||
|
||
|
||
def get_url_data(url: str, timesleep=0.5, data=None) -> json:
|
||
requests.adapters.DEFAULT_RETRIES = 5
|
||
s = requests.session()
|
||
s.keep_alive = False
|
||
headers = {'Connection': 'close'}
|
||
time.sleep(timesleep)
|
||
res = s.get(url=url, params=data, timeout=10, headers=headers).json()
|
||
return res
|
||
|
||
|
||
def main_handler(event, context):
|
||
set_jump_data()
|
||
time.sleep(5)
|
||
set_history_price()
|
||
time.sleep(1)
|
||
#set_discount()
|
||
#time.sleep(1)
|
||
set_game_price()
|
||
time.sleep(1)
|
||
set_gameinfo_ext()
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main_handler("", "")
|