diff --git a/config/config.py b/config/config.py index c87f84c..964c794 100644 --- a/config/config.py +++ b/config/config.py @@ -10,13 +10,7 @@ class Settings(BaseSettings): admin_email: str = "pengtao@kingsome.cn" items_per_user: int = 50 is_debug: bool = True - mongo_info = { - "host": 'localhost', - "user": "admin", - "pswd": "kingsome", - "port": 27017, - "db": "jump" - } + MONGODB_URL = "mongodb://admin:kingsome@localhost" platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3" all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset={offset}&platForm={platformid}&systemList=&type=4&version=3" diff --git a/main.py b/main.py index 3774085..4091530 100644 --- a/main.py +++ b/main.py @@ -1,20 +1,20 @@ # uvicorn main:app --host=127.0.0.1 --port=8030 --reload +from pydantic.fields import T from config.config import settings from fastapi import Depends, FastAPI, BackgroundTasks, Request +from fastapi.responses import JSONResponse from dependencies import get_token_header -from interface import route as interface_route from scripts.common.mongodb import get_mongo from typing import Optional from scripts.logger import logger from fastapi.middleware.cors import CORSMiddleware - +import starlette +import re +from pydantic import BaseModel, Field, EmailStr +import json # from apscheduler.events import EVENT_JOB_EXECUTED # from jobs.jobs import Schedule, job_execute tags_metadata = [ - # { - # "name": "common", - # "description": "Operations with users. The **login** logic is also here.", - # }, { "name": "common", "description": "Manage items. So _fancy_ they have their own docs.", @@ -29,7 +29,6 @@ tags_metadata = [ def create_app(): application = FastAPI(dependencies=[Depends(get_token_header)], openapi_tags=tags_metadata) - application.include_router(interface_route.router, prefix="/interface") return application @@ -43,11 +42,14 @@ app.add_middleware( ) +class FindArgs(BaseModel): + oldGameId: int + name: str + + @app.on_event("startup") async def startup_event(): app.state.mongo = await get_mongo() - # Schedule.start() - # Schedule.add_listener(job_execute, EVENT_JOB_EXECUTED) @app.on_event("shutdown") @@ -56,13 +58,106 @@ async def shutdown_event(): await app.state.mongo.wait_close() -@app.get("/") -async def root(request: Request): - # db = request.app.state.mongo - # keys = await db.get("online_devices") - logger.info("starting!") +@app.get("/getPlatform") +async def getPlatform(request: Request, platformAlias: str): + db = request.app.state.mongo + existing_platform = db["platform"].find_one( + {"platformAlias": platformAlias}, {"_id": 0}) + logger.info(f"starting get Platform with {platformAlias}!") + return JSONResponse( + status_code=starlette.status.HTTP_200_OK, + content=existing_platform, + ) - return {"message": "kajsk"} + +@app.get("/getgamelist") +async def getgamelist( + request: Request, + cutoff: bool, + isLowest: bool, + platform: int = 1, + skip: int = 0, + limit: int = 10, +): + db = request.app.state.mongo + + try: + find_args = {} + if cutoff: + find_args["cutOff"] = {"$ne": 0} + if isLowest: + find_args["isLowest"] = {"$ne": 0} + find_args["platform"] = platform + gamelist = db["gameinfo"].find(find_args, { + "_id": 0, + "oldGameId": 1, + "name": 1 + }).sort("_id").skip(skip).limit(limit) + logger.info(f"get gamelist with {find_args}!") + return JSONResponse( + status_code=starlette.status.HTTP_200_OK, + content=list(gamelist), + ) + except Exception as e: + return JSONResponse( + status_code=starlette.status.HTTP_500_INTERNAL_SERVER_ERROR, + content=e, + ) + + +@app.get("/getgameinfo") +async def getgameinfo(request: Request, oldGameId: int = 0, name: str = ""): + db = request.app.state.mongo + if oldGameId: + gameinfo = db["gameinfo"].find({"oldGameId": oldGameId}, {"_id": 0}) + elif name: + gameinfo = db["gameinfo"].find({"name": re.compile(name)}, {"_id": 0}) + else: + logger.error(f"get gameinfo with {oldGameId} {name}!") + return JSONResponse( + status_code=starlette.status.HTTP_400_BAD_REQUEST, + content={"message": "参数未提供"}, + ) + logger.info(f"get gameinfo with {oldGameId} {name}!") + return JSONResponse( + status_code=starlette.status.HTTP_200_OK, + content=list(gameinfo), + ) + + +@app.get("/getgameprice") +async def getgameprice(request: Request, oldGameId: int): + db = request.app.state.mongo + gameprice = db["gameprice"].find_one({"oldGameId": oldGameId}, {"_id": 0}) + logger.info(f"get gameprice with {oldGameId} !") + return JSONResponse( + status_code=starlette.status.HTTP_200_OK, + content=gameprice, + ) + + +@app.get("/getgameinfoext") +async def getgameinfoext(request: Request, oldGameId: int): + db = request.app.state.mongo + gameinfoext = db["gameinfoext"].find_one({"oldGameId": oldGameId}, + {"_id": 0}) + logger.info(f"get gameinfoext with {oldGameId} !") + return JSONResponse( + status_code=starlette.status.HTTP_200_OK, + content=gameinfoext, + ) + + +@app.get("/gethistoryprice") +async def gethistoryprice(request: Request, oldGameId: int): + db = request.app.state.mongo + history_price = db["history_price"].find_one({"oldGameId": oldGameId}, + {"_id": 0}) + logger.info(f"get historyprice with {oldGameId} !") + return JSONResponse( + status_code=starlette.status.HTTP_200_OK, + content=history_price, + ) if __name__ == '__main__': diff --git a/scripts/common/mongodb.py b/scripts/common/mongodb.py index b77f8da..71bd28e 100644 --- a/scripts/common/mongodb.py +++ b/scripts/common/mongodb.py @@ -1,88 +1,26 @@ # sample as https://github.com/mongodb-developer/mongodb-with-fastapi/blob/master/app.py import os -from fastapi import FastAPI, Body, HTTPException, status -from fastapi.responses import JSONResponse -from fastapi.encoders import jsonable_encoder -from pydantic import BaseModel, Field, EmailStr -from bson import ObjectId -from typing import Optional, List -import motor.motor_asyncio -from pydantic.types import Json -from config.config import settings +import pymongo async def get_mongo(): - client = motor.motor_asyncio.AsyncIOMotorClient(settings.MONGODB_URL) - db = client.jump - return db + if os.getenv('env') == 'dev': - -class PyObjectId(ObjectId): - @classmethod - def __get_validators__(cls): - yield cls.validate - - @classmethod - def validate(cls, v): - if not ObjectId.is_valid(v): - raise ValueError("Invalid objectid") - return ObjectId(v) - - @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="string") - - -class PlatformModel(BaseModel): - id: PyObjectId = Field(default_factory=PyObjectId, alias="_id") - moduleId: int = Field(...) - platformAlias: str = Field(...) - iconRes: str = Field(...) - gameNum: int = Field(...) - filter: list = [] - countryFilter: bool - - class Config: - allow_population_by_field_name = True - arbitrary_types_allowed = True - json_encoders = {ObjectId: str} - schema_extra = { - "example": { - "moduleId": - 4, - "platformAlias": - 'Steam', - "iconRes": - 'http://switch-cdn.vgjump.com/869270335915032576', - "gameNum": - 759, - "filter": [{ - 'termsId': 26, - 'terms': '精选' - }, { - 'termsId': 27, - 'terms': '全部' - }], - "countryFilter": - None - } - } - - -class UpdateStudentModel(BaseModel): - name: Optional[str] - email: Optional[EmailStr] - course: Optional[str] - gpa: Optional[float] - - class Config: - arbitrary_types_allowed = True - json_encoders = {ObjectId: str} - schema_extra = { - "example": { - "name": "Jane Doe", - "email": "jdoe@example.com", - "course": "Experiments, Science, and Fashion in Nanophotonics", - "gpa": "3.0", - } + mongo_info = { + "host": 'localhost', + "user": "admin", + "pswd": "kingsome", + "port": 27017, + "db": "jump" } + mongo_client = pymongo.MongoClient(mongo_info['host'], + mongo_info['port']) + mongo_auth = mongo_client['admin'] + mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd']) + mongo_db = mongo_client.jump + else: + mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"} + mongo_client = pymongo.MongoClient(mongo_info['host'], + mongo_info['port']) + mongo_db = mongo_client.jump + return mongo_db diff --git a/scripts/get_jump_data.py b/scripts/get_jump_data.py index 7f5459a..a53ef22 100644 --- a/scripts/get_jump_data.py +++ b/scripts/get_jump_data.py @@ -1,61 +1,67 @@ -from sys import int_info import pymongo import datetime -from bson.objectid import ObjectId -from bson import json_util -import copy -import re -import pdb import json import requests import time +import os -mongo_info = { - "host": 'localhost', - "user": "admin", - "pswd": "kingsome", - "port": 27017, - "db": "jump" -} + +def get_log(log_path_files="/data/logs/ops/ops.log"): + from loguru import logger + logger.add(log_path_files, rotation="500 MB", enqueue=True) + return logger + + +if os.getenv('env') == 'dev': + logger = get_log("get_jump.log") + mongo_info = { + "host": 'localhost', + "user": "admin", + "pswd": "kingsome", + "port": 27017, + "db": "jump" + } + mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port']) + mongo_auth = mongo_client['admin'] + mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd']) + mongo_db = mongo_client.jump +else: + logger = get_log("/data/logs/ops/get_jump.log") + mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"} + mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port']) + mongo_db = mongo_client.jump +# 即将推出:https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=1&version=16 +# 热门新游:https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=2&version=16 +# 正在流行:https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=3&version=16 platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3" +discount_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3" all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=30&offset={offset}&platForm={platformid}&systemList=&type=4&version=3" -# all_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3" - game_info_url = "https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id={gameid}&path=find_discount_gamedetail_switch&platform={platformid}&version=3" game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gameid}&platform={platformid}" game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all" -mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port']) -mongo_auth = mongo_client['admin'] -mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd']) -mongo_db = mongo_client.jump - def set_jump_data() -> bool: if not set_platform(): - print("get platform info failed!") + logger.error("get platform info failed!") else: if not set_all_game(): - print("get game list failed!") - + logger.error("get game list failed!") return True def set_platform() -> bool: info = get_url_data(platform_url) for platform in info.get('data'): - platform['uptime'] = datetime.datetime.strftime( - datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") - platformAlias = platform.get('platformAlias') - existing_platform = mongo_db["platform"].find_one( - {"platformAlias": platformAlias}) - if not existing_platform: - new_platform = mongo_db["platform"].insert_one(platform) - created_platform = mongo_db["platform"].find_one( - {"_id": new_platform.inserted_id}) - print(f"insert platform {created_platform}") - else: - print(f"key found with {existing_platform}") + try: + platform['uptime'] = datetime.datetime.strftime( + datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") + platformAlias = platform.get('platformAlias') + mongo_db['platform'].update_one({'platformAlias': platformAlias}, + {'$set': platform}, + upsert=True) + except: + logger.error(f"some error with set platform with {platform}") return True @@ -68,37 +74,110 @@ def set_all_game() -> bool: for id in ids: platformid = id.get("moduleId") - nums = int(id.get("gameNum")) ii = 0 - i = 0 while 1: url = all_game_url.format(offset=ii, platformid=platformid) info = get_url_data(url) - print(f"url={url}") - if info.get("data"): - for item in info.get("data"): - i += 1 - item['uptime'] = datetime.datetime.strftime( - datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") - gameid = item.get("oldGameId") - mongo_db['history_price'].update_one({'oldGameId': gameid}, - {'$set': item}, - upsert=True) - else: - print(info) - if ii > nums: - break ii += 30 - print(f"total={i}") + try: + if info.get("data"): + for item in info.get("data"): + item['uptime'] = datetime.datetime.strftime( + datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") + gameid = item.get("oldGameId") + cutOff = item.get("cutOff") + m_cutOff = mongo_db['gameinfo'].find_one( + {"oldGameId": gameid}, { + "_id": 0, + "cutOff": 1, + "cuttime": 1 + }) + if not m_cutOff: + mm_cutOff = 0 + else: + mm_cutOff = m_cutOff.get('cutOff') + # 判断是否打折,修改cuttime数值 + if not cutOff: + item['cuttime'] = 0 + elif mm_cutOff == 0 and cutOff > 0: + item['cuttime'] = item['uptime'] + else: + pass + + mongo_db['gameinfo'].update_one({'oldGameId': gameid}, + {'$set': item}, + upsert=True) + else: + logger.debug(f"get {url} {info}") + break + except: + logger.error(f"get gameinfo failed with {info}") + + logger.info(f"platformid = {id} \ttotal={ii}") return True +def set_gameinfo_ext() -> bool: + gameid_info = mongo_db['gameinfo'].find({}, { + "_id": 0, + "oldGameId": 1, + "platform": 1, + }) + i = 0 + for line in gameid_info: + try: + oldGameId = line.get("oldGameId", 0) + platform = line.get("platform", 0) + url = game_info_url.format(gameid=oldGameId, platformid=platform) + + data = get_url_data(url=url, timesleep=1).get('data') + #print(data) + data['oldGameId'] = oldGameId + data['uptime'] = datetime.datetime.strftime( + datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") + mongo_db['gameinfoext'].update_one({'oldGameId': oldGameId}, + {'$set': data}, + upsert=True) + i += 1 + logger.info(f"collect gameinfoext {i}") + except Exception as e: + logger.error(f"get game info error with {line} {url} {e}") + logger.info(f"get game ext info total= {i}!") + + +def set_game_price() -> bool: + gameid_info = mongo_db['gameinfo'].find({}, { + "_id": 0, + "oldGameId": 1, + "platform": 1, + }) + i = 0 + for line in gameid_info: + try: + oldGameId = line.get("oldGameId", 0) + platform = line.get("platform", 0) + url = game_price_url.format(gameid=oldGameId, platformid=platform) + + data = get_url_data(url).get('data') + data['oldGameId'] = oldGameId + data['uptime'] = datetime.datetime.strftime( + datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") + mongo_db['gameprice'].update_one({'oldGameId': oldGameId}, + {'$set': data}, + upsert=True) + i += 1 + except: + logger.error(f"get game price error with {line} {url}") + logger.info(f"get game price total={i} ") + + def set_history_price() -> bool: gameid_info = mongo_db['gameinfo'].find({}, { "_id": 0, "oldGameId": 1, "platform": 1, }) + i = 0 for line in gameid_info: try: oldGameId = line.get("oldGameId", 0) @@ -113,20 +192,67 @@ def set_history_price() -> bool: mongo_db['history_price'].update_one({'oldGameId': oldGameId}, {'$set': data}, upsert=True) + i += 1 except: - print(f"get history price error with {line} {url}") + logger.error(f"get history price error with {line} {url}") + logger.info(f"get history price total={i}") -def get_url_data(url: str, data=None) -> json: +def set_discount() -> bool: + collections = mongo_db.list_collection_names() + if "discount_price" in collections: + logger.debug(f"find discount_price remove it \n {collections}") + mongo_db.drop_collection("discount_price") + ids = mongo_db['platform'].find({}, {"_id": 0, "moduleId": 1}) + #ids = [{"moduleId": 1}] + for id in ids: + platformid = id.get("moduleId") + i = 0 + while 1: + url = discount_game_url.format(offset=i, platformid=platformid) + info = get_url_data(url) + i += 10 + try: + if info.get("data"): + for item in info.get("data"): + item['uptime'] = datetime.datetime.strftime( + datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") + gameid = item.get("oldGameId") + if item.get("platform") == 53: + item["platform"] = 52 + mongo_db['discount_price'].update_one( + {'oldGameId': gameid}, {'$set': item}, upsert=True) + else: + logger.debug(f"get {url} {info}") + break + except: + logger.error(f"get gameinfo failed with {info}") + + logger.info(f"platformid = {id} \ttotal={i}") + return True + + +def get_url_data(url: str, timesleep=0.5, data=None) -> json: requests.adapters.DEFAULT_RETRIES = 5 s = requests.session() s.keep_alive = False headers = {'Connection': 'close'} - time.sleep(1) + time.sleep(timesleep) res = s.get(url=url, params=data, timeout=10, headers=headers).json() return res -if __name__ == "__main__": - #set_jump_data() - set_history_price() \ No newline at end of file +def main_handler(event, context): + set_jump_data() + time.sleep(5) + set_history_price() + time.sleep(1) + #set_discount() + #time.sleep(1) + set_game_price() + time.sleep(1) + set_gameinfo_ext() + + +if __name__ == '__main__': + main_handler("", "")