fix bug

config/config.py

@@ -10,13 +10,7 @@ class Settings(BaseSettings):
     admin_email: str = "pengtao@kingsome.cn"
     items_per_user: int = 50
     is_debug: bool = True
-    mongo_info = {
+
-        "host": 'localhost',
-        "user": "admin",
-        "pswd": "kingsome",
-        "port": 27017,
-        "db": "jump"
-    }
     MONGODB_URL = "mongodb://admin:kingsome@localhost"
     platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3"
     all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset={offset}&platForm={platformid}&systemList=&type=4&version=3"

main.py

@@ -1,20 +1,20 @@
 # uvicorn main:app --host=127.0.0.1 --port=8030 --reload
+from pydantic.fields import T
 from config.config import settings
 from fastapi import Depends, FastAPI, BackgroundTasks, Request
+from fastapi.responses import JSONResponse
 from dependencies import get_token_header
-from interface import route as interface_route
 from scripts.common.mongodb import get_mongo
 from typing import Optional
 from scripts.logger import logger
 from fastapi.middleware.cors import CORSMiddleware
-
+import starlette
+import re
+from pydantic import BaseModel, Field, EmailStr
+import json
 # from apscheduler.events import EVENT_JOB_EXECUTED
 # from jobs.jobs import Schedule, job_execute
 tags_metadata = [
-    # {
-    #     "name": "common",
-    #     "description": "Operations with users. The **login** logic is also here.",
-    # },
     {
         "name": "common",
         "description": "Manage items. So _fancy_ they have their own docs.",
@@ -29,7 +29,6 @@ tags_metadata = [
 def create_app():
     application = FastAPI(dependencies=[Depends(get_token_header)],
                           openapi_tags=tags_metadata)
-    application.include_router(interface_route.router, prefix="/interface")
     return application
 
 
@@ -43,11 +42,14 @@ app.add_middleware(
 )
 
 
+class FindArgs(BaseModel):
+    oldGameId: int
+    name: str
+
+
 @app.on_event("startup")
 async def startup_event():
     app.state.mongo = await get_mongo()
-    # Schedule.start()
-    # Schedule.add_listener(job_execute, EVENT_JOB_EXECUTED)
 
 
 @app.on_event("shutdown")
@@ -56,13 +58,106 @@ async def shutdown_event():
     await app.state.mongo.wait_close()
 
 
-@app.get("/")
+@app.get("/getPlatform")
-async def root(request: Request):
+async def getPlatform(request: Request, platformAlias: str):
-    # db = request.app.state.mongo
+    db = request.app.state.mongo
-    # keys = await db.get("online_devices")
+    existing_platform = db["platform"].find_one(
-    logger.info("starting!")
+        {"platformAlias": platformAlias}, {"_id": 0})
+    logger.info(f"starting get Platform with {platformAlias}!")
+    return JSONResponse(
+        status_code=starlette.status.HTTP_200_OK,
+        content=existing_platform,
+    )
 
-    return {"message": "kajsk"}
+
+@app.get("/getgamelist")
+async def getgamelist(
+    request: Request,
+    cutoff: bool,
+    isLowest: bool,
+    platform: int = 1,
+    skip: int = 0,
+    limit: int = 10,
+):
+    db = request.app.state.mongo
+
+    try:
+        find_args = {}
+        if cutoff:
+            find_args["cutOff"] = {"$ne": 0}
+        if isLowest:
+            find_args["isLowest"] = {"$ne": 0}
+        find_args["platform"] = platform
+        gamelist = db["gameinfo"].find(find_args, {
+            "_id": 0,
+            "oldGameId": 1,
+            "name": 1
+        }).sort("_id").skip(skip).limit(limit)
+        logger.info(f"get gamelist with {find_args}!")
+        return JSONResponse(
+            status_code=starlette.status.HTTP_200_OK,
+            content=list(gamelist),
+        )
+    except Exception as e:
+        return JSONResponse(
+            status_code=starlette.status.HTTP_500_INTERNAL_SERVER_ERROR,
+            content=e,
+        )
+
+
+@app.get("/getgameinfo")
+async def getgameinfo(request: Request, oldGameId: int = 0, name: str = ""):
+    db = request.app.state.mongo
+    if oldGameId:
+        gameinfo = db["gameinfo"].find({"oldGameId": oldGameId}, {"_id": 0})
+    elif name:
+        gameinfo = db["gameinfo"].find({"name": re.compile(name)}, {"_id": 0})
+    else:
+        logger.error(f"get gameinfo with {oldGameId} {name}!")
+        return JSONResponse(
+            status_code=starlette.status.HTTP_400_BAD_REQUEST,
+            content={"message": "参数未提供"},
+        )
+    logger.info(f"get gameinfo with {oldGameId} {name}!")
+    return JSONResponse(
+        status_code=starlette.status.HTTP_200_OK,
+        content=list(gameinfo),
+    )
+
+
+@app.get("/getgameprice")
+async def getgameprice(request: Request, oldGameId: int):
+    db = request.app.state.mongo
+    gameprice = db["gameprice"].find_one({"oldGameId": oldGameId}, {"_id": 0})
+    logger.info(f"get gameprice with {oldGameId} !")
+    return JSONResponse(
+        status_code=starlette.status.HTTP_200_OK,
+        content=gameprice,
+    )
+
+
+@app.get("/getgameinfoext")
+async def getgameinfoext(request: Request, oldGameId: int):
+    db = request.app.state.mongo
+    gameinfoext = db["gameinfoext"].find_one({"oldGameId": oldGameId},
+                                             {"_id": 0})
+    logger.info(f"get gameinfoext with {oldGameId} !")
+    return JSONResponse(
+        status_code=starlette.status.HTTP_200_OK,
+        content=gameinfoext,
+    )
+
+
+@app.get("/gethistoryprice")
+async def gethistoryprice(request: Request, oldGameId: int):
+    db = request.app.state.mongo
+    history_price = db["history_price"].find_one({"oldGameId": oldGameId},
+                                                 {"_id": 0})
+    logger.info(f"get historyprice with {oldGameId} !")
+    return JSONResponse(
+        status_code=starlette.status.HTTP_200_OK,
+        content=history_price,
+    )
 
 
 if __name__ == '__main__':

scripts/common/mongodb.py

@@ -1,88 +1,26 @@
 # sample as https://github.com/mongodb-developer/mongodb-with-fastapi/blob/master/app.py
 import os
-from fastapi import FastAPI, Body, HTTPException, status
+import pymongo
-from fastapi.responses import JSONResponse
-from fastapi.encoders import jsonable_encoder
-from pydantic import BaseModel, Field, EmailStr
-from bson import ObjectId
-from typing import Optional, List
-import motor.motor_asyncio
-from pydantic.types import Json
-from config.config import settings
 
 
 async def get_mongo():
-    client = motor.motor_asyncio.AsyncIOMotorClient(settings.MONGODB_URL)
+    if os.getenv('env') == 'dev':
-    db = client.jump
-    return db
 
-
+        mongo_info = {
-class PyObjectId(ObjectId):
+            "host": 'localhost',
-    @classmethod
+            "user": "admin",
-    def __get_validators__(cls):
+            "pswd": "kingsome",
-        yield cls.validate
+            "port": 27017,
-
+            "db": "jump"
-    @classmethod
-    def validate(cls, v):
-        if not ObjectId.is_valid(v):
-            raise ValueError("Invalid objectid")
-        return ObjectId(v)
-
-    @classmethod
-    def __modify_schema__(cls, field_schema):
-        field_schema.update(type="string")
-
-
-class PlatformModel(BaseModel):
-    id: PyObjectId = Field(default_factory=PyObjectId, alias="_id")
-    moduleId: int = Field(...)
-    platformAlias: str = Field(...)
-    iconRes: str = Field(...)
-    gameNum: int = Field(...)
-    filter: list = []
-    countryFilter: bool
-
-    class Config:
-        allow_population_by_field_name = True
-        arbitrary_types_allowed = True
-        json_encoders = {ObjectId: str}
-        schema_extra = {
-            "example": {
-                "moduleId":
-                4,
-                "platformAlias":
-                'Steam',
-                "iconRes":
-                'http://switch-cdn.vgjump.com/869270335915032576',
-                "gameNum":
-                759,
-                "filter": [{
-                    'termsId': 26,
-                    'terms': '精选'
-                }, {
-                    'termsId': 27,
-                    'terms': '全部'
-                }],
-                "countryFilter":
-                None
-            }
-        }
-
-
-class UpdateStudentModel(BaseModel):
-    name: Optional[str]
-    email: Optional[EmailStr]
-    course: Optional[str]
-    gpa: Optional[float]
-
-    class Config:
-        arbitrary_types_allowed = True
-        json_encoders = {ObjectId: str}
-        schema_extra = {
-            "example": {
-                "name": "Jane Doe",
-                "email": "jdoe@example.com",
-                "course": "Experiments, Science, and Fashion in Nanophotonics",
-                "gpa": "3.0",
-            }
         }
+        mongo_client = pymongo.MongoClient(mongo_info['host'],
+                                           mongo_info['port'])
+        mongo_auth = mongo_client['admin']
+        mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd'])
+        mongo_db = mongo_client.jump
+    else:
+        mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"}
+        mongo_client = pymongo.MongoClient(mongo_info['host'],
+                                           mongo_info['port'])
+        mongo_db = mongo_client.jump
+    return mongo_db

scripts/get_jump_data.py

@@ -1,61 +1,67 @@
-from sys import int_info
 import pymongo
 import datetime
-from bson.objectid import ObjectId
-from bson import json_util
-import copy
-import re
-import pdb
 import json
 import requests
 import time
+import os
 
-mongo_info = {
+
-    "host": 'localhost',
+def get_log(log_path_files="/data/logs/ops/ops.log"):
-    "user": "admin",
+    from loguru import logger
-    "pswd": "kingsome",
+    logger.add(log_path_files, rotation="500 MB", enqueue=True)
-    "port": 27017,
+    return logger
-    "db": "jump"
+
-}
+
+if os.getenv('env') == 'dev':
+    logger = get_log("get_jump.log")
+    mongo_info = {
+        "host": 'localhost',
+        "user": "admin",
+        "pswd": "kingsome",
+        "port": 27017,
+        "db": "jump"
+    }
+    mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
+    mongo_auth = mongo_client['admin']
+    mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd'])
+    mongo_db = mongo_client.jump
+else:
+    logger = get_log("/data/logs/ops/get_jump.log")
+    mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"}
+    mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
+    mongo_db = mongo_client.jump
+# 即将推出：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=1&version=16
+# 热门新游：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=2&version=16
+# 正在流行：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=3&version=16
 platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3"
+discount_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3"
 all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=30&offset={offset}&platForm={platformid}&systemList=&type=4&version=3"
-# all_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3"
-
 game_info_url = "https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id={gameid}&path=find_discount_gamedetail_switch&platform={platformid}&version=3"
 game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gameid}&platform={platformid}"
 game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all"
 
-mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
-mongo_auth = mongo_client['admin']
-mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd'])
-mongo_db = mongo_client.jump
-
 
 def set_jump_data() -> bool:
     if not set_platform():
-        print("get platform info failed!")
+        logger.error("get platform info failed!")
     else:
         if not set_all_game():
-            print("get game list failed!")
+            logger.error("get game list failed!")
-
     return True
 
 
 def set_platform() -> bool:
     info = get_url_data(platform_url)
     for platform in info.get('data'):
-        platform['uptime'] = datetime.datetime.strftime(
+        try:
-            datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
+            platform['uptime'] = datetime.datetime.strftime(
-        platformAlias = platform.get('platformAlias')
+                datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
-        existing_platform = mongo_db["platform"].find_one(
+            platformAlias = platform.get('platformAlias')
-            {"platformAlias": platformAlias})
+            mongo_db['platform'].update_one({'platformAlias': platformAlias},
-        if not existing_platform:
+                                            {'$set': platform},
-            new_platform = mongo_db["platform"].insert_one(platform)
+                                            upsert=True)
-            created_platform = mongo_db["platform"].find_one(
+        except:
-                {"_id": new_platform.inserted_id})
+            logger.error(f"some error with set platform with {platform}")
-            print(f"insert platform {created_platform}")
-        else:
-            print(f"key found with {existing_platform}")
     return True
 
 
@@ -68,37 +74,110 @@ def set_all_game() -> bool:
 
     for id in ids:
         platformid = id.get("moduleId")
-        nums = int(id.get("gameNum"))
         ii = 0
-        i = 0
         while 1:
             url = all_game_url.format(offset=ii, platformid=platformid)
             info = get_url_data(url)
-            print(f"url={url}")
-            if info.get("data"):
-                for item in info.get("data"):
-                    i += 1
-                    item['uptime'] = datetime.datetime.strftime(
-                        datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
-                    gameid = item.get("oldGameId")
-                    mongo_db['history_price'].update_one({'oldGameId': gameid},
-                                                         {'$set': item},
-                                                         upsert=True)
-            else:
-                print(info)
-            if ii > nums:
-                break
             ii += 30
-            print(f"total={i}")
+            try:
+                if info.get("data"):
+                    for item in info.get("data"):
+                        item['uptime'] = datetime.datetime.strftime(
+                            datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
+                        gameid = item.get("oldGameId")
+                        cutOff = item.get("cutOff")
+                        m_cutOff = mongo_db['gameinfo'].find_one(
+                            {"oldGameId": gameid}, {
+                                "_id": 0,
+                                "cutOff": 1,
+                                "cuttime": 1
+                            })
+                        if not m_cutOff:
+                            mm_cutOff = 0
+                        else:
+                            mm_cutOff = m_cutOff.get('cutOff')
+                        # 判断是否打折，修改cuttime数值
+                        if not cutOff:
+                            item['cuttime'] = 0
+                        elif mm_cutOff == 0 and cutOff > 0:
+                            item['cuttime'] = item['uptime']
+                        else:
+                            pass
+
+                        mongo_db['gameinfo'].update_one({'oldGameId': gameid},
+                                                        {'$set': item},
+                                                        upsert=True)
+                else:
+                    logger.debug(f"get {url} {info}")
+                    break
+            except:
+                logger.error(f"get gameinfo failed with {info}")
+
+        logger.info(f"platformid = {id} \ttotal={ii}")
     return True
 
 
+def set_gameinfo_ext() -> bool:
+    gameid_info = mongo_db['gameinfo'].find({}, {
+        "_id": 0,
+        "oldGameId": 1,
+        "platform": 1,
+    })
+    i = 0
+    for line in gameid_info:
+        try:
+            oldGameId = line.get("oldGameId", 0)
+            platform = line.get("platform", 0)
+            url = game_info_url.format(gameid=oldGameId, platformid=platform)
+
+            data = get_url_data(url=url, timesleep=1).get('data')
+            #print(data)
+            data['oldGameId'] = oldGameId
+            data['uptime'] = datetime.datetime.strftime(
+                datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
+            mongo_db['gameinfoext'].update_one({'oldGameId': oldGameId},
+                                               {'$set': data},
+                                               upsert=True)
+            i += 1
+            logger.info(f"collect gameinfoext {i}")
+        except Exception as e:
+            logger.error(f"get game info error with {line} {url} {e}")
+    logger.info(f"get game ext info total= {i}!")
+
+
+def set_game_price() -> bool:
+    gameid_info = mongo_db['gameinfo'].find({}, {
+        "_id": 0,
+        "oldGameId": 1,
+        "platform": 1,
+    })
+    i = 0
+    for line in gameid_info:
+        try:
+            oldGameId = line.get("oldGameId", 0)
+            platform = line.get("platform", 0)
+            url = game_price_url.format(gameid=oldGameId, platformid=platform)
+
+            data = get_url_data(url).get('data')
+            data['oldGameId'] = oldGameId
+            data['uptime'] = datetime.datetime.strftime(
+                datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
+            mongo_db['gameprice'].update_one({'oldGameId': oldGameId},
+                                             {'$set': data},
+                                             upsert=True)
+            i += 1
+        except:
+            logger.error(f"get game price error with {line} {url}")
+    logger.info(f"get game price total={i} ")
+
+
 def set_history_price() -> bool:
     gameid_info = mongo_db['gameinfo'].find({}, {
         "_id": 0,
         "oldGameId": 1,
         "platform": 1,
     })
+    i = 0
     for line in gameid_info:
         try:
             oldGameId = line.get("oldGameId", 0)
@@ -113,20 +192,67 @@ def set_history_price() -> bool:
             mongo_db['history_price'].update_one({'oldGameId': oldGameId},
                                                  {'$set': data},
                                                  upsert=True)
+            i += 1
         except:
-            print(f"get history price error with {line} {url}")
+            logger.error(f"get history price error with {line} {url}")
+    logger.info(f"get history price total={i}")
 
 
-def get_url_data(url: str, data=None) -> json:
+def set_discount() -> bool:
+    collections = mongo_db.list_collection_names()
+    if "discount_price" in collections:
+        logger.debug(f"find discount_price remove it \n {collections}")
+        mongo_db.drop_collection("discount_price")
+    ids = mongo_db['platform'].find({}, {"_id": 0, "moduleId": 1})
+    #ids = [{"moduleId": 1}]
+    for id in ids:
+        platformid = id.get("moduleId")
+        i = 0
+        while 1:
+            url = discount_game_url.format(offset=i, platformid=platformid)
+            info = get_url_data(url)
+            i += 10
+            try:
+                if info.get("data"):
+                    for item in info.get("data"):
+                        item['uptime'] = datetime.datetime.strftime(
+                            datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
+                        gameid = item.get("oldGameId")
+                        if item.get("platform") == 53:
+                            item["platform"] = 52
+                        mongo_db['discount_price'].update_one(
+                            {'oldGameId': gameid}, {'$set': item}, upsert=True)
+                else:
+                    logger.debug(f"get {url} {info}")
+                    break
+            except:
+                logger.error(f"get gameinfo failed with {info}")
+
+        logger.info(f"platformid = {id} \ttotal={i}")
+    return True
+
+
+def get_url_data(url: str, timesleep=0.5, data=None) -> json:
     requests.adapters.DEFAULT_RETRIES = 5
     s = requests.session()
     s.keep_alive = False
     headers = {'Connection': 'close'}
-    time.sleep(1)
+    time.sleep(timesleep)
     res = s.get(url=url, params=data, timeout=10, headers=headers).json()
     return res
 
 
-if __name__ == "__main__":
+def main_handler(event, context):
-    #set_jump_data()
+    set_jump_data()
-    set_history_price()
+    time.sleep(5)
+    set_history_price()
+    time.sleep(1)
+    #set_discount()
+    #time.sleep(1)
+    set_game_price()
+    time.sleep(1)
+    set_gameinfo_ext()
+
+
+if __name__ == '__main__':
+    main_handler("", "")