From 3b9e293083afa17a52aabb9739f6b19c3c946439 Mon Sep 17 00:00:00 2001 From: pengtao Date: Tue, 23 Nov 2021 18:04:58 +0800 Subject: [PATCH] fix --- __init__.py | 0 config/__init__.py | 0 config/config.py | 34 ++++++++++++++ dependencies.py | 19 ++++++++ interface/__init__.py | 0 interface/route.py | 31 +++++++++++++ jobs.sqlite | Bin 0 -> 16384 bytes main.py | 74 ++++++++++++++++++++++++++++++ readme.md | 89 +++++++++++++++++++++++++++++++++++++ requirements.txt | 9 ++++ scripts/__init__.py | 0 scripts/common/__init__.py | 0 scripts/common/mongodb.py | 88 ++++++++++++++++++++++++++++++++++++ scripts/logger.py | 20 +++++++++ scripts/sync_data.py | 64 ++++++++++++++++++++++++++ test.py | 30 +++++++++++++ 16 files changed, 458 insertions(+) create mode 100644 __init__.py create mode 100644 config/__init__.py create mode 100644 config/config.py create mode 100644 dependencies.py create mode 100644 interface/__init__.py create mode 100644 interface/route.py create mode 100644 jobs.sqlite create mode 100644 main.py create mode 100644 readme.md create mode 100644 requirements.txt create mode 100644 scripts/__init__.py create mode 100644 scripts/common/__init__.py create mode 100644 scripts/common/mongodb.py create mode 100644 scripts/logger.py create mode 100644 scripts/sync_data.py create mode 100644 test.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/config.py b/config/config.py new file mode 100644 index 0000000..964c794 --- /dev/null +++ b/config/config.py @@ -0,0 +1,34 @@ +from pydantic import BaseSettings +from typing import Optional +import os + +from pydantic.fields import T + + +class Settings(BaseSettings): + app_name: str = "Kingsome API" + admin_email: str = "pengtao@kingsome.cn" + items_per_user: int = 50 + is_debug: bool = True + + MONGODB_URL = "mongodb://admin:kingsome@localhost" + platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3" + all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset={offset}&platForm={platformid}&systemList=&type=4&version=3" + game_info_url = "https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id={gameid}&path=find_discount_gamedetail_switch&platform={platformid}&version=3" + game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gameid}&platform={platformid}" + game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all" + + x_token: str = "abc" + + root_path_in_servers: Optional[bool] = False + root_path: str = '/api/v1' + + origins: list = [ + "http://*.kingsome.cn", + "https://*.kingsome.cn", + "http://localhost", + "http://localhost:8080", + ] + + +settings = Settings() diff --git a/dependencies.py b/dependencies.py new file mode 100644 index 0000000..5e51c0c --- /dev/null +++ b/dependencies.py @@ -0,0 +1,19 @@ +from fastapi import Header, HTTPException +from pydantic.fields import T +from starlette.requests import Request +import pdb +from config.config import settings + + +async def get_token_header(request: Request, x_token: str = Header(...)): + dd = settings.x_token + if x_token == settings.x_token or x_token == dd: + return True + else: + raise HTTPException(status_code=400, detail="X-Token header invalid") + + +async def get_query_token(token: str): + if token != "jessica": + raise HTTPException(status_code=400, + detail="No Jessica token provided") diff --git a/interface/__init__.py b/interface/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/interface/route.py b/interface/route.py new file mode 100644 index 0000000..44aa411 --- /dev/null +++ b/interface/route.py @@ -0,0 +1,31 @@ +from fastapi import APIRouter, BackgroundTasks, UploadFile, File, Form +from typing import List +from pydantic import BaseModel +from starlette.requests import Request +from starlette.responses import JSONResponse +from config.config import settings +from scripts.sync_data import get_jump_data + +router = APIRouter() + + +@router.get("/sync_data") +async def get_lists(request: Request) -> bool: + db = request.app.state.mongo + await get_jump_data(db) + return True + + +@router.get("/get_list") +async def get_lists(request: Request, keys: str) -> JSONResponse: + return {"url": keys} + + +@router.get("/get_allprice") +async def get_allprice(request: Request, keys: str) -> JSONResponse: + return {"url": keys} + + +@router.get("/get_gameinfo") +async def get_gameinfo(request: Request, id: int) -> JSONResponse: + return {"url": 'keys'} \ No newline at end of file diff --git a/jobs.sqlite b/jobs.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..b369870f5723090fc577bb494091ce879a2f2701 GIT binary patch literal 16384 zcmeI&&2Jk;6aeu3s=c;jZ&DFVgd+BWWJ|zTMsa zwBj>r?{&j{!tATcqN1wGN4BjfN(N>F=INxt%Ivl}J#fx4qZDdiq^;+Qo{1DIW_=F} zoKOG-Pyhu`00mG01yBG5Pyhw~9|DJqdOTZOQx6?KXwiW`a7iy}dbHK=(9pfz-HqIU z4wxJE1DE;xbpEKZQ7;u6C3~}at#s4A@N~OYwdXHk=T6j+KP;&6?An_8mBq+z=TwdJ zr)!fIG>Yq$(is;|Cj6Ftqgda#UaaSw_no|5tu^fGXO+r3b~16IYWw3#t=N#eq2q4E z2qfd{mD>7=Gd`(rZWZgd>~iUro%37yWd1`fmfc=cm62d{(19G@C4JT%ZJ+jZ=6j&S zP&4IlOT(-!43*5ig<;BfbLq>W%m(hqhJ3pna zKQO-aC!f*26geKhOT)_Un}qC0}e zS?IUhG!%Sg=2^cm*&7vFkXQ(wx(18Jq~wvv9}1|}Q=vLjW?SPzrd-IxOYj$~M`9xx z|1Fli{JNBY*7WYN`+{q3PaHwZ)5XXqSGGM8v_0aBtp~lb!u5^qhG3#DzJ|HK%S}3< zO-NFpk?|JYBmEA83z4w>@Ibq?WIVd*I6FVre*u(V&qL81=U8KTMq_IAE*yTrv3e$l zA21r;B^|-7&B^K+EE+eZ!~Jdm78msRj|17`%GS8YJPD~>$o_%o%7VU}e)|XESvha< z3^z~ZV<>>XuoDXQLHD{GwT^_;DiDwfC4Ch0w{n2D1ZVe zfC4Ch0x0lG31p_Pavb@&KTUcueMxm3Ev2u*i^cJ4mvR38RI#30Pp#iysqSH8Q2+%{ p00mG01yBG5Pyhu`00mG01?B}3@_!@U)D2bsW241&BO!me_zO#NWT*fD literal 0 HcmV?d00001 diff --git a/main.py b/main.py new file mode 100644 index 0000000..3774085 --- /dev/null +++ b/main.py @@ -0,0 +1,74 @@ +# uvicorn main:app --host=127.0.0.1 --port=8030 --reload +from config.config import settings +from fastapi import Depends, FastAPI, BackgroundTasks, Request +from dependencies import get_token_header +from interface import route as interface_route +from scripts.common.mongodb import get_mongo +from typing import Optional +from scripts.logger import logger +from fastapi.middleware.cors import CORSMiddleware + +# from apscheduler.events import EVENT_JOB_EXECUTED +# from jobs.jobs import Schedule, job_execute +tags_metadata = [ + # { + # "name": "common", + # "description": "Operations with users. The **login** logic is also here.", + # }, + { + "name": "common", + "description": "Manage items. So _fancy_ they have their own docs.", + "externalDocs": { + "description": "Items external docs", + "url": "https://fastapi.tiangolo.com/", + }, + }, +] + + +def create_app(): + application = FastAPI(dependencies=[Depends(get_token_header)], + openapi_tags=tags_metadata) + application.include_router(interface_route.router, prefix="/interface") + return application + + +app = create_app() +app.add_middleware( + CORSMiddleware, + allow_origins=settings.origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.on_event("startup") +async def startup_event(): + app.state.mongo = await get_mongo() + # Schedule.start() + # Schedule.add_listener(job_execute, EVENT_JOB_EXECUTED) + + +@app.on_event("shutdown") +async def shutdown_event(): + app.state.mongo.close() + await app.state.mongo.wait_close() + + +@app.get("/") +async def root(request: Request): + # db = request.app.state.mongo + # keys = await db.get("online_devices") + logger.info("starting!") + + return {"message": "kajsk"} + + +if __name__ == '__main__': + import uvicorn + uvicorn.run(app='main:app', + host="0.0.0.0", + port=8030, + reload=True, + debug=True) diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..b686e20 --- /dev/null +++ b/readme.md @@ -0,0 +1,89 @@ +### 项目简介： + +从jump接口获得游戏列表及历史价格，信息，写入mongodb，提供接口返回数据 + +new project with fastapi + +https://fastapi.tiangolo.com/zh/tutorial/bigger-applications/ + +采用异步方式实现，考虑并发性能 + +### 功能 + +获得平台信息：包含区域->国家 + +platform_url="https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3" + +获得游戏列表 + +https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=1&platForm=1&systemList=&type=4&version=3 + +获得游戏信息: + +id=**oldGameId** + +https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id=33113&path=find_discount_gamedetail_switch&platform=1&version=3 + +获得全区游戏价格： + +https://switch.jumpvg.com/jump/price/getAllPriceByGame?id=20160&platform=1 + +获得历史价格: + +https://switch.jumpvg.com/switch/getDiscount?appid=20160&platform=1&zone=all + +获得史低信息： + +程序判断 实时价格vs历史价格 + + + +> https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3 +> +> **platformAlias** +> +> 游戏列表 +> termsId 18:史低 17:全部 16:精选 +> https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset=0&platform=1&session=1637580994841&size=10&termsId=18&version=3 +> +> +> +> 游戏详情 +> https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id=20160&path=find_discount_gamedetail_switch&platform=1&version=3 +> 所有商店价格 +> https://switch.jumpvg.com/jump/price/getAllPriceByGame?id=20160&platform=1 +> 历史价格 +> 全区:all 港区:HK 欧区:UK 日区:JP 美区:US +> https://switch.jumpvg.com/switch/getDiscount?appid=20160&platform=1&zone=all +> +> 查询游戏 +> https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=%E6%8A%98%E6%89%A3%3B%E5%8C%85%E5%90%AB%E8%AF%95%E7%8E%A9&limit=10&offset=0&platForm=1&systemList=&type=5&version=3 +> +> featureList 折扣 试玩等标签 +> categoryList 是 多个中文标签(urlencode) 以 %3B分割 +> limit 最大30, 一般10个 +> +> switch 最新发布 +> https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=4&version=3 +> 正在流行 +> https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=3&version=3 +> 即将推出 +> https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=1&version=3 +> 热门新游 +> https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=2&version=3 +> 好评游戏 +> https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=6&version=3 +> 高分 +> https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=5&version=3 +> +> +> 搜索游戏 +> 综合1 游戏3 评测2 用户4 +> platform: switch 1 steam 4 xbox 8 ps4 51 +> https://switch.jumpvg.com/jump/searchGame/list?limit=10&offset=0&searchName=%E6%9A%97%E9%BB%91&type=1&version=3 +> https://switch.jumpvg.com/jump/searchGame/list?limit=10&offset=0&searchName=%E6%9A%97%E9%BB%91&type=2&version=3 +> https://switch.jumpvg.com/jump/searchGame/list?limit=10&offset=0&platForm=&searchName=%E6%9A%97%E9%BB%91&type=3&version=3 +> https://switch.jumpvg.com/jump/searchGame/list?limit=10&offset=0&searchName=%E6%9A%97%E9%BB%91&type=4&version=3 +> +> + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..98ab485 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.65.2 +fastapi-mail==0.3.7 +GitPython==3.1.18 +pydantic==1.8.2 +uvicorn==0.15.0 +oss2==2.15.0 +motor==2.3.0 +mypy-extensions==0.4.3 +pymongo==3.11.1 \ No newline at end of file diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/common/__init__.py b/scripts/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/common/mongodb.py b/scripts/common/mongodb.py new file mode 100644 index 0000000..b77f8da --- /dev/null +++ b/scripts/common/mongodb.py @@ -0,0 +1,88 @@ +# sample as https://github.com/mongodb-developer/mongodb-with-fastapi/blob/master/app.py +import os +from fastapi import FastAPI, Body, HTTPException, status +from fastapi.responses import JSONResponse +from fastapi.encoders import jsonable_encoder +from pydantic import BaseModel, Field, EmailStr +from bson import ObjectId +from typing import Optional, List +import motor.motor_asyncio +from pydantic.types import Json +from config.config import settings + + +async def get_mongo(): + client = motor.motor_asyncio.AsyncIOMotorClient(settings.MONGODB_URL) + db = client.jump + return db + + +class PyObjectId(ObjectId): + @classmethod + def __get_validators__(cls): + yield cls.validate + + @classmethod + def validate(cls, v): + if not ObjectId.is_valid(v): + raise ValueError("Invalid objectid") + return ObjectId(v) + + @classmethod + def __modify_schema__(cls, field_schema): + field_schema.update(type="string") + + +class PlatformModel(BaseModel): + id: PyObjectId = Field(default_factory=PyObjectId, alias="_id") + moduleId: int = Field(...) + platformAlias: str = Field(...) + iconRes: str = Field(...) + gameNum: int = Field(...) + filter: list = [] + countryFilter: bool + + class Config: + allow_population_by_field_name = True + arbitrary_types_allowed = True + json_encoders = {ObjectId: str} + schema_extra = { + "example": { + "moduleId": + 4, + "platformAlias": + 'Steam', + "iconRes": + 'http://switch-cdn.vgjump.com/869270335915032576', + "gameNum": + 759, + "filter": [{ + 'termsId': 26, + 'terms': '精选' + }, { + 'termsId': 27, + 'terms': '全部' + }], + "countryFilter": + None + } + } + + +class UpdateStudentModel(BaseModel): + name: Optional[str] + email: Optional[EmailStr] + course: Optional[str] + gpa: Optional[float] + + class Config: + arbitrary_types_allowed = True + json_encoders = {ObjectId: str} + schema_extra = { + "example": { + "name": "Jane Doe", + "email": "jdoe@example.com", + "course": "Experiments, Science, and Fashion in Nanophotonics", + "gpa": "3.0", + } + } diff --git a/scripts/logger.py b/scripts/logger.py new file mode 100644 index 0000000..4f6ab0c --- /dev/null +++ b/scripts/logger.py @@ -0,0 +1,20 @@ +import os +import time +from loguru import logger + +basedir = os.path.dirname( + os.path.dirname(os.path.abspath(__file__))) + +# print(f"log basedir{basedir}") # /xxx/python_code/FastAdmin/backend/app +# 定位到log日志文件 +log_path = os.path.join(basedir, 'logs') + +if not os.path.exists(log_path): + os.mkdir(log_path) + +log_path_error = os.path.join( + log_path, f'{time.strftime("%Y-%m-%d")}_error.log') + +# 日志简单配置 +# 具体其他配置 可自行参考 https://github.com/Delgan/loguru +logger.add(log_path_error, rotation="12:00", retention="5 days", enqueue=True) diff --git a/scripts/sync_data.py b/scripts/sync_data.py new file mode 100644 index 0000000..034fde7 --- /dev/null +++ b/scripts/sync_data.py @@ -0,0 +1,64 @@ +import json +from pydantic.fields import T +from config.config import settings +import requests +from starlette.requests import Request +from starlette.responses import JSONResponse +from fastapi.encoders import jsonable_encoder +from scripts.logger import logger + + +async def get_jump_data(db) -> bool: + platform_info = get_platform() + platforms = platform_info.get('data') + for platform in platforms: + platformAlias = platform.get('platformAlias') + existing_platform = await db["platform"].find_one( + {"platformAlias": platformAlias}) + if not existing_platform: + platform = jsonable_encoder(a) + new_platform = await db["platform"].insert_one(platform) + created_platform = await db["platform"].find_one( + {"_id": new_platform.inserted_id}) + logger.info(f"insert platform {created_platform}") + else: + logger.info(f"key found with {existing_platform}") + + # all_game = get_all_game() + # #gameids = all_game['data'] + # print(all_game) + return True + # for gameid in gameids: + # game_info = get_game_info(gameid) + + +def get_platform() -> json: + info = get_url_data(settings.platform_url) + return info + + +def get_all_game() -> json: + info = get_url_data(settings.all_game_url) + print(info) + return None + + +def get_game_info(gameid: int) -> dict: + pass + + +def get_game_price(gameid: int) -> dict: + pass + + +def get_history_price(gameid) -> dict: + pass + + +def get_url_data(url: str, data=None) -> json: + requests.adapters.DEFAULT_RETRIES = 5 + s = requests.session() + s.keep_alive = False + headers = {'Connection': 'close'} + res = s.get(url=url, params=data, timeout=10, headers=headers).json() + return res diff --git a/test.py b/test.py new file mode 100644 index 0000000..445e376 --- /dev/null +++ b/test.py @@ -0,0 +1,30 @@ +from fastapi.testclient import TestClient +from ops.common.common import EmailSchema +from main import app +import time +import pdb +import json +client = TestClient(app) + + +def test_read_main(): + response = client.get("/") + assert response.status_code == 200 + assert response.json() == {"message": "Hello Bigger Applications!"} + + +def test_post_mail(): + data = {"email": ['pengtao@kingsome.cn'], "body": 'test message!', + "subject": 'ttt {}'.format(time.time())} + response = client.post("/common/email", headers={"X-Token": "fake-super-scret-token", "accept": "application/json", "Content-Type": "application/json"}, + data=json.dumps(data), + ) + # data=EmailSchema(**data).json() + print(response) + # pdb.set_trace() + assert response.status_code == 200 + assert response.json() == {"message": "email has been sent"} + + +if __name__ == "__main__": + test_post_mail()