ops/jump_collect
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix bug

Browse Source
This commit is contained in:
pengtao 2021-12-15 13:45:52 +08:00
parent 5844564b25
commit 8b52c58f62
4 changed files with 314 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
config/config.py
View File

@ -10,13 +10,7 @@ class Settings(BaseSettings):
admin_email: str = "pengtao@kingsome.cn"
items_per_user: int = 50
is_debug: bool = True
mongo_info = {
"host": 'localhost',
"user": "admin",
"pswd": "kingsome",
"port": 27017,
"db": "jump"
}
MONGODB_URL = "mongodb://admin:kingsome@localhost"
platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3"
all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset={offset}&platForm={platformid}&systemList=&type=4&version=3"

125
main.py
View File

@ -1,20 +1,20 @@
# uvicorn main:app --host=127.0.0.1 --port=8030 --reload
from pydantic.fields import T
from config.config import settings
from fastapi import Depends, FastAPI, BackgroundTasks, Request
from fastapi.responses import JSONResponse
from dependencies import get_token_header
from interface import route as interface_route
from scripts.common.mongodb import get_mongo
from typing import Optional
from scripts.logger import logger
from fastapi.middleware.cors import CORSMiddleware
import starlette
import re
from pydantic import BaseModel, Field, EmailStr
import json
# from apscheduler.events import EVENT_JOB_EXECUTED
# from jobs.jobs import Schedule, job_execute
tags_metadata = [
# {
# "name": "common",
# "description": "Operations with users. The **login** logic is also here.",
# },
{
"name": "common",
"description": "Manage items. So _fancy_ they have their own docs.",
@ -29,7 +29,6 @@ tags_metadata = [
def create_app():
application = FastAPI(dependencies=[Depends(get_token_header)],
openapi_tags=tags_metadata)
application.include_router(interface_route.router, prefix="/interface")
return application
@ -43,11 +42,14 @@ app.add_middleware(
)
class FindArgs(BaseModel):
oldGameId: int
name: str
@app.on_event("startup")
async def startup_event():
app.state.mongo = await get_mongo()
# Schedule.start()
# Schedule.add_listener(job_execute, EVENT_JOB_EXECUTED)
@app.on_event("shutdown")
@ -56,13 +58,106 @@ async def shutdown_event():
await app.state.mongo.wait_close()
@app.get("/")
async def root(request: Request):
# db = request.app.state.mongo
# keys = await db.get("online_devices")
logger.info("starting!")
@app.get("/getPlatform")
async def getPlatform(request: Request, platformAlias: str):
db = request.app.state.mongo
existing_platform = db["platform"].find_one(
{"platformAlias": platformAlias}, {"_id": 0})
logger.info(f"starting get Platform with {platformAlias}!")
return JSONResponse(
status_code=starlette.status.HTTP_200_OK,
content=existing_platform,
)
return {"message": "kajsk"}
@app.get("/getgamelist")
async def getgamelist(
request: Request,
cutoff: bool,
isLowest: bool,
platform: int = 1,
skip: int = 0,
limit: int = 10,
):
db = request.app.state.mongo
try:
find_args = {}
if cutoff:
find_args["cutOff"] = {"$ne": 0}
if isLowest:
find_args["isLowest"] = {"$ne": 0}
find_args["platform"] = platform
gamelist = db["gameinfo"].find(find_args, {
"_id": 0,
"oldGameId": 1,
"name": 1
}).sort("_id").skip(skip).limit(limit)
logger.info(f"get gamelist with {find_args}!")
return JSONResponse(
status_code=starlette.status.HTTP_200_OK,
content=list(gamelist),
)
except Exception as e:
return JSONResponse(
status_code=starlette.status.HTTP_500_INTERNAL_SERVER_ERROR,
content=e,
)
@app.get("/getgameinfo")
async def getgameinfo(request: Request, oldGameId: int = 0, name: str = ""):
db = request.app.state.mongo
if oldGameId:
gameinfo = db["gameinfo"].find({"oldGameId": oldGameId}, {"_id": 0})
elif name:
gameinfo = db["gameinfo"].find({"name": re.compile(name)}, {"_id": 0})
else:
logger.error(f"get gameinfo with {oldGameId} {name}!")
return JSONResponse(
status_code=starlette.status.HTTP_400_BAD_REQUEST,
content={"message": "参数未提供"},
)
logger.info(f"get gameinfo with {oldGameId} {name}!")
return JSONResponse(
status_code=starlette.status.HTTP_200_OK,
content=list(gameinfo),
)
@app.get("/getgameprice")
async def getgameprice(request: Request, oldGameId: int):
db = request.app.state.mongo
gameprice = db["gameprice"].find_one({"oldGameId": oldGameId}, {"_id": 0})
logger.info(f"get gameprice with {oldGameId} !")
return JSONResponse(
status_code=starlette.status.HTTP_200_OK,
content=gameprice,
)
@app.get("/getgameinfoext")
async def getgameinfoext(request: Request, oldGameId: int):
db = request.app.state.mongo
gameinfoext = db["gameinfoext"].find_one({"oldGameId": oldGameId},
{"_id": 0})
logger.info(f"get gameinfoext with {oldGameId} !")
return JSONResponse(
status_code=starlette.status.HTTP_200_OK,
content=gameinfoext,
)
@app.get("/gethistoryprice")
async def gethistoryprice(request: Request, oldGameId: int):
db = request.app.state.mongo
history_price = db["history_price"].find_one({"oldGameId": oldGameId},
{"_id": 0})
logger.info(f"get historyprice with {oldGameId} !")
return JSONResponse(
status_code=starlette.status.HTTP_200_OK,
content=history_price,
)
if __name__ == '__main__':

100
scripts/common/mongodb.py
View File

@ -1,88 +1,26 @@
# sample as https://github.com/mongodb-developer/mongodb-with-fastapi/blob/master/app.py
import os
from fastapi import FastAPI, Body, HTTPException, status
from fastapi.responses import JSONResponse
from fastapi.encoders import jsonable_encoder
from pydantic import BaseModel, Field, EmailStr
from bson import ObjectId
from typing import Optional, List
import motor.motor_asyncio
from pydantic.types import Json
from config.config import settings
import pymongo
async def get_mongo():
client = motor.motor_asyncio.AsyncIOMotorClient(settings.MONGODB_URL)
db = client.jump
return db
if os.getenv('env') == 'dev':
class PyObjectId(ObjectId):
@classmethod
def __get_validators__(cls):
yield cls.validate
@classmethod
def validate(cls, v):
if not ObjectId.is_valid(v):
raise ValueError("Invalid objectid")
return ObjectId(v)
@classmethod
def __modify_schema__(cls, field_schema):
field_schema.update(type="string")
class PlatformModel(BaseModel):
id: PyObjectId = Field(default_factory=PyObjectId, alias="_id")
moduleId: int = Field(...)
platformAlias: str = Field(...)
iconRes: str = Field(...)
gameNum: int = Field(...)
filter: list = []
countryFilter: bool
class Config:
allow_population_by_field_name = True
arbitrary_types_allowed = True
json_encoders = {ObjectId: str}
schema_extra = {
"example": {
"moduleId":
4,
"platformAlias":
'Steam',
"iconRes":
'http://switch-cdn.vgjump.com/869270335915032576',
"gameNum":
759,
"filter": [{
'termsId': 26,
'terms': '精选'
}, {
'termsId': 27,
'terms': '全部'
}],
"countryFilter":
None
}
}
class UpdateStudentModel(BaseModel):
name: Optional[str]
email: Optional[EmailStr]
course: Optional[str]
gpa: Optional[float]
class Config:
arbitrary_types_allowed = True
json_encoders = {ObjectId: str}
schema_extra = {
"example": {
"name": "Jane Doe",
"email": "jdoe@example.com",
"course": "Experiments, Science, and Fashion in Nanophotonics",
"gpa": "3.0",
}
mongo_info = {
"host": 'localhost',
"user": "admin",
"pswd": "kingsome",
"port": 27017,
"db": "jump"
}
mongo_client = pymongo.MongoClient(mongo_info['host'],
mongo_info['port'])
mongo_auth = mongo_client['admin']
mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd'])
mongo_db = mongo_client.jump
else:
mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"}
mongo_client = pymongo.MongoClient(mongo_info['host'],
mongo_info['port'])
mongo_db = mongo_client.jump
return mongo_db

242
scripts/get_jump_data.py
View File

@ -1,61 +1,67 @@
from sys import int_info
import pymongo
import datetime
from bson.objectid import ObjectId
from bson import json_util
import copy
import re
import pdb
import json
import requests
import time
import os
mongo_info = {
"host": 'localhost',
"user": "admin",
"pswd": "kingsome",
"port": 27017,
"db": "jump"
}
def get_log(log_path_files="/data/logs/ops/ops.log"):
from loguru import logger
logger.add(log_path_files, rotation="500 MB", enqueue=True)
return logger
if os.getenv('env') == 'dev':
logger = get_log("get_jump.log")
mongo_info = {
"host": 'localhost',
"user": "admin",
"pswd": "kingsome",
"port": 27017,
"db": "jump"
}
mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
mongo_auth = mongo_client['admin']
mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd'])
mongo_db = mongo_client.jump
else:
logger = get_log("/data/logs/ops/get_jump.log")
mongo_info = {"host": '10.10.5.6', "port": 27017, "db": "jump"}
mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
mongo_db = mongo_client.jump
# 即将推出：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=1&version=16
# 热门新游：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=2&version=16
# 正在流行：https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=10&offset=0&platForm=1&systemList=&type=3&version=16
platform_url = "https://switch.jumpvg.com/jump/platform/order/v2?needCount=1&needFilter=1&version=3"
discount_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3"
all_game_url = "https://switch.jumpvg.com/jump/findGame/list?categoryList=&featureList=&limit=30&offset={offset}&platForm={platformid}&systemList=&type=4&version=3"
# all_game_url = "https://switch.jumpvg.com/jump/discount/find4Discount/5/v2?offset={offset}&platform={platformid}&size=10&termsId=17&version=3"
game_info_url = "https://switch.jumpvg.com/jump/game/detail?clickFrom=-1&id={gameid}&path=find_discount_gamedetail_switch&platform={platformid}&version=3"
game_price_url = "https://switch.jumpvg.com/jump/price/getAllPriceByGame?id={gameid}&platform={platformid}"
game_history_price_url = "https://switch.jumpvg.com/switch/getDiscount?appid={gameid}&platform={platformid}&zone=all"
mongo_client = pymongo.MongoClient(mongo_info['host'], mongo_info['port'])
mongo_auth = mongo_client['admin']
mongo_auth.authenticate(mongo_info['user'], mongo_info['pswd'])
mongo_db = mongo_client.jump
def set_jump_data() -> bool:
if not set_platform():
print("get platform info failed!")
logger.error("get platform info failed!")
else:
if not set_all_game():
print("get game list failed!")
logger.error("get game list failed!")
return True
def set_platform() -> bool:
info = get_url_data(platform_url)
for platform in info.get('data'):
platform['uptime'] = datetime.datetime.strftime(
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
platformAlias = platform.get('platformAlias')
existing_platform = mongo_db["platform"].find_one(
{"platformAlias": platformAlias})
if not existing_platform:
new_platform = mongo_db["platform"].insert_one(platform)
created_platform = mongo_db["platform"].find_one(
{"_id": new_platform.inserted_id})
print(f"insert platform {created_platform}")
else:
print(f"key found with {existing_platform}")
try:
platform['uptime'] = datetime.datetime.strftime(
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
platformAlias = platform.get('platformAlias')
mongo_db['platform'].update_one({'platformAlias': platformAlias},
{'$set': platform},
upsert=True)
except:
logger.error(f"some error with set platform with {platform}")
return True
@ -68,37 +74,110 @@ def set_all_game() -> bool:
for id in ids:
platformid = id.get("moduleId")
nums = int(id.get("gameNum"))
ii = 0
i = 0
while 1:
url = all_game_url.format(offset=ii, platformid=platformid)
info = get_url_data(url)
print(f"url={url}")
if info.get("data"):
for item in info.get("data"):
i += 1
item['uptime'] = datetime.datetime.strftime(
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
gameid = item.get("oldGameId")
mongo_db['history_price'].update_one({'oldGameId': gameid},
{'$set': item},
upsert=True)
else:
print(info)
if ii > nums:
break
ii += 30
print(f"total={i}")
try:
if info.get("data"):
for item in info.get("data"):
item['uptime'] = datetime.datetime.strftime(
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
gameid = item.get("oldGameId")
cutOff = item.get("cutOff")
m_cutOff = mongo_db['gameinfo'].find_one(
{"oldGameId": gameid}, {
"_id": 0,
"cutOff": 1,
"cuttime": 1
})
if not m_cutOff:
mm_cutOff = 0
else:
mm_cutOff = m_cutOff.get('cutOff')
# 判断是否打折，修改cuttime数值
if not cutOff:
item['cuttime'] = 0
elif mm_cutOff == 0 and cutOff > 0:
item['cuttime'] = item['uptime']
else:
pass
mongo_db['gameinfo'].update_one({'oldGameId': gameid},
{'$set': item},
upsert=True)
else:
logger.debug(f"get {url} {info}")
break
except:
logger.error(f"get gameinfo failed with {info}")
logger.info(f"platformid = {id} \ttotal={ii}")
return True
def set_gameinfo_ext() -> bool:
gameid_info = mongo_db['gameinfo'].find({}, {
"_id": 0,
"oldGameId": 1,
"platform": 1,
})
i = 0
for line in gameid_info:
try:
oldGameId = line.get("oldGameId", 0)
platform = line.get("platform", 0)
url = game_info_url.format(gameid=oldGameId, platformid=platform)
data = get_url_data(url=url, timesleep=1).get('data')
#print(data)
data['oldGameId'] = oldGameId
data['uptime'] = datetime.datetime.strftime(
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
mongo_db['gameinfoext'].update_one({'oldGameId': oldGameId},
{'$set': data},
upsert=True)
i += 1
logger.info(f"collect gameinfoext {i}")
except Exception as e:
logger.error(f"get game info error with {line} {url} {e}")
logger.info(f"get game ext info total= {i}!")
def set_game_price() -> bool:
gameid_info = mongo_db['gameinfo'].find({}, {
"_id": 0,
"oldGameId": 1,
"platform": 1,
})
i = 0
for line in gameid_info:
try:
oldGameId = line.get("oldGameId", 0)
platform = line.get("platform", 0)
url = game_price_url.format(gameid=oldGameId, platformid=platform)
data = get_url_data(url).get('data')
data['oldGameId'] = oldGameId
data['uptime'] = datetime.datetime.strftime(
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
mongo_db['gameprice'].update_one({'oldGameId': oldGameId},
{'$set': data},
upsert=True)
i += 1
except:
logger.error(f"get game price error with {line} {url}")
logger.info(f"get game price total={i} ")
def set_history_price() -> bool:
gameid_info = mongo_db['gameinfo'].find({}, {
"_id": 0,
"oldGameId": 1,
"platform": 1,
})
i = 0
for line in gameid_info:
try:
oldGameId = line.get("oldGameId", 0)
@ -113,20 +192,67 @@ def set_history_price() -> bool:
mongo_db['history_price'].update_one({'oldGameId': oldGameId},
{'$set': data},
upsert=True)
i += 1
except:
print(f"get history price error with {line} {url}")
logger.error(f"get history price error with {line} {url}")
logger.info(f"get history price total={i}")
def get_url_data(url: str, data=None) -> json:
def set_discount() -> bool:
collections = mongo_db.list_collection_names()
if "discount_price" in collections:
logger.debug(f"find discount_price remove it \n {collections}")
mongo_db.drop_collection("discount_price")
ids = mongo_db['platform'].find({}, {"_id": 0, "moduleId": 1})
#ids = [{"moduleId": 1}]
for id in ids:
platformid = id.get("moduleId")
i = 0
while 1:
url = discount_game_url.format(offset=i, platformid=platformid)
info = get_url_data(url)
i += 10
try:
if info.get("data"):
for item in info.get("data"):
item['uptime'] = datetime.datetime.strftime(
datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")
gameid = item.get("oldGameId")
if item.get("platform") == 53:
item["platform"] = 52
mongo_db['discount_price'].update_one(
{'oldGameId': gameid}, {'$set': item}, upsert=True)
else:
logger.debug(f"get {url} {info}")
break
except:
logger.error(f"get gameinfo failed with {info}")
logger.info(f"platformid = {id} \ttotal={i}")
return True
def get_url_data(url: str, timesleep=0.5, data=None) -> json:
requests.adapters.DEFAULT_RETRIES = 5
s = requests.session()
s.keep_alive = False
headers = {'Connection': 'close'}
time.sleep(1)
time.sleep(timesleep)
res = s.get(url=url, params=data, timeout=10, headers=headers).json()
return res
if __name__ == "__main__":
#set_jump_data()
set_history_price()
def main_handler(event, context):
set_jump_data()
time.sleep(5)
set_history_price()
time.sleep(1)
#set_discount()
#time.sleep(1)
set_game_price()
time.sleep(1)
set_gameinfo_ext()
if __name__ == '__main__':
main_handler("", "")