228 lines
9.6 KiB
Python
228 lines
9.6 KiB
Python
# -*- coding: utf-8 -*-
|
||
from ops.mtga import FromTga
|
||
from ops.plog import define_logger
|
||
from ops.mmysql import MysqlBase
|
||
import logging
|
||
import datetime
|
||
import pdb
|
||
import sys
|
||
from bson.objectid import ObjectId
|
||
import json
|
||
|
||
define_logger("/data/logs/ops/taptap_report.log")
|
||
log = logging.getLogger(__name__)
|
||
import sys
|
||
|
||
|
||
class TapTapReport:
|
||
def __init__(self, day):
|
||
self.db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'}
|
||
self.url = "http://10.10.3.17:8992/querySql"
|
||
self.api_secret = "n9H4R32ZcjtSeN89ljCY6ESzTmOlnwwnOB3r4YsggnP5M1AXLtKtiS4sS1KKLOEQ"
|
||
self.tga = FromTga(url=self.url, token=self.api_secret)
|
||
self.day = day
|
||
self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
|
||
self.bbday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=2)).strftime(
|
||
'%Y-%m-%d')
|
||
|
||
def clear_noice(self):
|
||
# 结合前后2天的数据，清理diff_order >200的数据
|
||
pass
|
||
|
||
|
||
def build_report(self):
|
||
mydb = MysqlBase(**self.db_conf)
|
||
all = dict()
|
||
gameid_order = dict()
|
||
# get 最新的gameid info数据
|
||
gameid_info = self.get_gameid_info(self.bday)
|
||
for item in (self.day, self.bday, self.bbday):
|
||
b_d = self.split_order_data(item)
|
||
for key in b_d:
|
||
gameid_order.setdefault(key, {}).update(b_d[key])
|
||
for key in gameid_order.keys():
|
||
all.setdefault(key, {})['gameid'] = key.split("#")[0]
|
||
all[key]['catename'] = key.split('#')[1]
|
||
all[key]['date'] = self.bday
|
||
|
||
if gameid_order[key].get(self.bday, 0) and gameid_order[key].get(self.bbday, 0) and gameid_order[key].get(
|
||
self.day, 0):
|
||
all[key]['diff_order'] = gameid_order[key].get(self.bbday, 0) - gameid_order[key].get(self.bday, 0)
|
||
all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.bday, 0)
|
||
elif gameid_order[key].get(self.bday, 0) and gameid_order[key].get(self.bbday, 0):
|
||
all[key]['diff_order'] = gameid_order[key].get(self.bbday, 0) - gameid_order[key].get(self.bday, 0)
|
||
all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.bday, 0)
|
||
elif gameid_order[key].get(self.bday, 0) and gameid_order[key].get(self.day, 0):
|
||
all[key]['diff_order'] = "新晋榜"
|
||
all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.bday, 0)
|
||
elif gameid_order[key].get(self.bbday, 0) and gameid_order[key].get(self.day, 0):
|
||
all[key]['diff_order'] = gameid_order[key].get(self.bbday, 0) - gameid_order[key].get(self.day, 0)
|
||
all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.day, 0)
|
||
else:
|
||
# 三天中有2天没数据，写入日志不予处理
|
||
log.error(f"{key} missing 2 days data,{self.bday}={gameid_order[key].get(self.bday, 0)},{self.bbday}={gameid_order[key].get(self.bbday, 0)},{self.day}={gameid_order[key].get(self.day, 0)}")
|
||
|
||
for key in gameid_info.keys():
|
||
all[key]['title'] = gameid_info[key]['title']
|
||
all[key]['cate'] = gameid_info[key]['cate']
|
||
all[key]['topic'] = gameid_info[key]['topic']
|
||
all[key]['score'] = gameid_info[key]['score']
|
||
all[key]['reserve'] = gameid_info[key]['reserve']
|
||
all[key]['watch'] = gameid_info[key]['watch']
|
||
all[key]['download'] = gameid_info[key]['download']
|
||
all[key]['sell'] = gameid_info[key]['sell']
|
||
all[key]['review'] = gameid_info[key]['review']
|
||
all[key]['tags'] = gameid_info[key]['tags']
|
||
|
||
b_game_info = self.get_gameid_info(self.bbday)
|
||
n_game_info = self.get_gameid_info(self.day)
|
||
for key in all.keys():
|
||
if not all[key].get('title', None):
|
||
try:
|
||
all[key]['title'] = b_game_info.get(key,{}).get('title',None) or n_game_info.get(key,{}).get('title',None)
|
||
all[key]['cate'] = b_game_info.get(key,{}).get('cate',None) or n_game_info.get(key,{}).get('cate',None)
|
||
all[key]['topic'] = b_game_info.get(key,{}).get('topic',None) or n_game_info.get(key,{}).get('topic',None)
|
||
all[key]['score'] = b_game_info.get(key, {}).get('score', 0) or n_game_info.get(key, {}).get(
|
||
'score', 0)
|
||
all[key]['reserve'] = b_game_info.get(key, {}).get('reserve', 0) or n_game_info.get(key, {}).get(
|
||
'reserve', 0)
|
||
all[key]['watch'] = b_game_info.get(key, {}).get('watch', 0) or n_game_info.get(key, {}).get(
|
||
'watch', 0)
|
||
all[key]['download'] = b_game_info.get(key, {}).get('download', 0) or n_game_info.get(key, {}).get(
|
||
'download', 0)
|
||
all[key]['sell'] = b_game_info.get(key, {}).get('sell', 0) or n_game_info.get(key, {}).get('sell',
|
||
0)
|
||
all[key]['review'] = b_game_info.get(key, {}).get('review', 0) or n_game_info.get(key, {}).get(
|
||
'review', 0)
|
||
all[key]['tags'] = b_game_info.get(key,{}).get('tags',None) or n_game_info.get(key,{}).get('tags',None)
|
||
except Exception:
|
||
log.error(f"0={key} 1={all.get(key, 0)} 2={b_game_info.get(key, 0)} 3={n_game_info.get(key, 0)}",
|
||
exc_info=True)
|
||
|
||
for line in all.values():
|
||
try:
|
||
#print(line)
|
||
if line.get('current_order', 0) > 0:
|
||
mydb.insert("taptap_data_new", line)
|
||
else:
|
||
log.error(f"some value is zore ,{line}!")
|
||
except Exception:
|
||
log.error(f"install 2 db failed ,values={line}", exc_info=True)
|
||
|
||
|
||
def get_gameid_info(self, day):
|
||
all_data = dict()
|
||
sql = f"""SELECT
|
||
gameid,
|
||
catename,
|
||
title,
|
||
cate,
|
||
topic,
|
||
score,
|
||
reserve,
|
||
watch,
|
||
download,
|
||
sell,
|
||
review ,
|
||
tags
|
||
FROM
|
||
v_event_25
|
||
where
|
||
"$part_date"='{day}'"""
|
||
data = self.tga.get_data(sql)
|
||
if data:
|
||
for line in data:
|
||
try:
|
||
gameid, catename, title, cate, topic, score, reserve, watch, download, sell, review, tags = line
|
||
key = f"{str(int(gameid))}#{catename}"
|
||
all_data.setdefault(key, {})["key"] = key
|
||
all_data.setdefault(key, {})["catename"] = catename
|
||
all_data.setdefault(key, {})["title"] = title
|
||
all_data.setdefault(key, {})["cate"] = cate
|
||
all_data.setdefault(key, {})["topic"] = topic
|
||
all_data.setdefault(key, {})["score"] = score
|
||
all_data.setdefault(key, {})["reserve"] = reserve
|
||
all_data.setdefault(key, {})["watch"] = watch
|
||
all_data.setdefault(key, {})["download"] = download
|
||
all_data.setdefault(key, {})["sell"] = sell
|
||
all_data.setdefault(key, {})["review"] = review
|
||
all_data.setdefault(key, {})["tags"] = tags
|
||
except Exception:
|
||
log.error(f"split {line} failed", exc_info=True)
|
||
return all_data
|
||
|
||
|
||
def split_order_data(self, day):
|
||
all_data = dict()
|
||
mydb = MysqlBase(**self.db_conf)
|
||
sql = f"select gameid,catename,`order`,date from taptap_order WHERE date='{day}'"
|
||
data = mydb.query(sql)
|
||
if data:
|
||
for line in data:
|
||
try:
|
||
gameid, catename, order, date = line
|
||
key = f"{str(gameid)}#{catename}"
|
||
temp = {}
|
||
temp[date] = order
|
||
all_data[key] = temp
|
||
except Exception:
|
||
log.error(f"split line failed,values={line}", exc_info=True)
|
||
return all_data
|
||
|
||
|
||
def get_order(self):
|
||
sql = f"""
|
||
SELECT
|
||
gameid,
|
||
catename,
|
||
"order"
|
||
FROM
|
||
v_event_25
|
||
WHERE
|
||
"$part_date"='{self.day}'
|
||
|
||
"""
|
||
data = self.tga.get_data(sql)
|
||
if data:
|
||
self.order2mysql(data)
|
||
return True
|
||
else:
|
||
return False
|
||
|
||
def order2mysql(self, data):
|
||
mydb = MysqlBase(**self.db_conf)
|
||
table_name = "taptap_order"
|
||
for line in data:
|
||
temp = {}
|
||
try:
|
||
temp["gameid"], temp["catename"], temp["order"] = line
|
||
temp["date"] = self.day
|
||
mydb.insert(table_name, temp)
|
||
except Exception:
|
||
log.error(f"instert {line} 2 db failed", exc_info=True)
|
||
|
||
|
||
def main():
|
||
if len(sys.argv) == 3:
|
||
day = sys.argv[2]
|
||
methods = sys.argv[1]
|
||
else:
|
||
methods = sys.argv[1]
|
||
day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
|
||
if not (day and methods):
|
||
raise Exception(f"PLs input day={day} methods={methods}")
|
||
tap = TapTapReport(day)
|
||
if methods == 'data':
|
||
tap.get_order()
|
||
elif methods == "report":
|
||
tap.build_report()
|
||
elif methods == 'all':
|
||
# tap.get_order()
|
||
tap.build_report()
|
||
else:
|
||
print("PLS input methods in ('data','report')")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|