From 25267f44ff3603d301261b36661aaf8ae37e2877 Mon Sep 17 00:00:00 2001 From: pengtao Date: Wed, 6 Nov 2019 14:35:33 +0800 Subject: [PATCH] add tags field --- taptap/taptap_report.py | 10 +- taptap/taptap_report_new.py | 202 ++++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 taptap/taptap_report_new.py diff --git a/taptap/taptap_report.py b/taptap/taptap_report.py index 5209d6c..35f2a0c 100644 --- a/taptap/taptap_report.py +++ b/taptap/taptap_report.py @@ -23,9 +23,8 @@ class TapTapReport: self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime('%Y-%m-%d') def clear_noice(self): - # 结合前后2天的数据，清理diff_order >200的数据 pass - + def build_report(self): @@ -38,7 +37,7 @@ class TapTapReport: b_d = self.split_order_data(item) for key in b_d: gameid_order.setdefault(key, {}).update(b_d[key]) - pdb.set_trace() + for key in gameid_order.keys(): all.setdefault(key, {})['gameid'] = key.split("#")[0] all[key]['catename'] = key.split('#')[1] @@ -77,8 +76,7 @@ class TapTapReport: for line in all.values(): try: - print(line) - # mydb.insert("taptap_data", line) + mydb.insert("taptap_data", line) except Exception: log.error(f"install 2 db failed ,values={line}", exc_info=True) @@ -191,7 +189,7 @@ def main(): elif methods == "report": tap.build_report() elif methods == 'all': - #tap.get_order() + tap.get_order() tap.build_report() else: print("PLS input methods in ('data','report')") diff --git a/taptap/taptap_report_new.py b/taptap/taptap_report_new.py new file mode 100644 index 0000000..52ba0d8 --- /dev/null +++ b/taptap/taptap_report_new.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +from ops.mtga import FromTga +from ops.plog import define_logger +from ops.mmysql import MysqlBase +import logging +import datetime +import pdb +import sys +from bson.objectid import ObjectId +import json + +define_logger("/data/logs/ops/taptap_report.log") +log = logging.getLogger(__name__) +import sys + + +class TapTapReport: + def __init__(self, day): + self.db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'} + self.url = "http://10.10.3.17:8992/querySql" + self.api_secret = "n9H4R32ZcjtSeN89ljCY6ESzTmOlnwwnOB3r4YsggnP5M1AXLtKtiS4sS1KKLOEQ" + self.tga = FromTga(url=self.url, token=self.api_secret) + self.day = day + self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime('%Y-%m-%d') + self.bbday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=2)).strftime( + '%Y-%m-%d') + + def clear_noice(self): + # 结合前后2天的数据，清理diff_order >200的数据 + pass + + + def build_report(self): + mydb = MysqlBase(**self.db_conf) + all = dict() + gameid_order = dict() + # get 最新的gameid info数据 + gameid_info = self.get_gameid_info(self.day) + for item in (self.day, self.bday, self.bbday): + b_d = self.split_order_data(item) + for key in b_d: + gameid_order.setdefault(key, {}).update(b_d[key]) + pdb.set_trace() + for key in gameid_order.keys(): + all.setdefault(key, {})['gameid'] = key.split("#")[0] + all[key]['catename'] = key.split('#')[1] + all[key]['date'] = self.day + all[key]['diff_order'] = gameid_order[key].get(self.bday, 301) - gameid_order[key].get(self.day, 301) + all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.day, 301) + + for key in gameid_info.keys(): + all[key]['title'] = gameid_info[key]['title'] + all[key]['cate'] = gameid_info[key]['cate'] + all[key]['topic'] = gameid_info[key]['topic'] + all[key]['score'] = gameid_info[key]['score'] + all[key]['reserve'] = gameid_info[key]['reserve'] + all[key]['watch'] = gameid_info[key]['watch'] + all[key]['download'] = gameid_info[key]['download'] + all[key]['sell'] = gameid_info[key]['sell'] + all[key]['review'] = gameid_info[key]['review'] + all[key]['tags'] = gameid_info[key]['tags'] + + b_game_info = self.get_gameid_info(self.bday) + for key in all.keys(): + if not all[key].get('title', None): + try: + all[key]['title'] = b_game_info[key]['title'] + all[key]['cate'] = b_game_info[key]['cate'] + all[key]['topic'] = b_game_info[key]['topic'] + all[key]['score'] = b_game_info[key]['score'] + all[key]['reserve'] = b_game_info[key]['reserve'] + all[key]['watch'] = b_game_info[key]['watch'] + all[key]['download'] = b_game_info[key]['download'] + all[key]['sell'] = b_game_info[key]['sell'] + all[key]['review'] = b_game_info[key]['review'] + all[key]['tags'] = b_game_info[key]['tags'] + except Exception: + print(f"0={key}") + + for line in all.values(): + try: + print(line) + # mydb.insert("taptap_data", line) + except Exception: + log.error(f"install 2 db failed ,values={line}", exc_info=True) + + + def get_gameid_info(self, day): + all_data = dict() + sql = f"""SELECT + gameid, + catename, + title, + cate, + topic, + score, + reserve, + watch, + download, + sell, + review , + tags + FROM + v_event_25 + where + "$part_date"='{day}'""" + data = self.tga.get_data(sql) + if data: + for line in data: + try: + gameid, catename, title, cate, topic, score, reserve, watch, download, sell, review, tags = line + key = f"{str(int(gameid))}#{catename}" + all_data.setdefault(key, {})["key"] = key + all_data.setdefault(key, {})["catename"] = catename + all_data.setdefault(key, {})["title"] = title + all_data.setdefault(key, {})["cate"] = cate + all_data.setdefault(key, {})["topic"] = topic + all_data.setdefault(key, {})["score"] = score + all_data.setdefault(key, {})["reserve"] = reserve + all_data.setdefault(key, {})["watch"] = watch + all_data.setdefault(key, {})["download"] = download + all_data.setdefault(key, {})["sell"] = sell + all_data.setdefault(key, {})["review"] = review + all_data.setdefault(key, {})["tags"] = tags + except Exception: + log.error(f"split {line} failed", exc_info=True) + return all_data + + + def split_order_data(self, day): + all_data = dict() + mydb = MysqlBase(**self.db_conf) + sql = f"select gameid,catename,`order`,date from taptap_order WHERE date='{day}'" + data = mydb.query(sql) + if data: + for line in data: + try: + gameid, catename, order, date = line + key = f"{str(gameid)}#{catename}" + temp = {} + temp[date] = order + all_data[key] = temp + except Exception: + log.error(f"split line failed,values={line}", exc_info=True) + return all_data + + + def get_order(self): + sql = f""" + SELECT + gameid, + catename, + "order" + FROM + v_event_25 + WHERE + "$part_date"='{self.day}' + + """ + data = self.tga.get_data(sql) + if data: + self.order2mysql(data) + return True + else: + return False + + def order2mysql(self, data): + mydb = MysqlBase(**self.db_conf) + table_name = "taptap_order" + for line in data: + temp = {} + try: + temp["gameid"], temp["catename"], temp["order"] = line + temp["date"] = self.day + mydb.insert(table_name, temp) + except Exception: + log.error(f"instert {line} 2 db failed", exc_info=True) + + +def main(): + if len(sys.argv) == 3: + day = sys.argv[2] + methods = sys.argv[1] + else: + methods = sys.argv[1] + day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') + if not (day and methods): + raise Exception(f"PLs input day={day} methods={methods}") + tap = TapTapReport(day) + if methods == 'data': + tap.get_order() + elif methods == "report": + tap.build_report() + elif methods == 'all': + # tap.get_order() + tap.build_report() + else: + print("PLS input methods in ('data','report')") + + +if __name__ == "__main__": + main()