# -*- coding: utf-8 -*- from ops.mtga import FromTga from ops.plog import define_logger from ops.mmysql import MysqlBase import logging import datetime import pdb import sys from bson.objectid import ObjectId import json define_logger("/data/logs/ops/taptap_collect.log") log = logging.getLogger(__name__) import sys class TapTapReport: def __init__(self, day): self.db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'} self.url = "http://10.10.3.17:8992/querySql" self.api_secret = "n9H4R32ZcjtSeN89ljCY6ESzTmOlnwwnOB3r4YsggnP5M1AXLtKtiS4sS1KKLOEQ" self.tga = FromTga(url=self.url, token=self.api_secret) self.day = day self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime('%Y-%m-%d') self.bbday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=2)).strftime( '%Y-%m-%d') def clear_noice(self): # 结合前后2天的数据,清理diff_order >200的数据 pass def diff_item(self, all, item): for key in self.gameid_info_1.keys(): all.setdefault(key, {})['gameid'] = key.split("#")[0] all[key]['catename'] = key.split('#')[1] all[key]['date'] = self.bday diff_key = f"diff_{item}" if self.gameid_info_1.get(key, {}).get(item, -1)>=0 and self.gameid_info_2.get(key, {}).get(item, -1)>=0: all[key][diff_key] = int(self.gameid_info_1[key].get(item, 0) - self.gameid_info_2[key].get(item, 0)) all.setdefault(key, {})[item] = self.gameid_info_1[key].get(item, 0) elif self.gameid_info_1.get(key, {}).get(item, -1)>=0 and self.gameid_info_0.get(key, {}).get(item, -1)>=0: all[key][diff_key] = "新晋榜" all.setdefault(key, {})[item] = self.gameid_info_1[key].get(item, 0) elif self.gameid_info_0.get(key, {}).get(item, -1)>=0 and self.gameid_info_2.get(key, {}).get(item, -1)>=0: all[key][diff_key] = int(self.gameid_info_0[key].get(item, 0) - self.gameid_info_2[key].get(item, 0)) all.setdefault(key, {})[item] = self.gameid_info_0[key].get(item, 0) else: pass # 三天中有2天没数据,写入日志不予处理 # log.error(f"{all[key]} {item} {self.gameid_info_0.get(key,{})} {self.gameid_info_1.get(key, # {})} {self.gameid_info_2.get(key,{})} missing 2 days data") if item == "order": try: all[key][diff_key] = -int(all[key][diff_key]) except: pass return all def build_report(self): mydb = MysqlBase(**self.db_conf) all = dict() gameid_order = dict() # get 最新的gameid info数据 self.gameid_info_0 = self.get_gameid_info(self.day) self.gameid_info_1 = self.get_gameid_info(self.bday) self.gameid_info_2 = self.get_gameid_info(self.bbday) for key in self.gameid_info_1.keys(): all.setdefault(key, {})['gameid'] = key.split("#")[0] all[key]['catename'] = key.split('#')[1] all[key]['date'] = self.bday try: # ("order", "watch", "download", "sell", "review", "reserve", "topic") all[key]['title'] = self.gameid_info_1.get(key,{}).get('title', None) or self.gameid_info_0.get(key,{}).get('title',None) or self.gameid_info_2.get(key,{}).get('title', None) all[key]['cate'] = self.gameid_info_1.get(key,{}).get('cate', None) or self.gameid_info_0.get(key,{}).get('cate',None) or self.gameid_info_2.get(key,{}).get('cate', None) all[key]['score'] = self.gameid_info_1.get(key,{}).get('score', 0) or self.gameid_info_0.get(key,{}).get('score',0) or self.gameid_info_2.get(key,{}).get('score', 0) all[key]['tags'] = self.gameid_info_1.get(key,{}).get('tags', None) or self.gameid_info_0.get(key,{}).get('tags',None) or self.gameid_info_2.get(key,{}).get('tags', None) except Exception: log.error(f"split {all[key]} error ", exc_info=True) items = ("order", "watch", "download", "sell", "review", "reserve", "topic") for item in items: all = self.diff_item(all, item) i = 0 for line in all.values(): try: #print(line) if line.get("order", 0) >= 1: mydb.insert("taptap_collect", line) i += 1 except Exception: log.error(f"install 2 db failed ,values={line}", exc_info=True) log.info(f"insert {i} 2 mysql!") def get_gameid_info(self, day): all_data = dict() sql = f"""SELECT gameid, catename, title, cate, "order", topic, score, reserve, watch, download, sell, review , tags FROM v_event_25 where "$part_date"='{day}'""" data = self.tga.get_data(sql) if data: for line in data: try: gameid, catename, title, cate, order, topic, score, reserve, watch, download, sell, review, tags = line key = f"{str(int(gameid))}#{catename}" all_data.setdefault(key, {})["key"] = key all_data.setdefault(key, {})["catename"] = catename all_data.setdefault(key, {})["title"] = title all_data.setdefault(key, {})["cate"] = cate all_data.setdefault(key, {})["order"] = order all_data.setdefault(key, {})["topic"] = topic all_data.setdefault(key, {})["score"] = score all_data.setdefault(key, {})["reserve"] = reserve all_data.setdefault(key, {})["watch"] = watch all_data.setdefault(key, {})["download"] = download all_data.setdefault(key, {})["sell"] = sell all_data.setdefault(key, {})["review"] = review all_data.setdefault(key, {})["tags"] = tags except Exception: log.error(f"split {line} failed", exc_info=True) return all_data def main(): if len(sys.argv) == 2: day = sys.argv[1] else: day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') tap = TapTapReport(day) tap.build_report() if __name__ == "__main__": main()