add tags field

taptap/taptap_report.py

@@ -23,7 +23,6 @@ class TapTapReport:
         self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
 
     def clear_noice(self):
-        # 结合前后2天的数据，清理diff_order >200的数据
         pass
     
 
@@ -38,7 +37,7 @@ class TapTapReport:
             b_d = self.split_order_data(item)
             for key in b_d:
                 gameid_order.setdefault(key, {}).update(b_d[key])
-        pdb.set_trace()
+
         for key in gameid_order.keys():
             all.setdefault(key, {})['gameid'] = key.split("#")[0]
             all[key]['catename'] = key.split('#')[1]
@@ -77,8 +76,7 @@ class TapTapReport:
 
         for line in all.values():
             try:
-                print(line)
+                mydb.insert("taptap_data", line)
-            # mydb.insert("taptap_data", line)
             except Exception:
                 log.error(f"install 2 db failed ,values={line}", exc_info=True)
 
@@ -191,7 +189,7 @@ def main():
     elif methods == "report":
         tap.build_report()
     elif methods == 'all':
-        #tap.get_order()
+        tap.get_order()
         tap.build_report()
     else:
         print("PLS input methods in ('data','report')")

taptap/taptap_report_new.py

@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+from ops.mtga import FromTga
+from ops.plog import define_logger
+from ops.mmysql import MysqlBase
+import logging
+import datetime
+import pdb
+import sys
+from bson.objectid import ObjectId
+import json
+
+define_logger("/data/logs/ops/taptap_report.log")
+log = logging.getLogger(__name__)
+import sys
+
+
+class TapTapReport:
+    def __init__(self, day):
+        self.db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'}
+        self.url = "http://10.10.3.17:8992/querySql"
+        self.api_secret = "n9H4R32ZcjtSeN89ljCY6ESzTmOlnwwnOB3r4YsggnP5M1AXLtKtiS4sS1KKLOEQ"
+        self.tga = FromTga(url=self.url, token=self.api_secret)
+        self.day = day
+        self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+        self.bbday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=2)).strftime(
+            '%Y-%m-%d')
+
+    def clear_noice(self):
+        # 结合前后2天的数据，清理diff_order >200的数据
+        pass
+
+
+    def build_report(self):
+        mydb = MysqlBase(**self.db_conf)
+        all = dict()
+        gameid_order = dict()
+        # get 最新的gameid info数据
+        gameid_info = self.get_gameid_info(self.day)
+        for item in (self.day, self.bday, self.bbday):
+            b_d = self.split_order_data(item)
+            for key in b_d:
+                gameid_order.setdefault(key, {}).update(b_d[key])
+        pdb.set_trace()
+        for key in gameid_order.keys():
+            all.setdefault(key, {})['gameid'] = key.split("#")[0]
+            all[key]['catename'] = key.split('#')[1]
+            all[key]['date'] = self.day
+            all[key]['diff_order'] = gameid_order[key].get(self.bday, 301) - gameid_order[key].get(self.day, 301)
+            all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.day, 301)
+
+        for key in gameid_info.keys():
+            all[key]['title'] = gameid_info[key]['title']
+            all[key]['cate'] = gameid_info[key]['cate']
+            all[key]['topic'] = gameid_info[key]['topic']
+            all[key]['score'] = gameid_info[key]['score']
+            all[key]['reserve'] = gameid_info[key]['reserve']
+            all[key]['watch'] = gameid_info[key]['watch']
+            all[key]['download'] = gameid_info[key]['download']
+            all[key]['sell'] = gameid_info[key]['sell']
+            all[key]['review'] = gameid_info[key]['review']
+            all[key]['tags'] = gameid_info[key]['tags']
+
+        b_game_info = self.get_gameid_info(self.bday)
+        for key in all.keys():
+            if not all[key].get('title', None):
+                try:
+                    all[key]['title'] = b_game_info[key]['title']
+                    all[key]['cate'] = b_game_info[key]['cate']
+                    all[key]['topic'] = b_game_info[key]['topic']
+                    all[key]['score'] = b_game_info[key]['score']
+                    all[key]['reserve'] = b_game_info[key]['reserve']
+                    all[key]['watch'] = b_game_info[key]['watch']
+                    all[key]['download'] = b_game_info[key]['download']
+                    all[key]['sell'] = b_game_info[key]['sell']
+                    all[key]['review'] = b_game_info[key]['review']
+                    all[key]['tags'] = b_game_info[key]['tags']
+                except Exception:
+                    print(f"0={key}")
+
+        for line in all.values():
+            try:
+                print(line)
+            # mydb.insert("taptap_data", line)
+            except Exception:
+                log.error(f"install 2 db failed ,values={line}", exc_info=True)
+
+
+    def get_gameid_info(self, day):
+        all_data = dict()
+        sql = f"""SELECT
+            gameid,
+            catename,
+            title,
+            cate,
+            topic,
+            score,
+            reserve,
+            watch,
+            download,
+            sell,
+            review ,
+            tags
+        FROM
+            v_event_25 
+        where
+            "$part_date"='{day}'"""
+        data = self.tga.get_data(sql)
+        if data:
+            for line in data:
+                try:
+                    gameid, catename, title, cate, topic, score, reserve, watch, download, sell, review, tags = line
+                    key = f"{str(int(gameid))}#{catename}"
+                    all_data.setdefault(key, {})["key"] = key
+                    all_data.setdefault(key, {})["catename"] = catename
+                    all_data.setdefault(key, {})["title"] = title
+                    all_data.setdefault(key, {})["cate"] = cate
+                    all_data.setdefault(key, {})["topic"] = topic
+                    all_data.setdefault(key, {})["score"] = score
+                    all_data.setdefault(key, {})["reserve"] = reserve
+                    all_data.setdefault(key, {})["watch"] = watch
+                    all_data.setdefault(key, {})["download"] = download
+                    all_data.setdefault(key, {})["sell"] = sell
+                    all_data.setdefault(key, {})["review"] = review
+                    all_data.setdefault(key, {})["tags"] = tags
+                except Exception:
+                    log.error(f"split {line} failed", exc_info=True)
+        return all_data
+
+
+    def split_order_data(self, day):
+        all_data = dict()
+        mydb = MysqlBase(**self.db_conf)
+        sql = f"select gameid,catename,`order`,date from taptap_order WHERE date='{day}'"
+        data = mydb.query(sql)
+        if data:
+            for line in data:
+                try:
+                    gameid, catename, order, date = line
+                    key = f"{str(gameid)}#{catename}"
+                    temp = {}
+                    temp[date] = order
+                    all_data[key] = temp
+                except Exception:
+                    log.error(f"split line failed,values={line}", exc_info=True)
+        return all_data
+
+
+    def get_order(self):
+        sql = f"""
+                SELECT
+                    gameid,
+                    catename,
+                    "order" 
+                FROM
+                    v_event_25 
+                WHERE
+                    "$part_date"='{self.day}'
+                
+                """
+        data = self.tga.get_data(sql)
+        if data:
+            self.order2mysql(data)
+            return True
+        else:
+            return False
+
+    def order2mysql(self, data):
+        mydb = MysqlBase(**self.db_conf)
+        table_name = "taptap_order"
+        for line in data:
+            temp = {}
+            try:
+                temp["gameid"], temp["catename"], temp["order"] = line
+                temp["date"] = self.day
+                mydb.insert(table_name, temp)
+            except Exception:
+                log.error(f"instert {line} 2 db failed", exc_info=True)
+
+
+def main():
+    if len(sys.argv) == 3:
+        day = sys.argv[2]
+        methods = sys.argv[1]
+    else:
+        methods = sys.argv[1]
+        day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+    if not (day and methods):
+        raise Exception(f"PLs input day={day} methods={methods}")
+    tap = TapTapReport(day)
+    if methods == 'data':
+        tap.get_order()
+    elif methods == "report":
+        tap.build_report()
+    elif methods == 'all':
+        # tap.get_order()
+        tap.build_report()
+    else:
+        print("PLS input methods in ('data','report')")
+
+
+if __name__ == "__main__":
+    main()