remove no use

# Conflicts:
#	daily_report/templates/tap_weekly.html

.gitignore

@@ -0,0 +1,10 @@
+.idea/datacollect.iml
+.idea/dataSources.local.xml
+.idea/dataSources.xml
+.idea/encodings.xml
+.idea/misc.xml
+.idea/modules.xml
+.idea/vcs.xml
+.idea/workspace.xml
+.idea/codeStyles/codeStyleConfig.xml
+.idea/dataSources/ded72fb6-4b0d-46ce-b5cb-564dd047df87.xml

1.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+import requests
+
+
+class MpGameList:
+    def __init__(self):
+        self.mp_games_url = "https://mp.kingsome.cn/api/open/games/list"
+
+    def get_gameid(self):
+        r = requests.get(self.mp_games_url)
+        all = []
+        if r.status_code == requests.codes.ok:
+            try:
+                data = r.json().get("gameList")
+            except Exception:
+                print(f"get data failed!")
+            try:
+                for line in data:  # {"game":"一起织娃娃","game_id":8002,"game_name":"knitting","platform":"weixin","platform_id":6001,"platform_name":"微信"}
+                    if line.get("platform") == "test":
+                        continue
+                    else:
+                        temp = {}
+                        temp['name_cn'] = line.get("game", "")
+                        temp['game_id'] = line.get("game_id", 0)
+                        temp['platform_name'] = line.get("platform_name", "")
+                        temp['platform_id'] = line.get("platform_id", 0)
+                        all.append(temp)
+            except Exception:
+                print(f"split data with{r.json()} failed")
+            return all
+        else:
+            return None
+
+
+if __name__ == "__main__":
+    mm = MpGameList()
+    data = mm.get_gameid()
+    print(data)

daily_report/daily_report_cron.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+import datetime
+import sys
+import requests
+from ops.plog import define_logger
+import logging
+
+
+define_logger("/data/logs/ops/daily_report_cron.log")
+
+log = logging.getLogger(__name__)
+
+
+def get_url(url):
+    r = requests.get(url)
+    if r.status_code == requests.codes.ok:
+        return True
+    else:
+        log.error(f"connect {url} failed, {r.content}", exc_info=True)
+        return False
+
+
+def main():
+    try:
+        project = sys.argv[1]
+    except Exception:
+        log.error("PLS inpit project!")
+        raise Exception("PLS inpit project")
+    if len(sys.argv) != 3:
+        times = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+    else:
+        times = sys.argv[2]
+
+    if str(project) == '2001':
+        url = f"http://10.10.3.10:6700/send-dailyreport?day={times}&project=2001"
+    else:
+        url = f"http://10.10.3.10:6700/send-dailyreport?day={times}"
+
+    if get_url(url):
+        log.info(f"connect {project} {times} success!")
+    else:
+        log.error(f"send report  {project} {times} failed!")
+
+
+if __name__ == "__main__":
+    main()

daily_report/dreport.py

@@ -10,6 +10,13 @@ from collections import defaultdict
 from ops.plog import define_logger
 import logging
 import datetime
+import requests
+from ops.mmysql import MysqlBase
+import copy
+import json
+from collections import defaultdict
+db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'}
+
 
 define_logger("/data/logs/ops/daily_report.log")
 import pdb
@@ -27,9 +34,13 @@ app.config['MAIL_PASSWORD'] = 'bX8cfBAyj9MBqH22'
 
 mail = Mail(app)
 
-recipients = ["pengtao@kingsome.cn","yulixing@kingsome.cn"]
+#recipients_mini = ["pengtao@kingsome.cn"]
+recipients_mini = ["pengtao@kingsome.cn", "tangwenjing@kingsome.cn", "yuexin@kingsome.cn", "yuetao@kingsome.cn"]
 
 
+
+recipients_2001 = ["pengtao@kingsome.cn", "chenliang@kingsome.cn", "yuexin@kingsome.cn", "yuetao@kingsome.cn"]
+#recipients_2001 = ["pengtao@kingsome.cn"]
 # FROMAPPID_CN = {"wxdb103a128e118619": "拯救熊猫泡泡", "wxc137c93eedeab6f2": "爆冰达人"}
 
 
@@ -60,32 +71,65 @@ def send_async_email(app, msg):
 
 @app.route('/send-dailyreport')
 def send_dailyreport():
-    title = "主题"
+    title = "OPS报表"
-    msg = Message(title, sender=sender, recipients=recipients)
-
-
     day = request.args.get('day')
     project = request.args.get('project') or 'mini_games'
     if not (project and day):
         return jsonify("PLS input arfs")
+    if str(project) == '2001':
+        recipients = recipients_2001
+    else:
+        recipients = recipients_mini
+
+    msg = Message(title, sender=sender, recipients=recipients)
     rp = Report(day, project)
     data = rp.run()
     print(data)
     #data[day] = day
-    msg.subject = f"{project}_{day}_数据"
-    msg.html = render_template('report.html', data=data, day=day)
 
-    thread = Thread(target=send_async_email, args=[app, msg])
+    if str(project) == '2001':
-    thread.start()
+        msg.subject = f"求生之岛_{day}_游戏日报"
+    else:
+        msg.subject = f"休闲游戏_{day}_游戏日报"
+    if data:
+        msg.html = render_template('report.html', data=data, day=day)
+
+        thread = Thread(target=send_async_email, args=[app, msg])
+        thread.start()
+
+        return jsonify("邮件发送成功")
+    else:
+        return jsonify("get Data Failed!")
+
+
+@app.route('/send-tapweekly')
+def send_tapweekly():
+    title = "TAP周报"
+    day = request.args.get('day')
+    recipients = ["pengtao@kingsome.cn", "yuexin@kingsome.cn", "yuetao@kingsome.cn"]
+    msg = Message(title, sender=sender, recipients=recipients)
+    twr = TapWeeklyReport(day)
+    data, tags = twr.run()
+    # print(data)
+    # data[day] = day
+    msg.subject = f"TAPTAP_{day}_游戏周报"
+    if data:
+        msg.html = render_template('tap_weekly.html', data=data, day=day, tags=tags)
+
+        thread = Thread(target=send_async_email, args=[app, msg])
+        thread.start()
+
+        return jsonify("邮件发送成功")
+    else:
+        return jsonify("get Data Failed!")
 
-    return jsonify("邮件发送成功")
 
 
 class Report:
     def __init__(self, day, project):
         self.day = day
         self.project = project
-
+        self.not_minigames = (2001, 2002,2003)
 
     def get_all_data(self, **args):
         activa_sql = f"""SELECT
@@ -94,7 +138,7 @@ class Report:
                     v_event_{args['suffix']} 
                 where
                     gameid='{args['gameid']}' 
-                    and channel='{args['channelid']}' 
+                    and "channel"='{args['channelid']}' 
                     and "$part_event"='event_11_1' 
                     and "$part_date"='{self.day}'"""
 
@@ -104,7 +148,7 @@ class Report:
                     v_event_{args['suffix']} 
                 where
                     gameid='{args['gameid']}' 
-                    and channel='{args['channelid']}' 
+                    and "channel"='{args['channelid']}' 
                     and "$part_event"='event_11_1' 
                     and account_register_date between timestamp'{self.day} 00:00:00' and timestamp'{self.day} 23:59:59'"""
 
@@ -115,7 +159,7 @@ class Report:
                 where
                     "$part_event"='event_11_10' 
                     and  gameid='{args['gameid']}' 
-                    and channel='{args['channelid']}' 
+                    and "channel"='{args['channelid']}' 
                     and "$part_event"='event_11_1' 
                     and "$part_date"='{self.day}'"""
 
@@ -133,10 +177,11 @@ class Report:
         timeonlie_sql = f"""SELECT
             sum(cast(online_duration as int)) 
         FROM
-            v_event_19 
+            v_event_{args['suffix']} 
         where
             "$part_event"='event_21_2' 
             and gameid='{args['gameid']}' 
+            and "channel"='{args['channelid']}' 
             and "$part_date"='{self.day}' """
 
         ad_101_sql = f"""SELECT
@@ -148,6 +193,7 @@ class Report:
             and gameid='{args['gameid']}' 
             and "channel"='{args['channelid']}' 
             AND activity_id=101   
+            AND adv_id_state='0'
             and "$part_date"='{self.day}'  """
 
         ad_1_sql = f"""SELECT
@@ -159,6 +205,7 @@ class Report:
             and gameid='{args['gameid']}' 
             and "channel"='{args['channelid']}' 
             AND activity_id=1   
+            AND adv_id_state='0'
             and "$part_date"='{self.day}'  """
 
         ad_201_sql = f"""SELECT
@@ -171,24 +218,47 @@ class Report:
             and "channel"='{args['channelid']}' 
             AND activity_id=201   
             and "$part_date"='{self.day}'  """
-        jumpout_sql = f"""SELECT
+        if args['gameid'] not in self.not_minigames:
-                      count(distinct "#account_id")
+            jumpout_sql = f"""SELECT
-                FROM
+                          count(distinct "#account_id")
-                    v_event_19 
+                    FROM
-                where
+                        v_event_{args['suffix']} 
-                    "$part_event"='event_1_4' 
+                    where
+                        "$part_event"='event_1_4' 
+                        and gameid='{args['gameid']}' 
+                        and "channel"='{args['channelid']}'
+                        and "$part_date"='{self.day}'
+                        and "jump_result"=1 """
+        else:
+            jumpout_sql = f"""SELECT
+                          count(distinct "#account_id")
+                    FROM
+                        v_event_{args['suffix']} 
+                    where
+                        "$part_event"='event_11_31'
+                    and gameid='{args['gameid']}' 
+                    and "channel"='{args['channelid']}'
                     and "$part_date"='{self.day}'
-                    and "jump_result"=1 """
+                    and "button_name" like 'wx%_success'"""
+
         try:
-            activa = args['tga'].get_data(activa_sql)[0][0]
+            activa = args['tga'].get_data(activa_sql)[0][0] or 0
-            new = args['tga'].get_data(new_sql)[0][0]
+            new = args['tga'].get_data(new_sql)[0][0] or 0
-            share = args['tga'].get_data(share_sql)[0][0]
+            share = args['tga'].get_data(share_sql)[0][0] or 0
-            timeonlie = args['tga'].get_data(timeonlie_sql)[0][0] or 0
+            jumpout = args['tga'].get_data(jumpout_sql)[0][0] or 0
-            ad_101 = args['tga'].get_data(ad_101_sql)[0][0]
+            if activa:
-            ad_1 = args['tga'].get_data(ad_1_sql)[0][0]
+                timeonlie = round(int(args['tga'].get_data(timeonlie_sql)[0][0] or 0) / activa, 2)
-            ad_201 = args['tga'].get_data(ad_201_sql)[0][0]
+                jump_per = round((100 * jumpout) / activa, 2)
-            jumpout = args['tga'].get_data(jumpout_sql)[0][0]
+            else:
-            jump_per = round((100 * jumpout) / activa, 2)
+                timeonlie = 0
+                jump_per = 0
+
+            if args['gameid'] not in self.not_minigames:
+                ad_101 = args['tga'].get_data(ad_101_sql)[0][0] or 0
+                ad_1 = args['tga'].get_data(ad_1_sql)[0][0] or 0
+                ad_201 = args['tga'].get_data(ad_201_sql)[0][0] or 0
+            else:
+                ad_101 = ad_1 = ad_201 = 0
             return [activa, new, share, timeonlie, ad_1, ad_101, ad_201, jump_per]
         except Exception:
             log.error(f"get data from tga failed ,{args['gameid']}", exc_info=True)
@@ -202,7 +272,7 @@ class Report:
             v_event_{args['suffix']} 
         where
             gameid='{args['gameid']}' 
-            and channel='{args['channelid']}' 
+            and "channel"='{args['channelid']}' 
             and "$part_event"='event_11_1' 
             and "$part_date"='{self.day}'
             and from_appid='{args['fromappid']}'"""
@@ -212,7 +282,7 @@ class Report:
             v_event_{args['suffix']} 
         where
             gameid='{args['gameid']}' 
-            and channel='{args['channelid']}' 
+            and "channel"='{args['channelid']}' 
             and "$part_event"='event_11_1' 
             and account_register_date between timestamp'{self.day} 00:00:00' and timestamp'{self.day} 23:59:59'
             and from_appid='{args['fromappid']}'   """
@@ -224,7 +294,7 @@ class Report:
         where
             "$part_event"='event_11_10' 
             and  gameid='{args['gameid']}' 
-            and channel='{args['channelid']}' 
+            and "channel"='{args['channelid']}' 
             and "$part_event"='event_11_1' 
             and "$part_date"='{self.day}'
             and from_appid='{args['fromappid']}'  """
@@ -236,7 +306,7 @@ class Report:
                 where
                     "$part_event"='event_11_11' 
                     and  gameid='{args['gameid']}' 
-                    and channel='{args['channelid']}' 
+                    and "channel"='{args['channelid']}' 
                     and "$part_event"='event_11_1' 
                     and "$part_date"='{self.day}'
                     and from_appid='{args['fromappid']}'  """
@@ -249,49 +319,154 @@ class Report:
                 k = 0
             else:
                 k = round((100 * byshare / (activa - byshare)), 2)
-            return (args['fromappid'], args['fromappid_cn'], activa, new, share, k)
+            return (args['fromappid_cn'], activa, new, share, k)
         except Exception:
             log.error(f"collect input failed {args['gameid']} {args['fromappid']}", exc_info=True)
             return None
 
 
     def get_output_fromappid(self, **args):
-        jump_sql = f"""SELECT
+        if args['gameid'] not in self.not_minigames:
-              count("#account_id"),count(distinct "#account_id")
+            jump_sql = f"""SELECT
-        FROM
+                              count("#account_id"),count(distinct "#account_id")
-            v_event_19 
+                        FROM
-        where
+                            v_event_{args['suffix']}
-            "$part_event"='event_1_4' 
+                        where
-            and "$part_date"='{self.day}'
+                            "$part_event"='event_1_4' 
-            and "jump_appid"='{args['fromappid']}'
+                            and  gameid='{args['gameid']}' 
-            and "jump_result"=1 """
+                            and "channel"='{args['channelid']}'
+                            and "$part_date"='{self.day}'
+                            and "jump_appid"='{args['fromappid']}'
+                            and "jump_result"=1 """
+        else:
+            jump_sql = f"""SELECT
+                  count("#account_id"),count(distinct "#account_id")
+            FROM
+                v_event_{args['suffix']}
+            where
+                   "$part_event"='event_11_31'
+                    and gameid='{args['gameid']}' 
+                    and "channel"='{args['channelid']}'
+                    and "$part_date"='{self.day}'
+                    and "button_name"='{args['fromappid']}_success'"""
 
         data = args['tga'].get_data(jump_sql)
         if data:
             try:
                 jump_num, jump_pre = data[0]
-                return (args['fromappid'], args['fromappid_cn'], jump_num, jump_pre)
+                return (args['fromappid_cn'], jump_num, jump_pre)
             except Exception:
                 log.error(f"get data from output by {args['gameid']} {args['fromappid']} failed", exc_info=True)
                 return None
         else:
             return None
 
+    def get_args_cn(self):
+        gameids_cn = dict()
+        channel_cn = dict()
+        url = "http://10.10.5.4:2333/api/open/games/list"
+        r = requests.get(url)
+        if r.status_code == requests.codes.ok:
+            data = r.json().get('gameList')
+            for line in data:
+                try:
+                    gameids_cn[line.get('game_id')] = line.get('game')
+                    channel_cn[line.get('platform_id')] = line.get('platform_name')
+                except Exception:
+                    log.error(f"split {line} failed!", exc_info=True)
+            return (gameids_cn, channel_cn)
+        else:
+            return None
+
+    def get_ss_input_fromappid(self, args):
+        if args['gameid'] not in self.not_minigames:
+            sql = f"""select distinct from_appid                        FROM
+                                        v_event_{args['suffix']}
+                                    where
+                                        "$part_event"='event_11_1' 
+                                        and  gameid='{args['gameid']}' 
+                                        and "channel"='{args['channelid']}'
+                                        and "$part_date"='{self.day}'"""
+            data = args['tga'].get_data(sql)
+            input_fromappids = list()
+            if data:
+                try:
+                    for line in data:
+                        if line and line[0] != '':
+                            input_fromappids.append(line[0])
+                except Exception:
+                    log.error(f"split {line} error", exc_info=True)
+            return input_fromappids
+        else:
+            return None
+
+
+    def get_ss_output_fromappid(self, args):
+        output_fromappids = list()
+        if args['gameid'] not in self.not_minigames:
+            sql = f"""select distinct jump_appid                        FROM
+                                        v_event_{args['suffix']}
+                                    where
+                                        "$part_event"='event_1_4' 
+                                        and  gameid='{args['gameid']}' 
+                                        and "channel"='{args['channelid']}'
+                                        and "jump_result"=1
+                                        and "$part_date"='{self.day}'"""
+            data = args['tga'].get_data(sql)
+            if data:
+                try:
+                    for line in data:
+                        if line and line[0] != '':
+                            output_fromappids.append(line[0].split('_success')[0])
+                except Exception:
+                    log.error(f"split {line} error", exc_info=True)
+            return output_fromappids
+        else:
+            sql = f"""select distinct button_name                        FROM
+                                                    v_event_{args['suffix']}
+                                                where
+                                                    "$part_event"='event_11_31' 
+                                                    and  gameid='{args['gameid']}' 
+                                                    and "channel"='{args['channelid']}'
+                                                    and "$part_date"='{self.day}'
+                                                     and button_name like 'wx%_success'"""
+
+            data = args['tga'].get_data(sql)
+            if data:
+                try:
+                    for line in data:
+                        if line and line[0] != '':
+                            output_fromappids.append(line[0])
+                except Exception:
+                    log.error(f"split {line} error", exc_info=True)
+            return output_fromappids
+
+
+
     def run(self):
         if self.project == 'mini_games':
-            parms = [(1004, 6001), (1011, 6001), (1001, 6001)]
+            parms = [(1004, 6001), (1011, 6001), (1001, 6001), (1013, 6001), (1016, 6001), (2020, 6001), (2021, 6001),
-        else:
+                     (2022, 6001)]
+        elif self.project == '2001':
             parms = [(2001, 6001), (2002, 6001)]
+        else:
+            return None
+
+        cn = self.get_args_cn()
+        if not cn:
+            return None
+        else:
+            game_cn, chanel_cn = cn
+
 
         data = list()
         for item in parms:
             args = {}
             args['gameid'], args['channelid'] = item
-            # key = f"{args['gameid']}#{args['channelid']}"
 
             temp = {}
-            temp['gameid'] = args['gameid']
+            temp['gameid'] = game_cn.get(args['gameid'], None)
-            temp['channelid'] = args['channelid']
+            temp['channelid'] = chanel_cn.get(args['channelid'], None)
             temp['input'] = defaultdict(list)
             temp['output'] = defaultdict(list)
             temp['input'] = []
@@ -305,29 +480,163 @@ class Report:
             args['tga'] = tga
 
             temp['all'] = self.get_all_data(**args)
-            mp = MpInterface()
+            if temp['all'] and self.project == '2001':
-            fromappids = mp.get_fromappid_cn(args['gameid'], args['channelid'])
+                ss_input_fromappids = self.get_ss_input_fromappid(args)
-            f_keys = fromappids.keys()
+                ss_output_fromappids = self.get_ss_output_fromappid(args)
-            if not f_keys:
+
-                continue
+                mp = MpInterface()
-            for f_key in f_keys:
+                fromappids = mp.get_fromappid_cn(args['gameid'], args['channelid'])
-                args['fromappid'] = f_key
+                # log.info(f"1={fromappids} 2={ss_input_fromappids} 3={ss_output_fromappids}")
-                args['fromappid_cn'] = fromappids.get(f_key, None) or "未知"
+                if fromappids and ss_input_fromappids:
-                temp['input'].append(self.get_input_fromappid(**args))
+                    for item in ss_input_fromappids:
-                temp['output'].append(self.get_output_fromappid(**args))
+                        args['fromappid'] = item
-                print(temp)
+                        args['fromappid_cn'] = fromappids.get(item, None) or item
-            data.append(temp)
+                        temp['input'].append(self.get_input_fromappid(**args))
+
+                if fromappids and ss_output_fromappids:
+                    for item in ss_output_fromappids:
+                        if args['gameid'] in self.not_minigames:
+                            args['fromappid'] = item.split('_success')[0]
+                        else:
+                            args['fromappid'] = item
+                        args['fromappid_cn'] = fromappids.get(item.split('_success')[0], None) or item
+                        temp['output'].append(self.get_output_fromappid(**args))
+                print(f"current_data={temp}")
+                try:
+                    if temp['all'][0] > 5:
+                        data.append(temp)
+                except Exception:
+                    log.error(f"get active user failed,{temp['all']}", exc_info=True)
+            elif temp['all'] and self.project == 'mini_games':
+                try:
+                    if temp['all'][0] > 5:
+                        data.append(temp)
+                except Exception:
+                    log.error(f"get active user failed,{temp['all']}", exc_info=True)
+            else:
+                log.error(f"someting was wrong with {temp}")
         return data
 
 
 
 
+class TapWeeklyReport:
+    def __init__(self, day):
+        self.day = day
+        self.db_conn = MysqlBase(**db_conf)
+        self.limit = 20
+
+        self.all_type = {"download": {"name": "热门榜", "row": ["download"]}, "new": {"name": "新品榜", "row": ["download"]},
+                         "reserve":  {"name": "预约榜", "row": ["reserve"]}, "sell": {"name": "热卖榜", "row": ["sell"]},
+                         "played":   {"name": "热玩榜", "row": ["download"]}}
+
+        self.base_head = ["diff_order","title", "order", "min_order", "max_order","watch"]
+        self.base_end = [ "score", "tags"]
+        self.row_cn = { "title": "游戏名称", "order": "当前排名", "min_order": "最高排名", "max_order": "最低排名",
+                       "diff_order": "排名变动", "watch":"关注数","download": "下载数", "reserve": "预约数", "sell": "购买数",
+                        "score": "评分","tags": "游戏标签"}
+
+
+    def struct_data(self, data):
+        r_data = list()
+        for key in self.all_type.keys():
+            try:
+                data_key = data.get(key, [])
+                temp = {}
+                temp["name"] = self.all_type.get(key).get("name")
+                rows = self.base_head + self.all_type.get(key).get("row") + self.base_end
+                temp["row_name"] = []
+                for k in rows:
+                    temp["row_name"].append(self.row_cn.get(k))
+                temp["row_data"] = []
+                for line in data_key[0]:
+                    tt = {}
+                    # (30802, '死战骑士团', 56, 56, 56, 0, 5.1, '付费,策略,单机', 8984, 0, 7794, 0, 272, 34)
+                    tt['gameid'], tt['title'], tt['order'], tt['min_order'], tt['max_order'], tt['diff_order'],tt['score'], tt['tags'], tt['watch'], tt['download'], tt['sell'], tt['reserve'], tt['review'],tt['topic'] = line
+
+                    new_tt = copy.deepcopy(tt)
+                    for item in tt.keys():
+                        if item not in rows:
+                            new_tt.pop(item)
+                    ttt = list()
+                    for k in rows:
+                        ttt.append(new_tt.get(k))
+                    temp["row_data"].append(ttt)
+                r_data.append(temp)
+            except Exception:
+                log.error(f"struct data with {line} Failed", exc_info=True)
+        new_tags = dict()
+        if r_data:
+            new_tags = self.collect_tags(r_data)
+        return r_data, new_tags
+
+    def collect_tags(self, data):
+        row_tags = dict()
+        new_data=dict()
+        for line in data:
+            try:
+                data_type = line.get("name")
+                row_tags[data_type]={}
+                for items in line.get("row_data"):
+                        for tag in items[-1].split(','):
+                            row_tags[data_type][tag] = row_tags[data_type].get(tag, 0) + 1
+                new_tags = sorted(row_tags[data_type].items(), key=lambda d: d[1], reverse=True)
+                new_data[data_type] = list()
+                for i in range(0, 3):
+                    new_data[data_type].append(new_tags[i])
+
+            except Exception:
+                log.error(f"collect tags failed with {line}", exc_info=True)
+
+        return new_data
+
+    def build(self):
+        # weekly_day = json.dumps(self.get_weekly_days()).strip('[]')
+        # tap_types = ("new", "download", "reserve", "sell", "played")
+        all_data = defaultdict(list)
+        for tap_type in self.all_type.keys():
+            sql = f"""select
+                gameid,
+                title as "名称",
+                `order` as "当前排名",
+            min_order as "最高排名",
+            max_order as "最低排名",
+            diff_order as "排名变动",
+            score as "评分",
+            tags as "游戏标签",
+            watch as "关注数",
+            download as "下载数",
+            sell as "购买数",
+            reserve as "预约数",
+            review as "评论数",
+            topic as "话题"                 
+            from
+                tap_weekly                  
+            where
+                catename='{tap_type}'                      
+                and date = '{self.day}' 
+                and order_posi = 1                          
+            order by
+                `diff_order`  desc limit {self.limit};"""
+
+            data = self.db_conn.query(sql)
+            #log.info(f"sql={sql}")
+            if data:
+                all_data[tap_type].append(data)
+        return all_data
+
+    def run(self):
+        data = self.build()
+        r_data, tags = self.struct_data(data)
+        #print(f"rr={r_data},tags={tags}")
+        return r_data, tags
+
 def main():
     day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
     project = 'mini_games'
     cc = Report(day, project)
     data = cc.run()
-    print(data)
+    #print(data)
 
 
 if __name__ == "__main__":

daily_report/get_grafana.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+import requests
+import pdb
+# Head = {"Accept": "application/json",
+#     "Authorization": "Bearer eyJrIjoiczVHc2lpWkw1a2Jsb0RxeXNKaEhPczQ3QzNtSUtRNHIiLCJuIjoibWlsZXMiLCJpZCI6MX0="}
+# url = "http://grafana.kingsome.cn/d/oN5uGwEik/kingsomeren-shu-qu-xian?orgId=1&from=1572226084591&to=1572247684591"
+
+import os, stat
+import urllib.request
+from datetime import datetime, date, time, timedelta
+import time as Time
+
+
+# 获取七日前0点与今日0点的时间戳
+def last_seven_day():
+    midnight = datetime.combine(date.today(), time.min)
+    yesterday_mid = midnight - timedelta(days=7)  # 想要此前几天的，就改这个参数
+    epoch = datetime.utcfromtimestamp(0)
+    midnight = midnight - timedelta(seconds=1)
+    midnight = int((midnight - epoch).total_seconds() * 1000.0)
+    yesterday_mid = int((yesterday_mid - epoch).total_seconds() * 1000.0)
+    return str(yesterday_mid), str(midnight)
+
+
+# 下载指定的dashboard
+def download_db():
+    # 组装url，跑代码之前现在浏览器试试
+    dbuid = "p5aRo00Wz"
+    # dbuid = "mI5F3QnWk"
+    grafana_server = "https://grafana-test.kingsome.cn"
+    # grafana_server = "http://grafana.kingsome.cn"
+    # url = f"""{grafana_server}/render/d/{dbuid}?from{last_seven_day()[0]}&to={last_seven_day()[
+    #     1]}&&var-datasource=xxx&width=1500&height=700&tz=UTC%2B08%3A00"""
+    url="http://grafana.kingsome.cn/render/d-solo/hzR9LcoZk/taptappai-ming-fen-xi?orgId=1&from=1572413361151&to=1572434961151&var-gameid=10056&var-date=All&var-catename=download&panelId=2&width=1000&height=1500&tz=Asia%2FShanghai"
+    # url="http://grafana.kingsome.cn/d/e2zRV1emk/shu-ju-yan-shi?orgId=1&from=1571715151698&to=1572319951698&var-name=All&var-date=All"
+
+    # header = {"Accept": "application/json", "Content-Type": "application/json",
+    #     "Authorization": "Bearer eyJrIjoiczVHc2lpWkw1a2Jsb0RxeXNKaEhPczQ3QzNtSUtRNHIiLCJuIjoibWlsZXMiLCJpZCI6MX0="}
+    keys="eyJrIjoiczVHc2lpWkw1a2Jsb0RxeXNKaEhPczQ3QzNtSUtRNHIiLCJuIjoibWlsZXMiLCJpZCI6MX0="
+    #keys = "eyJrIjoiWGFFMXp4YTlaODFHUFpVQmxBcHVDWUZNSUFvb2doNDEiLCJuIjoidGVzdCIsImlkIjoyfQ=="
+    header = {"Accept": "application/json", "Content-Type": "application/json", "Authorization": f"Bearer {keys}"}
+    print(url)
+    request = urllib.request.Request(url, headers=header)
+    #pdb.set_trace()
+    try:
+        # 访问并下载面板图
+        response = urllib.request.urlopen(request)
+        time_now = int(Time.time())
+        time_local = Time.localtime(time_now)
+        dt = Time.strftime("%Y-%m-%d", time_local)
+        img_name = f"img{dt}.png"
+
+        filename = img_name
+        # print(response.getcode())
+        if (int(response.getcode()) == 200):
+            with open(filename, "wb") as f:
+                f.write(response.read())
+            return filename
+        else:
+            return "failed"
+    except:
+        return "failed"
+
+
+if __name__ == "__main__":
+    download_db()

daily_report/templates/index.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Index</title>
+</head>
+<body>
+<h1>tttt</h1>
+<img src="{{ user_image }}" alt="test Image">
+
+<img src="https://stackabuse.com/assets/images/logo-dropbox.svg?v=47a80c143d" alt="222">
+</body>
+</html>

daily_report/templates/report.html

@@ -106,13 +106,13 @@
         <table>
           <thead>
             <tr>
-              <th>活跃</th>
+              <th>活跃人数</th>
-              <th>新增</th>
+              <th>新增人数</th>
-              <th>分享</th>
+              <th>分享人数</th>
-              <th>时长</th>
+              <th>人均时长（秒）</th>
-              <th>视屏显示</th>
+              <th>视频广告展示次数</th>
-              <th>banner显示</th>
+              <th>banner展示次数</th>
-              <th>插屏显示</th>
+              <th>插屏广告展示次数</th>
               <th>导出率</th>
             </tr>
           </thead>
@@ -126,17 +126,18 @@
         </table>
       </div>
 
+      {% if line.input %}
+
       <div class="panel">
-        <h2>导入数据</h2>
+        <h2>买量数据</h2>
         <table>
           <thead>
             <tr>
               <!--('wxdb103a128e118619', '拯救熊猫泡泡', 2, 1, 0, 0.0)-->
-              <th>渠道ID</th>
               <th>游戏名称</th>
-              <th>活跃</th>
+              <th>活跃人数</th>
-              <th>新增</th>
+              <th>新增人数</th>
-              <th>分享</th>
+              <th>分享人数</th>
               <th>K值</th>
             </tr>
           </thead>
@@ -151,13 +152,14 @@
           </tbody>
         </table>
       </div>
+      {% endif %}
 
+      {% if line.output %}
       <div class="panel">
-        <h2>导出数据：</h2>
+        <h2>卖量数据：</h2>
         <table>
           <thead>
             <tr>
-              <th>渠道ID</th>
               <th>跳转目标</th>
               <th>跳转次数</th>
               <th>跳转人数</th>
@@ -174,6 +176,7 @@
           </tbody>
         </table>
       </div>
+      {% endif %}
     </div>
     {% endfor %}
   </body>

daily_report/templates/tap_weekly.html

@@ -0,0 +1,152 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta charset="UTF-8" />
+		<title>TAP周报</title>
+		<style>
+			* {
+				margin: 0;
+				padding: 0;
+			}
+
+			html {
+				background-color: #fdfdfd;
+			}
+
+			body {
+				padding: 0 20px;
+				color: #333;
+			}
+
+			h1.title {
+				margin: 30px 0 20px;
+				font-size: 32px !important;
+				font-weight: 500;
+				text-align: center !important;
+			}
+
+			.date {
+				margin: 20px 0;
+				text-align: center;
+			}
+
+			.section {
+				border-top: 1px dashed #ccc;
+			}
+
+			.info {
+				margin: 60px 0 30px;
+				font-size: 16px;
+				text-align: center;
+			}
+
+			.info > span {
+				margin-right: 20px;
+			}
+
+			.info .gameid,
+			.info .channelid {
+				font-weight: 700;
+			}
+
+			.panel {
+				width: 80%;
+				margin: 0 auto 60px;
+			}
+
+			.panel h2 {
+				height: 36px;
+				padding-left: 16px;
+				margin: 20px 0;
+				font-size: 24px;
+				font-weight: 500;
+				border-left: #909090 3px solid;
+			}
+
+			.panel table {
+				/* width: 100%; */
+				line-height: 2;
+				text-align: left;
+				border-collapse: collapse;
+			}
+
+			.panel table th {
+				font-weight: 500;
+				color: #909090;
+			}
+
+			.panel table td,
+			.panel table th {
+				padding: 0 16px;
+				border-bottom: 1px solid #f0f0f0;
+				font-size: 16px;
+			}
+
+			.panel table tr:nth-child(even) {
+				background-color: #f3f3f3;
+			}
+
+			.panel .part {
+				margin-bottom: 16px;
+			}
+			.panel .part .title {
+				margin-bottom: 8px;
+				font-weight: 700;
+			}
+			.panel .part .content span:nth-child(2n) {
+				margin-right: 16px;
+			}
+		</style>
+	</head>
+	<body>
+		<h1 class="title">TAPTAP数据周报· OPS</h1>
+		<p class="date">{{ day }}</p>
+
+		<div class="section">
+			<div class="info">
+				<span>{{ "各榜单游戏类别分布(前三)" }}</span>
+			</div>
+			<div class="panel">
+				{% for key in tags.keys() %}
+				<div class="part">
+					<div class="title">{{ key }}</div>
+					<div class="content">
+						{% for one in tags.get(key) %}
+						<span>{{ one[0] }}</span>[
+						<span>{{ one[1] }}</span>]
+						{% endfor %}
+					</div>
+				</div>
+				{% endfor %}
+			</div>
+		</div>
+
+		{% for key in data %}
+		<div class="section">
+			<div class="info">
+				<span>报表类型：<span class="gameid">{{ key.get('name') }}</span></span>
+			</div>
+			<div class="panel">
+				<table>
+					<thead>
+						<tr>
+							{% for row_name_one in key.get('row_name') %}
+							<td>{{ row_name_one }}</td>
+							{% endfor %}
+						</tr>
+					</thead>
+					<tbody>
+						{% for one_data in key.get('row_data') %}
+						<tr>
+							{% for item in one_data %}
+							<td>{{ item }}</td>
+							{% endfor %}
+						</tr>
+						{% endfor %}
+					</tbody>
+				</table>
+			</div>
+			{% endfor %}
+		</div>
+	</body>
+</html>

daily_report/test01.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+</head>
+<body>
+<iframe src="http://grafana.kingsome.cn/d/hzR9LcoZk/taptappai-ming-fen-xi?from=1572224823823&to=1572246423823&orgId=1&var-gameid=10491&var-gameid=10498&var-gameid=12492&var-date=2019-10-27&var-date=2019-10-26&var-date=2019-10-25&var-date=2019-10-24&var-date=2019-10-23&var-date=2019-10-22&var-date=2019-10-21&var-catename=download"
+        width="650" height="300" frameborder="0"></iframe>
+</body>
+</html>

daily_report/test_report.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+from flask import Flask, render_template, jsonify,url_for
+# from flask_mail import Mail, Message
+import os
+from threading import Thread
+from email.mime.text import MIMEText
+from email.mime.image import MIMEImage
+from email.mime.multipart import MIMEMultipart
+from email.header import Header
+import smtplib
+
+PEOPLE_FOLDER = os.path.join(os.path.abspath('.'), '/static/images')
+
+app = Flask(__name__)
+app.config['UPLOAD_FOLDER'] = PEOPLE_FOLDER
+sender = "ops@kingsome.cn"
+app.config['MAIL_SERVER'] = 'smtp.exmail.qq.com'
+app.config['MAIL_PORT'] = '465'
+app.config['MAIL_USE_SSL'] = True
+app.config['MAIL_USE_TLS'] = False  ## 默认就是 false, 加上警示自己
+app.config['MAIL_USERNAME'] = sender
+app.config['MAIL_PASSWORD'] = 'bX8cfBAyj9MBqH22'
+# mail = Mail(app)
+recipients = ['pengtao@kingsome.cn']
+
+
+def add_img(file, imgid, msg):
+    fp = open(file, 'rb')
+    img = MIMEImage(fp.read())
+    fp.close()
+    img.add_header("Content-ID", f'<{imgid}>')
+    html = f"<html><body><br><img src=\"cid:{imgid}\" border=\"1\"</br></body></html>"
+    content = MIMEText(html, 'html', 'utf-8')
+    msg.attach(content)
+    msg.attach(img)
+    return msg
+
+
+@app.route('/')
+def show_index():
+    title = "test报表"
+    # msg = Message(title, sender=sender, recipients=recipients)
+    msg = MIMEMultipart('related')
+    msg['from'] = sender
+    mail_to = "pengtao@kingsome.cn"
+    msg['Subject'] = Header(title, 'utf-8')
+    full_filename = os.path.join(app.config['UPLOAD_FOLDER'], 'img2019-10-31.png')
+    msg = add_img(full_filename, 'Active', msg)
+    msgText = MIMEText("邮件正文", 'html', 'utf-8')
+    msg.attach(msgText)
+
+    mail_user = sender
+    mail_passwd = 'bX8cfBAyj9MBqH22'
+
+    try:
+        smtObj = smtplib.SMTP_SSL()
+        smtObj.connect('smtp.exmail.qq.com', 465)
+        smtObj.login(mail_user, mail_passwd)
+        COMMSAPACE = ','
+        smtObj.sendmail(sender, mail_to.split(','), msg.as_string())
+    except smtplib.SMTPException:
+        return jsonify("Error")
+
+    # full_filename = url_for("static",filename= 'images/img2019-10-31.png')
+    # filename = "/root/miles/test/static/images/img2019-10-31.png"
+    #     import base64
+    # img_file = open(filename, 'rb')
+    # base64_data = base64.b64encode(img_file.read())
+    # html = f"""<img src="data: image/png;base64, {base64_data}" alt="image1">"""
+    #
+    # # print(full_filename)
+    # msg.html = render_template('index.html', user_image=full_filename)
+    # msg.html += html
+
+
+    return jsonify("邮件发送成功")
+
+
+if __name__ == "__main__":
+    # show_index()  # main()  #
+    app.run(host='0.0.0.0', port=9700, debug=False)

daily_report/weekly_report_cron.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+import datetime
+import sys
+import requests
+from ops.plog import define_logger
+import logging
+
+
+define_logger("/data/logs/ops/weekly_report_cron.log")
+
+log = logging.getLogger(__name__)
+
+
+def get_url(url):
+    r = requests.get(url)
+    if r.status_code == requests.codes.ok:
+        return True
+    else:
+        log.error(f"connect {url} failed, {r.content}", exc_info=True)
+        return False
+
+
+def main():
+
+    if len(sys.argv) == 2:
+        times = sys.argv[2]
+    else:
+        times = (datetime.date.today() - datetime.timedelta(days=2)).strftime('%Y-%m-%d')
+
+    url = f"http://10.10.3.10:6700/send-tapweekly?day={times}"
+
+    if get_url(url):
+        log.info(f"connect  {times} success!")
+    else:
+        log.error(f"send report  {times} failed!")
+
+
+if __name__ == "__main__":
+    main()

ops/minterface.py

@@ -18,3 +18,33 @@ class MpInterface:
         key = "fromappid_cn"
         url = f"{self.base_url}channelid={channelid}&gameid={gameid}&key={key}"
         return self.get_data(url)
+
+
+class MpGameList:
+    def __init__(self):
+        self.mp_games_url = "https://mp.kingsome.cn/api/open/games/list"
+
+    def get_gameid(self):
+        r = requests.get(self.mp_games_url)
+        all = []
+        if r.status_code == requests.codes.ok:
+            try:
+                data = r.json().get("gameList")
+            except Exception:
+                print(f"get data failed!")
+            try:
+                for line in data:  # {"game":"一起织娃娃","game_id":8002,"game_name":"knitting","platform":"weixin","platform_id":6001,"platform_name":"微信"}
+                    if line.get("platform") == "test":
+                        continue
+                    else:
+                        temp = {}
+                        temp['name_cn'] = line.get("game", "")
+                        temp['game_id'] = line.get("game_id", 0)
+                        temp['platform_name'] = line.get("platform_name", "")
+                        temp['platform_id'] = line.get("platform_id", 0)
+                        all.append(temp)
+            except Exception:
+                print(f"split data with{r.json()} failed")
+            return all
+        else:
+            return None

ops/plog.py

@@ -11,7 +11,7 @@ def define_logger(filename="/data/logs/aa.log",debug=True):
         logger.setLevel(logging.ERROR)
 
     # 设置输出格式
-    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(lineno)s - %(levelname)s - %(message)s')
 
     # 设置日志文件处理器
     fh = logging.FileHandler(filename)

taptap/tap2influxdb.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+
+import pdb
+from influxdb import InfluxDBClient, exceptions
+from ops.mtga import FromTga
+from ops.plog import define_logger
+import logging
+import datetime
+import sys
+
+define_logger("/data/logs/ops/tap2influxdb.log")
+log = logging.getLogger(__name__)
+
+
+class TAP2Influx:
+    def __init__(self, day):
+        url = "http://10.10.3.17:8992/querySql"
+        api_secret = "n9H4R32ZcjtSeN89ljCY6ESzTmOlnwwnOB3r4YsggnP5M1AXLtKtiS4sS1KKLOEQ"
+        self.tga = FromTga(url=url, token=api_secret)
+        self.day = day
+        self.influx = InfluxDBClient('10.10.3.19', '8086', 'miles', 'aspect', 'tap_data')
+        self.measurement = "tapdata"
+
+    def get_tga_data(self):
+        sql = f"""SELECT
+            gameid,
+            catename,
+            cate,
+            title,
+            score,
+            "order",
+            tags,
+            reserve,
+            watch,
+            download,
+            sell,
+            review,
+            topic,
+            "#event_time" 
+        FROM
+            v_event_25 
+        WHERE
+            "$part_date"='{self.day}'  """
+
+        body = list()
+        data = self.tga.get_data(sql)
+        if data:
+            try:
+                for line in data:
+                    gameid, catename, cate, title, score, order, tag, reserve, watch, download, sell, review, topic, createdat = line
+                    tags = {"gameid": int(gameid), "catename": catename}
+                    fields = {"title":    title, "score": score, "tag": tag, "reserve": reserve, "watch": watch,
+                              "download": download, "sell": sell, "review": review, "topic": topic, "cate": cate,
+                              "catedat":  createdat, "order": order}
+                    temp = {"measurement": self.measurement, "tags": tags, "fields": fields}
+                    body.append(temp)
+            except Exception:
+                log.error(f"split data failed ,{line}", exc_info=True)
+            try:
+                #print(body)
+                self.influx.write_points(body)
+            except Exception:
+                log.error(f"write 2 influx failed,data={body}", exc_info=True)
+        else:
+            print(f"sql={sql}")
+
+
+    def run(self):
+        self.get_tga_data()
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 2:
+        try:
+            d = sys.argv[1]
+        except Exception:
+            log.error(f"split args with start time failed ,args={d}", exc_info=True)
+            raise Exception(f"split args with start time failed ,args={d}")
+    else:
+        d = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+    tt = TAP2Influx(d)
+    tt.run()

taptap/tap_split_tags.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+from ops.mmysql import MysqlBase
+from ops.mtga import FromTga
+from ops.plog import define_logger
+import logging
+import datetime
+import pdb
+define_logger("/data/logs/ops/split_tags.log")
+log = logging.getLogger(__name__)
+
+db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'}
+
+
+class SplitTapTags:
+    def __init__(self, day):
+        self.db_conn = MysqlBase(**db_conf)
+        self.url = "http://10.10.3.17:8992/querySql"
+        self.api_secret = "n9H4R32ZcjtSeN89ljCY6ESzTmOlnwwnOB3r4YsggnP5M1AXLtKtiS4sS1KKLOEQ"
+        self.tga = FromTga(url=self.url, token=self.api_secret)
+        self.type = ['new', 'reserve']
+        self.day = day
+
+
+    def get_data(self):
+        all = dict()
+        for types in self.type:
+            all[types] = {}
+            sql = f"""SELECT tags FROM v_event_25 where catename='{types}' and "$part_date"='{self.day}' """
+            tags = self.tga.get_data(sql)
+            if tags:
+                for line in tags:
+                    if line:
+                        try:
+                            for item in line[0].split(","):
+                                all[types][item] = all[types].get(item, 0) + 1
+                        except Exception:
+                            log.error(f"split {line} failed!", exc_info=True)
+        #print(all)
+        return all
+
+    def write2db(self, data):
+        db_data = {}
+        tables_name = "tap_tags_counts"
+        for key in data.keys():
+            for k in data[key]:
+                db_data['type'] = key
+                db_data['name'] = k
+                db_data['date'] = self.day
+                db_data['nums'] = data[key][k]
+                try:
+                    self.db_conn.insert(tables_name, db_data)
+                except Exception:
+                    log.error(f"insert {db_data} failed!", exc_info=True)
+
+
+    def run(self):
+        data = self.get_data()
+        #pdb.set_trace()
+        self.write2db(data)
+
+
+def main():
+    import sys
+    if len(sys.argv) == 2:
+        day = sys.argv[1]
+    else:
+        day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+    stt = SplitTapTags(day)
+    stt.run()
+
+
+if __name__ == "__main__":
+    main()

taptap/tap_weekly_mail.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+from ops.mmysql import MysqlBase
+from ops.plog import define_logger
+import logging
+import datetime
+import json
+from collections import defaultdict
+import pdb
+import sys
+define_logger("/data/logs/ops/tap_weekly.log")
+log = logging.getLogger(__name__)
+db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'}
+
+
+class TapWeeklyReport:
+    def __init__(self, day):
+        self.day = day
+        self.db_conn = MysqlBase(**db_conf)
+        self.limit = 20
+
+    def get_weekly_days(self):
+        weekly_days = []
+        for i in range(0, 7):
+            current_day = (datetime.datetime.strptime(self.day, '%Y-%m-%d') - datetime.timedelta(days=i)).strftime(
+                '%Y-%m-%d')
+            weekly_days.append(current_day)
+        return weekly_days
+
+
+    def build(self):
+
+        weekly_day = json.dumps(self.get_weekly_days()).strip('[]')
+        tap_types = ("new", "download", "reserve", "sell", "played")
+        all_data = dict()
+        for tap_type in tap_types:
+            sql = f"""select
+                    gameid,
+                    min(`order`), max(`order`)
+                from
+                    taptap_collect 
+                where
+                    catename='{tap_type}' 
+                    and date in ({weekly_day})
+                group by
+                    gameid 
+                """
+
+            data = self.db_conn.query(sql)
+            if data:
+                try:
+                    for line in data:
+                        gameid, min_order, max_order = line
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["min_order"] = min_order
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["max_order"] = max_order
+
+                # all_data[tap_type].append(data)
+                except Exception:
+                    log.error(f"get {tap_type} min/max order failed!", exc_info=True)
+
+            sql02 = f"""select gameid,title,score,tags,`order`,watch,download,sell,review,reserve,topic from 
+                    taptap_collect 
+                where
+                    catename='{tap_type}' 
+                    and date ='{self.day}'
+           """
+            data02 = self.db_conn.query(sql02)
+            if data02:
+                try:
+                    for line in data02:
+                        gameid, title, score, tags, order, watch, download, sell, review, reserve, topic = line
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["title"] = title
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["score"] = score
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["tags"] = ",".join(
+                            tags.split(',')[0:3])
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["order"] = order
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["watch"] = watch
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["download"] = download
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["sell"] = sell
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["review"] = review
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["reserve"] = reserve
+                        all_data.setdefault(gameid, {}).setdefault(tap_type, {})["topic"] = topic
+
+                except Exception:
+                    log.error(f"get {tap_type} details  failed!", exc_info=True)
+        # check data if not title ,get from last day
+        for gameid in all_data.keys():
+            for catename in all_data[gameid]:
+                if not all_data[gameid][catename].get('title'):
+                    all_data[gameid][catename]["title"], all_data[gameid][catename]["score"], \
+                    all_data[gameid][catename]["tags"], all_data[gameid][catename]["order"], all_data[gameid][catename][
+                        "watch"], all_data[gameid][catename]["download"], all_data[gameid][catename]["sell"], \
+                    all_data[gameid][catename]["review"], all_data[gameid][catename]["reserve"], \
+                    all_data[gameid][catename]["topic"] = self.get_last_data(gameid, catename)
+                    if not all_data[gameid][catename]["title"]:
+                        all_data[gameid].pop(catename)
+                    else:
+                        all_data[gameid][catename]["tags"] = ",".join(
+                            all_data[gameid][catename]["tags"].split(',')[0:3])
+
+        return all_data
+
+    def get_last_data(self, gameid, catename):
+        sql = f"""select title,score,tags,`order`,watch,download,sell,review,reserve,topic from 
+                            taptap_collect 
+                        where
+                            catename='{catename}' 
+                            and gameid ={gameid} order by date desc limit 1 """
+        data = self.db_conn.query(sql)
+
+        try:
+            title, score, tags, order, watch, download, sell, review, reserve, topic = data[0]
+        except Exception:
+            title, score, tags, order, watch, download, sell, review, reserve, topic = None
+            log.error(f"get last info data failed", exc_info=True)
+        return (title, score, tags, order, watch, download, sell, review, reserve, topic)
+
+    def run(self):
+        table_name = "tap_weekly"
+        all = dict()
+        data = self.build()
+        for gameid in data.keys():
+            for catename in data[gameid]:
+                all['gameid'] = gameid
+                all['catename'] = catename
+                all['min_order'] = data[gameid][catename].get('min_order', 0)
+                all['max_order'] = data[gameid][catename].get('max_order', 0)
+                all['diff_order'] = int(all['max_order']) - int(all['min_order'])
+                all['title'] = data[gameid][catename].get('title', "")
+                all['score'] = data[gameid][catename].get('score', 0)
+                all['tags'] = data[gameid][catename].get('tags', "")
+                all['order'] = data[gameid][catename].get('order', 0)
+                all['watch'] = data[gameid][catename].get('watch', 0)
+                all['download'] = data[gameid][catename].get('download', 0)
+                all['sell'] = data[gameid][catename].get('sell', 0)
+                all['review'] = data[gameid][catename].get('review', 0)
+                all['reserve'] = data[gameid][catename].get('reserve', 0)
+                all['topic'] = data[gameid][catename].get('topic', 0)
+                all['date'] = self.day
+                if abs(all['order'] - all['min_order']) <= abs(all['order'] - all['max_order']):
+                    all["order_posi"] = 1
+                else:
+                    all["order_posi"] = 0
+
+                self.db_conn.insert(table_name, all)
+                print(all)
+
+
+def main():
+    if len(sys.argv) == 2:
+        day = sys.argv[1]
+    else:
+        day = (datetime.date.today() - datetime.timedelta(days=2)).strftime('%Y-%m-%d')
+    twr = TapWeeklyReport(day)
+    twr.run()
+
+
+if __name__ == "__main__":
+    main()

taptap/taptap_daily_tags.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+from ops.mtga import FromTga
+from ops.plog import define_logger
+from ops.mmysql import MysqlBase
+import logging
+import datetime
+import pdb
+import sys
+import json
+
+define_logger("/data/logs/ops/taptap_daily_taps.log")
+log = logging.getLogger(__name__)
+import sys
+
+
+class TapTapReport:
+    def __init__(self, day):
+        self.db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'}
+        self.url = "http://10.10.3.17:8992/querySql"
+        self.api_secret = "n9H4R32ZcjtSeN89ljCY6ESzTmOlnwwnOB3r4YsggnP5M1AXLtKtiS4sS1KKLOEQ"
+        self.tga = FromTga(url=self.url, token=self.api_secret)
+        self.day = day  # self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime(  # '%Y-%m-%d')
+
+
+    def get_tags(self):
+        sql = f"""SELECT
+            catename,
+            tags,
+            "order" 
+        FROM
+            v_event_25 
+        where
+            "$part_date"='{self.day}'"""
+        data = self.tga.get_data(sql)
+        tags_data = {}
+        if data:
+            for line in data:
+                try:
+                    catename, tags, order = line
+                    if catename not in tags_data.keys():
+                        tags_data[catename] = {}
+                    temp_dict = tags_data.get(catename, {})
+                    #print(f"base temp={temp_dict}")
+                    for tag in tags.split(","):
+                        temp_dict[tag] = temp_dict.get(tag, 0) + self.order2score(order)
+                    #print(f"old={tags_data},type={catename},new={temp_dict}")
+                    # pdb.set_trace()
+                    tags_data[catename].update(temp_dict)
+                except Exception:
+                    log.error(f"split {line} failed", exc_info=True)
+        return tags_data
+
+    def write2db(self, data):
+        table_name = "taptap_tags_daily"
+        db = MysqlBase(**self.db_conf)
+        for catename in data.keys():
+            temp = {}
+            temp["catename"] = catename
+            temp["day"] = self.day
+            for name in data.get(catename).keys():
+                temp["score"] = data.get(catename)[name]
+                temp["type"] = name
+                db.insert(table_name, temp)
+
+
+
+
+    def order2score(self, order):
+        try:
+            order_int = int(order)
+            if order_int <= 10 and order_int >= 1:
+                return 5
+            elif order_int > 10 and order_int <= 50:
+                return 3
+            elif order_int <= 200:
+                return 2
+            else:
+                return 1
+        except Exception:
+            log.error(f"PLS check order={order} ", exc_info=True)
+            return 0
+
+
+def get_days(bengin):
+    all_days = []
+    # end = (datetime.date.today() - datetime.timedelta(days=2)).strftime('%Y-%m-%d')
+
+    for i in range(1, 100):
+        n = i + 2
+        next_day = (datetime.date.today() - datetime.timedelta(days=n)).strftime('%Y-%m-%d')
+        if next_day == bengin:
+            return all_days
+        else:
+            all_days.append(next_day)
+
+
+def main():
+    if len(sys.argv) == 2:
+        day = sys.argv[1]
+    else:
+        day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+
+    # begin = "2019-10-15"
+    # all_days = get_days(begin)
+    # print(all_days)
+
+    tap = TapTapReport(day)
+    data = tap.get_tags()
+    tap.write2db(data)
+
+
+
+if __name__ == "__main__":
+    main()
+

taptap/taptap_data_collect.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+from ops.mtga import FromTga
+from ops.plog import define_logger
+from ops.mmysql import MysqlBase
+import logging
+import datetime
+import pdb
+import sys
+from bson.objectid import ObjectId
+import json
+
+define_logger("/data/logs/ops/taptap_collect.log")
+log = logging.getLogger(__name__)
+import sys
+
+
+class TapTapReport:
+    def __init__(self, day):
+        self.db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'}
+        self.url = "http://10.10.3.17:8992/querySql"
+        self.api_secret = "n9H4R32ZcjtSeN89ljCY6ESzTmOlnwwnOB3r4YsggnP5M1AXLtKtiS4sS1KKLOEQ"
+        self.tga = FromTga(url=self.url, token=self.api_secret)
+        self.day = day
+        self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+        self.bbday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=2)).strftime(
+            '%Y-%m-%d')
+
+    def clear_noice(self):
+        # 结合前后2天的数据，清理diff_order >200的数据
+        pass
+
+
+    def diff_item(self, all, item):
+        for key in self.gameid_info_1.keys():
+            all.setdefault(key, {})['gameid'] = key.split("#")[0]
+            all[key]['catename'] = key.split('#')[1]
+            all[key]['date'] = self.bday
+            diff_key = f"diff_{item}"
+            if self.gameid_info_1.get(key, {}).get(item, -1)>=0 and self.gameid_info_2.get(key, {}).get(item, -1)>=0:
+                all[key][diff_key] = int(self.gameid_info_1[key].get(item, 0) - self.gameid_info_2[key].get(item, 0))
+                all.setdefault(key, {})[item] = self.gameid_info_1[key].get(item, 0)
+            elif self.gameid_info_1.get(key, {}).get(item, -1)>=0 and self.gameid_info_0.get(key, {}).get(item, -1)>=0:
+                all[key][diff_key] = "新晋榜"
+                all.setdefault(key, {})[item] = self.gameid_info_1[key].get(item, 0)
+            elif self.gameid_info_0.get(key, {}).get(item, -1)>=0 and self.gameid_info_2.get(key, {}).get(item, -1)>=0:
+                all[key][diff_key] = int(self.gameid_info_0[key].get(item, 0) - self.gameid_info_2[key].get(item, 0))
+                all.setdefault(key, {})[item] = self.gameid_info_0[key].get(item, 0)
+            else:
+                pass
+                # 三天中有2天没数据，写入日志不予处理  # log.error(f"{all[key]} {item} {self.gameid_info_0.get(key,{})} {self.gameid_info_1.get(key,
+            # {})} {self.gameid_info_2.get(key,{})} missing 2 days data")
+
+            if item == "order":
+                try:
+                    all[key][diff_key] = -int(all[key][diff_key])
+                except:
+                    pass
+        return all
+
+    def build_report(self):
+        mydb = MysqlBase(**self.db_conf)
+        all = dict()
+        gameid_order = dict()
+        # get 最新的gameid info数据
+        self.gameid_info_0 = self.get_gameid_info(self.day)
+        self.gameid_info_1 = self.get_gameid_info(self.bday)
+        self.gameid_info_2 = self.get_gameid_info(self.bbday)
+
+        for key in self.gameid_info_1.keys():
+            all.setdefault(key, {})['gameid'] = key.split("#")[0]
+            all[key]['catename'] = key.split('#')[1]
+            all[key]['date'] = self.bday
+            try:
+                #  ("order", "watch", "download", "sell", "review", "reserve", "topic")
+                all[key]['title'] = self.gameid_info_1.get(key,{}).get('title', None) or self.gameid_info_0.get(key,{}).get('title',None) or self.gameid_info_2.get(key,{}).get('title', None)
+                all[key]['cate'] = self.gameid_info_1.get(key,{}).get('cate', None) or self.gameid_info_0.get(key,{}).get('cate',None) or self.gameid_info_2.get(key,{}).get('cate', None)
+                all[key]['score'] = self.gameid_info_1.get(key,{}).get('score', 0) or self.gameid_info_0.get(key,{}).get('score',0) or self.gameid_info_2.get(key,{}).get('score', 0)
+                all[key]['tags'] = self.gameid_info_1.get(key,{}).get('tags', None) or self.gameid_info_0.get(key,{}).get('tags',None) or self.gameid_info_2.get(key,{}).get('tags', None)
+            except Exception:
+                log.error(f"split {all[key]} error ", exc_info=True)
+
+        items = ("order", "watch", "download", "sell", "review", "reserve", "topic")
+        for item in items:
+            all = self.diff_item(all, item)
+
+        i = 0
+        for line in all.values():
+            try:
+                #print(line)
+
+                if line.get("order", 0) >= 1:
+                    mydb.insert("taptap_collect", line)
+                    i += 1
+            except Exception:
+                log.error(f"install 2 db failed ,values={line}", exc_info=True)
+        log.info(f"insert {i} 2 mysql!")
+
+    def get_gameid_info(self, day):
+        all_data = dict()
+        sql = f"""SELECT
+            gameid,
+            catename,
+            title,
+            cate,
+            "order",
+            topic,
+            score,
+            reserve,
+            watch,
+            download,
+            sell,
+            review ,
+            tags
+        FROM
+            v_event_25 
+        where
+            "$part_date"='{day}'"""
+        data = self.tga.get_data(sql)
+        if data:
+            for line in data:
+                try:
+                    gameid, catename, title, cate, order, topic, score, reserve, watch, download, sell, review, tags = line
+                    key = f"{str(int(gameid))}#{catename}"
+                    all_data.setdefault(key, {})["key"] = key
+                    all_data.setdefault(key, {})["catename"] = catename
+                    all_data.setdefault(key, {})["title"] = title
+                    all_data.setdefault(key, {})["cate"] = cate
+                    all_data.setdefault(key, {})["order"] = order
+                    all_data.setdefault(key, {})["topic"] = topic
+                    all_data.setdefault(key, {})["score"] = score
+                    all_data.setdefault(key, {})["reserve"] = reserve
+                    all_data.setdefault(key, {})["watch"] = watch
+                    all_data.setdefault(key, {})["download"] = download
+                    all_data.setdefault(key, {})["sell"] = sell
+                    all_data.setdefault(key, {})["review"] = review
+                    all_data.setdefault(key, {})["tags"] = tags
+                except Exception:
+                    log.error(f"split {line} failed", exc_info=True)
+        return all_data
+
+
+def main():
+    if len(sys.argv) == 2:
+        day = sys.argv[1]
+    else:
+        day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+
+    tap = TapTapReport(day)
+    tap.build_report()
+
+
+if __name__ == "__main__":
+    main()

taptap/taptap_report.py

@@ -22,6 +22,10 @@ class TapTapReport:
         self.day = day
         self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
 
+    def clear_noice(self):
+        pass
+    
+
 
     def build_report(self):
         mydb = MysqlBase(**self.db_conf)
@@ -51,6 +55,7 @@ class TapTapReport:
             all[key]['download'] = gameid_info[key]['download']
             all[key]['sell'] = gameid_info[key]['sell']
             all[key]['review'] = gameid_info[key]['review']
+            all[key]['tags'] = gameid_info[key]['tags']
 
         b_game_info = self.get_gameid_info(self.bday)
         for key in all.keys():
@@ -65,6 +70,7 @@ class TapTapReport:
                     all[key]['download'] = b_game_info[key]['download']
                     all[key]['sell'] = b_game_info[key]['sell']
                     all[key]['review'] = b_game_info[key]['review']
+                    all[key]['tags'] = b_game_info[key]['tags']
                 except Exception:
                     print(f"0={key}")
 
@@ -88,7 +94,8 @@ class TapTapReport:
             watch,
             download,
             sell,
-            review 
+            review ,
+            tags
         FROM
             v_event_25 
         where
@@ -97,7 +104,7 @@ class TapTapReport:
         if data:
             for line in data:
                 try:
-                    gameid, catename, title, cate, topic, score, reserve, watch, download, sell, review = line
+                    gameid, catename, title, cate, topic, score, reserve, watch, download, sell, review, tags = line
                     key = f"{str(int(gameid))}#{catename}"
                     all_data.setdefault(key, {})["key"] = key
                     all_data.setdefault(key, {})["catename"] = catename
@@ -110,6 +117,7 @@ class TapTapReport:
                     all_data.setdefault(key, {})["download"] = download
                     all_data.setdefault(key, {})["sell"] = sell
                     all_data.setdefault(key, {})["review"] = review
+                    all_data.setdefault(key, {})["tags"] = tags
                 except Exception:
                     log.error(f"split {line} failed", exc_info=True)
         return all_data

taptap/taptap_report_new.py

@@ -0,0 +1,227 @@
+# -*- coding: utf-8 -*-
+from ops.mtga import FromTga
+from ops.plog import define_logger
+from ops.mmysql import MysqlBase
+import logging
+import datetime
+import pdb
+import sys
+from bson.objectid import ObjectId
+import json
+
+define_logger("/data/logs/ops/taptap_report.log")
+log = logging.getLogger(__name__)
+import sys
+
+
+class TapTapReport:
+    def __init__(self, day):
+        self.db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'}
+        self.url = "http://10.10.3.17:8992/querySql"
+        self.api_secret = "n9H4R32ZcjtSeN89ljCY6ESzTmOlnwwnOB3r4YsggnP5M1AXLtKtiS4sS1KKLOEQ"
+        self.tga = FromTga(url=self.url, token=self.api_secret)
+        self.day = day
+        self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+        self.bbday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=2)).strftime(
+            '%Y-%m-%d')
+
+    def clear_noice(self):
+        # 结合前后2天的数据，清理diff_order >200的数据
+        pass
+
+
+    def build_report(self):
+        mydb = MysqlBase(**self.db_conf)
+        all = dict()
+        gameid_order = dict()
+        # get 最新的gameid info数据
+        gameid_info = self.get_gameid_info(self.bday)
+        for item in (self.day, self.bday, self.bbday):
+            b_d = self.split_order_data(item)
+            for key in b_d:
+                gameid_order.setdefault(key, {}).update(b_d[key])
+        for key in gameid_order.keys():
+            all.setdefault(key, {})['gameid'] = key.split("#")[0]
+            all[key]['catename'] = key.split('#')[1]
+            all[key]['date'] = self.bday
+
+            if gameid_order[key].get(self.bday, 0) and gameid_order[key].get(self.bbday, 0) and gameid_order[key].get(
+                self.day, 0):
+                all[key]['diff_order'] = gameid_order[key].get(self.bbday, 0) - gameid_order[key].get(self.bday, 0)
+                all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.bday, 0)
+            elif gameid_order[key].get(self.bday, 0) and gameid_order[key].get(self.bbday, 0):
+                all[key]['diff_order'] = gameid_order[key].get(self.bbday, 0) - gameid_order[key].get(self.bday, 0)
+                all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.bday, 0)
+            elif gameid_order[key].get(self.bday, 0) and gameid_order[key].get(self.day, 0):
+                all[key]['diff_order'] = "新晋榜"
+                all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.bday, 0)
+            elif gameid_order[key].get(self.bbday, 0) and gameid_order[key].get(self.day, 0):
+                all[key]['diff_order'] = gameid_order[key].get(self.bbday, 0) - gameid_order[key].get(self.day, 0)
+                all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.day, 0)
+            else:
+                # 三天中有2天没数据，写入日志不予处理
+                log.error(f"{key} missing 2 days data,{self.bday}={gameid_order[key].get(self.bday, 0)},{self.bbday}={gameid_order[key].get(self.bbday, 0)},{self.day}={gameid_order[key].get(self.day, 0)}")
+
+        for key in gameid_info.keys():
+            all[key]['title'] = gameid_info[key]['title']
+            all[key]['cate'] = gameid_info[key]['cate']
+            all[key]['topic'] = gameid_info[key]['topic']
+            all[key]['score'] = gameid_info[key]['score']
+            all[key]['reserve'] = gameid_info[key]['reserve']
+            all[key]['watch'] = gameid_info[key]['watch']
+            all[key]['download'] = gameid_info[key]['download']
+            all[key]['sell'] = gameid_info[key]['sell']
+            all[key]['review'] = gameid_info[key]['review']
+            all[key]['tags'] = gameid_info[key]['tags']
+
+        b_game_info = self.get_gameid_info(self.bbday)
+        n_game_info = self.get_gameid_info(self.day)
+        for key in all.keys():
+            if not all[key].get('title', None):
+                try:
+                    all[key]['title'] = b_game_info.get(key,{}).get('title',None) or n_game_info.get(key,{}).get('title',None)
+                    all[key]['cate'] = b_game_info.get(key,{}).get('cate',None) or n_game_info.get(key,{}).get('cate',None)
+                    all[key]['topic'] = b_game_info.get(key,{}).get('topic',None) or n_game_info.get(key,{}).get('topic',None)
+                    all[key]['score'] = b_game_info.get(key, {}).get('score', 0) or n_game_info.get(key, {}).get(
+                        'score', 0)
+                    all[key]['reserve'] = b_game_info.get(key, {}).get('reserve', 0) or n_game_info.get(key, {}).get(
+                        'reserve', 0)
+                    all[key]['watch'] = b_game_info.get(key, {}).get('watch', 0) or n_game_info.get(key, {}).get(
+                        'watch', 0)
+                    all[key]['download'] = b_game_info.get(key, {}).get('download', 0) or n_game_info.get(key, {}).get(
+                        'download', 0)
+                    all[key]['sell'] = b_game_info.get(key, {}).get('sell', 0) or n_game_info.get(key, {}).get('sell',
+                        0)
+                    all[key]['review'] = b_game_info.get(key, {}).get('review', 0) or n_game_info.get(key, {}).get(
+                        'review', 0)
+                    all[key]['tags'] = b_game_info.get(key,{}).get('tags',None) or n_game_info.get(key,{}).get('tags',None)
+                except Exception:
+                    log.error(f"0={key} 1={all.get(key, 0)} 2={b_game_info.get(key, 0)} 3={n_game_info.get(key, 0)}",
+                        exc_info=True)
+
+        for line in all.values():
+            try:
+                #print(line)
+                if line.get('current_order', 0) > 0:
+                    mydb.insert("taptap_data_new", line)
+                else:
+                    log.error(f"some value is zore ,{line}!")
+            except Exception:
+                log.error(f"install 2 db failed ,values={line}", exc_info=True)
+
+
+    def get_gameid_info(self, day):
+        all_data = dict()
+        sql = f"""SELECT
+            gameid,
+            catename,
+            title,
+            cate,
+            topic,
+            score,
+            reserve,
+            watch,
+            download,
+            sell,
+            review ,
+            tags
+        FROM
+            v_event_25 
+        where
+            "$part_date"='{day}'"""
+        data = self.tga.get_data(sql)
+        if data:
+            for line in data:
+                try:
+                    gameid, catename, title, cate, topic, score, reserve, watch, download, sell, review, tags = line
+                    key = f"{str(int(gameid))}#{catename}"
+                    all_data.setdefault(key, {})["key"] = key
+                    all_data.setdefault(key, {})["catename"] = catename
+                    all_data.setdefault(key, {})["title"] = title
+                    all_data.setdefault(key, {})["cate"] = cate
+                    all_data.setdefault(key, {})["topic"] = topic
+                    all_data.setdefault(key, {})["score"] = score
+                    all_data.setdefault(key, {})["reserve"] = reserve
+                    all_data.setdefault(key, {})["watch"] = watch
+                    all_data.setdefault(key, {})["download"] = download
+                    all_data.setdefault(key, {})["sell"] = sell
+                    all_data.setdefault(key, {})["review"] = review
+                    all_data.setdefault(key, {})["tags"] = tags
+                except Exception:
+                    log.error(f"split {line} failed", exc_info=True)
+        return all_data
+
+
+    def split_order_data(self, day):
+        all_data = dict()
+        mydb = MysqlBase(**self.db_conf)
+        sql = f"select gameid,catename,`order`,date from taptap_order WHERE date='{day}'"
+        data = mydb.query(sql)
+        if data:
+            for line in data:
+                try:
+                    gameid, catename, order, date = line
+                    key = f"{str(gameid)}#{catename}"
+                    temp = {}
+                    temp[date] = order
+                    all_data[key] = temp
+                except Exception:
+                    log.error(f"split line failed,values={line}", exc_info=True)
+        return all_data
+
+
+    def get_order(self):
+        sql = f"""
+                SELECT
+                    gameid,
+                    catename,
+                    "order" 
+                FROM
+                    v_event_25 
+                WHERE
+                    "$part_date"='{self.day}'
+                
+                """
+        data = self.tga.get_data(sql)
+        if data:
+            self.order2mysql(data)
+            return True
+        else:
+            return False
+
+    def order2mysql(self, data):
+        mydb = MysqlBase(**self.db_conf)
+        table_name = "taptap_order"
+        for line in data:
+            temp = {}
+            try:
+                temp["gameid"], temp["catename"], temp["order"] = line
+                temp["date"] = self.day
+                mydb.insert(table_name, temp)
+            except Exception:
+                log.error(f"instert {line} 2 db failed", exc_info=True)
+
+
+def main():
+    if len(sys.argv) == 3:
+        day = sys.argv[2]
+        methods = sys.argv[1]
+    else:
+        methods = sys.argv[1]
+        day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+    if not (day and methods):
+        raise Exception(f"PLs input day={day} methods={methods}")
+    tap = TapTapReport(day)
+    if methods == 'data':
+        tap.get_order()
+    elif methods == "report":
+        tap.build_report()
+    elif methods == 'all':
+        # tap.get_order()
+        tap.build_report()
+    else:
+        print("PLS input methods in ('data','report')")
+
+
+if __name__ == "__main__":
+    main()