datacollect/taptap/taptap_report.py

# -*- coding: utf-8 -*-
from ops.mtga import FromTga
from ops.plog import define_logger
from ops.mmysql import MysqlBase
import logging
import datetime
import pdb
import sys
from bson.objectid import ObjectId
import json

define_logger("/data/logs/ops/taptap_report.log")
log = logging.getLogger(__name__)
import sys

class TapTapReport:
    def __init__(self, day):
        self.db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'}
        self.url = "http://10.10.3.17:8992/querySql"
        self.api_secret = "n9H4R32ZcjtSeN89ljCY6ESzTmOlnwwnOB3r4YsggnP5M1AXLtKtiS4sS1KKLOEQ"
        self.tga = FromTga(url=self.url, token=self.api_secret)
        self.day = day
        self.bday = (datetime.datetime.strptime(self.day, "%Y-%m-%d") - datetime.timedelta(days=1)).strftime('%Y-%m-%d')

    def clear_noice(self):
        pass
    


    def build_report(self):
        mydb = MysqlBase(**self.db_conf)
        all = dict()
        gameid_order = dict()
        # get 最新的gameid info数据
        gameid_info = self.get_gameid_info(self.day)
        for item in (self.day, self.bday):
            b_d = self.split_order_data(item)
            for key in b_d:
                gameid_order.setdefault(key, {}).update(b_d[key])

        for key in gameid_order.keys():
            all.setdefault(key, {})['gameid'] = key.split("#")[0]
            all[key]['catename'] = key.split('#')[1]
            all[key]['date'] = self.day
            all[key]['diff_order'] = gameid_order[key].get(self.bday, 301) - gameid_order[key].get(self.day, 301)
            all.setdefault(key, {})['current_order'] = gameid_order[key].get(self.day, 301)

        for key in gameid_info.keys():
            all[key]['title'] = gameid_info[key]['title']
            all[key]['cate'] = gameid_info[key]['cate']
            all[key]['topic'] = gameid_info[key]['topic']
            all[key]['score'] = gameid_info[key]['score']
            all[key]['reserve'] = gameid_info[key]['reserve']
            all[key]['watch'] = gameid_info[key]['watch']
            all[key]['download'] = gameid_info[key]['download']
            all[key]['sell'] = gameid_info[key]['sell']
            all[key]['review'] = gameid_info[key]['review']
            all[key]['tags'] = gameid_info[key]['tags']

        b_game_info = self.get_gameid_info(self.bday)
        for key in all.keys():
            if not all[key].get('title', None):
                try:
                    all[key]['title'] = b_game_info[key]['title']
                    all[key]['cate'] = b_game_info[key]['cate']
                    all[key]['topic'] = b_game_info[key]['topic']
                    all[key]['score'] = b_game_info[key]['score']
                    all[key]['reserve'] = b_game_info[key]['reserve']
                    all[key]['watch'] = b_game_info[key]['watch']
                    all[key]['download'] = b_game_info[key]['download']
                    all[key]['sell'] = b_game_info[key]['sell']
                    all[key]['review'] = b_game_info[key]['review']
                    all[key]['tags'] = b_game_info[key]['tags']
                except Exception:
                    print(f"0={key}")

        for line in all.values():
            try:
                mydb.insert("taptap_data", line)
            except Exception:
                log.error(f"install 2 db failed ,values={line}", exc_info=True)


    def get_gameid_info(self, day):
        all_data = dict()
        sql = f"""SELECT
            gameid,
            catename,
            title,
            cate,
            topic,
            score,
            reserve,
            watch,
            download,
            sell,
            review ,
            tags
        FROM
            v_event_25 
        where
            "$part_date"='{day}'"""
        data = self.tga.get_data(sql)
        if data:
            for line in data:
                try:
                    gameid, catename, title, cate, topic, score, reserve, watch, download, sell, review, tags = line
                    key = f"{str(int(gameid))}#{catename}"
                    all_data.setdefault(key, {})["key"] = key
                    all_data.setdefault(key, {})["catename"] = catename
                    all_data.setdefault(key, {})["title"] = title
                    all_data.setdefault(key, {})["cate"] = cate
                    all_data.setdefault(key, {})["topic"] = topic
                    all_data.setdefault(key, {})["score"] = score
                    all_data.setdefault(key, {})["reserve"] = reserve
                    all_data.setdefault(key, {})["watch"] = watch
                    all_data.setdefault(key, {})["download"] = download
                    all_data.setdefault(key, {})["sell"] = sell
                    all_data.setdefault(key, {})["review"] = review
                    all_data.setdefault(key, {})["tags"] = tags
                except Exception:
                    log.error(f"split {line} failed", exc_info=True)
        return all_data


    def split_order_data(self, day):
        all_data = dict()
        mydb = MysqlBase(**self.db_conf)
        sql = f"select gameid,catename,`order`,date from taptap_order WHERE date='{day}'"
        data = mydb.query(sql)
        if data:
            for line in data:
                try:
                    gameid, catename, order, date = line
                    key = f"{str(gameid)}#{catename}"
                    temp = {}
                    temp[date] = order
                    all_data[key] = temp
                except Exception:
                    log.error(f"split line failed,values={line}", exc_info=True)
        return all_data



    def get_order(self):
        sql = f"""
                SELECT
                    gameid,
                    catename,
                    "order" 
                FROM
                    v_event_25 
                WHERE
                    "$part_date"='{self.day}'
                
                """
        data = self.tga.get_data(sql)
        if data:
            self.order2mysql(data)
            return True
        else:
            return False

    def order2mysql(self, data):
        mydb = MysqlBase(**self.db_conf)
        table_name = "taptap_order"
        for line in data:
            temp = {}
            try:
                temp["gameid"], temp["catename"], temp["order"] = line
                temp["date"] = self.day
                mydb.insert(table_name, temp)
            except Exception:
                log.error(f"instert {line} 2 db failed", exc_info=True)


def main():
    if len(sys.argv) == 3:
        day = sys.argv[2]
        methods = sys.argv[1]
    else:
        methods = sys.argv[1]
        day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
    if not (day and methods):
        raise Exception(f"PLs input day={day} methods={methods}")
    tap = TapTapReport(day)
    if methods == 'data':
        tap.get_order()
    elif methods == "report":
        tap.build_report()
    elif methods == 'all':
        tap.get_order()
        tap.build_report()
    else:
        print("PLS input methods in ('data','report')")
if __name__ == "__main__":
    main()