diff --git a/config/config.py b/config/config.py index 2fe7203..ef7a886 100644 --- a/config/config.py +++ b/config/config.py @@ -7,16 +7,16 @@ DB = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'games adv_id_state = {'1': '重新拉取', '0': '显示'} ad_type = {'1': '视屏启动', '2': '视频错误回调', '101': 'banner显示', '201': '插屏显示'} -if not is_debug: - event_list = ( - (1016, 6001, 'shop_1016'), (1016, 6001, 'shop_new_1016')) -else: - event_list = ( - (1004, 6001, 'tap_1004'), (1004, 6001, 'tap_new_1004'), (1004, 6001, 'share'), (1004, 6001, 'share_new'), - (1004, 6001, 'stage'), (1004, 6001, 'stage_new'), (1004, 6001, 'ad_video'), (1004, 6001, 'ad_video_new'), - (1016, 6001, 'shop_1016'), (1016, 6001, 'shop_new_1016'), (1016, 6001, 'tap_1016'), - (1016, 6001, 'tap_new_1016'), (2001, 6001, 'tap_2001'), (2001, 6001, 'tap_new_2001'), - (1004, 6001, 'items_produce'), (1004, 6001, 'items_consum'), (1004, 6001, 'share_map')) +# if not is_debug: +# event_list = ( +# (1016, 6001, 'shop_1016'), (1016, 6001, 'shop_new_1016')) +# else: +# event_list = ( +# (1004, 6001, 'tap_1004'), (1004, 6001, 'tap_new_1004'), (1004, 6001, 'share'), (1004, 6001, 'share_new'), +# (1004, 6001, 'stage'), (1004, 6001, 'stage_new'), (1004, 6001, 'ad_video'), (1004, 6001, 'ad_video_new'), +# (1016, 6001, 'shop_1016'), (1016, 6001, 'shop_new_1016'), (1016, 6001, 'tap_1016'), +# (1016, 6001, 'tap_new_1016'), (2001, 6001, 'tap_2001'), (2001, 6001, 'tap_new_2001'), +# (1004, 6001, 'items_produce'), (1004, 6001, 'items_consum'), (1004, 6001, 'share_map')) # event_list = ((1004, 6001, 'items_produce'), (1004, 6001, 'items_consum'),(1004, 6001, 'share_map')) diff --git a/mytask/tga.py b/mytask/tga.py index 9b7eb5c..fde9c67 100644 --- a/mytask/tga.py +++ b/mytask/tga.py @@ -23,7 +23,7 @@ class GetFromTga: item = g.get_api_key(self.gameid) self.url = item['url'] if is_debug: - self.tgaid = 123 + self.tgaid = 1234 else: self.tgaid = item['tgaid'] self.suffix = item.get('suffix', 0) diff --git a/report_interface.py b/report_interface.py index 4b1c6af..ec21fd9 100644 --- a/report_interface.py +++ b/report_interface.py @@ -14,6 +14,7 @@ from tornado import gen import tornado.options import json from config.config import * +from ops.mmysql import MysqlBase #from apscheduler.schedulers.tornado import TornadoScheduler from tasks import run_tasks from urllib.parse import unquote @@ -38,11 +39,44 @@ class DispatchHandler(tornado.web.RequestHandler): self.write("pls check args!") def _selfgetmethods(self): - methods = [] - for line in event_list: - _, _, method = line - methods.append(method) - self.write({'errcode': 0, "errmsg": "", "result": json.dumps(methods, ensure_ascii=False)}) + gameid = self.get_query_argument('gameid') + channelid = self.get_query_argument('channelid') + + sql = f"""SELECT + a.method_name, + b.fields, + b.comment + FROM + (SELECT + method_name + FROM + methods + WHERE + in_used=1 + AND gameid={gameid} + AND channelid={channelid})a , + (SELECT + method_name, + fields, + comment + FROM + methods_detail )b + WHERE a.method_name=b.method_name""" + my_db = MysqlBase(**DB) + data = my_db.query(sql) + output = [] + if data: + for line in data: + temp = {} + try: + temp['methods'], temp['fields'], temp['comment'] = line + output.append(temp) + except Exception: + log.error(f"split {line} error, get data from methods!", exc_info=True) + self.write( + {'errcode': 1, "errmsg": f"split {line} error, get data from methods!", "result": output}) + self.write({'errcode': 0, "errmsg": "", "result": output}) + def _selfCheckingHandler(self): self.write(json.dumps({'errcode': 0, 'errmsg': '', 'healthy': 1, 'max_rundelay': 10}, separators=(',', ':'))) @@ -54,15 +88,15 @@ class DispatchHandler(tornado.web.RequestHandler): self.url = gc.url self.tga = FromTga(url=self.url, token=self.api_key) - gameid = self.get_arguments('gameid')[0] + gameid = self.get_query_argument('gameid') print(f"gameid was {gameid}") - times = self.get_arguments('times')[0] + times = self.get_query_argument('times') try: - channel=self.get_arguments('channel')[0] + channel = self.get_query_argument('channel') except: channel = 6001 - methods = self.get_arguments('methods')[0] + methods = self.get_query_argument('methods') if not (gameid and times and channel and methods): result = {'errcode': 2, "errmsg": f"get args failed`"} @@ -70,14 +104,14 @@ class DispatchHandler(tornado.web.RequestHandler): self.write(result) try: - all = self._get_tap_data(gameid, times, channel, methods) + all = self._get_dailyreport_data(gameid, times, channel, methods) return self.write({'errcode': 0, "errmsg": "", "result": json.dumps(all, ensure_ascii=False)}) except Exception: log.error(f"run self._get_tap_data(**kwargs) failed", exc_info=True) return self.write({'errcode': 2, "errmsg": "get data failed`"}) - def _get_tap_data(self, gameid, times, channel, methods): + def _get_dailyreport_data(self, gameid, times, channel, methods): sql = f"""SELECT result FROM v_event_{self.suffix} where "$part_event"='rep_{methods}'and date='{times}' and "gameid"={gameid} and channelid={channel} """ data = self.tga.get_data(sql) diff --git a/tasks.py b/tasks.py index b4dc9e3..37d007f 100644 --- a/tasks.py +++ b/tasks.py @@ -15,46 +15,50 @@ define_logger("/data/logs/data_collect.log") log = logging.getLogger(__name__) +def get_event(): + sql = f"""SELECT gameid,channelid,method_name FROM methods WHERE in_used=1""" + db = MysqlBase(**DB) + data = db.query(sql) + events = [] + if data: + for line in data: + if line: + # gameid, channelid, method_name = line + events.append(line) + return events def run_tasks(): - args = event_list - pool = Pool(processes=cpu_count()) - pool.map(simple_work, args) - pool.close() - pool.join() + args = get_event() + # args = event_list + pool = Pool(processes=cpu_count()) + pool.map(simple_work, args) + pool.close() + pool.join() class CollectGameData(): - def __init__(self, times): - self.times = times + def __init__(self, times): + self.times = times - def workflow(self, line): - kwargs = {} - kwargs['gameid'], kwargs['channelid'], kwargs['event_type'] = line - kwargs['date'] = self.times + def workflow(self, line): + kwargs = {} + kwargs['gameid'], kwargs['channelid'], kwargs['event_type'] = line + kwargs['date'] = self.times - func = f"run_event_{kwargs['event_type']}(kwargs)" - log.info(f"run {func} kwargs={kwargs}!") - eval(func) + func = f"run_event_{kwargs['event_type']}(kwargs)" + log.info(f"run {func} kwargs={kwargs}!") + eval(func) def simple_work(line): - try: - times = sys.argv[1] - except: - times = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') - cc = CollectGameData(times) - cc.workflow(line) - + try: + times = sys.argv[1] + except: + times = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') + cc = CollectGameData(times) + cc.workflow(line) if __name__ == "__main__": - run_tasks() - # try: - # times = sys.argv[1] - # except: - # times = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') - # for line in event_list: - # cc = CollectGameData(times) - # cc.workflow(line) + run_tasks() # try: # times = sys.argv[1] # except: # times = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') # for line in event_list: # cc = CollectGameData(times) # cc.workflow(line)