ops/datacollect
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复空文件生成的BUG

Browse Source
This commit is contained in:
pengtao 2019-08-14 14:31:41 +08:00
parent 217662584b
commit 18ba47e255
4 changed files with 89 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
config/config.py
View File

@ -7,16 +7,16 @@ DB = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'games
adv_id_state = {'1': '重新拉取', '0': '显示'}
ad_type = {'1': '视屏启动', '2': '视频错误回调', '101': 'banner显示', '201': '插屏显示'}
if not is_debug:
event_list = (
(1016, 6001, 'shop_1016'), (1016, 6001, 'shop_new_1016'))
else:
event_list = (
(1004, 6001, 'tap_1004'), (1004, 6001, 'tap_new_1004'), (1004, 6001, 'share'), (1004, 6001, 'share_new'),
(1004, 6001, 'stage'), (1004, 6001, 'stage_new'), (1004, 6001, 'ad_video'), (1004, 6001, 'ad_video_new'),
(1016, 6001, 'shop_1016'), (1016, 6001, 'shop_new_1016'), (1016, 6001, 'tap_1016'),
(1016, 6001, 'tap_new_1016'), (2001, 6001, 'tap_2001'), (2001, 6001, 'tap_new_2001'),
(1004, 6001, 'items_produce'), (1004, 6001, 'items_consum'), (1004, 6001, 'share_map'))
# if not is_debug:
# event_list = (
# (1016, 6001, 'shop_1016'), (1016, 6001, 'shop_new_1016'))
# else:
# event_list = (
# (1004, 6001, 'tap_1004'), (1004, 6001, 'tap_new_1004'), (1004, 6001, 'share'), (1004, 6001, 'share_new'),
# (1004, 6001, 'stage'), (1004, 6001, 'stage_new'), (1004, 6001, 'ad_video'), (1004, 6001, 'ad_video_new'),
# (1016, 6001, 'shop_1016'), (1016, 6001, 'shop_new_1016'), (1016, 6001, 'tap_1016'),
# (1016, 6001, 'tap_new_1016'), (2001, 6001, 'tap_2001'), (2001, 6001, 'tap_new_2001'),
# (1004, 6001, 'items_produce'), (1004, 6001, 'items_consum'), (1004, 6001, 'share_map'))
# event_list = ((1004, 6001, 'items_produce'), (1004, 6001, 'items_consum'),(1004, 6001, 'share_map'))

2
mytask/tga.py
View File

@ -23,7 +23,7 @@ class GetFromTga:
item = g.get_api_key(self.gameid)
self.url = item['url']
if is_debug:
self.tgaid = 123
self.tgaid = 1234
else:
self.tgaid = item['tgaid']
self.suffix = item.get('suffix', 0)

56
report_interface.py
View File

@ -14,6 +14,7 @@ from tornado import gen
import tornado.options
import json
from config.config import *
from ops.mmysql import MysqlBase
#from apscheduler.schedulers.tornado import TornadoScheduler
from tasks import run_tasks
from urllib.parse import unquote
@ -38,11 +39,44 @@ class DispatchHandler(tornado.web.RequestHandler):
self.write("pls check args!")
def _selfgetmethods(self):
methods = []
for line in event_list:
_, _, method = line
methods.append(method)
self.write({'errcode': 0, "errmsg": "", "result": json.dumps(methods, ensure_ascii=False)})
gameid = self.get_query_argument('gameid')
channelid = self.get_query_argument('channelid')
sql = f"""SELECT
a.method_name,
b.fields,
b.comment
FROM
(SELECT
method_name
FROM
methods
WHERE
in_used=1
AND gameid={gameid}
AND channelid={channelid})a ,
(SELECT
method_name,
fields,
comment
FROM
methods_detail )b
WHERE a.method_name=b.method_name"""
my_db = MysqlBase(**DB)
data = my_db.query(sql)
output = []
if data:
for line in data:
temp = {}
try:
temp['methods'], temp['fields'], temp['comment'] = line
output.append(temp)
except Exception:
log.error(f"split {line} error, get data from methods!", exc_info=True)
self.write(
{'errcode': 1, "errmsg": f"split {line} error, get data from methods!", "result": output})
self.write({'errcode': 0, "errmsg": "", "result": output})
def _selfCheckingHandler(self):
self.write(json.dumps({'errcode': 0, 'errmsg': '', 'healthy': 1, 'max_rundelay': 10}, separators=(',', ':')))
@ -54,15 +88,15 @@ class DispatchHandler(tornado.web.RequestHandler):
self.url = gc.url
self.tga = FromTga(url=self.url, token=self.api_key)
gameid = self.get_arguments('gameid')[0]
gameid = self.get_query_argument('gameid')
print(f"gameid was {gameid}")
times = self.get_arguments('times')[0]
times = self.get_query_argument('times')
try:
channel=self.get_arguments('channel')[0]
channel = self.get_query_argument('channel')
except:
channel = 6001
methods = self.get_arguments('methods')[0]
methods = self.get_query_argument('methods')
if not (gameid and times and channel and methods):
result = {'errcode': 2, "errmsg": f"get args failed`"}
@ -70,14 +104,14 @@ class DispatchHandler(tornado.web.RequestHandler):
self.write(result)
try:
all = self._get_tap_data(gameid, times, channel, methods)
all = self._get_dailyreport_data(gameid, times, channel, methods)
return self.write({'errcode': 0, "errmsg": "", "result": json.dumps(all, ensure_ascii=False)})
except Exception:
log.error(f"run self._get_tap_data(**kwargs) failed", exc_info=True)
return self.write({'errcode': 2, "errmsg": "get data failed`"})
def _get_tap_data(self, gameid, times, channel, methods):
def _get_dailyreport_data(self, gameid, times, channel, methods):
sql = f"""SELECT result FROM v_event_{self.suffix} where "$part_event"='rep_{methods}'and date='{times}'
and "gameid"={gameid} and channelid={channel} """
data = self.tga.get_data(sql)

62
tasks.py
View File

@ -15,46 +15,50 @@ define_logger("/data/logs/data_collect.log")
log = logging.getLogger(__name__)
def get_event():
sql = f"""SELECT gameid,channelid,method_name FROM methods WHERE in_used=1"""
db = MysqlBase(**DB)
data = db.query(sql)
events = []
if data:
for line in data:
if line:
# gameid, channelid, method_name = line
events.append(line)
return events
def run_tasks():
args = event_list
pool = Pool(processes=cpu_count())
pool.map(simple_work, args)
pool.close()
pool.join()
args = get_event()
# args = event_list
pool = Pool(processes=cpu_count())
pool.map(simple_work, args)
pool.close()
pool.join()
class CollectGameData():
def __init__(self, times):
self.times = times
def __init__(self, times):
self.times = times
def workflow(self, line):
kwargs = {}
kwargs['gameid'], kwargs['channelid'], kwargs['event_type'] = line
kwargs['date'] = self.times
def workflow(self, line):
kwargs = {}
kwargs['gameid'], kwargs['channelid'], kwargs['event_type'] = line
kwargs['date'] = self.times
func = f"run_event_{kwargs['event_type']}(kwargs)"
log.info(f"run {func} kwargs={kwargs}!")
eval(func)
func = f"run_event_{kwargs['event_type']}(kwargs)"
log.info(f"run {func} kwargs={kwargs}!")
eval(func)
def simple_work(line):
try:
times = sys.argv[1]
except:
times = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
cc = CollectGameData(times)
cc.workflow(line)
try:
times = sys.argv[1]
except:
times = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
cc = CollectGameData(times)
cc.workflow(line)
if __name__ == "__main__":
run_tasks()
# try:
# times = sys.argv[1]
# except:
# times = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
# for line in event_list:
# cc = CollectGameData(times)
# cc.workflow(line)
run_tasks() # try: # times = sys.argv[1] # except: # times = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') # for line in event_list: # cc = CollectGameData(times) # cc.workflow(line)