2019-12-11 15:08:38 +08:00

633 lines
24 KiB
Python

# -*- coding: utf-8 -*-
# from __future__ import absolute_import
from ops.mtga import GetTgaConfig, FromTga
#from ops.minterface import MpInterface
import os
from flask import Flask, render_template, request, jsonify
from flask_mail import Mail, Message
from threading import Thread
from collections import defaultdict
from ops.plog import define_logger
import logging
import datetime
import requests
from ops.mmysql import MysqlBase
import copy
import json
from collections import defaultdict
db_conf = {'user': 'mytga', 'pswd': 'gzVwh4HGR68G', 'host': '10.10.3.5', 'db': 'external_data'}
define_logger("/data/logs/ops/daily_report.log")
import pdb
log = logging.getLogger(__name__)
sender = "ops@kingsome.cn"
app = Flask(__name__)
app.config['MAIL_SERVER'] = 'smtp.exmail.qq.com'
app.config['MAIL_PORT'] = '465'
app.config['MAIL_USE_SSL'] = True
app.config['MAIL_USE_TLS'] = False ## 默认就是 false, 加上警示自己
app.config['MAIL_USERNAME'] = sender
app.config['MAIL_PASSWORD'] = 'bX8cfBAyj9MBqH22'
mail = Mail(app)
recipients_mini = ["pengtao@kingsome.cn"]
#recipients_mini = ["pengtao@kingsome.cn", "tangwenjing@kingsome.cn", "yuexin@kingsome.cn", "yuetao@kingsome.cn"]
#recipients_2001 = ["pengtao@kingsome.cn", "chenliang@kingsome.cn", "yuexin@kingsome.cn", "yuetao@kingsome.cn"]
recipients_2001 = ["pengtao@kingsome.cn"]
# FROMAPPID_CN = {"wxdb103a128e118619": "拯救熊猫泡泡", "wxc137c93eedeab6f2": "爆冰达人"}
class MpInterface:
def __init__(self):
self.base_url = "https://mp.kingsome.cn/api/open/cfg/all?"
def get_data(self, url):
import requests
r = requests.get(url)
if r.status_code == requests.codes.ok:
return r.json().get('result')
else:
return None
def get_fromappid_cn(self, gameid, channelid):
key = "fromappid_cn"
url = f"{self.base_url}channelid={channelid}&gameid={gameid}&key={key}"
return self.get_data(url)
def send_async_email(app, msg):
with app.app_context():
mail.send(msg)
@app.route('/send-dailyreport')
def send_dailyreport():
title = "OPS报表"
day = request.args.get('day')
project = request.args.get('project') or 'mini_games'
if not (project and day):
return jsonify("PLS input arfs")
if str(project) == '2001':
recipients = recipients_2001
else:
recipients = recipients_mini
msg = Message(title, sender=sender, recipients=recipients)
rp = Report(day, project)
data = rp.run()
print(data)
#data[day] = day
if str(project) == '2001':
msg.subject = f"求生之岛_{day}_游戏日报"
else:
msg.subject = f"休闲游戏_{day}_游戏日报"
if data:
msg.html = render_template('report.html', data=data, day=day)
thread = Thread(target=send_async_email, args=[app, msg])
thread.start()
return jsonify("邮件发送成功")
else:
return jsonify("get Data Failed!")
@app.route('/send-tapweekly')
def send_tapweekly():
title = "TAP周报"
day = request.args.get('day')
recipients = ["pengtao@kingsome.cn", "yuexin@kingsome.cn", "yuetao@kingsome.cn"]
msg = Message(title, sender=sender, recipients=recipients)
twr = TapWeeklyReport(day)
data, tags = twr.run()
# print(data)
# data[day] = day
msg.subject = f"TAPTAP_{day}_游戏周报"
if data:
msg.html = render_template('tap_weekly.html', data=data, day=day, tags=tags)
thread = Thread(target=send_async_email, args=[app, msg])
thread.start()
return jsonify("邮件发送成功")
else:
return jsonify("get Data Failed!")
class Report:
def __init__(self, day, project):
self.day = day
self.project = project
self.not_minigames = (2001, 2002,2003)
def get_all_data(self, **args):
activa_sql = f"""SELECT
count(distinct "#account_id")
FROM
v_event_{args['suffix']}
where
gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_event"='event_11_1'
and "$part_date"='{self.day}'"""
new_sql = f"""SELECT
count(distinct "#account_id")
FROM
v_event_{args['suffix']}
where
gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_event"='event_11_1'
and account_register_date between timestamp'{self.day} 00:00:00' and timestamp'{self.day} 23:59:59'"""
share_sql = f"""SELECT
count(distinct \"#account_id\")
FROM
v_event_{args['suffix']}
where
"$part_event"='event_11_10'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_event"='event_11_1'
and "$part_date"='{self.day}'"""
# byshare_sql = f"""SELECT
# count(distinct \"#account_id\")
# FROM
# v_event_{args['suffix']}
# where
# "$part_event"='event_11_11'
# and gameid='{args['gameid']}'
# and channel='{args['channelid']}'
# and "$part_event"='event_11_1'
# and "$part_date"='{self.day}'"""
timeonlie_sql = f"""SELECT
sum(cast(online_duration as int))
FROM
v_event_{args['suffix']}
where
"$part_event"='event_21_2'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_date"='{self.day}' """
ad_101_sql = f"""SELECT
count(1)
FROM
v_event_{args['suffix']}
WHERE
"$part_event"='event_11_21'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
AND activity_id=101
AND adv_id_state='0'
and "$part_date"='{self.day}' """
ad_1_sql = f"""SELECT
count(1)
FROM
v_event_{args['suffix']}
WHERE
"$part_event"='event_11_21'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
AND activity_id=1
AND adv_id_state='0'
and "$part_date"='{self.day}' """
ad_201_sql = f"""SELECT
count(1)
FROM
v_event_{args['suffix']}
WHERE
"$part_event"='event_11_21'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
AND activity_id=201
and "$part_date"='{self.day}' """
if args['gameid'] not in self.not_minigames:
jumpout_sql = f"""SELECT
count(distinct "#account_id")
FROM
v_event_{args['suffix']}
where
"$part_event"='event_1_4'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_date"='{self.day}'
and "jump_result"=1 """
else:
jumpout_sql = f"""SELECT
count(distinct "#account_id")
FROM
v_event_{args['suffix']}
where
"$part_event"='event_11_31'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_date"='{self.day}'
and "button_name" like 'wx%_success'"""
try:
activa = args['tga'].get_data(activa_sql)[0][0] or 0
new = args['tga'].get_data(new_sql)[0][0] or 0
share = args['tga'].get_data(share_sql)[0][0] or 0
timeonlie = round(int(args['tga'].get_data(timeonlie_sql)[0][0] or 0) / activa, 2)
jumpout = args['tga'].get_data(jumpout_sql)[0][0] or 0
jump_per = round((100 * jumpout) / activa, 2)
if args['gameid'] not in self.not_minigames:
ad_101 = args['tga'].get_data(ad_101_sql)[0][0] or 0
ad_1 = args['tga'].get_data(ad_1_sql)[0][0] or 0
ad_201 = args['tga'].get_data(ad_201_sql)[0][0] or 0
else:
ad_101 = ad_1 = ad_201 = 0
return [activa, new, share, timeonlie, ad_1, ad_101, ad_201, jump_per]
except Exception:
log.error(f"get data from tga failed ,{args['gameid']}", exc_info=True)
return None
def get_input_fromappid(self, **args):
activa_sql = f"""SELECT
count(distinct "#account_id")
FROM
v_event_{args['suffix']}
where
gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_event"='event_11_1'
and "$part_date"='{self.day}'
and from_appid='{args['fromappid']}'"""
new_sql = f"""SELECT
count(distinct "#account_id")
FROM
v_event_{args['suffix']}
where
gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_event"='event_11_1'
and account_register_date between timestamp'{self.day} 00:00:00' and timestamp'{self.day} 23:59:59'
and from_appid='{args['fromappid']}' """
share_sql = f"""SELECT
count(distinct \"#account_id\")
FROM
v_event_{args['suffix']}
where
"$part_event"='event_11_10'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_event"='event_11_1'
and "$part_date"='{self.day}'
and from_appid='{args['fromappid']}' """
byshare_sql = f"""SELECT
count(distinct \"#account_id\")
FROM
v_event_{args['suffix']}
where
"$part_event"='event_11_11'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_event"='event_11_1'
and "$part_date"='{self.day}'
and from_appid='{args['fromappid']}' """
try:
activa = args['tga'].get_data(activa_sql)[0][0] or 0
new = args['tga'].get_data(new_sql)[0][0] or 0
share = args['tga'].get_data(share_sql)[0][0] or 0
byshare = args['tga'].get_data(byshare_sql)[0][0] or 0
if activa == 0:
k = 0
else:
k = round((100 * byshare / (activa - byshare)), 2)
return (args['fromappid_cn'], activa, new, share, k)
except Exception:
log.error(f"collect input failed {args['gameid']} {args['fromappid']}", exc_info=True)
return None
def get_output_fromappid(self, **args):
if args['gameid'] not in self.not_minigames:
jump_sql = f"""SELECT
count("#account_id"),count(distinct "#account_id")
FROM
v_event_{args['suffix']}
where
"$part_event"='event_1_4'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_date"='{self.day}'
and "jump_appid"='{args['fromappid']}'
and "jump_result"=1 """
else:
jump_sql = f"""SELECT
count("#account_id"),count(distinct "#account_id")
FROM
v_event_{args['suffix']}
where
"$part_event"='event_11_31'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_date"='{self.day}'
and "button_name"='{args['fromappid']}_success'"""
data = args['tga'].get_data(jump_sql)
if data:
try:
jump_num, jump_pre = data[0]
return (args['fromappid_cn'], jump_num, jump_pre)
except Exception:
log.error(f"get data from output by {args['gameid']} {args['fromappid']} failed", exc_info=True)
return None
else:
return None
def get_args_cn(self):
gameids_cn = dict()
channel_cn = dict()
url = "http://10.10.5.4:2333/api/open/games/list"
r = requests.get(url)
if r.status_code == requests.codes.ok:
data = r.json().get('gameList')
for line in data:
try:
gameids_cn[line.get('game_id')] = line.get('game')
channel_cn[line.get('platform_id')] = line.get('platform_name')
except Exception:
log.error(f"split {line} failed!", exc_info=True)
return (gameids_cn, channel_cn)
else:
return None
def get_ss_input_fromappid(self, args):
if args['gameid'] not in self.not_minigames:
sql = f"""select distinct from_appid FROM
v_event_{args['suffix']}
where
"$part_event"='event_11_1'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_date"='{self.day}'"""
data = args['tga'].get_data(sql)
input_fromappids = list()
if data:
try:
for line in data:
if line and line[0] != '':
input_fromappids.append(line[0])
except Exception:
log.error(f"split {line} error", exc_info=True)
return input_fromappids
else:
return None
def get_ss_output_fromappid(self, args):
output_fromappids = list()
if args['gameid'] not in self.not_minigames:
sql = f"""select distinct jump_appid FROM
v_event_{args['suffix']}
where
"$part_event"='event_1_4'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "jump_result"=1
and "$part_date"='{self.day}'"""
data = args['tga'].get_data(sql)
if data:
try:
for line in data:
if line and line[0] != '':
output_fromappids.append(line[0].split('_success')[0])
except Exception:
log.error(f"split {line} error", exc_info=True)
return output_fromappids
else:
sql = f"""select distinct button_name FROM
v_event_{args['suffix']}
where
"$part_event"='event_11_31'
and gameid='{args['gameid']}'
and "channel"='{args['channelid']}'
and "$part_date"='{self.day}'
and button_name like 'wx%_success'"""
data = args['tga'].get_data(sql)
if data:
try:
for line in data:
if line and line[0] != '':
output_fromappids.append(line[0])
except Exception:
log.error(f"split {line} error", exc_info=True)
return output_fromappids
def run(self):
if self.project == 'mini_games':
parms = [(1004, 6001), (1011, 6001), (1001, 6001), (1013, 6001), (1016, 6001), (2020, 6001), (2021, 6001),
(2022, 6001)]
elif self.project == '2001':
parms = [(2001, 6001), (2002, 6001),(2003, 6001)]
else:
return None
cn = self.get_args_cn()
if not cn:
return None
else:
game_cn, chanel_cn = cn
data = list()
for item in parms:
args = {}
args['gameid'], args['channelid'] = item
temp = {}
temp['gameid'] = game_cn.get(args['gameid'], None)
temp['channelid'] = chanel_cn.get(args['channelid'], None)
temp['input'] = defaultdict(list)
temp['output'] = defaultdict(list)
temp['input'] = []
temp['output'] = []
g = GetTgaConfig()
item = g.get_api_key(args['gameid'])
url = item['url']
args['suffix'] = item.get('suffix', 0)
api_key = item.get('api_key', None)
tga = FromTga(url, api_key)
args['tga'] = tga
temp['all'] = self.get_all_data(**args)
if temp['all']:
ss_input_fromappids = self.get_ss_input_fromappid(args)
ss_output_fromappids = self.get_ss_output_fromappid(args)
mp = MpInterface()
fromappids = mp.get_fromappid_cn(args['gameid'], args['channelid'])
log.info(f"1={fromappids} 2={ss_input_fromappids} 3={ss_output_fromappids}")
if fromappids and ss_input_fromappids:
for item in ss_input_fromappids:
args['fromappid'] = item
args['fromappid_cn'] = fromappids.get(item, None) or item
temp['input'].append(self.get_input_fromappid(**args))
if fromappids and ss_output_fromappids:
for item in ss_output_fromappids:
if args['gameid'] in self.not_minigames:
args['fromappid'] = item.split('_success')[0]
else:
args['fromappid'] = item
args['fromappid_cn'] = fromappids.get(item.split('_success')[0], None) or item
temp['output'].append(self.get_output_fromappid(**args))
print(f"current_data={temp}")
try:
if temp['all'][0] > 10:
data.append(temp)
except Exception:
log.error(f"get active user failed,{temp['all']}", exc_info=True)
return data
class TapWeeklyReport:
def __init__(self, day):
self.day = day
self.db_conn = MysqlBase(**db_conf)
self.limit = 20
self.all_type = {"download": {"name": "热门榜", "row": ["download"]}, "new": {"name": "新品榜", "row": ["download"]},
"reserve": {"name": "预约榜", "row": ["reserve"]}, "sell": {"name": "热卖榜", "row": ["sell"]},
"played": {"name": "热玩榜", "row": ["download"]}}
self.base_head = ["diff_order","title", "order", "min_order", "max_order","watch"]
self.base_end = [ "score", "tags"]
self.row_cn = { "title": "游戏名称", "order": "当前排名", "min_order": "最高排名", "max_order": "最低排名",
"diff_order": "排名变动", "watch":"关注数","download": "下载数", "reserve": "预约数", "sell": "购买数",
"score": "评分","tags": "游戏标签"}
def struct_data(self, data):
r_data = list()
for key in self.all_type.keys():
try:
data_key = data.get(key, [])
temp = {}
temp["name"] = self.all_type.get(key).get("name")
rows = self.base_head + self.all_type.get(key).get("row") + self.base_end
temp["row_name"] = []
for k in rows:
temp["row_name"].append(self.row_cn.get(k))
temp["row_data"] = []
for line in data_key[0]:
tt = {}
# (30802, '死战骑士团', 56, 56, 56, 0, 5.1, '付费,策略,单机', 8984, 0, 7794, 0, 272, 34)
tt['gameid'], tt['title'], tt['order'], tt['min_order'], tt['max_order'], tt['diff_order'],tt['score'], tt['tags'], tt['watch'], tt['download'], tt['sell'], tt['reserve'], tt['review'],tt['topic'] = line
new_tt = copy.deepcopy(tt)
for item in tt.keys():
if item not in rows:
new_tt.pop(item)
ttt = list()
for k in rows:
ttt.append(new_tt.get(k))
temp["row_data"].append(ttt)
r_data.append(temp)
except Exception:
log.error(f"struct data with {line} Failed", exc_info=True)
new_tags = dict()
if r_data:
new_tags = self.collect_tags(r_data)
return r_data, new_tags
def collect_tags(self, data):
row_tags = dict()
new_data=dict()
for line in data:
try:
data_type = line.get("name")
row_tags[data_type]={}
for items in line.get("row_data"):
for tag in items[-1].split(','):
row_tags[data_type][tag] = row_tags[data_type].get(tag, 0) + 1
new_tags = sorted(row_tags[data_type].items(), key=lambda d: d[1], reverse=True)
new_data[data_type] = list()
for i in range(0, 3):
new_data[data_type].append(new_tags[i])
except Exception:
log.error(f"collect tags failed with {line}", exc_info=True)
return new_data
def build(self):
# weekly_day = json.dumps(self.get_weekly_days()).strip('[]')
# tap_types = ("new", "download", "reserve", "sell", "played")
all_data = defaultdict(list)
for tap_type in self.all_type.keys():
sql = f"""select
gameid,
title as "名称",
`order` as "当前排名",
min_order as "最高排名",
max_order as "最低排名",
diff_order as "排名变动",
score as "评分",
tags as "游戏标签",
watch as "关注数",
download as "下载数",
sell as "购买数",
reserve as "预约数",
review as "评论数",
topic as "话题"
from
tap_weekly
where
catename='{tap_type}'
and date = '{self.day}'
and order_posi = 1
order by
`diff_order` desc limit {self.limit};"""
data = self.db_conn.query(sql)
#log.info(f"sql={sql}")
if data:
all_data[tap_type].append(data)
return all_data
def run(self):
data = self.build()
r_data, tags = self.struct_data(data)
#print(f"rr={r_data},tags={tags}")
return r_data, tags
def main():
day = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
project = 'mini_games'
cc = Report(day, project)
data = cc.run()
#print(data)
if __name__ == "__main__":
#main()
app.run(host='0.0.0.0', port=6700, debug=False)