add news
This commit is contained in:
pengtao
parent
a2466e610c
commit
6faa712586
23
.editorconfig
Normal file
23
.editorconfig
Normal file
@ -0,0 +1,23 @@
|
||||
[*]
|
||||
charset=utf-8
|
||||
end_of_line=crlf
|
||||
insert_final_newline=false
|
||||
indent_style=space
|
||||
indent_size=4
|
||||
|
||||
[*.json]
|
||||
indent_style=space
|
||||
indent_size=2
|
||||
|
||||
[{*.pyw,*.py}]
|
||||
indent_style=tab
|
||||
tab_width=4
|
||||
|
||||
[*.pyi]
|
||||
indent_style=tab
|
||||
tab_width=4
|
||||
|
||||
[{*.yml,*.yaml}]
|
||||
indent_style=space
|
||||
indent_size=2
|
||||
|
||||
165
common/common.py
165
common/common.py
@ -5,117 +5,134 @@ import functools
|
||||
|
||||
|
||||
def tarce(f):
|
||||
@functools.wraps(f)
|
||||
def decorated_function(*args, **kwargs):
|
||||
print(f"func name was {__name__}, args was {f}, {args}, {kwargs}")
|
||||
result = f(*args, **kwargs)
|
||||
print(f"return was {result}!")
|
||||
@functools.wraps(f)
|
||||
def decorated_function(*args, **kwargs):
|
||||
print(f"func name was {__name__}, args was {f}, {args}, {kwargs}")
|
||||
result = f(*args, **kwargs)
|
||||
print(f"return was {result}!")
|
||||
|
||||
return decorated_function
|
||||
return decorated_function
|
||||
|
||||
|
||||
def send_ops_mail(sub, body, sendto):
|
||||
import requests
|
||||
mail_url = 'http://10.10.3.10:5011/mail/api/v2.0/alarm'
|
||||
data = {"sub": sub, "content": body, "sendto": sendto}
|
||||
r = requests.post(url=mail_url, data=data)
|
||||
if r.status_code == 200:
|
||||
log.info("send mail success!")
|
||||
import requests
|
||||
mail_url = 'http://10.10.3.10:5011/mail/api/v2.0/alarm'
|
||||
data = {"sub": sub, "content": body, "sendto": sendto}
|
||||
r = requests.post(url=mail_url, data=data)
|
||||
if r.status_code == 200:
|
||||
log.info("send mail success!")
|
||||
|
||||
else:
|
||||
err_msg = "send mail failed ,output was %s" % r.content
|
||||
log.error("send mail failed ,{0}".format(err_msg))
|
||||
else:
|
||||
err_msg = "send mail failed ,output was %s" % r.content
|
||||
log.error("send mail failed ,{0}".format(err_msg))
|
||||
|
||||
|
||||
def send_alter_mail(sub, body, sendto):
|
||||
import requests
|
||||
mail_url = 'http://10.10.3.16:50010/mail/api/v2.0/send'
|
||||
data = {"sub": sub, "content": body, "sendto": sendto}
|
||||
r = requests.post(url=mail_url, data=data)
|
||||
if r.status_code == 200:
|
||||
log.info("send mail success!")
|
||||
else:
|
||||
err_msg = "send mail failed ,output was %s" % r.content
|
||||
log.error("send mail failed ,{0}".format(err_msg))
|
||||
import requests
|
||||
mail_url = 'http://10.10.3.16:50010/mail/api/v2.0/send'
|
||||
data = {"sub": sub, "content": body, "sendto": sendto}
|
||||
r = requests.post(url=mail_url, data=data)
|
||||
if r.status_code == 200:
|
||||
log.info("send mail success!")
|
||||
else:
|
||||
err_msg = "send mail failed ,output was %s" % r.content
|
||||
log.error("send mail failed ,{0}".format(err_msg))
|
||||
|
||||
|
||||
def run_cmd(cmd):
|
||||
import subprocess
|
||||
msg = "Starting run: %s " % cmd
|
||||
log.info("run_cmd {0}".format(msg))
|
||||
cmdref = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
|
||||
output, error_info = cmdref.communicate()
|
||||
if cmdref.returncode != 0:
|
||||
if isinstance(error_info, list) or isinstance(error_info, tuple):
|
||||
error_info = error_info[0]
|
||||
msg = "RUN %s ERROR,error info: %s" % (cmd, error_info)
|
||||
log.error(msg)
|
||||
return False, error_info
|
||||
import subprocess
|
||||
msg = "Starting run: %s " % cmd
|
||||
log.info("run_cmd {0}".format(msg))
|
||||
cmdref = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
|
||||
output, error_info = cmdref.communicate()
|
||||
if cmdref.returncode != 0:
|
||||
if isinstance(error_info, list) or isinstance(error_info, tuple):
|
||||
error_info = error_info[0]
|
||||
msg = "RUN %s ERROR,error info: %s" % (cmd, error_info)
|
||||
log.error(msg)
|
||||
return False, error_info
|
||||
|
||||
else:
|
||||
msg = "run %s success" % cmd
|
||||
log.info(msg)
|
||||
# print "Run Success!!"
|
||||
return True, output
|
||||
else:
|
||||
msg = "run %s success" % cmd
|
||||
log.info(msg)
|
||||
# print "Run Success!!"
|
||||
return True, output
|
||||
|
||||
|
||||
def str_datetime(step=0):
|
||||
import datetime
|
||||
return (datetime.datetime.now() - datetime.timedelta(int(step))).strftime("%Y-%m-%d %H:%M:%S")
|
||||
import datetime
|
||||
return (datetime.datetime.now() - datetime.timedelta(int(step))).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def str_short_time(step=0):
|
||||
import datetime
|
||||
return (datetime.datetime.now() - datetime.timedelta(int(step))).strftime("%Y%m%d%H%M%S")
|
||||
import datetime
|
||||
return (datetime.datetime.now() - datetime.timedelta(int(step))).strftime("%Y%m%d%H%M%S")
|
||||
|
||||
|
||||
def write_files(filename, data):
|
||||
with open(filename, 'a+') as f:
|
||||
f.write(data)
|
||||
f.write('\n')
|
||||
with open(filename, 'a+') as f:
|
||||
f.write(data)
|
||||
f.write('\n')
|
||||
|
||||
|
||||
def read_files(filename):
|
||||
with open(filename, 'r') as f:
|
||||
data = f.read().strip()
|
||||
return data
|
||||
with open(filename, 'r') as f:
|
||||
data = f.read().strip()
|
||||
return data
|
||||
|
||||
|
||||
def cal_crc32(data):
|
||||
import binascii
|
||||
return binascii.crc32(data.encode())
|
||||
import binascii
|
||||
return binascii.crc32(data.encode())
|
||||
|
||||
|
||||
def cal_db_num(data, db=16):
|
||||
import binascii
|
||||
data_crc = binascii.crc32(data.encode())
|
||||
return int(data_crc % db) + 1
|
||||
import binascii
|
||||
data_crc = binascii.crc32(data.encode())
|
||||
return int(data_crc % db) + 1
|
||||
|
||||
|
||||
def to_str(bytes_or_str):
|
||||
if isinstance(bytes_or_str, bytes):
|
||||
value = bytes_or_str.decode('utf-8')
|
||||
else:
|
||||
value = bytes_or_str
|
||||
return value
|
||||
if isinstance(bytes_or_str, bytes):
|
||||
value = bytes_or_str.decode('utf-8')
|
||||
else:
|
||||
value = bytes_or_str
|
||||
return value
|
||||
|
||||
|
||||
# 返回格式化时间的下一天
|
||||
def next_day(day, i=1):
|
||||
import datetime
|
||||
try:
|
||||
begin = datetime.datetime.strptime(day, '%Y-%m-%d')
|
||||
except:
|
||||
raise Exception("PLS input time as '2019-03-01!'")
|
||||
next_day = (begin + datetime.timedelta(days=i))
|
||||
return datetime.datetime.strftime(next_day, "%Y-%m-%d")
|
||||
import datetime
|
||||
try:
|
||||
begin = datetime.datetime.strptime(day, '%Y-%m-%d')
|
||||
except:
|
||||
raise Exception("PLS input time as '2019-03-01!'")
|
||||
next_day = (begin + datetime.timedelta(days=i))
|
||||
return datetime.datetime.strftime(next_day, "%Y-%m-%d")
|
||||
|
||||
|
||||
def class2dict(a):
|
||||
try:
|
||||
dd = dict((name, getattr(a, name)) for name in dir(a) if
|
||||
(not name.startswith('_') and not name.startswith('__') and not callable(getattr(a, name))))
|
||||
except Exception as e:
|
||||
print("{} 转换到dict失败 ,{}".format(a, e))
|
||||
return None
|
||||
return dd # a=open(filename,'r').read() # time.strftime("%Y%m%d %H:%M:%S") # print( '20180807 15:23:29') # # time.strptime("20180702","%Y%m%d") # time.struct_time(tm_year=2018, tm_mon=7, tm_mday=2, tm_hour=0, tm_min=0, tm_sec=0, tm_wday=0, tm_yday=183, tm_isdst=-1)
|
||||
try:
|
||||
dd = dict((name, getattr(a, name)) for name in dir(a) if
|
||||
(not name.startswith('_') and not name.startswith('__') and not callable(getattr(a, name))))
|
||||
except Exception as e:
|
||||
print("{} 转换到dict失败 ,{}".format(a, e))
|
||||
return None
|
||||
return dd # a=open(filename,'r').read() # time.strftime("%Y%m%d %H:%M:%S") # print( '20180807 15:23:29') # # time.strptime("20180702","%Y%m%d") # time.struct_time(tm_year=2018, tm_mon=7, tm_mday=2, tm_hour=0, tm_min=0, tm_sec=0, tm_wday=0, tm_yday=183, tm_isdst=-1)
|
||||
|
||||
|
||||
def is_number(s):
|
||||
try:
|
||||
float(s)
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import unicodedata
|
||||
unicodedata.numeric(s)
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
@ -0,0 +1,17 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
mysql_conn = {"host": "192.168.100.30", "user": "miles", "pswd": "aspect", "db": "wjtx_log01"}
|
||||
|
||||
db_tables = (
|
||||
"act_recharge", "arena_$date", "cailiao_$date", "chat_$date", "copy_end_$date", "exp_flow_$date", "first_charge",
|
||||
"gift_bag", "item_change_$date", "mail_$date", "match_$date", "money_change_$date", "multi_player_boss",
|
||||
"offline_award_$date", "pet_hh", "player_info_$date", "player_login_$date", "player_logout_$date",
|
||||
"power_detail_$date",
|
||||
"qy_answer_$date", "qy_question_$date", "role_online_count_$date", "sev_jinjie", "skill_upgrade", "sqhw_$date",
|
||||
"strength_equip_$date", "task_flow_$date", "team_gard", "tower", "treasure_$date", "tribe_battle_$date",
|
||||
"tribe_copy",
|
||||
"tricks_$date", "vip_change_$date$date")
|
||||
tga_url = "http://10.10.3.17:8992/querySql"
|
||||
appid = "f325f667ceba4ce58a2a6553c977977d"
|
||||
key = "SLZlX1IfdTkT7zYsnjJHaDuz9J4UraP65os8gr2fPoOzruPjHIILBxDMNd6W2743"
|
||||
tgaid = 1009
|
||||
db_row = {"act_recharge": ('name', 'acount_id', 'nickname'), "arena": ('name', 'acount_id', 'nickname')}
|
||||
86
wjtx_log2tga.py
Normal file
86
wjtx_log2tga.py
Normal file
@ -0,0 +1,86 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
-------------------------------------------------
|
||||
File Name： wjtx_log2tga
|
||||
Description :
|
||||
Author : pengtao
|
||||
date： 2020/2/12
|
||||
-------------------------------------------------
|
||||
Change Activity:
|
||||
2020/2/12:
|
||||
-------------------------------------------------
|
||||
"""
|
||||
__author__ = 'pengtao'
|
||||
|
||||
from common.plog import define_logger
|
||||
from common.mmysql import MysqlBase
|
||||
from common.mtga import FromTga
|
||||
from common.common import is_number
|
||||
import logging
|
||||
from conf.wjtx_log import *
|
||||
import datetime
|
||||
|
||||
define_logger("/data/logs/ops/wjtx_log2tga.log")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Db2TGA:
|
||||
def __init__(self, day):
|
||||
self.tga = FromTga(tga_url, key)
|
||||
self.tga.init_tga_write(tgaid)
|
||||
self.mysql = MysqlBase(**mysql_conn)
|
||||
self.day = day
|
||||
|
||||
def write2tga(self, data, event_name):
|
||||
self.tga.put_event_data(data, event_name=event_name)
|
||||
|
||||
def check_data(self, data):
|
||||
return data
|
||||
|
||||
def collect_table(self, table_name, rows):
|
||||
data = []
|
||||
row_name = " ".jion(rows)
|
||||
sql = f"select {row_name} from {table_name} where time between '{self.day} 00:00:00' and '{self.day} 23:59:59'"
|
||||
out = self.mysql.query(sql)
|
||||
if out:
|
||||
for line in out:
|
||||
try:
|
||||
temp = {}
|
||||
for i in range(0, len(rows)):
|
||||
temp[rows[i]] = line[i]
|
||||
data.append(temp)
|
||||
except Exception:
|
||||
log.error(f"split {line} failed!", exc_info=True)
|
||||
else:
|
||||
log.error(f"get data from {table_name} failed!")
|
||||
return data
|
||||
|
||||
def run(self):
|
||||
if not datetime.datetime.strptime(self.day, "%Y%m%d"):
|
||||
log.error(f"{self.day} unformat!", exc_info=True)
|
||||
return False
|
||||
for table in db_tables:
|
||||
if table.split('_')[-1] == "$date":
|
||||
table_name = f"{table.split('_')[0]}_{self.day}"
|
||||
rows = db_row.get(table.split('_')[0], None)
|
||||
event_name = table.split('_')[0]
|
||||
else:
|
||||
table_name = table
|
||||
rows = db_row.get(table)
|
||||
event_name = table
|
||||
data = self.collect_table(table_name, rows)
|
||||
new = self.check_data(data)
|
||||
self.write2tga(new, event_name)
|
||||
|
||||
|
||||
def main():
|
||||
import sys
|
||||
day = sys.argv[1]
|
||||
if not day:
|
||||
raise Exception(f"pls check inputs days={day}")
|
||||
db2tga = Db2TGA(day)
|
||||
db2tga.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user