diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..3bc9557 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,23 @@ +[*] +charset=utf-8 +end_of_line=crlf +insert_final_newline=false +indent_style=space +indent_size=4 + +[*.json] +indent_style=space +indent_size=2 + +[{*.pyw,*.py}] +indent_style=tab +tab_width=4 + +[*.pyi] +indent_style=tab +tab_width=4 + +[{*.yml,*.yaml}] +indent_style=space +indent_size=2 + diff --git a/common/common.py b/common/common.py index 827b092..0c82f5a 100644 --- a/common/common.py +++ b/common/common.py @@ -5,117 +5,134 @@ import functools def tarce(f): - @functools.wraps(f) - def decorated_function(*args, **kwargs): - print(f"func name was {__name__}, args was {f}, {args}, {kwargs}") - result = f(*args, **kwargs) - print(f"return was {result}!") + @functools.wraps(f) + def decorated_function(*args, **kwargs): + print(f"func name was {__name__}, args was {f}, {args}, {kwargs}") + result = f(*args, **kwargs) + print(f"return was {result}!") - return decorated_function + return decorated_function def send_ops_mail(sub, body, sendto): - import requests - mail_url = 'http://10.10.3.10:5011/mail/api/v2.0/alarm' - data = {"sub": sub, "content": body, "sendto": sendto} - r = requests.post(url=mail_url, data=data) - if r.status_code == 200: - log.info("send mail success!") + import requests + mail_url = 'http://10.10.3.10:5011/mail/api/v2.0/alarm' + data = {"sub": sub, "content": body, "sendto": sendto} + r = requests.post(url=mail_url, data=data) + if r.status_code == 200: + log.info("send mail success!") - else: - err_msg = "send mail failed ,output was %s" % r.content - log.error("send mail failed ,{0}".format(err_msg)) + else: + err_msg = "send mail failed ,output was %s" % r.content + log.error("send mail failed ,{0}".format(err_msg)) def send_alter_mail(sub, body, sendto): - import requests - mail_url = 'http://10.10.3.16:50010/mail/api/v2.0/send' - data = {"sub": sub, "content": body, "sendto": sendto} - r = requests.post(url=mail_url, data=data) - if r.status_code == 200: - log.info("send mail success!") - else: - err_msg = "send mail failed ,output was %s" % r.content - log.error("send mail failed ,{0}".format(err_msg)) + import requests + mail_url = 'http://10.10.3.16:50010/mail/api/v2.0/send' + data = {"sub": sub, "content": body, "sendto": sendto} + r = requests.post(url=mail_url, data=data) + if r.status_code == 200: + log.info("send mail success!") + else: + err_msg = "send mail failed ,output was %s" % r.content + log.error("send mail failed ,{0}".format(err_msg)) def run_cmd(cmd): - import subprocess - msg = "Starting run: %s " % cmd - log.info("run_cmd {0}".format(msg)) - cmdref = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - output, error_info = cmdref.communicate() - if cmdref.returncode != 0: - if isinstance(error_info, list) or isinstance(error_info, tuple): - error_info = error_info[0] - msg = "RUN %s ERROR,error info: %s" % (cmd, error_info) - log.error(msg) - return False, error_info + import subprocess + msg = "Starting run: %s " % cmd + log.info("run_cmd {0}".format(msg)) + cmdref = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + output, error_info = cmdref.communicate() + if cmdref.returncode != 0: + if isinstance(error_info, list) or isinstance(error_info, tuple): + error_info = error_info[0] + msg = "RUN %s ERROR,error info: %s" % (cmd, error_info) + log.error(msg) + return False, error_info - else: - msg = "run %s success" % cmd - log.info(msg) - # print "Run Success!!" - return True, output + else: + msg = "run %s success" % cmd + log.info(msg) + # print "Run Success!!" + return True, output def str_datetime(step=0): - import datetime - return (datetime.datetime.now() - datetime.timedelta(int(step))).strftime("%Y-%m-%d %H:%M:%S") + import datetime + return (datetime.datetime.now() - datetime.timedelta(int(step))).strftime("%Y-%m-%d %H:%M:%S") def str_short_time(step=0): - import datetime - return (datetime.datetime.now() - datetime.timedelta(int(step))).strftime("%Y%m%d%H%M%S") + import datetime + return (datetime.datetime.now() - datetime.timedelta(int(step))).strftime("%Y%m%d%H%M%S") def write_files(filename, data): - with open(filename, 'a+') as f: - f.write(data) - f.write('\n') + with open(filename, 'a+') as f: + f.write(data) + f.write('\n') def read_files(filename): - with open(filename, 'r') as f: - data = f.read().strip() - return data + with open(filename, 'r') as f: + data = f.read().strip() + return data def cal_crc32(data): - import binascii - return binascii.crc32(data.encode()) + import binascii + return binascii.crc32(data.encode()) def cal_db_num(data, db=16): - import binascii - data_crc = binascii.crc32(data.encode()) - return int(data_crc % db) + 1 + import binascii + data_crc = binascii.crc32(data.encode()) + return int(data_crc % db) + 1 def to_str(bytes_or_str): - if isinstance(bytes_or_str, bytes): - value = bytes_or_str.decode('utf-8') - else: - value = bytes_or_str - return value + if isinstance(bytes_or_str, bytes): + value = bytes_or_str.decode('utf-8') + else: + value = bytes_or_str + return value # 返回格式化时间的下一天 def next_day(day, i=1): - import datetime - try: - begin = datetime.datetime.strptime(day, '%Y-%m-%d') - except: - raise Exception("PLS input time as '2019-03-01!'") - next_day = (begin + datetime.timedelta(days=i)) - return datetime.datetime.strftime(next_day, "%Y-%m-%d") + import datetime + try: + begin = datetime.datetime.strptime(day, '%Y-%m-%d') + except: + raise Exception("PLS input time as '2019-03-01!'") + next_day = (begin + datetime.timedelta(days=i)) + return datetime.datetime.strftime(next_day, "%Y-%m-%d") def class2dict(a): - try: - dd = dict((name, getattr(a, name)) for name in dir(a) if - (not name.startswith('_') and not name.startswith('__') and not callable(getattr(a, name)))) - except Exception as e: - print("{} 转换到dict失败 ,{}".format(a, e)) - return None - return dd # a=open(filename,'r').read() # time.strftime("%Y%m%d %H:%M:%S") # print( '20180807 15:23:29') # # time.strptime("20180702","%Y%m%d") # time.struct_time(tm_year=2018, tm_mon=7, tm_mday=2, tm_hour=0, tm_min=0, tm_sec=0, tm_wday=0, tm_yday=183, tm_isdst=-1) + try: + dd = dict((name, getattr(a, name)) for name in dir(a) if + (not name.startswith('_') and not name.startswith('__') and not callable(getattr(a, name)))) + except Exception as e: + print("{} 转换到dict失败 ,{}".format(a, e)) + return None + return dd # a=open(filename,'r').read() # time.strftime("%Y%m%d %H:%M:%S") # print( '20180807 15:23:29') # # time.strptime("20180702","%Y%m%d") # time.struct_time(tm_year=2018, tm_mon=7, tm_mday=2, tm_hour=0, tm_min=0, tm_sec=0, tm_wday=0, tm_yday=183, tm_isdst=-1) + + +def is_number(s): + try: + float(s) + return True + except ValueError: + pass + + try: + import unicodedata + unicodedata.numeric(s) + return True + except (TypeError, ValueError): + pass + + return False diff --git a/conf/wjtx_log.py b/conf/wjtx_log.py index e69de29..588d431 100644 --- a/conf/wjtx_log.py +++ b/conf/wjtx_log.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +mysql_conn = {"host": "192.168.100.30", "user": "miles", "pswd": "aspect", "db": "wjtx_log01"} + +db_tables = ( + "act_recharge", "arena_$date", "cailiao_$date", "chat_$date", "copy_end_$date", "exp_flow_$date", "first_charge", + "gift_bag", "item_change_$date", "mail_$date", "match_$date", "money_change_$date", "multi_player_boss", + "offline_award_$date", "pet_hh", "player_info_$date", "player_login_$date", "player_logout_$date", + "power_detail_$date", + "qy_answer_$date", "qy_question_$date", "role_online_count_$date", "sev_jinjie", "skill_upgrade", "sqhw_$date", + "strength_equip_$date", "task_flow_$date", "team_gard", "tower", "treasure_$date", "tribe_battle_$date", + "tribe_copy", + "tricks_$date", "vip_change_$date$date") +tga_url = "http://10.10.3.17:8992/querySql" +appid = "f325f667ceba4ce58a2a6553c977977d" +key = "SLZlX1IfdTkT7zYsnjJHaDuz9J4UraP65os8gr2fPoOzruPjHIILBxDMNd6W2743" +tgaid = 1009 +db_row = {"act_recharge": ('name', 'acount_id', 'nickname'), "arena": ('name', 'acount_id', 'nickname')} diff --git a/wjtx_log2tga.py b/wjtx_log2tga.py new file mode 100644 index 0000000..4a507c0 --- /dev/null +++ b/wjtx_log2tga.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name: wjtx_log2tga + Description : + Author : pengtao + date: 2020/2/12 +------------------------------------------------- + Change Activity: + 2020/2/12: +------------------------------------------------- +""" +__author__ = 'pengtao' + +from common.plog import define_logger +from common.mmysql import MysqlBase +from common.mtga import FromTga +from common.common import is_number +import logging +from conf.wjtx_log import * +import datetime + +define_logger("/data/logs/ops/wjtx_log2tga.log") +log = logging.getLogger(__name__) + + +class Db2TGA: + def __init__(self, day): + self.tga = FromTga(tga_url, key) + self.tga.init_tga_write(tgaid) + self.mysql = MysqlBase(**mysql_conn) + self.day = day + + def write2tga(self, data, event_name): + self.tga.put_event_data(data, event_name=event_name) + + def check_data(self, data): + return data + + def collect_table(self, table_name, rows): + data = [] + row_name = " ".jion(rows) + sql = f"select {row_name} from {table_name} where time between '{self.day} 00:00:00' and '{self.day} 23:59:59'" + out = self.mysql.query(sql) + if out: + for line in out: + try: + temp = {} + for i in range(0, len(rows)): + temp[rows[i]] = line[i] + data.append(temp) + except Exception: + log.error(f"split {line} failed!", exc_info=True) + else: + log.error(f"get data from {table_name} failed!") + return data + + def run(self): + if not datetime.datetime.strptime(self.day, "%Y%m%d"): + log.error(f"{self.day} unformat!", exc_info=True) + return False + for table in db_tables: + if table.split('_')[-1] == "$date": + table_name = f"{table.split('_')[0]}_{self.day}" + rows = db_row.get(table.split('_')[0], None) + event_name = table.split('_')[0] + else: + table_name = table + rows = db_row.get(table) + event_name = table + data = self.collect_table(table_name, rows) + new = self.check_data(data) + self.write2tga(new, event_name) + + +def main(): + import sys + day = sys.argv[1] + if not day: + raise Exception(f"pls check inputs days={day}") + db2tga = Db2TGA(day) + db2tga.run() + + +if __name__ == "__main__": + main()