From 645165bb3220d31d27f19a5e51ca620d03304491 Mon Sep 17 00:00:00 2001 From: aozhiwei Date: Fri, 10 Jan 2020 14:36:14 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/rankserver/delrank.py | 101 -------------------------- tools/rankserver/delrank_cmd.py | 110 ----------------------------- tools/rankserver/game2003rank.py | 109 +++++++++------------------- tools/rankserver/rankserver_cmd.py | 76 ++++++++++++++++++++ 4 files changed, 111 insertions(+), 285 deletions(-) delete mode 100644 tools/rankserver/delrank.py delete mode 100644 tools/rankserver/delrank_cmd.py create mode 100644 tools/rankserver/rankserver_cmd.py diff --git a/tools/rankserver/delrank.py b/tools/rankserver/delrank.py deleted file mode 100644 index 40d730e..0000000 --- a/tools/rankserver/delrank.py +++ /dev/null @@ -1,101 +0,0 @@ -# -*- coding: utf-8 -*- -#!/usr/bin/python - -import pymysql -import hashlib -import json -import urllib.request -import base64 -import tornado.ioloop -import tornado.web -import time -import datetime -import redis -import os -import functools -import delrank_cmd -import sys - -CONFIG_DIR = '' - -def IsOnlineEnv(): - return os.getenv("SERVER_ENV"); - -if (IsOnlineEnv()): - CONFIG_DIR = '/var/data/conf_test/game2003api_rankserver/config' -else: - CONFIG_DIR = '../config' - -def info(msg): - print(str(datetime.datetime.now()) + '[INFO] ' + msg) - -def getRedis(): - redis_conf = json.loadsmysql_conf = json.loads(open(CONFIG_DIR + '/rankserver.redis.cluster.json', 'r').read()) - for conf in redis_conf: - r = redis.Redis(host = conf['host'], - port = conf['port'], - password = conf['passwd'], - charset = 'utf8' - ) - return r; - -#字符串排序 -def customCmp(a, b): - if len(a[4]) < len(b[4]): - return -1 - elif len(a[4]) > len(b[4]): - return 1 - #endif - if a[4] < b[4]: - return -1 - elif a[4] > b[4]: - return 1 - else: - return - -#删除排行榜数据 -def delMysqlData(): - pass_list = [] - coin_list = [] - r = getRedis() - updateRank(r, pass_list, coin_list) - -#更新排行榜 -def updateRank(r, pass_list, coin_list): - pass_list.sort(key=take_pass, reverse=True) - pass_rank = [] - for pass_index in range(min(50, len(pass_list))): - pass_rank.append(pass_list[pass_index]) - r.set("game2003api:pass_rank", json.dumps(pass_rank)) - - coin = sorted(coin_list, key=functools.cmp_to_key(customCmp), reverse = True) - coin_rank = [] - for coin_index in range(min(50, len(coin))): - coin_rank.append(coin[coin_index]) - r.set("game2003api:coin_rank", json.dumps(coin_rank)) - -class SelfCheckingHandler(tornado.web.RequestHandler): - - def get(self): - self.write(json.dumps({ - 'errcode': 0, - 'errmsg': '', - 'healthy': 1, - 'max_rundelay': 10 - })) - -def make_app(): - return tornado.web.Application([ - (r"/webapp/index[\.]php", SelfCheckingHandler), - ]) - -if __name__ == "__main__": - conf = json.loads(open(CONFIG_DIR + '/rankserver.json', 'r').read()) - redis_cluster = json.loads(open(CONFIG_DIR + '/rankserver.redis.cluster.json', 'r').read()) - mysql_cluster = json.loads(open(CONFIG_DIR + '/rankserver.mysql.cluster.json', 'r').read()) - if len(sys.argv) <= 1: - app = make_app() - app.listen(conf['listen_port']) - delMysqlData() - else: - delrank_cmd.processCmdLine(sys.argv[1], redis_cluster, mysql_cluster) diff --git a/tools/rankserver/delrank_cmd.py b/tools/rankserver/delrank_cmd.py deleted file mode 100644 index 71d2c9a..0000000 --- a/tools/rankserver/delrank_cmd.py +++ /dev/null @@ -1,110 +0,0 @@ -# -*- coding: utf-8 -*- -#!/usr/bin/python - -import pymysql -import hashlib -import json -import urllib.request -import base64 -import tornado.ioloop -import tornado.web -import time -import datetime -import redis -import os -import functools - -CONFIG_DIR = '' - -def take_pass(elem): - return elem[3] - -def safeDiv(a, b): - if b == 0: - return 0 - else: - return a / b - -def IsOnlineEnv(): - return os.getenv("SERVER_ENV"); - -if (IsOnlineEnv()): - CONFIG_DIR = '/var/data/conf_test/game2003api_rankserver/config' -else: - CONFIG_DIR = '../config' - -def info(msg): - print(str(datetime.datetime.now()) + '[INFO] ' + msg) - -def getRedis(): - redis_conf = json.loadsmysql_conf = json.loads(open(CONFIG_DIR + '/rankserver.redis.cluster.json', 'r').read()) - for conf in redis_conf: - r = redis.Redis(host = conf['host'], - port = conf['port'], - password = conf['passwd'], - charset = 'utf8' - ) - return r; - -#字符串排序 -def customCmp(a, b): - if len(a[4]) < len(b[4]): - return -1 - elif len(a[4]) > len(b[4]): - return 1 - #endif - if a[4] < b[4]: - return -1 - elif a[4] > b[4]: - return 1 - else: - return - -#删除排行榜数据 -def delMysqlData(): - pass_list = [] - coin_list = [] - r = getRedis() - updateRank(r, pass_list, coin_list) - -class SelfCheckingHandler(tornado.web.RequestHandler): - - def get(self): - self.write(json.dumps({ - 'errcode': 0, - 'errmsg': '', - 'healthy': 1, - 'max_rundelay': 10 - })) - -def _crcDel_cmd(): - pass_list = [] - coin_list = [] - r = getRedis() - updateRank(r, pass_list, coin_list) - -def updateRank(r, pass_list, coin_list): - pass_list.sort(key=take_pass, reverse=True) - pass_rank = [] - for pass_index in range(min(50, len(pass_list))): - pass_rank.append(pass_list[pass_index]) - r.set("game2003api:pass_rank", json.dumps(pass_rank)) - - coin = sorted(coin_list, key=functools.cmp_to_key(customCmp), reverse = True) - coin_rank = [] - for coin_index in range(min(50, len(coin))): - coin_rank.append(coin[coin_index]) - r.set("game2003api:coin_rank", json.dumps(coin_rank)) - -def processCmdLine(cmd, redis_cluster, mysql_cluster): - precmd_hash = { - 'crcDel': _crcDel_cmd - } - print('ok') - if cmd in precmd_hash: - precmd_hash[cmd]() - -def make_app(): - return tornado.web.Application([ - (r"/webapp/index[\.]php", SelfCheckingHandler), - ]) diff --git a/tools/rankserver/game2003rank.py b/tools/rankserver/game2003rank.py index be0a546..57bd569 100644 --- a/tools/rankserver/game2003rank.py +++ b/tools/rankserver/game2003rank.py @@ -39,6 +39,13 @@ def safeDiv(a, b): else: return a / b +#获取channel +def getChannel(a): + str_list = a.split('_') + if len(str_list) < 3: + return 0 + return str_list[0] + def getRedis(): redis_conf = json.loadsmysql_conf = json.loads(open(CONFIG_DIR + '/rankserver.redis.cluster.json', 'r').read()) for conf in redis_conf: @@ -49,6 +56,10 @@ def getRedis(): ) return r; +def getRedisConf(): + redis_conf = json.loads(open(CONFIG_DIR + '/rankserver.redis.cluster.json', 'r').read()) + return redis_conf; + def getDaySeconds(time_val, incdays): time_zone = 8 dayseconds = int((time_val + time_zone * 3600)/3600/24 + incdays) * 3600 * 24 - 3600 * time_zone; @@ -70,45 +81,17 @@ def refreshData(row, pass_list): if (len(pass_list) > 50): del pass_list[50:] -#刷新金币数据 -def refreshCoinData(row, coin_list): - coin_list.append((row[0], row[1].decode('utf-8'), row[2], row[3], row[4])) - coin = sorted(coin_list, key=functools.cmp_to_key(customCmp), reverse = True) - if (len(coin) > 50): - del coin[50:] - -#字符串排序 -def customCmp(a, b): - if len(a[4]) < len(b[4]): - return -1 - elif len(a[4]) > len(b[4]): - return 1 - #endif - if a[4] < b[4]: - return -1 - elif a[4] > b[4]: - return 1 - else: - return 0 - #更新排行榜 -def updateRank(r, pass_list, coin_list): +def updateRank(r, channel, pass_list): pass_list.sort(key=take_pass, reverse=True) pass_rank = [] for pass_index in range(min(50, len(pass_list))): pass_rank.append(pass_list[pass_index]) - r.set("game2003api:pass_rank", json.dumps(pass_rank)) - - coin = sorted(coin_list, key=functools.cmp_to_key(customCmp), reverse = True) - coin_rank = [] - for coin_index in range(min(50, len(coin))): - coin_rank.append(coin[coin_index]) - r.set("game2003api:coin_rank", json.dumps(coin_rank)) + r.set("game2003api:pass_rank_" + channel, json.dumps(pass_rank)) def internalDayReadMysqlData(): mysql_conf = json.loads(open(CONFIG_DIR + '/rankserver.mysql.cluster.json', 'r').read()) - pass_list = [] - coin_list = [] + rank_hash = {} for conf in mysql_conf: conn = pymysql.connect(host = conf['host'], port = conf['port'], @@ -121,24 +104,23 @@ def internalDayReadMysqlData(): last_idx = 0 temp_idx = 0 while 1: - cursor.execute('SELECT accountid, user_name, avatar_url, pass, cumul_coin, idx' - ' FROM user WHERE idx > %s LIMIT 0, 5000' % (last_idx)) - + cursor.execute('SELECT accountid, user_name, avatar_url, pass, cumul_coin, idx ' + 'FROM user WHERE idx > %s LIMIT 0, 5000' % (last_idx)) has_data = False for row in cursor: has_data = True #更新通关榜 - refreshData(row, pass_list) - #更新金钱榜 - refreshCoinData(row, coin_list) - temp_idx = int(row[5]) - if (temp_idx > last_idx) : - last_idx = int(row[5]) + channel = getChannel(row[0]) + if channel not in rank_hash: + rank_hash[channel] = [] + refreshData(row, rank_hash[channel]) + last_idx = max(row[5], last_idx) time.sleep(0.001); if not has_data: break r = getRedis() - updateRank(r, pass_list, coin_list) + for channel in rank_hash: + updateRank(r, channel, rank_hash[channel]) #每日定时读取mysql里的数据生成排行榜写入redis后php读取redis返回客户端显示 def dayReadMysqlData(rushtime): @@ -151,18 +133,7 @@ def dayReadMysqlData(rushtime): def readMysqlData(rushtime): mysql_conf = json.loads(open(CONFIG_DIR + '/rankserver.mysql.cluster.json', 'r').read()) r = getRedis() - pass_list_str = r.get("game2003api:pass_rank") - if (not pass_list_str): - pass_list = [] - else: - pass_list = json.loads(pass_list_str) - - coin_list_str = r.get("game2003api:coin_rank") - if (not coin_list_str): - coin_list = [] - else: - coin_list = json.loads(coin_list_str) - + rank_hash = {} for conf in mysql_conf: conn = pymysql.connect(host = conf['host'], port = conf['port'], @@ -180,33 +151,23 @@ def readMysqlData(rushtime): has_data = False for row in cursor: has_data = True + channel = getChannel(row[0]) + if channel not in rank_hash: + rank_list = r.get('game2003api:pass_rank_' + channel) + rank_hash[channel] = [] if not rank_list else json.loads(rank_list) #更新通关榜 - delRepeatData(row, pass_list) - refreshData(row, pass_list) + delRepeatData(row, rank_hash[channel]) + refreshData(row, rank_hash[channel]) temp_idx = int(row[5]) if (temp_idx > last_idx) : last_idx = int(row[5]) time.sleep(0.001); if not has_data: break - last_idx = 0 - temp_idx = 0 - while 1: - cursor.execute('SELECT accountid, user_name, avatar_url, pass, cumul_coin, idx, modify_time FROM user ' - ' WHERE modify_time > %s AND idx > %s LIMIT 0, 1000' % (time.time() - 300, last_idx)) - has_data = False - for row in cursor: - has_data = True - #更新金币榜 - delRepeatData(row, coin_list) - refreshCoinData(row, coin_list) - temp_idx = int(row[5]) - if (temp_idx > last_idx) : - last_idx = int(row[5]) - time.sleep(0.001); - if not has_data: - break - updateRank(r, pass_list, coin_list) + + for channel in rank_hash: + updateRank(r, channel, rank_hash[channel]) + tornado.ioloop.IOLoop.current().call_later(rushtime, lambda : readMysqlData(rushtime) ) @@ -231,7 +192,7 @@ if __name__ == "__main__": app = make_app() app.listen(conf['listen_port']) - conf['rushtime'] = 300 + conf['rushtime'] = 5 tornado.ioloop.IOLoop.current().call_later(conf['rushtime'], lambda : readMysqlData(conf['rushtime']) ) diff --git a/tools/rankserver/rankserver_cmd.py b/tools/rankserver/rankserver_cmd.py new file mode 100644 index 0000000..d0bf269 --- /dev/null +++ b/tools/rankserver/rankserver_cmd.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/python + +import sys +import pymysql +import hashlib +import json +import urllib.request +import base64 +import tornado.ioloop +import tornado.web +import time +import datetime +import redis +import os +import functools + +import game2003rank + +def scanRedisKey(conn, pattern): + result = {} + cursor, keys = conn.scan(0, pattern, 1000) + while cursor != 0 or len(keys) > 0: + for key in keys: + if key in result: + result[key] += result[key] + else: + result[key] = 0 + keys = [] + if cursor != 0: + cursor, keys = conn.scan(cursor, pattern, 1000) + return result + +def _updateRank_cmd(debug_info): + game2003rank.internalDayReadMysqlData() + +def _clearRank_cmd(debug_info): + for conf in game2003rank.getRedisConf(): + r = redis.Redis(host = conf['host'], + port = conf['port'], + password = conf['passwd'], + charset = 'utf8' + ) + pass_list = [] + scan_keys = scanRedisKey(r, "game2003api:pass_rank_*") + for key in scan_keys : + r.set(key, json.dumps(pass_list)) + +def processCmdLine(cmd): + cmd_hash = { + 'updateRank': _updateRank_cmd, + 'clearRank': _clearRank_cmd, + } + precmd_hash = { + } + postcmd_hash = { + } + debug_info = { + 'record_count': 0, + 'param1': 0, + 'param2': 0, + 'param3': 0, + 'param4': 0 + } + if cmd in precmd_hash: + precmd_hash[cmd](debug_info) + if cmd in cmd_hash: + cmd_hash[cmd](debug_info) + if cmd in postcmd_hash: + postcmd_hash[cmd](debug_info) + +if __name__ == "__main__": + if len(sys.argv) <= 1: + pass + else: + processCmdLine(sys.argv[1])