完成

tools/rankserver/delrank.py

@@ -1,101 +0,0 @@
-# -*- coding: utf-8 -*-
-#!/usr/bin/python
-
-import pymysql
-import hashlib
-import json
-import urllib.request
-import base64
-import tornado.ioloop
-import tornado.web
-import time
-import datetime
-import redis
-import os
-import functools
-import delrank_cmd
-import sys
-
-CONFIG_DIR = ''
-
-def IsOnlineEnv():
-    return os.getenv("SERVER_ENV");
-
-if (IsOnlineEnv()):
-    CONFIG_DIR = '/var/data/conf_test/game2003api_rankserver/config'
-else:
-    CONFIG_DIR = '../config'
-
-def info(msg):
-    print(str(datetime.datetime.now()) + '[INFO] ' + msg)
-
-def getRedis():
-    redis_conf = json.loadsmysql_conf = json.loads(open(CONFIG_DIR + '/rankserver.redis.cluster.json', 'r').read())
-    for conf in redis_conf:
-        r = redis.Redis(host = conf['host'],
-                        port = conf['port'],
-                        password = conf['passwd'],
-                        charset = 'utf8'
-        )
-    return r;
-
-#字符串排序
-def customCmp(a, b):
-    if len(a[4]) < len(b[4]):
-        return -1
-    elif len(a[4]) > len(b[4]):
-        return 1
-    #endif
-    if a[4] < b[4]:
-        return -1
-    elif a[4] > b[4]:
-        return 1
-    else:
-        return
-
-#删除排行榜数据
-def delMysqlData():
-    pass_list = []
-    coin_list = []
-    r = getRedis()
-    updateRank(r, pass_list, coin_list)
-
-#更新排行榜
-def updateRank(r, pass_list, coin_list):
-    pass_list.sort(key=take_pass, reverse=True)
-    pass_rank = []
-    for pass_index in range(min(50, len(pass_list))):
-        pass_rank.append(pass_list[pass_index])
-    r.set("game2003api:pass_rank", json.dumps(pass_rank))
-
-    coin = sorted(coin_list, key=functools.cmp_to_key(customCmp), reverse = True)
-    coin_rank = []
-    for coin_index in range(min(50, len(coin))):
-        coin_rank.append(coin[coin_index])
-    r.set("game2003api:coin_rank", json.dumps(coin_rank))
-
-class SelfCheckingHandler(tornado.web.RequestHandler):
-
-    def get(self):
-        self.write(json.dumps({
-            'errcode': 0,
-            'errmsg': '',
-            'healthy': 1,
-            'max_rundelay': 10
-        }))
-
-def make_app():
-    return tornado.web.Application([
-        (r"/webapp/index[\.]php", SelfCheckingHandler),
-    ])
-
-if __name__ == "__main__":
-    conf = json.loads(open(CONFIG_DIR + '/rankserver.json', 'r').read())
-    redis_cluster = json.loads(open(CONFIG_DIR + '/rankserver.redis.cluster.json', 'r').read())
-    mysql_cluster = json.loads(open(CONFIG_DIR + '/rankserver.mysql.cluster.json', 'r').read())
-    if len(sys.argv) <= 1:
-        app = make_app()
-        app.listen(conf['listen_port'])
-        delMysqlData()
-    else:
-        delrank_cmd.processCmdLine(sys.argv[1], redis_cluster, mysql_cluster)

tools/rankserver/delrank_cmd.py

@@ -1,110 +0,0 @@
-# -*- coding: utf-8 -*-
-#!/usr/bin/python
-
-import pymysql
-import hashlib
-import json
-import urllib.request
-import base64
-import tornado.ioloop
-import tornado.web
-import time
-import datetime
-import redis
-import os
-import functools
-
-CONFIG_DIR = ''
-
-def take_pass(elem):
-    return elem[3]
-
-def safeDiv(a, b):
-    if b == 0:
-        return 0
-    else:
-        return a / b
-
-def IsOnlineEnv():
-    return os.getenv("SERVER_ENV");
-
-if (IsOnlineEnv()):
-    CONFIG_DIR = '/var/data/conf_test/game2003api_rankserver/config'
-else:
-    CONFIG_DIR = '../config'
-
-def info(msg):
-    print(str(datetime.datetime.now()) + '[INFO] ' + msg)
-
-def getRedis():
-    redis_conf = json.loadsmysql_conf = json.loads(open(CONFIG_DIR + '/rankserver.redis.cluster.json', 'r').read())
-    for conf in redis_conf:
-        r = redis.Redis(host = conf['host'],
-                        port = conf['port'],
-                        password = conf['passwd'],
-                        charset = 'utf8'
-        )
-    return r;
-
-#字符串排序
-def customCmp(a, b):
-    if len(a[4]) < len(b[4]):
-        return -1
-    elif len(a[4]) > len(b[4]):
-        return 1
-    #endif
-    if a[4] < b[4]:
-        return -1
-    elif a[4] > b[4]:
-        return 1
-    else:
-        return
-
-#删除排行榜数据
-def delMysqlData():
-    pass_list = []
-    coin_list = []
-    r = getRedis()
-    updateRank(r, pass_list, coin_list)
-
-class SelfCheckingHandler(tornado.web.RequestHandler):
-
-    def get(self):
-        self.write(json.dumps({
-            'errcode': 0,
-            'errmsg': '',
-            'healthy': 1,
-            'max_rundelay': 10
-        }))
-
-def _crcDel_cmd():
-    pass_list = []
-    coin_list = []
-    r = getRedis()
-    updateRank(r, pass_list, coin_list)
-
-def updateRank(r, pass_list, coin_list):
-    pass_list.sort(key=take_pass, reverse=True)
-    pass_rank = []
-    for pass_index in range(min(50, len(pass_list))):
-        pass_rank.append(pass_list[pass_index])
-    r.set("game2003api:pass_rank", json.dumps(pass_rank))
-
-    coin = sorted(coin_list, key=functools.cmp_to_key(customCmp), reverse = True)
-    coin_rank = []
-    for coin_index in range(min(50, len(coin))):
-        coin_rank.append(coin[coin_index])
-    r.set("game2003api:coin_rank", json.dumps(coin_rank))
-
-def processCmdLine(cmd, redis_cluster, mysql_cluster):
-    precmd_hash = {
-        'crcDel': _crcDel_cmd
-    }
-    print('ok')
-    if cmd in precmd_hash:
-        precmd_hash[cmd]()
-
-def make_app():
-    return tornado.web.Application([
-        (r"/webapp/index[\.]php", SelfCheckingHandler),
-    ])

tools/rankserver/game2003rank.py

@@ -39,6 +39,13 @@ def safeDiv(a, b):
     else:
         return a / b
 
+#获取channel
+def getChannel(a):
+    str_list = a.split('_')
+    if len(str_list) < 3:
+        return 0
+    return str_list[0]
+
 def getRedis():
     redis_conf = json.loadsmysql_conf = json.loads(open(CONFIG_DIR + '/rankserver.redis.cluster.json', 'r').read())
     for conf in redis_conf:
@@ -49,6 +56,10 @@ def getRedis():
         )
     return r;
 
+def getRedisConf():
+    redis_conf = json.loads(open(CONFIG_DIR + '/rankserver.redis.cluster.json', 'r').read())
+    return redis_conf;
+
 def getDaySeconds(time_val, incdays):
     time_zone = 8
     dayseconds = int((time_val + time_zone * 3600)/3600/24 + incdays) * 3600 * 24 - 3600 * time_zone;
@@ -70,45 +81,17 @@ def refreshData(row, pass_list):
     if (len(pass_list) > 50):
         del pass_list[50:]
 
-#刷新金币数据
-def refreshCoinData(row, coin_list):
-    coin_list.append((row[0], row[1].decode('utf-8'), row[2], row[3], row[4]))
-    coin = sorted(coin_list, key=functools.cmp_to_key(customCmp), reverse = True)
-    if (len(coin) > 50):
-        del coin[50:]
-
-#字符串排序
-def customCmp(a, b):
-    if len(a[4]) < len(b[4]):
-        return -1
-    elif len(a[4]) > len(b[4]):
-        return 1
-    #endif
-    if a[4] < b[4]:
-        return -1
-    elif a[4] > b[4]:
-        return 1
-    else:
-        return 0
-
 #更新排行榜
-def updateRank(r, pass_list, coin_list):
+def updateRank(r, channel, pass_list):
     pass_list.sort(key=take_pass, reverse=True)
     pass_rank = []
     for pass_index in range(min(50, len(pass_list))):
         pass_rank.append(pass_list[pass_index])
-    r.set("game2003api:pass_rank", json.dumps(pass_rank))
+    r.set("game2003api:pass_rank_" + channel, json.dumps(pass_rank))
-
-    coin = sorted(coin_list, key=functools.cmp_to_key(customCmp), reverse = True)
-    coin_rank = []
-    for coin_index in range(min(50, len(coin))):
-        coin_rank.append(coin[coin_index])
-    r.set("game2003api:coin_rank", json.dumps(coin_rank))
 
 def internalDayReadMysqlData():
     mysql_conf = json.loads(open(CONFIG_DIR + '/rankserver.mysql.cluster.json', 'r').read())
-    pass_list = []
+    rank_hash = {}
-    coin_list = []
     for conf in mysql_conf:
         conn = pymysql.connect(host = conf['host'],
                                port = conf['port'],
@@ -123,22 +106,21 @@ def internalDayReadMysqlData():
         while 1:
             cursor.execute('SELECT accountid, user_name, avatar_url, pass, cumul_coin, idx '
                            'FROM user WHERE idx > %s LIMIT 0, 5000' % (last_idx))
-
             has_data = False
             for row in cursor:
                 has_data = True
                 #更新通关榜
-                refreshData(row, pass_list)
+                channel = getChannel(row[0])
-                #更新金钱榜
+                if channel not in rank_hash:
-                refreshCoinData(row, coin_list)
+                    rank_hash[channel] = []
-                temp_idx = int(row[5])
+                refreshData(row, rank_hash[channel])
-                if (temp_idx > last_idx) :
+                last_idx = max(row[5], last_idx)
-                    last_idx = int(row[5])
             time.sleep(0.001);
             if not has_data:
                 break
     r = getRedis()
-    updateRank(r, pass_list, coin_list)
+    for channel in rank_hash:
+        updateRank(r, channel, rank_hash[channel])
 
 #每日定时读取mysql里的数据生成排行榜写入redis后php读取redis返回客户端显示
 def dayReadMysqlData(rushtime):
@@ -151,18 +133,7 @@ def dayReadMysqlData(rushtime):
 def readMysqlData(rushtime):
     mysql_conf = json.loads(open(CONFIG_DIR + '/rankserver.mysql.cluster.json', 'r').read())
     r = getRedis()
-    pass_list_str = r.get("game2003api:pass_rank")
+    rank_hash = {}
-    if (not pass_list_str):
-        pass_list = []
-    else:
-        pass_list = json.loads(pass_list_str)
-
-    coin_list_str = r.get("game2003api:coin_rank")
-    if (not coin_list_str):
-        coin_list = []
-    else:
-        coin_list = json.loads(coin_list_str)
-
     for conf in mysql_conf:
         conn = pymysql.connect(host = conf['host'],
                                port = conf['port'],
@@ -180,33 +151,23 @@ def readMysqlData(rushtime):
             has_data = False
             for row in cursor:
                 has_data = True
+                channel = getChannel(row[0])
+                if channel not in rank_hash:
+                    rank_list = r.get('game2003api:pass_rank_' + channel)
+                    rank_hash[channel] = [] if not rank_list else json.loads(rank_list)
                 #更新通关榜
-                delRepeatData(row, pass_list)
+                delRepeatData(row, rank_hash[channel])
-                refreshData(row, pass_list)
+                refreshData(row, rank_hash[channel])
                 temp_idx = int(row[5])
                 if (temp_idx > last_idx) :
                     last_idx = int(row[5])
             time.sleep(0.001);
             if not has_data:
                 break
-        last_idx = 0
+
-        temp_idx = 0
+    for channel in rank_hash:
-        while 1:
+        updateRank(r, channel, rank_hash[channel])
-            cursor.execute('SELECT accountid, user_name, avatar_url, pass, cumul_coin, idx, modify_time FROM user '
+
-                           ' WHERE modify_time > %s AND idx > %s LIMIT 0, 1000' % (time.time() - 300, last_idx))
-            has_data = False
-            for row in cursor:
-                has_data = True
-                #更新金币榜
-                delRepeatData(row, coin_list)
-                refreshCoinData(row, coin_list)
-                temp_idx = int(row[5])
-                if (temp_idx > last_idx) :
-                    last_idx = int(row[5])
-            time.sleep(0.001);
-            if not has_data:
-                break
-    updateRank(r, pass_list, coin_list)
     tornado.ioloop.IOLoop.current().call_later(rushtime,
                                                lambda : readMysqlData(rushtime)
     )
@@ -231,7 +192,7 @@ if __name__ == "__main__":
 
     app = make_app()
     app.listen(conf['listen_port'])
-    conf['rushtime'] = 300
+    conf['rushtime'] = 5
     tornado.ioloop.IOLoop.current().call_later(conf['rushtime'],
                                                lambda : readMysqlData(conf['rushtime'])
     )

tools/rankserver/rankserver_cmd.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/python
+
+import sys
+import pymysql
+import hashlib
+import json
+import urllib.request
+import base64
+import tornado.ioloop
+import tornado.web
+import time
+import datetime
+import redis
+import os
+import functools
+
+import game2003rank
+
+def scanRedisKey(conn, pattern):
+    result = {}
+    cursor, keys = conn.scan(0, pattern, 1000)
+    while cursor != 0 or len(keys) > 0:
+        for key in keys:
+            if key in result:
+                result[key] += result[key]
+            else:
+                result[key] = 0
+        keys = []
+        if cursor != 0:
+            cursor, keys = conn.scan(cursor, pattern, 1000)
+    return result
+
+def _updateRank_cmd(debug_info):
+    game2003rank.internalDayReadMysqlData()
+
+def _clearRank_cmd(debug_info):
+    for conf in game2003rank.getRedisConf():
+        r = redis.Redis(host = conf['host'],
+                        port = conf['port'],
+                        password = conf['passwd'],
+                        charset = 'utf8'
+        )
+        pass_list = []
+        scan_keys = scanRedisKey(r, "game2003api:pass_rank_*")
+        for key in scan_keys :
+            r.set(key, json.dumps(pass_list))
+
+def processCmdLine(cmd):
+    cmd_hash = {
+        'updateRank': _updateRank_cmd,
+        'clearRank': _clearRank_cmd,
+    }
+    precmd_hash = {
+    }
+    postcmd_hash = {
+    }
+    debug_info = {
+        'record_count': 0,
+        'param1': 0,
+        'param2': 0,
+        'param3': 0,
+        'param4': 0
+    }
+    if cmd in precmd_hash:
+        precmd_hash[cmd](debug_info)
+    if cmd in cmd_hash:
+        cmd_hash[cmd](debug_info)
+    if cmd in postcmd_hash:
+        postcmd_hash[cmd](debug_info)
+
+if __name__ == "__main__":
+    if len(sys.argv) <= 1:
+        pass
+    else:
+        processCmdLine(sys.argv[1])