add migratedb

tools/migratedb/0/mysql.json

@@ -0,0 +1,9 @@
+[
+  {
+    "instance_id" : 1,
+    "host" : "127.0.0.1",
+    "port" : 3306,
+    "user" : "root",
+    "passwd" : "keji178"
+  }
+]

tools/migratedb/0/redis.json

@@ -0,0 +1,9 @@
+[
+  {
+    "instance_id" : 1,
+    "host" : "127.0.0.1",
+    "port" : 6379,
+    "user" : "root",
+    "passwd" : ""
+  }
+]

tools/migratedb/6000/mysql.json

@@ -0,0 +1,9 @@
+[
+  {
+    "instance_id" : 1,
+    "host" : "127.0.0.1",
+    "port" : 3306,
+    "user" : "root",
+    "passwd" : "keji178"
+  }
+]

tools/migratedb/6000/redis.json

@@ -0,0 +1,9 @@
+[
+  {
+    "instance_id" : 1,
+    "host" : "127.0.0.1",
+    "port" : 6379,
+    "user" : "root",
+    "passwd" : ""
+  }
+]

tools/migratedb/common/mysql_rule.json

@@ -0,0 +1,46 @@
+[
+  {
+    "table_name": "version",
+    "where": ""
+  },
+  {
+    "table_name": "user",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "bag",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "sign",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "quest",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "share_achievement",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "activity",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "passinfo",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "shop",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "equip",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "shop_data",
+    "where": "accountid LIKE '$channel_%'"
+  }
+]

tools/migratedb/common/redis_rule.json

@@ -0,0 +1,5 @@
+[
+  {
+    "key_prefix": "game2004"
+  }
+]

tools/migratedb/import_redis.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/python
+
+import os
+import sys
+import time
+import json
+import redis
+import datetime
+import pprint
+
+argv = sys.argv
+host = argv[1]
+port = argv[2]
+passwd = argv[3]
+script_name = argv[4]
+
+def writeToRedis(conn, cmdline):
+    assert len(cmdline) > 1
+    print(cmdline)
+    if cmdline[0] == 'hset':
+        conn.hset(cmdline[1], cmdline[2], cmdline[3])
+    elif cmdline[0] == 'zadd':
+        conn.zadd(cmdline[1], {cmdline[3]: cmdline[2]})
+    elif cmdline[0] == 'sadd':
+        conn.sadd(cmdline[1], cmdline[2])
+    elif cmdline[0] == 'set':
+        if cmdline[2] != None:
+            conn.set(cmdline[1], cmdline[2])
+        else:
+            print('[warning]', cmdline)
+    elif cmdline[0] == 'expire':
+        conn.expire(cmdline[1], cmdline[2])
+    else:
+        assert False
+
+conn = redis.Redis(host = host,
+                   port = port,
+                   password = passwd,
+                   db = 0,
+                   decode_responses = True
+)
+
+with open(script_name, 'r') as f:
+    for line in f:
+        line = line.strip()
+        if line == '':
+            continue
+        writeToRedis(conn, json.loads(line))

tools/migratedb/migratedb.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/python
+
+import pymysql
+import hashlib
+import json
+import urllib.request
+import base64
+import time
+import datetime
+import redis
+import os
+import sys
+import functools
+
+CONFIG_DIR = './'
+context = {
+    'channel': 0,
+    'dbname_prefix': ''
+}
+EXCLUDE_KEYS = {
+    'game2004api:kill_rank_6006': 1,
+    'game2004api:win_rank_6006': 1,
+    'game2004api:integral_rank_6006': 1
+}
+
+def scanKeys(r, key_prefix, matched_keys):
+    scan_key = key_prefix + '*'
+    keys = []
+    cursor, keys = r.scan(0, scan_key, 1000)
+    scan_count = 0
+    while cursor != 0 or len(keys) > 0:
+        for key in keys:
+            matched_keys[key] = key
+        #end
+        keys = []
+        if cursor != 0:
+            cursor, keys = r.scan(cursor, scan_key, 1000)
+        scan_count += 1
+        if scan_count > 10000:
+            break
+    #end while cursor
+
+def excludeKey(key):
+    global EXCLUDE_KEYS
+    return key in EXCLUDE_KEYS
+
+def saveKeys(r, keys, curr_file):
+    for key in keys:
+        if excludeKey(key):
+            print('[WARNING]exclude key ' + key)
+            continue
+        data = r.get(key)
+        expire = r.ttl(key)
+        curr_file.write(json.dumps(
+            [
+                'set',
+                key,
+                data
+            ]
+        ) + '\n')
+        curr_file.write(json.dumps(
+            [
+                'expire',
+                key,
+                expire
+            ]
+        ) + '\n')
+        if expire < 0:
+            print('[ERROR] ' + key + ' ttl < 0')
+        if expire > 3600 * 24:
+            print('[ERROR] ' + key + ' ttl > 3600 * 24')
+
+def exportRedis(context, redis_conf, redis_rule_conf):
+    redis_dir = CONFIG_DIR + 'out/' + context['channel'] + '/redis/'
+    if not os.path.exists(redis_dir):
+        os.mkdir(redis_dir)
+    curr_file = open(redis_dir + context['channel'] + '.redis', 'w')
+    for conf in redis_conf:
+        r = redis.Redis(host = conf['host'],
+                        port = conf['port'],
+                        password = conf['passwd'],
+                        decode_responses = True
+        )
+        for rule in redis_rule_conf:
+            matched_keys = {}
+            scanKeys(r, rule['key_prefix'], matched_keys)
+            saveKeys(r, matched_keys, curr_file)
+    #end for conf
+
+def exportMysql(context, db_conf, db_rule_conf):
+    dbsh_file = open(context['out_dir'] + 'exportdb.sh', 'w')
+    for conf in db_conf:
+        db_dir = CONFIG_DIR + 'out/' + context['channel'] + '/' + str(conf['instance_id'])
+        if not os.path.exists(db_dir):
+            os.mkdir(db_dir)
+        for rule in db_rule_conf:
+            line = ''
+            if rule['where'] == '':
+                line = 'mysqldump -h %s -P %d -u%s -p%s %s %s > %d/%s.sql' % (
+                    conf['host'],
+                    conf['port'],
+                    conf['user'],
+                    conf['passwd'],
+                    context['dbname_prefix'] + str(conf['instance_id']),
+                    rule['table_name'],
+                    conf['instance_id'],
+                    rule['table_name']
+                )
+            else:
+                line = 'mysqldump -h %s -P %d -u%s -p%s %s %s --where="%s"> %d/%s.sql' % (
+                    conf['host'],
+                    conf['port'],
+                    conf['user'],
+                    conf['passwd'],
+                    context['dbname_prefix'] + str(conf['instance_id']),
+                    rule['table_name'],
+                    rule['where'].replace('$channel', context['channel']),
+                    conf['instance_id'],
+                    rule['table_name']
+                )
+            dbsh_file.write(line + "\n")
+
+def main():
+    global context
+    if not os.path.exists(CONFIG_DIR + 'out/'):
+        os.mkdir(CONFIG_DIR + 'out/')
+    if not os.path.exists(CONFIG_DIR + 'out/' + context['channel']):
+        os.mkdir(CONFIG_DIR + 'out/' + context['channel'])
+    context['out_dir'] = CONFIG_DIR + 'out/' + context['channel'] + '/'
+    db_conf = json.loads(open(CONFIG_DIR + context['channel'] + '/mysql.json', 'r').read())
+    redis_conf = json.loads(open(CONFIG_DIR + context['channel'] + '/redis.json', 'r').read())
+    db_rule_conf = json.loads(open(CONFIG_DIR + 'common/mysql_rule.json', 'r').read())
+    redis_rule_conf = json.loads(open(CONFIG_DIR + 'common/redis_rule.json', 'r').read())
+    exportRedis(context, redis_conf, redis_rule_conf)
+    #exportMysql(context, db_conf, db_rule_conf)
+
+if __name__ == "__main__":
+    #context['channel'] = sys.argv[1]
+    #context['dbname_prefix'] = sys.argv[2]
+    context['channel'] = '0'
+    context['dbname_prefix'] = 'gamedb2004'
+    main()