diff --git a/tools/migratedb/0/mysql.json b/tools/migratedb/0/mysql.json new file mode 100644 index 0000000..c2754a2 --- /dev/null +++ b/tools/migratedb/0/mysql.json @@ -0,0 +1,9 @@ +[ + { + "instance_id" : 1, + "host" : "127.0.0.1", + "port" : 3306, + "user" : "root", + "passwd" : "keji178" + } +] diff --git a/tools/migratedb/0/redis.json b/tools/migratedb/0/redis.json new file mode 100644 index 0000000..e0d8e02 --- /dev/null +++ b/tools/migratedb/0/redis.json @@ -0,0 +1,9 @@ +[ + { + "instance_id" : 1, + "host" : "127.0.0.1", + "port" : 6379, + "user" : "root", + "passwd" : "" + } +] diff --git a/tools/migratedb/common/mysql_rule.json b/tools/migratedb/common/mysql_rule.json new file mode 100644 index 0000000..bf17216 --- /dev/null +++ b/tools/migratedb/common/mysql_rule.json @@ -0,0 +1,46 @@ +[ + { + "table_name": "version", + "where": "" + }, + { + "table_name": "user", + "where": "accountid LIKE '$channel_%'" + }, + { + "table_name": "bag", + "where": "accountid LIKE '$channel_%'" + }, + { + "table_name": "sign", + "where": "accountid LIKE '$channel_%'" + }, + { + "table_name": "quest", + "where": "accountid LIKE '$channel_%'" + }, + { + "table_name": "share_achievement", + "where": "accountid LIKE '$channel_%'" + }, + { + "table_name": "activity", + "where": "accountid LIKE '$channel_%'" + }, + { + "table_name": "passinfo", + "where": "accountid LIKE '$channel_%'" + }, + { + "table_name": "shop", + "where": "accountid LIKE '$channel_%'" + }, + { + "table_name": "equip", + "where": "accountid LIKE '$channel_%'" + }, + { + "table_name": "shop_data", + "where": "accountid LIKE '$channel_%'" + } +] diff --git a/tools/migratedb/common/redis_rule.json b/tools/migratedb/common/redis_rule.json new file mode 100644 index 0000000..327ee0f --- /dev/null +++ b/tools/migratedb/common/redis_rule.json @@ -0,0 +1,5 @@ +[ + { + "key_prefix": "game2001" + } +] diff --git a/tools/migratedb/import_redis.py b/tools/migratedb/import_redis.py new file mode 100644 index 0000000..45a6eeb --- /dev/null +++ b/tools/migratedb/import_redis.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/python + +import os +import sys +import time +import json +import redis +import datetime +import pprint + +argv = sys.argv +host = argv[1] +port = argv[2] +passwd = argv[3] +script_name = argv[4] + +def writeToRedis(conn, cmdline): + assert len(cmdline) > 1 + print(cmdline) + if cmdline[0] == 'hset': + conn.hset(cmdline[1], cmdline[2], cmdline[3]) + elif cmdline[0] == 'zadd': + conn.zadd(cmdline[1], {cmdline[3]: cmdline[2]}) + elif cmdline[0] == 'sadd': + conn.sadd(cmdline[1], cmdline[2]) + elif cmdline[0] == 'set': + if cmdline[2] != None: + conn.set(cmdline[1], cmdline[2]) + else: + print('[warning]', cmdline) + elif cmdline[0] == 'expire': + if int(cmdline[2]) >= 0: + conn.expire(cmdline[1], cmdline[2]) + else: + assert False + +conn = redis.Redis(host = host, + port = port, + password = passwd, + db = 0, + decode_responses = True +) + +with open(script_name, 'r') as f: + for line in f: + line = line.strip() + if line == '': + continue + writeToRedis(conn, json.loads(line)) diff --git a/tools/migratedb/migratedb.py b/tools/migratedb/migratedb.py new file mode 100644 index 0000000..06c1d74 --- /dev/null +++ b/tools/migratedb/migratedb.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/python + +import pymysql +import hashlib +import json +import urllib.request +import base64 +import time +import datetime +import redis +import os +import sys +import functools + +CONFIG_DIR = './' +context = { + 'channel': 0, + 'dbname_prefix': '' +} +EXCLUDE_KEYS = { +} + +def scanKeys(r, key_prefix, matched_keys): + scan_key = key_prefix + '*' + keys = [] + cursor, keys = r.scan(0, scan_key, 1000) + scan_count = 0 + while cursor != 0 or len(keys) > 0: + for key in keys: + matched_keys[key] = key + #end + keys = [] + if cursor != 0: + cursor, keys = r.scan(cursor, scan_key, 1000) + scan_count += 1 + if scan_count > 10000: + break + #end while cursor + +def excludeKey(key): + global EXCLUDE_KEYS + return key in EXCLUDE_KEYS + +def saveKeys(r, keys, curr_file): + for key in keys: + if excludeKey(key): + print('[WARNING]exclude key ' + key) + continue + data = r.get(key) + expire = r.ttl(key) + curr_file.write(json.dumps( + [ + 'set', + key, + data + ] + ) + '\n') + curr_file.write(json.dumps( + [ + 'expire', + key, + expire + ] + ) + '\n') + if expire < 0: + print('[WARNING] ' + key + ' ttl < 0') + if expire > 3600 * 24: + print('[WARNING] ' + key + ' ttl > 3600 * 24') + +def exportRedis(context, redis_conf, redis_rule_conf): + redis_dir = CONFIG_DIR + 'out/' + context['channel'] + '/redis/' + if not os.path.exists(redis_dir): + os.mkdir(redis_dir) + curr_file = open(redis_dir + context['channel'] + '.redis', 'w') + for conf in redis_conf: + r = redis.Redis(host = conf['host'], + port = conf['port'], + password = conf['passwd'], + decode_responses = True + ) + for rule in redis_rule_conf: + matched_keys = {} + scanKeys(r, rule['key_prefix'], matched_keys) + saveKeys(r, matched_keys, curr_file) + #end for conf + +def exportMysql(context, db_conf, db_rule_conf): + dbsh_file = open(context['out_dir'] + 'exportdb.sh', 'w') + for conf in db_conf: + db_dir = CONFIG_DIR + 'out/' + context['channel'] + '/' + str(conf['instance_id']) + if not os.path.exists(db_dir): + os.mkdir(db_dir) + for rule in db_rule_conf: + line = '' + if rule['where'] == '': + line = 'mysqldump -h %s -P %d -u%s -p%s %s %s > %d/%s.sql' % ( + conf['host'], + conf['port'], + conf['user'], + conf['passwd'], + context['dbname_prefix'] + str(conf['instance_id']), + rule['table_name'], + conf['instance_id'], + rule['table_name'] + ) + else: + line = 'mysqldump -h %s -P %d -u%s -p%s %s %s --where="%s"> %d/%s.sql' % ( + conf['host'], + conf['port'], + conf['user'], + conf['passwd'], + context['dbname_prefix'] + str(conf['instance_id']), + rule['table_name'], + rule['where'].replace('$channel', context['channel']), + conf['instance_id'], + rule['table_name'] + ) + dbsh_file.write(line + "\n") + +def main(): + global context + if not os.path.exists(CONFIG_DIR + 'out/'): + os.mkdir(CONFIG_DIR + 'out/') + if not os.path.exists(CONFIG_DIR + 'out/' + context['channel']): + os.mkdir(CONFIG_DIR + 'out/' + context['channel']) + context['out_dir'] = CONFIG_DIR + 'out/' + context['channel'] + '/' + db_conf = json.loads(open(CONFIG_DIR + context['channel'] + '/mysql.json', 'r').read()) + redis_conf = json.loads(open(CONFIG_DIR + context['channel'] + '/redis.json', 'r').read()) + db_rule_conf = json.loads(open(CONFIG_DIR + 'common/mysql_rule.json', 'r').read()) + redis_rule_conf = json.loads(open(CONFIG_DIR + 'common/redis_rule.json', 'r').read()) + exportRedis(context, redis_conf, redis_rule_conf) + #exportMysql(context, db_conf, db_rule_conf) + +if __name__ == "__main__": + #context['channel'] = sys.argv[1] + #context['dbname_prefix'] = sys.argv[2] + context['channel'] = '0' + context['dbname_prefix'] = 'gamedb2001' + main()