From 783638db51ec492d5dc149b833ae0834aef1cd02 Mon Sep 17 00:00:00 2001 From: aozhiwei Date: Mon, 31 Aug 2020 17:12:00 +0800 Subject: [PATCH 1/3] 1 --- tools/migratedb/migratedb.py | 64 +++++++++++++++++++++++++++++++++ tools/migratedb/redis_rule.json | 6 ++++ 2 files changed, 70 insertions(+) create mode 100644 tools/migratedb/migratedb.py create mode 100644 tools/migratedb/redis_rule.json diff --git a/tools/migratedb/migratedb.py b/tools/migratedb/migratedb.py new file mode 100644 index 0000000..de85e8c --- /dev/null +++ b/tools/migratedb/migratedb.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/python + +import pymysql +import hashlib +import json +import urllib.request +import base64 +import time +import datetime +import redis +import os +import functools + +CONFIG_DIR = '.' +context = { + 'channel': 0, + 'dbname_prefix': '' +} + +def md5Str(data): + m = hashlib.md5() + m.update(data) + return m.hexdigest() + +def fetchAllAccount(context, db_conf, all_account_hash): + for conf in db_conf: + conn = pymysql.connect(host = conf['host'], + port = conf['port'], + user = conf['user'], + passwd = conf['passwd'], + db = context['dbname_prefix'] + str(conf['instance_id']), + charset = 'utf8' + ) + cursor = conn.cursor() + last_idx = 0 + while True: + cursor.execute('SELECT idx, accountid FROM user ' + 'WHERE idx > %s AND accountid LIKE "%s_\%" LIMIT 0, 10000;' % + (last_idx, + context['dbname_prefix'] + )) + has_data = False + for row in cursor: + has_data = True + all_account_hash[row[1]] = { + 'md5' : md5Str(row[1]) + } + if row[0] > last_idx: + last_idx = row[0] + if not has_data: + break + +def main(): + global context + db_conf = json.loads(open(CONFIG_DIR + '/mysql_list.json', 'r').read()) + redis_conf = json.loads(open(CONFIG_DIR + '/redis_list.json', 'r').read()) + db_rule_conf = json.loads(open(CONFIG_DIR + '/mysql_rule.json', 'r').read()) + redis_rule_conf = json.loads(open(CONFIG_DIR + '/redis_rule.json', 'r').read()) + all_account_hash = {} + fetchAllAccount(context, db_conf, all_account_hash) + +if __name__ == "__main__": + main() diff --git a/tools/migratedb/redis_rule.json b/tools/migratedb/redis_rule.json new file mode 100644 index 0000000..ddc0dd3 --- /dev/null +++ b/tools/migratedb/redis_rule.json @@ -0,0 +1,6 @@ +[ + { + "key_prefix": "", + "pattern": "md5" + } +] From 3538701702841bc9a024dc34cc511c16c2398481 Mon Sep 17 00:00:00 2001 From: aozhiwei Date: Mon, 31 Aug 2020 17:44:02 +0800 Subject: [PATCH 2/3] 1 --- tools/migratedb/migratedb.py | 57 ++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/tools/migratedb/migratedb.py b/tools/migratedb/migratedb.py index de85e8c..3ce4f73 100644 --- a/tools/migratedb/migratedb.py +++ b/tools/migratedb/migratedb.py @@ -18,38 +18,32 @@ context = { 'dbname_prefix': '' } -def md5Str(data): - m = hashlib.md5() - m.update(data) - return m.hexdigest() +def scanKeys(r, key_prefix, keys): + scan_key = key_prefix + '*' + cursor, keys = r.scan(0, scan_key, 1000) + scan_count = 0 + while cursor != 0 or len(keys) > 0: + #end + keys = [] + if cursor != 0: + cursor, keys = r.scan(cursor, scan_key, 1000) + scan_count += 1 + if scan_count > 10000: + break + #end while cursor -def fetchAllAccount(context, db_conf, all_account_hash): - for conf in db_conf: - conn = pymysql.connect(host = conf['host'], - port = conf['port'], - user = conf['user'], - passwd = conf['passwd'], - db = context['dbname_prefix'] + str(conf['instance_id']), - charset = 'utf8' +def exportRedis(context, redis_conf, redis_rule_conf): + for conf in redis_conf: + r = redis.Redis(host = conf['host'], + port = conf['port'], + password = conf['passwd'], + charset = 'utf8' ) - cursor = conn.cursor() - last_idx = 0 - while True: - cursor.execute('SELECT idx, accountid FROM user ' - 'WHERE idx > %s AND accountid LIKE "%s_\%" LIMIT 0, 10000;' % - (last_idx, - context['dbname_prefix'] - )) - has_data = False - for row in cursor: - has_data = True - all_account_hash[row[1]] = { - 'md5' : md5Str(row[1]) - } - if row[0] > last_idx: - last_idx = row[0] - if not has_data: - break + for rule in redis_rule_conf: + keys = {} + scanKeys(r, rule['key_prefix'], keys) + #end for rule + #end for conf def main(): global context @@ -57,8 +51,7 @@ def main(): redis_conf = json.loads(open(CONFIG_DIR + '/redis_list.json', 'r').read()) db_rule_conf = json.loads(open(CONFIG_DIR + '/mysql_rule.json', 'r').read()) redis_rule_conf = json.loads(open(CONFIG_DIR + '/redis_rule.json', 'r').read()) - all_account_hash = {} - fetchAllAccount(context, db_conf, all_account_hash) + exportRedis(context, redis_conf, redis_rule_conf) if __name__ == "__main__": main() From d72a3ec5b1c7204865928675d5e7fdcf765356da Mon Sep 17 00:00:00 2001 From: aozhiwei Date: Mon, 31 Aug 2020 17:44:57 +0800 Subject: [PATCH 3/3] 1 --- tools/migratedb/migratedb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/migratedb/migratedb.py b/tools/migratedb/migratedb.py index 3ce4f73..876bc2e 100644 --- a/tools/migratedb/migratedb.py +++ b/tools/migratedb/migratedb.py @@ -27,7 +27,7 @@ def scanKeys(r, key_prefix, keys): keys = [] if cursor != 0: cursor, keys = r.scan(cursor, scan_key, 1000) - scan_count += 1 + scan_count += 1 if scan_count > 10000: break #end while cursor