diff --git a/tools/migratedb/migratedb.py b/tools/migratedb/migratedb.py index de85e8c..3ce4f73 100644 --- a/tools/migratedb/migratedb.py +++ b/tools/migratedb/migratedb.py @@ -18,38 +18,32 @@ context = { 'dbname_prefix': '' } -def md5Str(data): - m = hashlib.md5() - m.update(data) - return m.hexdigest() +def scanKeys(r, key_prefix, keys): + scan_key = key_prefix + '*' + cursor, keys = r.scan(0, scan_key, 1000) + scan_count = 0 + while cursor != 0 or len(keys) > 0: + #end + keys = [] + if cursor != 0: + cursor, keys = r.scan(cursor, scan_key, 1000) + scan_count += 1 + if scan_count > 10000: + break + #end while cursor -def fetchAllAccount(context, db_conf, all_account_hash): - for conf in db_conf: - conn = pymysql.connect(host = conf['host'], - port = conf['port'], - user = conf['user'], - passwd = conf['passwd'], - db = context['dbname_prefix'] + str(conf['instance_id']), - charset = 'utf8' +def exportRedis(context, redis_conf, redis_rule_conf): + for conf in redis_conf: + r = redis.Redis(host = conf['host'], + port = conf['port'], + password = conf['passwd'], + charset = 'utf8' ) - cursor = conn.cursor() - last_idx = 0 - while True: - cursor.execute('SELECT idx, accountid FROM user ' - 'WHERE idx > %s AND accountid LIKE "%s_\%" LIMIT 0, 10000;' % - (last_idx, - context['dbname_prefix'] - )) - has_data = False - for row in cursor: - has_data = True - all_account_hash[row[1]] = { - 'md5' : md5Str(row[1]) - } - if row[0] > last_idx: - last_idx = row[0] - if not has_data: - break + for rule in redis_rule_conf: + keys = {} + scanKeys(r, rule['key_prefix'], keys) + #end for rule + #end for conf def main(): global context @@ -57,8 +51,7 @@ def main(): redis_conf = json.loads(open(CONFIG_DIR + '/redis_list.json', 'r').read()) db_rule_conf = json.loads(open(CONFIG_DIR + '/mysql_rule.json', 'r').read()) redis_rule_conf = json.loads(open(CONFIG_DIR + '/redis_rule.json', 'r').read()) - all_account_hash = {} - fetchAllAccount(context, db_conf, all_account_hash) + exportRedis(context, redis_conf, redis_rule_conf) if __name__ == "__main__": main()