# -*- coding: utf-8 -*- #!/usr/bin/python import pymysql import hashlib import json import urllib.request import base64 import time import datetime import redis import os import sys import functools CONFIG_DIR = './' context = { 'channel': 0, 'dbname_prefix': '' } def scanKeys(r, key_prefix, matched_keys): scan_key = key_prefix + '*' keys = [] cursor, keys = r.scan(0, scan_key, 1000) scan_count = 0 while cursor != 0 or len(keys) > 0: for key in keys: matched_keys[key] = key #end keys = [] if cursor != 0: cursor, keys = r.scan(cursor, scan_key, 1000) scan_count += 1 if scan_count > 10000: break #end while cursor def saveKeys(r, keys, curr_file): for key in keys: data = r.get(key) expire = r.ttl(key) curr_file.write(json.dumps( { 'cmd': 'set', 'key': key, 'data':data } ) + '\n') curr_file.write(json.dumps( { 'cmd': 'expire', 'key': key, 'time': expire } ) + '\n') if expire < 0: print('[ERROR] ' + key + ' ttl < 0') if expire > 3600 * 24: print('[ERROR] ' + key + ' ttl > 3600 * 24') def exportRedis(context, redis_conf, redis_rule_conf): curr_file = open(context['out_dir'] + context['channel'] + '.redis', 'w') for conf in redis_conf: r = redis.Redis(host = conf['host'], port = conf['port'], password = conf['passwd'], decode_responses = True ) for rule in redis_rule_conf: matched_keys = {} scanKeys(r, rule['key_prefix'], matched_keys) saveKeys(r, matched_keys, curr_file) #end for conf def exportMysql(context, db_conf, db_rule_conf): dbsh_file = open(context['out_dir'] + context['channel'] + '.sh', 'w') for conf in db_conf: for rule in db_rule_conf: line = '' if rule['where'] == '': line = 'mysqldump -h %s -P %d -u%s -p%s %s %s > %s.sql' % ( conf['host'], conf['port'], conf['user'], conf['passwd'], context['dbname_prefix'] + str(conf['instance_id']), rule['table_name'], rule['table_name'] ) else: line = 'mysqldump -h %s -P %d -u%s -p%s %s %s --where="%s"> %s.sql' % ( conf['host'], conf['port'], conf['user'], conf['passwd'], context['dbname_prefix'] + str(conf['instance_id']), rule['table_name'], rule['where'].replace('$channel', context['channel']), rule['table_name'] ) dbsh_file.write(line + "\n") def main(): global context if not os.path.exists(CONFIG_DIR + 'out/'): os.mkdir(CONFIG_DIR + 'out/') if not os.path.exists(CONFIG_DIR + 'out/' + context['channel']): os.mkdir(CONFIG_DIR + 'out/' + context['channel']) context['out_dir'] = CONFIG_DIR + 'out/' + context['channel'] + '/' db_conf = json.loads(open(CONFIG_DIR + context['channel'] + '/mysql.json', 'r').read()) redis_conf = json.loads(open(CONFIG_DIR + context['channel'] + '/redis.json', 'r').read()) db_rule_conf = json.loads(open(CONFIG_DIR + 'common/mysql_rule.json', 'r').read()) redis_rule_conf = json.loads(open(CONFIG_DIR + 'common/redis_rule.json', 'r').read()) exportRedis(context, redis_conf, redis_rule_conf) exportMysql(context, db_conf, db_rule_conf) if __name__ == "__main__": context['channel'] = sys.argv[1] context['dbname_prefix'] = sys.argv[2] main()