144 lines
4.6 KiB
Python
144 lines
4.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
#!/usr/bin/python
|
|
|
|
import pymysql
|
|
import hashlib
|
|
import json
|
|
import urllib.request
|
|
import base64
|
|
import time
|
|
import datetime
|
|
import redis
|
|
import os
|
|
import sys
|
|
import functools
|
|
|
|
CONFIG_DIR = './'
|
|
context = {
|
|
'channel': 0,
|
|
'dbname_prefix': ''
|
|
}
|
|
EXCLUDE_KEYS = {
|
|
'game2004api:kill_rank_6006': 1,
|
|
'game2004api:win_rank_6006': 1,
|
|
'game2004api:integral_rank_6006': 1
|
|
}
|
|
|
|
def scanKeys(r, key_prefix, matched_keys):
|
|
scan_key = key_prefix + '*'
|
|
keys = []
|
|
cursor, keys = r.scan(0, scan_key, 1000)
|
|
scan_count = 0
|
|
while cursor != 0 or len(keys) > 0:
|
|
for key in keys:
|
|
matched_keys[key] = key
|
|
#end
|
|
keys = []
|
|
if cursor != 0:
|
|
cursor, keys = r.scan(cursor, scan_key, 1000)
|
|
scan_count += 1
|
|
if scan_count > 10000:
|
|
break
|
|
#end while cursor
|
|
|
|
def excludeKey(key):
|
|
global EXCLUDE_KEYS
|
|
return key in EXCLUDE_KEYS
|
|
|
|
def saveKeys(r, keys, curr_file):
|
|
for key in keys:
|
|
if excludeKey(key):
|
|
print('[WARNING]exclude key ' + key)
|
|
continue
|
|
data = r.get(key)
|
|
expire = r.ttl(key)
|
|
curr_file.write(json.dumps(
|
|
[
|
|
'set',
|
|
key,
|
|
data
|
|
]
|
|
) + '\n')
|
|
curr_file.write(json.dumps(
|
|
[
|
|
'expire',
|
|
key,
|
|
expire
|
|
]
|
|
) + '\n')
|
|
if expire < 0:
|
|
print('[ERROR] ' + key + ' ttl < 0')
|
|
if expire > 3600 * 24:
|
|
print('[ERROR] ' + key + ' ttl > 3600 * 24')
|
|
|
|
def exportRedis(context, redis_conf, redis_rule_conf):
|
|
redis_dir = CONFIG_DIR + 'out/' + context['channel'] + '/redis/'
|
|
if not os.path.exists(redis_dir):
|
|
os.mkdir(redis_dir)
|
|
curr_file = open(redis_dir + context['channel'] + '.redis', 'w')
|
|
for conf in redis_conf:
|
|
r = redis.Redis(host = conf['host'],
|
|
port = conf['port'],
|
|
password = conf['passwd'],
|
|
decode_responses = True
|
|
)
|
|
for rule in redis_rule_conf:
|
|
matched_keys = {}
|
|
scanKeys(r, rule['key_prefix'], matched_keys)
|
|
saveKeys(r, matched_keys, curr_file)
|
|
#end for conf
|
|
|
|
def exportMysql(context, db_conf, db_rule_conf):
|
|
dbsh_file = open(context['out_dir'] + 'exportdb.sh', 'w')
|
|
for conf in db_conf:
|
|
db_dir = CONFIG_DIR + 'out/' + context['channel'] + '/' + str(conf['instance_id'])
|
|
if not os.path.exists(db_dir):
|
|
os.mkdir(db_dir)
|
|
for rule in db_rule_conf:
|
|
line = ''
|
|
if rule['where'] == '':
|
|
line = 'mysqldump -h %s -P %d -u%s -p%s %s %s > %d/%s.sql' % (
|
|
conf['host'],
|
|
conf['port'],
|
|
conf['user'],
|
|
conf['passwd'],
|
|
context['dbname_prefix'] + str(conf['instance_id']),
|
|
rule['table_name'],
|
|
conf['instance_id'],
|
|
rule['table_name']
|
|
)
|
|
else:
|
|
line = 'mysqldump -h %s -P %d -u%s -p%s %s %s --where="%s"> %d/%s.sql' % (
|
|
conf['host'],
|
|
conf['port'],
|
|
conf['user'],
|
|
conf['passwd'],
|
|
context['dbname_prefix'] + str(conf['instance_id']),
|
|
rule['table_name'],
|
|
rule['where'].replace('$channel', context['channel']),
|
|
conf['instance_id'],
|
|
rule['table_name']
|
|
)
|
|
dbsh_file.write(line + "\n")
|
|
|
|
def main():
|
|
global context
|
|
if not os.path.exists(CONFIG_DIR + 'out/'):
|
|
os.mkdir(CONFIG_DIR + 'out/')
|
|
if not os.path.exists(CONFIG_DIR + 'out/' + context['channel']):
|
|
os.mkdir(CONFIG_DIR + 'out/' + context['channel'])
|
|
context['out_dir'] = CONFIG_DIR + 'out/' + context['channel'] + '/'
|
|
db_conf = json.loads(open(CONFIG_DIR + context['channel'] + '/mysql.json', 'r').read())
|
|
redis_conf = json.loads(open(CONFIG_DIR + context['channel'] + '/redis.json', 'r').read())
|
|
db_rule_conf = json.loads(open(CONFIG_DIR + 'common/mysql_rule.json', 'r').read())
|
|
redis_rule_conf = json.loads(open(CONFIG_DIR + 'common/redis_rule.json', 'r').read())
|
|
exportRedis(context, redis_conf, redis_rule_conf)
|
|
#exportMysql(context, db_conf, db_rule_conf)
|
|
|
|
if __name__ == "__main__":
|
|
#context['channel'] = sys.argv[1]
|
|
#context['dbname_prefix'] = sys.argv[2]
|
|
context['channel'] = '0'
|
|
context['dbname_prefix'] = 'gamedb2004'
|
|
main()
|