From 574db618277efe06f10193efcdb947a4115b3635 Mon Sep 17 00:00:00 2001
From: aozhiwei <aozhiwei@kingsome.cn>
Date: Wed, 4 Nov 2020 14:21:23 +0800
Subject: [PATCH 1/6] add migratedb

---
 tools/migratedb/0/mysql.json           |   9 ++
 tools/migratedb/0/redis.json           |   9 ++
 tools/migratedb/6000/mysql.json        |   9 ++
 tools/migratedb/6000/redis.json        |   9 ++
 tools/migratedb/common/mysql_rule.json |  46 ++++++++
 tools/migratedb/common/redis_rule.json |   5 +
 tools/migratedb/import_redis.py        |  49 +++++++++
 tools/migratedb/migratedb.py           | 143 +++++++++++++++++++++++++
 8 files changed, 279 insertions(+)
 create mode 100644 tools/migratedb/0/mysql.json
 create mode 100644 tools/migratedb/0/redis.json
 create mode 100644 tools/migratedb/6000/mysql.json
 create mode 100644 tools/migratedb/6000/redis.json
 create mode 100644 tools/migratedb/common/mysql_rule.json
 create mode 100644 tools/migratedb/common/redis_rule.json
 create mode 100644 tools/migratedb/import_redis.py
 create mode 100644 tools/migratedb/migratedb.py

diff --git a/tools/migratedb/0/mysql.json b/tools/migratedb/0/mysql.json
new file mode 100644
index 0000000..c2754a2
--- /dev/null
+++ b/tools/migratedb/0/mysql.json
@@ -0,0 +1,9 @@
+[
+  {
+    "instance_id" : 1,
+    "host" : "127.0.0.1",
+    "port" : 3306,
+    "user" : "root",
+    "passwd" : "keji178"
+  }
+]
diff --git a/tools/migratedb/0/redis.json b/tools/migratedb/0/redis.json
new file mode 100644
index 0000000..e0d8e02
--- /dev/null
+++ b/tools/migratedb/0/redis.json
@@ -0,0 +1,9 @@
+[
+  {
+    "instance_id" : 1,
+    "host" : "127.0.0.1",
+    "port" : 6379,
+    "user" : "root",
+    "passwd" : ""
+  }
+]
diff --git a/tools/migratedb/6000/mysql.json b/tools/migratedb/6000/mysql.json
new file mode 100644
index 0000000..c2754a2
--- /dev/null
+++ b/tools/migratedb/6000/mysql.json
@@ -0,0 +1,9 @@
+[
+  {
+    "instance_id" : 1,
+    "host" : "127.0.0.1",
+    "port" : 3306,
+    "user" : "root",
+    "passwd" : "keji178"
+  }
+]
diff --git a/tools/migratedb/6000/redis.json b/tools/migratedb/6000/redis.json
new file mode 100644
index 0000000..e0d8e02
--- /dev/null
+++ b/tools/migratedb/6000/redis.json
@@ -0,0 +1,9 @@
+[
+  {
+    "instance_id" : 1,
+    "host" : "127.0.0.1",
+    "port" : 6379,
+    "user" : "root",
+    "passwd" : ""
+  }
+]
diff --git a/tools/migratedb/common/mysql_rule.json b/tools/migratedb/common/mysql_rule.json
new file mode 100644
index 0000000..bf17216
--- /dev/null
+++ b/tools/migratedb/common/mysql_rule.json
@@ -0,0 +1,46 @@
+[
+  {
+    "table_name": "version",
+    "where": ""
+  },
+  {
+    "table_name": "user",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "bag",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "sign",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "quest",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "share_achievement",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "activity",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "passinfo",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "shop",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "equip",
+    "where": "accountid LIKE '$channel_%'"
+  },
+  {
+    "table_name": "shop_data",
+    "where": "accountid LIKE '$channel_%'"
+  }
+]
diff --git a/tools/migratedb/common/redis_rule.json b/tools/migratedb/common/redis_rule.json
new file mode 100644
index 0000000..bd340bd
--- /dev/null
+++ b/tools/migratedb/common/redis_rule.json
@@ -0,0 +1,5 @@
+[
+  {
+    "key_prefix": "game2004"
+  }
+]
diff --git a/tools/migratedb/import_redis.py b/tools/migratedb/import_redis.py
new file mode 100644
index 0000000..52888c2
--- /dev/null
+++ b/tools/migratedb/import_redis.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/python
+
+import os
+import sys
+import time
+import json
+import redis
+import datetime
+import pprint
+
+argv = sys.argv
+host = argv[1]
+port = argv[2]
+passwd = argv[3]
+script_name = argv[4]
+
+def writeToRedis(conn, cmdline):
+    assert len(cmdline) > 1
+    print(cmdline)
+    if cmdline[0] == 'hset':
+        conn.hset(cmdline[1], cmdline[2], cmdline[3])
+    elif cmdline[0] == 'zadd':
+        conn.zadd(cmdline[1], {cmdline[3]: cmdline[2]})
+    elif cmdline[0] == 'sadd':
+        conn.sadd(cmdline[1], cmdline[2])
+    elif cmdline[0] == 'set':
+        if cmdline[2] != None:
+            conn.set(cmdline[1], cmdline[2])
+        else:
+            print('[warning]', cmdline)
+    elif cmdline[0] == 'expire':
+        conn.expire(cmdline[1], cmdline[2])
+    else:
+        assert False
+
+conn = redis.Redis(host = host,
+                   port = port,
+                   password = passwd,
+                   db = 0,
+                   decode_responses = True
+)
+
+with open(script_name, 'r') as f:
+    for line in f:
+        line = line.strip()
+        if line == '':
+            continue
+        writeToRedis(conn, json.loads(line))
diff --git a/tools/migratedb/migratedb.py b/tools/migratedb/migratedb.py
new file mode 100644
index 0000000..d88c716
--- /dev/null
+++ b/tools/migratedb/migratedb.py
@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/python
+
+import pymysql
+import hashlib
+import json
+import urllib.request
+import base64
+import time
+import datetime
+import redis
+import os
+import sys
+import functools
+
+CONFIG_DIR = './'
+context = {
+    'channel': 0,
+    'dbname_prefix': ''
+}
+EXCLUDE_KEYS = {
+    'game2004api:kill_rank_6006': 1,
+    'game2004api:win_rank_6006': 1,
+    'game2004api:integral_rank_6006': 1
+}
+
+def scanKeys(r, key_prefix, matched_keys):
+    scan_key = key_prefix + '*'
+    keys = []
+    cursor, keys = r.scan(0, scan_key, 1000)
+    scan_count = 0
+    while cursor != 0 or len(keys) > 0:
+        for key in keys:
+            matched_keys[key] = key
+        #end
+        keys = []
+        if cursor != 0:
+            cursor, keys = r.scan(cursor, scan_key, 1000)
+        scan_count += 1
+        if scan_count > 10000:
+            break
+    #end while cursor
+
+def excludeKey(key):
+    global EXCLUDE_KEYS
+    return key in EXCLUDE_KEYS
+
+def saveKeys(r, keys, curr_file):
+    for key in keys:
+        if excludeKey(key):
+            print('[WARNING]exclude key ' + key)
+            continue
+        data = r.get(key)
+        expire = r.ttl(key)
+        curr_file.write(json.dumps(
+            [
+                'set',
+                key,
+                data
+            ]
+        ) + '\n')
+        curr_file.write(json.dumps(
+            [
+                'expire',
+                key,
+                expire
+            ]
+        ) + '\n')
+        if expire < 0:
+            print('[ERROR] ' + key + ' ttl < 0')
+        if expire > 3600 * 24:
+            print('[ERROR] ' + key + ' ttl > 3600 * 24')
+
+def exportRedis(context, redis_conf, redis_rule_conf):
+    redis_dir = CONFIG_DIR + 'out/' + context['channel'] + '/redis/'
+    if not os.path.exists(redis_dir):
+        os.mkdir(redis_dir)
+    curr_file = open(redis_dir + context['channel'] + '.redis', 'w')
+    for conf in redis_conf:
+        r = redis.Redis(host = conf['host'],
+                        port = conf['port'],
+                        password = conf['passwd'],
+                        decode_responses = True
+        )
+        for rule in redis_rule_conf:
+            matched_keys = {}
+            scanKeys(r, rule['key_prefix'], matched_keys)
+            saveKeys(r, matched_keys, curr_file)
+    #end for conf
+
+def exportMysql(context, db_conf, db_rule_conf):
+    dbsh_file = open(context['out_dir'] + 'exportdb.sh', 'w')
+    for conf in db_conf:
+        db_dir = CONFIG_DIR + 'out/' + context['channel'] + '/' + str(conf['instance_id'])
+        if not os.path.exists(db_dir):
+            os.mkdir(db_dir)
+        for rule in db_rule_conf:
+            line = ''
+            if rule['where'] == '':
+                line = 'mysqldump -h %s -P %d -u%s -p%s %s %s > %d/%s.sql' % (
+                    conf['host'],
+                    conf['port'],
+                    conf['user'],
+                    conf['passwd'],
+                    context['dbname_prefix'] + str(conf['instance_id']),
+                    rule['table_name'],
+                    conf['instance_id'],
+                    rule['table_name']
+                )
+            else:
+                line = 'mysqldump -h %s -P %d -u%s -p%s %s %s --where="%s"> %d/%s.sql' % (
+                    conf['host'],
+                    conf['port'],
+                    conf['user'],
+                    conf['passwd'],
+                    context['dbname_prefix'] + str(conf['instance_id']),
+                    rule['table_name'],
+                    rule['where'].replace('$channel', context['channel']),
+                    conf['instance_id'],
+                    rule['table_name']
+                )
+            dbsh_file.write(line + "\n")
+
+def main():
+    global context
+    if not os.path.exists(CONFIG_DIR + 'out/'):
+        os.mkdir(CONFIG_DIR + 'out/')
+    if not os.path.exists(CONFIG_DIR + 'out/' + context['channel']):
+        os.mkdir(CONFIG_DIR + 'out/' + context['channel'])
+    context['out_dir'] = CONFIG_DIR + 'out/' + context['channel'] + '/'
+    db_conf = json.loads(open(CONFIG_DIR + context['channel'] + '/mysql.json', 'r').read())
+    redis_conf = json.loads(open(CONFIG_DIR + context['channel'] + '/redis.json', 'r').read())
+    db_rule_conf = json.loads(open(CONFIG_DIR + 'common/mysql_rule.json', 'r').read())
+    redis_rule_conf = json.loads(open(CONFIG_DIR + 'common/redis_rule.json', 'r').read())
+    exportRedis(context, redis_conf, redis_rule_conf)
+    #exportMysql(context, db_conf, db_rule_conf)
+
+if __name__ == "__main__":
+    #context['channel'] = sys.argv[1]
+    #context['dbname_prefix'] = sys.argv[2]
+    context['channel'] = '0'
+    context['dbname_prefix'] = 'gamedb2004'
+    main()

From ed2de758bd69e9b150b354418f2162d99ade832e Mon Sep 17 00:00:00 2001
From: aozhiwei <aozhiwei@kingsome.cn>
Date: Wed, 4 Nov 2020 14:30:45 +0800
Subject: [PATCH 2/6] 1

---
 tools/migratedb/migratedb.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tools/migratedb/migratedb.py b/tools/migratedb/migratedb.py
index d88c716..07add44 100644
--- a/tools/migratedb/migratedb.py
+++ b/tools/migratedb/migratedb.py
@@ -19,9 +19,6 @@ context = {
     'dbname_prefix': ''
 }
 EXCLUDE_KEYS = {
-    'game2004api:kill_rank_6006': 1,
-    'game2004api:win_rank_6006': 1,
-    'game2004api:integral_rank_6006': 1
 }
 
 def scanKeys(r, key_prefix, matched_keys):
@@ -139,5 +136,5 @@ if __name__ == "__main__":
     #context['channel'] = sys.argv[1]
     #context['dbname_prefix'] = sys.argv[2]
     context['channel'] = '0'
-    context['dbname_prefix'] = 'gamedb2004'
+    context['dbname_prefix'] = 'gamedb2001'
     main()

From df11c553ffcc377e38d0fd99422c2b128f855a71 Mon Sep 17 00:00:00 2001
From: aozhiwei <aozhiwei@kingsome.cn>
Date: Wed, 4 Nov 2020 14:32:44 +0800
Subject: [PATCH 3/6] 1

---
 tools/migratedb/6000/mysql.json | 9 ---------
 tools/migratedb/6000/redis.json | 9 ---------
 2 files changed, 18 deletions(-)
 delete mode 100644 tools/migratedb/6000/mysql.json
 delete mode 100644 tools/migratedb/6000/redis.json

diff --git a/tools/migratedb/6000/mysql.json b/tools/migratedb/6000/mysql.json
deleted file mode 100644
index c2754a2..0000000
--- a/tools/migratedb/6000/mysql.json
+++ /dev/null
@@ -1,9 +0,0 @@
-[
-  {
-    "instance_id" : 1,
-    "host" : "127.0.0.1",
-    "port" : 3306,
-    "user" : "root",
-    "passwd" : "keji178"
-  }
-]
diff --git a/tools/migratedb/6000/redis.json b/tools/migratedb/6000/redis.json
deleted file mode 100644
index e0d8e02..0000000
--- a/tools/migratedb/6000/redis.json
+++ /dev/null
@@ -1,9 +0,0 @@
-[
-  {
-    "instance_id" : 1,
-    "host" : "127.0.0.1",
-    "port" : 6379,
-    "user" : "root",
-    "passwd" : ""
-  }
-]

From 3247aeb95b44c07c7abeb8a1e2a98cb4c3f79f39 Mon Sep 17 00:00:00 2001
From: aozhiwei <aozhiwei@kingsome.cn>
Date: Wed, 4 Nov 2020 14:34:23 +0800
Subject: [PATCH 4/6] 1

---
 tools/migratedb/common/redis_rule.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/migratedb/common/redis_rule.json b/tools/migratedb/common/redis_rule.json
index bd340bd..327ee0f 100644
--- a/tools/migratedb/common/redis_rule.json
+++ b/tools/migratedb/common/redis_rule.json
@@ -1,5 +1,5 @@
 [
   {
-    "key_prefix": "game2004"
+    "key_prefix": "game2001"
   }
 ]

From e5014dbae898c6340a4f7f6a2716483deb65fdd6 Mon Sep 17 00:00:00 2001
From: aozhiwei <aozhiwei@kingsome.cn>
Date: Wed, 4 Nov 2020 14:40:14 +0800
Subject: [PATCH 5/6] 1

---
 tools/migratedb/migratedb.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/migratedb/migratedb.py b/tools/migratedb/migratedb.py
index 07add44..06c1d74 100644
--- a/tools/migratedb/migratedb.py
+++ b/tools/migratedb/migratedb.py
@@ -64,9 +64,9 @@ def saveKeys(r, keys, curr_file):
             ]
         ) + '\n')
         if expire < 0:
-            print('[ERROR] ' + key + ' ttl < 0')
+            print('[WARNING] ' + key + ' ttl < 0')
         if expire > 3600 * 24:
-            print('[ERROR] ' + key + ' ttl > 3600 * 24')
+            print('[WARNING] ' + key + ' ttl > 3600 * 24')
 
 def exportRedis(context, redis_conf, redis_rule_conf):
     redis_dir = CONFIG_DIR + 'out/' + context['channel'] + '/redis/'

From a86cd02af57fb7c7c9bd1391a94a8bc7d9a3fffa Mon Sep 17 00:00:00 2001
From: aozhiwei <aozhiwei@kingsome.cn>
Date: Thu, 5 Nov 2020 14:47:45 +0800
Subject: [PATCH 6/6] 1

---
 tools/migratedb/import_redis.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/migratedb/import_redis.py b/tools/migratedb/import_redis.py
index 52888c2..45a6eeb 100644
--- a/tools/migratedb/import_redis.py
+++ b/tools/migratedb/import_redis.py
@@ -30,7 +30,8 @@ def writeToRedis(conn, cmdline):
         else:
             print('[warning]', cmdline)
     elif cmdline[0] == 'expire':
-        conn.expire(cmdline[1], cmdline[2])
+        if int(cmdline[2]) >= 0:
+            conn.expire(cmdline[1], cmdline[2])
     else:
         assert False