add convert_newjson.py

scripts/md_csv/convert_newjson.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+import os
+import re
+import json
+import pprint
+
+OLD_PATH = '/root/opensource/tools/scripts/md_csv/data/old/'
+NEW_PATH = '/root/opensource/tools/scripts/md_csv/data/new/'
+
+log_dict = json.loads(open('dict.json', 'r').read())
+
+def getFiles(path):
+    files = []
+    for f in os.listdir(path):
+        if os.path.isfile(path + '/' + f) and f.find('upload_prod-bj') >= 0:
+            files.append(f)
+    return files
+
+def convertOldEvent(oldJson):
+    assert(oldJson['#event_name'] != '')
+    event_name = oldJson['#event_name']
+    strings = event_name.split('_')
+    assert(len(strings) == 3)
+    node = log_dict[strings[1] + '-' + strings[2]]
+    assert(node != None)
+    properties = oldJson['properties']
+    for i in range(1, 20):
+        strKey = 'str' + str(i)
+        numKey = 'num' + str(i)
+        if strKey in properties:
+            assert(strKey in node)
+            properties[node[strKey]['en_name']] = properties[strKey]
+            del properties[strKey]
+        if numKey in properties:
+            if numKey not in node:
+                print(numKey, node, oldJson)
+            assert(numKey in node)
+            properties[node[numKey]['en_name']] = properties[numKey]
+            del properties[numKey]
+
+def convert_newjson(gameid):
+    files = getFiles(OLD_PATH + gameid)
+    for f in files:
+        print(f)
+        new_file = open(NEW_PATH + gameid + '/' + f, 'w')
+        for line in open(OLD_PATH + gameid + '/' + f, 'r'):
+            jsonObj = json.loads(line)
+            assert(jsonObj['#type'] in ['track', 'user_add', 'user_set', 'user_setOnce'])
+            if jsonObj['#type'] == 'track':
+                convertOldEvent(jsonObj)
+                new_file.write(json.dumps(jsonObj))
+            else:
+                new_file.write(line)
+
+convert_newjson('1004')
+convert_newjson('1011')