添加爬虫数据写入数数脚本

taptap/collect_mongodb.py

@@ -86,6 +86,9 @@ define_logger("/data/logs/ops/reptile2ss.log")
 log = logging.getLogger(__name__)
 MONGOINFO = {'host': '10.10.5.6', 'port': 27017}
 
+conver_float = ('score')
+conver_int = ('order', 'gameid', 'watch', 'reserve', 'sell', 'download', 'review', 'topic')
+
 
 class CollectMongo():
     def __init__(self, item):
@@ -119,9 +122,30 @@ class CollectMongo():
                         line[key] = str(line[key])  # line[key]=ObjectId(base64.b64decode(key))
                     elif isinstance(line[key], (list, tuple, dict)):
                         line[key] = json.dumps(line[key])
+
                 line['gameid'] = self.gameid
                 line['account_id'] = str(line['object_id'])
-                pdb.set_trace()
+
+                try:
+                    line['tags'] = ",".join(json.loads(line['tags']))
+                except Exception:
+                    log.error(f"correct {line['tags']} Failed")
+                    line['tags'] = ""
+
+                for item in conver_float:
+                    try:
+                        line[item] = float(line[item])
+                    except Exception:
+                        log.error(f"correct {line[item]} Failed")
+                        line[item] = 0
+
+                for item in conver_int:
+                    try:
+                        line[item] = int(line[item])
+                    except Exception:
+                        log.error(f"correct {line[item]} Failed")
+                        line[item] = 0
+
                 if not self.tga.put_event_data(line, event_name):
                     log.error("write {}_{} event error! {}\n".format(event_name, self.tgaid, line))
                     run_false = True