修正缓存生产脚本

ad_interface_tornado.py

@@ -80,7 +80,7 @@ class DispatchHandler(tornado.web.RequestHandler):
         if ids:
             for id in ids:
                 if id:
-                    key = f"ad::{id}::num"
+                    key = f"adnum::{id}::num"
                     my_redis.incr(key, amount=1)
             self.write({'errcode': 0, "errmsg": '', "message": f"{ids} incr success!"})
         else:
@@ -142,7 +142,7 @@ class DispatchHandler(tornado.web.RequestHandler):
                             id_list.append(new)
                             break
                 for id in id_list:
-                    temp = my_redis.hgetall(f"ad::{id}::info")
+                    temp = my_redis.hgetall(f"adinfo::{id}::info")
                     info.append(temp)
                 result = {'errcode': 0, "errmsg": '', "message": {"totoal": len(info), "result": info}}
             except Exception as e:

ad_tasks.py

@@ -15,35 +15,45 @@ limit = 100
 
 BEGIN = '1999-01-01'
 END = '3000-01-01'
+mydb = MysqlBase(**mysql_promotion_config)
+remove_list = ('ad_num', 'gameid')
+
+
+def get_area_by_locationid(ldid):
+    area = []
+    sql = f"select area from location where id={ldid}"
+    data = mydb.query(sql)
+    if data:
+        for line in data:
+            area.append(line[0])
+    else:
+        log.error(f"get area from location failed!,sql={sql}")
+    return area
+
+
+def split_ad_info(localtionid, line):
+    # 从locationid获取位置相关属性，与ad信息打包，产生提供给接口用的数据
+    location_sql = f"select area,x,y,x_offset,y_offset,type,mode,ld_property from location WHERE in_used=1 and id={localtionid}"
+    data = mydb.query(location_sql)
+    try:
+        line['area'], line['x'], line['y'], line['x_offset'], line['y_offset'], line['type'], line['mode'], line[
+            'ld_property'] = data[0]
+    except Exception:
+        log.error(f"get localtion info by sql={location_sql} error!", exc_info=True)
+
+    for item in remove_list:
+        line.pop(item)
+    return line
+
 
 
 
 def send_cache_data():
-    mydb = MysqlBase(**mysql_promotion_config)
     log.info("start update cache !")
     now = datetime.datetime.today().strftime("%Y-%m-%d %H:%M:%S")
     all = []
-
+    ad_add_list = []
     get_data_sql = f"""select
-        a.id,
-        a.name,
-        a.ad_num,
-        a.ad_title,
-        a.ad_body,
-        a.ad_image,
-        a.jump_param,
-        a.ad_sort,
-        a.companyid,
-        a.gameid,
-        a.channelid,
-        a.jump_status,
-        a.ad_property,
-        b.area,
-        b.type,
-        b.mode,
-        a.locationid 
-    from
-        (select
             id,
             name,
             ad_num,
@@ -64,37 +74,18 @@ def send_cache_data():
             status=1 
             and in_used=1 
             and '{now}'> begin_time 
-            and '{now}'<end_time ) a ,
+            and '{now}'<end_time )
-        (select
+        """
-            id,
-            area,
-            type,
-            mode 
-        from
-            location) b 
-    where
-        a.locationid=b.id"""
 
     data = mydb.query(get_data_sql)
     if data:
         for line in data:
-
             if line:
                 item = {}
                 try:
                     item['id'], item['name'], item['ad_num'], item['ad_title'], item['ad_body'], item['ad_image'], item[
                         'jump_param'], item['ad_sort'], item['companyid'], item['gameid'], item['channelid'], item[
-                        'jump_status'], item['ad_property'], item['area'], item['type'], item['mode'], item[
+                        'jump_status'], item['locationid'], item['ad_property'] = line
-                        'locationid'] = line
-                    try:
-                        item['x_offset'] = item['area'].split(',')[-2].strip()
-                    except:
-                        item['x_offset'] = 0
-                    try:
-                        item['y_offset'] = item['area'].split(',')[-1].strip().strip(')')
-                    except:
-                        item['y_offset'] = 0
-
                     if item.get('jump_param', "") and (item['ad_property'].find("jump_param") == -1):
                         try:
                             temp = {}
@@ -111,7 +102,6 @@ def send_cache_data():
                             item['ad_property'] = json.dumps(item['ad_property'])
                         except Exception:
                             log.error(f"write {item}", exc_info=True)
-
                     all.append(item)
                 except Exception:
                     log.error("split data failed", exc_info=True)
@@ -119,68 +109,52 @@ def send_cache_data():
     if all:
         # log.info(f"get data was {all}!\n")
         for line in all:
-            key = f"ad::{line.get('gameid', 0)}::{line.get('channelid', 0)}::{line.get('area').split(',')[0].strip()}::{line.get('locationid', 0)}"
+            locationid = json.loads(line.get('locationid'))
-            if int(line['ad_num']) > 0:
+            for item in locationid:
-                num = my_redis.get(f"ad::{line['id']}::num")
+                area = get_area_by_locationid(item)
-                if not num:
+                for one in area:
-                    num = 0
+                    key = f"ad::{line.get('gameid', 0)}::{line.get('channelid', 0)}::{one}::{item}"
-                if int(line['ad_num']) > int(num):
+                    if int(line['ad_num']) == 0:
-                    my_redis.sadd(key, line['id'])
+                        ad_add_list.append(line)
-                    my_redis.expire(key, 120)
+                    elif int(line['ad_num']) > 0:
-                    n = int(line['ad_num']) - int(num)
+                        num = my_redis.get(f"adnum::{line['id']}::num")
-                    log.info(f"add {line['id']} to {key} ,num was {line['ad_num']},limit was {n}!")
+                        if not num:
-                else:
+                            num = 0
-                    if my_redis.sismember(key, line['id']):
+                        if int(line['ad_num']) > int(num):
-                        my_redis.srem(key, line['id'])
+                            my_redis.sadd(key, line['id'])
-                        log.info(f"remove {line['id']} from {key}!")
+                            my_redis.expire(key, 120)
-            elif int(line['ad_num']) == 0:
+                            info_key = f"adinfo::{line['id']}::info"
-                my_redis.sadd(key, line['id'])
+                            my_redis.hmset(info_key, split_ad_info(item, line))
-                my_redis.expire(key, 120)
+                            my_redis.expire(info_key, 60 * 60)
-                log.info(f"add {line['id']} to {key} ,num was unlimit !")
+                        else:
-            else:
+                            if my_redis.sismember(key, line['id']):
-                log.error(f"get ad_num from mysql failed! ad_num={line['ad_num']}")
+                                my_redis.srem(key, line['id'])
-
+                                log.info(f"remove {line['id']} from {key}!")
-            if not my_redis.exists(f"ad::{line['id']}::info"):
+                    else:
-                # remove some filed from adinfo
+                        log.error(f"get ad_num from mysql failed! ad_num={line['ad_num']}")
-                remove_list = ('ad_num', 'gameid')
-                for item in remove_list:
-                    line.pop(item)
-                my_redis.hmset(f"ad::{line['id']}::info", line)
-                # my_redis.expire(f"ad::{line['id']}::info", 60)
-                my_redis.expire(f"ad::{line['id']}::info", 3600 * 24)
-                log.info(f"add ad::{line['id']}::info to redis!")
 
     # 删除过期的数据
     log.info("remove expire data from cache!")
-
     expire_sql = f"""select
-            a.id,a.gameid,a.channelid,a.locationid,a.status,b.area
-        from
-            (select
                 id,gameid,channelid,locationid,status
             from
                 ad 
             where
                  '{now}'> end_time 
-                or status in (3,4) ) a ,
+                or status in (3,4) """
-            (select
-                id,
-                area
-            from
-                location) b 
-        where
-            a.locationid=b.id"""
     remove_data = mydb.query(expire_sql)
     if remove_data:
         for line in remove_data:
             try:
-                id, gameid, channelid, locationid, _, area = line
+                id, gameid, channelid, locationid = line
-                key = f"ad::{gameid}::{channelid}::{area.split(',')[0].strip()}::{locationid}"
+                for item in locationid:
-                # key = f"{gameid}:{locationid}"
+                    area = get_area_by_locationid(item)
-                if my_redis.sismember(key, id):
+                    for one in area:
-                    my_redis.srem(key, id)
+                        key = f"ad::{line.get('gameid', 0)}::{line.get('channelid', 0)}::{one}::{item}"
-                    my_redis.expire(f"ad::{id}::info", 1)
+                        if my_redis.sismember(key, id):
-                    log.info(f"remove {id} from {key} success!")
+                            my_redis.srem(key, id)
+                            my_redis.expire(f"ad::{id}::info", 1)
+                            log.info(f"remove {id} from {key} success!")
             except Exception:
                 log.error("拆解过期数据出错!", exc_info=True)