修正缓存生产脚本
This commit is contained in:
pengtao
parent
ad40659f91
commit
a28ae9d771
@ -80,7 +80,7 @@ class DispatchHandler(tornado.web.RequestHandler):
|
||||
if ids:
|
||||
for id in ids:
|
||||
if id:
|
||||
key = f"ad::{id}::num"
|
||||
key = f"adnum::{id}::num"
|
||||
my_redis.incr(key, amount=1)
|
||||
self.write({'errcode': 0, "errmsg": '', "message": f"{ids} incr success!"})
|
||||
else:
|
||||
@ -142,7 +142,7 @@ class DispatchHandler(tornado.web.RequestHandler):
|
||||
id_list.append(new)
|
||||
break
|
||||
for id in id_list:
|
||||
temp = my_redis.hgetall(f"ad::{id}::info")
|
||||
temp = my_redis.hgetall(f"adinfo::{id}::info")
|
||||
info.append(temp)
|
||||
result = {'errcode': 0, "errmsg": '', "message": {"totoal": len(info), "result": info}}
|
||||
except Exception as e:
|
||||
|
||||
160
ad_tasks.py
160
ad_tasks.py
@ -15,35 +15,45 @@ limit = 100
|
||||
|
||||
BEGIN = '1999-01-01'
|
||||
END = '3000-01-01'
|
||||
mydb = MysqlBase(**mysql_promotion_config)
|
||||
remove_list = ('ad_num', 'gameid')
|
||||
|
||||
|
||||
def get_area_by_locationid(ldid):
|
||||
area = []
|
||||
sql = f"select area from location where id={ldid}"
|
||||
data = mydb.query(sql)
|
||||
if data:
|
||||
for line in data:
|
||||
area.append(line[0])
|
||||
else:
|
||||
log.error(f"get area from location failed!,sql={sql}")
|
||||
return area
|
||||
|
||||
|
||||
def split_ad_info(localtionid, line):
|
||||
# 从locationid获取位置相关属性，与ad信息打包，产生提供给接口用的数据
|
||||
location_sql = f"select area,x,y,x_offset,y_offset,type,mode,ld_property from location WHERE in_used=1 and id={localtionid}"
|
||||
data = mydb.query(location_sql)
|
||||
try:
|
||||
line['area'], line['x'], line['y'], line['x_offset'], line['y_offset'], line['type'], line['mode'], line[
|
||||
'ld_property'] = data[0]
|
||||
except Exception:
|
||||
log.error(f"get localtion info by sql={location_sql} error!", exc_info=True)
|
||||
|
||||
for item in remove_list:
|
||||
line.pop(item)
|
||||
return line
|
||||
|
||||
|
||||
|
||||
|
||||
def send_cache_data():
|
||||
mydb = MysqlBase(**mysql_promotion_config)
|
||||
log.info("start update cache !")
|
||||
now = datetime.datetime.today().strftime("%Y-%m-%d %H:%M:%S")
|
||||
all = []
|
||||
|
||||
ad_add_list = []
|
||||
get_data_sql = f"""select
|
||||
a.id,
|
||||
a.name,
|
||||
a.ad_num,
|
||||
a.ad_title,
|
||||
a.ad_body,
|
||||
a.ad_image,
|
||||
a.jump_param,
|
||||
a.ad_sort,
|
||||
a.companyid,
|
||||
a.gameid,
|
||||
a.channelid,
|
||||
a.jump_status,
|
||||
a.ad_property,
|
||||
b.area,
|
||||
b.type,
|
||||
b.mode,
|
||||
a.locationid
|
||||
from
|
||||
(select
|
||||
id,
|
||||
name,
|
||||
ad_num,
|
||||
@ -64,37 +74,18 @@ def send_cache_data():
|
||||
status=1
|
||||
and in_used=1
|
||||
and '{now}'> begin_time
|
||||
and '{now}'<end_time ) a ,
|
||||
(select
|
||||
id,
|
||||
area,
|
||||
type,
|
||||
mode
|
||||
from
|
||||
location) b
|
||||
where
|
||||
a.locationid=b.id"""
|
||||
and '{now}'<end_time )
|
||||
"""
|
||||
|
||||
data = mydb.query(get_data_sql)
|
||||
if data:
|
||||
for line in data:
|
||||
|
||||
if line:
|
||||
item = {}
|
||||
try:
|
||||
item['id'], item['name'], item['ad_num'], item['ad_title'], item['ad_body'], item['ad_image'], item[
|
||||
'jump_param'], item['ad_sort'], item['companyid'], item['gameid'], item['channelid'], item[
|
||||
'jump_status'], item['ad_property'], item['area'], item['type'], item['mode'], item[
|
||||
'locationid'] = line
|
||||
try:
|
||||
item['x_offset'] = item['area'].split(',')[-2].strip()
|
||||
except:
|
||||
item['x_offset'] = 0
|
||||
try:
|
||||
item['y_offset'] = item['area'].split(',')[-1].strip().strip(')')
|
||||
except:
|
||||
item['y_offset'] = 0
|
||||
|
||||
'jump_status'], item['locationid'], item['ad_property'] = line
|
||||
if item.get('jump_param', "") and (item['ad_property'].find("jump_param") == -1):
|
||||
try:
|
||||
temp = {}
|
||||
@ -111,7 +102,6 @@ def send_cache_data():
|
||||
item['ad_property'] = json.dumps(item['ad_property'])
|
||||
except Exception:
|
||||
log.error(f"write {item}", exc_info=True)
|
||||
|
||||
all.append(item)
|
||||
except Exception:
|
||||
log.error("split data failed", exc_info=True)
|
||||
@ -119,68 +109,52 @@ def send_cache_data():
|
||||
if all:
|
||||
# log.info(f"get data was {all}!\n")
|
||||
for line in all:
|
||||
key = f"ad::{line.get('gameid', 0)}::{line.get('channelid', 0)}::{line.get('area').split(',')[0].strip()}::{line.get('locationid', 0)}"
|
||||
if int(line['ad_num']) > 0:
|
||||
num = my_redis.get(f"ad::{line['id']}::num")
|
||||
if not num:
|
||||
num = 0
|
||||
if int(line['ad_num']) > int(num):
|
||||
my_redis.sadd(key, line['id'])
|
||||
my_redis.expire(key, 120)
|
||||
n = int(line['ad_num']) - int(num)
|
||||
log.info(f"add {line['id']} to {key} ,num was {line['ad_num']},limit was {n}!")
|
||||
else:
|
||||
if my_redis.sismember(key, line['id']):
|
||||
my_redis.srem(key, line['id'])
|
||||
log.info(f"remove {line['id']} from {key}!")
|
||||
elif int(line['ad_num']) == 0:
|
||||
my_redis.sadd(key, line['id'])
|
||||
my_redis.expire(key, 120)
|
||||
log.info(f"add {line['id']} to {key} ,num was unlimit !")
|
||||
else:
|
||||
log.error(f"get ad_num from mysql failed! ad_num={line['ad_num']}")
|
||||
|
||||
if not my_redis.exists(f"ad::{line['id']}::info"):
|
||||
# remove some filed from adinfo
|
||||
remove_list = ('ad_num', 'gameid')
|
||||
for item in remove_list:
|
||||
line.pop(item)
|
||||
my_redis.hmset(f"ad::{line['id']}::info", line)
|
||||
# my_redis.expire(f"ad::{line['id']}::info", 60)
|
||||
my_redis.expire(f"ad::{line['id']}::info", 3600 * 24)
|
||||
log.info(f"add ad::{line['id']}::info to redis!")
|
||||
locationid = json.loads(line.get('locationid'))
|
||||
for item in locationid:
|
||||
area = get_area_by_locationid(item)
|
||||
for one in area:
|
||||
key = f"ad::{line.get('gameid', 0)}::{line.get('channelid', 0)}::{one}::{item}"
|
||||
if int(line['ad_num']) == 0:
|
||||
ad_add_list.append(line)
|
||||
elif int(line['ad_num']) > 0:
|
||||
num = my_redis.get(f"adnum::{line['id']}::num")
|
||||
if not num:
|
||||
num = 0
|
||||
if int(line['ad_num']) > int(num):
|
||||
my_redis.sadd(key, line['id'])
|
||||
my_redis.expire(key, 120)
|
||||
info_key = f"adinfo::{line['id']}::info"
|
||||
my_redis.hmset(info_key, split_ad_info(item, line))
|
||||
my_redis.expire(info_key, 60 * 60)
|
||||
else:
|
||||
if my_redis.sismember(key, line['id']):
|
||||
my_redis.srem(key, line['id'])
|
||||
log.info(f"remove {line['id']} from {key}!")
|
||||
else:
|
||||
log.error(f"get ad_num from mysql failed! ad_num={line['ad_num']}")
|
||||
|
||||
# 删除过期的数据
|
||||
log.info("remove expire data from cache!")
|
||||
|
||||
expire_sql = f"""select
|
||||
a.id,a.gameid,a.channelid,a.locationid,a.status,b.area
|
||||
from
|
||||
(select
|
||||
id,gameid,channelid,locationid,status
|
||||
from
|
||||
ad
|
||||
where
|
||||
'{now}'> end_time
|
||||
or status in (3,4) ) a ,
|
||||
(select
|
||||
id,
|
||||
area
|
||||
from
|
||||
location) b
|
||||
where
|
||||
a.locationid=b.id"""
|
||||
or status in (3,4) """
|
||||
remove_data = mydb.query(expire_sql)
|
||||
if remove_data:
|
||||
for line in remove_data:
|
||||
try:
|
||||
id, gameid, channelid, locationid, _, area = line
|
||||
key = f"ad::{gameid}::{channelid}::{area.split(',')[0].strip()}::{locationid}"
|
||||
# key = f"{gameid}:{locationid}"
|
||||
if my_redis.sismember(key, id):
|
||||
my_redis.srem(key, id)
|
||||
my_redis.expire(f"ad::{id}::info", 1)
|
||||
log.info(f"remove {id} from {key} success!")
|
||||
id, gameid, channelid, locationid = line
|
||||
for item in locationid:
|
||||
area = get_area_by_locationid(item)
|
||||
for one in area:
|
||||
key = f"ad::{line.get('gameid', 0)}::{line.get('channelid', 0)}::{one}::{item}"
|
||||
if my_redis.sismember(key, id):
|
||||
my_redis.srem(key, id)
|
||||
my_redis.expire(f"ad::{id}::info", 1)
|
||||
log.info(f"remove {id} from {key} success!")
|
||||
except Exception:
|
||||
log.error("拆解过期数据出错!", exc_info=True)
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user