diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..132821e --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +*.pyc +*.log +*.old +.idea/ +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +Icon? +ehthumbs.db +Thumbs.db +.pem +test_* +*.pid +*.log.* +.DS_Store +.vscode/settings.json +.gitignore +.DS_Store +*.txt +*.csv + diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..15a15b2 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..79a4365 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..b95155a --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/ops_interface.iml b/.idea/ops_interface.iml new file mode 100644 index 0000000..6f63a63 --- /dev/null +++ b/.idea/ops_interface.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/config/__pycache__/__init__.cpython-37.pyc b/config/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..240ca3b Binary files /dev/null and b/config/__pycache__/__init__.cpython-37.pyc differ diff --git a/config/__pycache__/config.cpython-37.pyc b/config/__pycache__/config.cpython-37.pyc new file mode 100644 index 0000000..1342fa3 Binary files /dev/null and b/config/__pycache__/config.cpython-37.pyc differ diff --git a/data_channel/__pycache__/__init__.cpython-37.pyc b/data_channel/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..19d0968 Binary files /dev/null and b/data_channel/__pycache__/__init__.cpython-37.pyc differ diff --git a/data_channel/__pycache__/gamelog_external.cpython-37.pyc b/data_channel/__pycache__/gamelog_external.cpython-37.pyc new file mode 100644 index 0000000..c0140c3 Binary files /dev/null and b/data_channel/__pycache__/gamelog_external.cpython-37.pyc differ diff --git a/data_channel/__pycache__/mp2shushu.cpython-37.pyc b/data_channel/__pycache__/mp2shushu.cpython-37.pyc new file mode 100644 index 0000000..5cecb14 Binary files /dev/null and b/data_channel/__pycache__/mp2shushu.cpython-37.pyc differ diff --git a/data_channel/gamelog_external.py b/data_channel/gamelog_external.py index 0c7f196..dcc55b6 100644 --- a/data_channel/gamelog_external.py +++ b/data_channel/gamelog_external.py @@ -2,8 +2,8 @@ #被mp系统调用，用于同步(第三方传递日志需要)appkey到gamelog的相应配置文件 import requests import logging -from ops.mansible import AnsibleAPI -from ops.mmysql import MysqlBase +from myops.mansible import AnsibleAPI +from myops.mmysql import MysqlBase import time from config.config import args log = logging.getLogger(__name__) diff --git a/data_channel/mp2shushu.py b/data_channel/mp2shushu.py index 3db6818..f72583b 100644 --- a/data_channel/mp2shushu.py +++ b/data_channel/mp2shushu.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import requests import logging -#from ops.ss_virtual_create import SS_Virtual_command + import json log = logging.getLogger(__name__) import pdb @@ -103,7 +103,7 @@ class SS_Virtual_command: class Mp2shushu(): def __init__(self, gameid, channel): self.mp_url = f"http://10.10.5.4:2333/api/open/promotion/co-list?channelid={channel}&gameid={gameid}" - self.filename = "/data/git/ops_interface/ops/csv/2001_b_n.csv" + self.filename = "/data/git/ops_interface/myops/csv/2001_b_n.csv" def get_ad_list(self): r = requests.get(self.mp_url) diff --git a/data_channel/produce_ss_csv.py b/data_channel/produce_ss_csv.py new file mode 100644 index 0000000..203542e --- /dev/null +++ b/data_channel/produce_ss_csv.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +import pdb +import requests + + +def get_mp_interface(gameid, channelid): + base_url = "http://10.10.5.4:2333/api/open/cfg/all?" + key = "name-table" + url = f"{base_url}channelid={channelid}&gameid={gameid}&key={key}" + r = requests.get(url) + if r.status_code == requests.codes.ok: + return r.json().get('result') + else: + return None + + +class ProduceSSCsv: + def __init__(self): + # self.base_path = "/data/git/ops_interface/myops/csv" + self.base_path = "/tmp" + self.gameid_list = [1004, 1001, 2001, 1011,2002] + self.channelid = 6001 + + def write2csv(self, data): + filename = f"{self.base_path}/all__button_name.csv" + with open(filename, 'w') as f: + f.write("button_name,button_name_cn\n") + print(data) + for key, values in data.items(): + f.write(f"{key},{values}\n") + + + def run(self): + all = dict() + for gameid in self.gameid_list: + data = get_mp_interface(gameid, self.channelid) + all.update(data) + self.write2csv(all) + + +if __name__ == "__main__": + pp = ProduceSSCsv() + pp.run() diff --git a/ops/ss_virtual_create.py b/data_channel/ss_virtual_create.py similarity index 94% rename from ops/ss_virtual_create.py rename to data_channel/ss_virtual_create.py index 6311ea4..1c43f9d 100644 --- a/ops/ss_virtual_create.py +++ b/data_channel/ss_virtual_create.py @@ -34,7 +34,7 @@ class SS_Virtual_command: def upload_gameid(self): - filename = "/data/git/ops_interface/ops/csv/gameid.csv" + filename = "/data/git/ops_interface/myops/csv/gameid.csv" projectId = int(19) createParam = {"commonHeader": {"projectId": 19}, "mainColumn": {"property": {"columnName": "gameid", "tableType": 0}}, @@ -54,7 +54,7 @@ class SS_Virtual_command: def upload_2001_b_n(self): - filename = "/data/git/ops_interface/ops/csv/2001_b_n.csv" + filename = "/data/git/ops_interface/myops/csv/2001_b_n.csv" projectId = int(22) createParam = {"commonHeader": {"projectId": 22}, "mainColumn": {"property": {"columnName": "#vp@b_n", "tableType": 0, "subTableType": "vprop_sql"}}, @@ -73,7 +73,7 @@ class SS_Virtual_command: return False def upload_1004_button_name(self): - filename = "/data/git/ops_interface/ops/csv/1004_button_name.csv" + filename = "/data/git/ops_interface/myops/csv/1004_button_name.csv" projectId = int(19) createParam = {"commonHeader": {"projectId": 19}, "mainColumn": {"property": {"columnName": "#vp@button_name", "tableType": 0, "subTableType": "vprop_sql"}}, diff --git a/ops/__init__.py b/deploy/__init__.py similarity index 100% rename from ops/__init__.py rename to deploy/__init__.py diff --git a/deploy/deploy_keys.py b/deploy/deploy_keys.py new file mode 100644 index 0000000..4713d07 --- /dev/null +++ b/deploy/deploy_keys.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# python3 deploy_keys.py cloud_php +# python3 deploy_keys.py loginserver_php +from ops.mmysql import MysqlBase +from ops.mansible import AnsibleAPI +from ops.mlog import define_logger +from ops.common import run_cmd +import logging +import datetime +import os +import tarfile +import svn +import shutil +import pdb +define_logger("/data/logs/ops/deploy_keys.log") +log = logging.getLogger(__name__) + +dbargs = dict() +dbargs['host'] = '10.10.3.5' +dbargs['user'] = 'ops' +dbargs['pswd'] = 'deploy2018' +dbargs['db'] = 'ywplatform' + + +class DeployKeys: + def __init__(self, project): + self.env='prod' + self.project = project + self.svn_base = f"/data/publish/svn/{self.env}" + self.svn_paths = f"{self.svn_base}/{self.project}" + + + + def run(self): + if not os.path.isdir(self.svn_paths): + self.svn_paths = f"/data/publish/svn/{self.env}/global/{self.project}" + svn_path, ipadr = self.get_config() + if ipadr: + args = dict() + tar_file = self.build_svn_tar() + if tar_file: + hostfile = self.build_hostfile(ipadr) + args['hostfile'] = hostfile + args['tar_file'] = tar_file[0] + if self.ansible_deploy(args): + return True + else: + log.error(f"tar file no found with {self.project}") + return False + else: + raise Exception(f"{self.project} config not found in manager table!") + + + def get_config(self): + sql = f"""SELECT + config_address, + run_server + FROM + `deploy_projectmanage` + WHERE + project_name="{self.project}" + and env='{self.env}'""" + + mydb = MysqlBase(**dbargs) + data = mydb.query(sql) + ip_addr = list() + if data: + for line in data: + try: + svn_path = line[0] + ip_addr.append(line[1]) + except Exception: + log.error(f"split {self.project} config failed", exc_info=True) + svn_path = None + + return (svn_path, ip_addr) + + + def build_svn_tar(self): + try: + log.info("svn export config start") + cmd = f"cd {self.svn_paths} && svn cleanup && svn up && tar zcvf keys.tar.gz keys " + status, output = run_cmd(cmd) + if status: + log.info("svn export config success") + else: + raise Exception("update svn failed!") + except Exception: + log.error(f"get remote {self.project} svn failed", exc_info=True) + + # project_paths = os.path.join(self.svn_paths, self.project) + # tar = tarfile.open(self.svn_paths + '.tar.gz', 'w:gz') + # os.chdir(self.svn_paths) + # key_dirs = os.path.join(self.svn_paths, "keys") + # for f in os.listdir(key_dirs): + # tar.add(f) + # tar.close() + + # 判断是否有生成tar包 + list_dir = os.listdir(self.svn_paths) + tar_files = list() + for i in list_dir: + if i.split('.')[-1] == "gz" and os.path.isfile(os.path.join(self.svn_paths, i)): + tar_files.append(os.path.join(self.svn_paths, i)) + if len(tar_files) == 1: + log.info("tarfile {0} config to tar.gz success".format(self.project)) + return tar_files + else: + return None + + + def build_hostfile(self, ip_addr): + now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + host_files = f"/tmp/host_{now}" + with open(host_files, 'w') as f: + for line in ip_addr: + f.writelines(line + '\n') + return host_files + + def ansible_deploy(self, args): + an = AnsibleAPI(args['hostfile']) + tar_files = args['tar_file'].split('/')[-1] + data = {'dest_filename': tar_files, 'source': args['tar_file'], 'project': self.project} + resule = an.run_playbook('deploy_keys.yml', **data) + if not (resule['failed'] or resule['unreachable']): + log.info(f"deploy keys with {self.project} success!") + return True + else: + log.error(f"{self.project} deploy keys failed,{resule}", exc_info=True) + return False + + +def main(): + projects = ['loginserver_php', 'cloud_php'] + for project in projects: + dk = DeployKeys(project) + if not dk.run(): + msg = f"deploy {project} failed" + log.error(msg, exc_info=True) + raise Exception(msg) + + +if __name__ == "__main__": + main() diff --git a/deploy/deploy_keys.yml b/deploy/deploy_keys.yml new file mode 100644 index 0000000..faa7463 --- /dev/null +++ b/deploy/deploy_keys.yml @@ -0,0 +1,11 @@ +- hosts: all + tasks: + + - name: rsync deploy file + copy: src={{ source }} dest=/data/apps/{{ project }}/{{ dest_filename }} + + - name: back old config file + shell: "rsync -avcp /data/apps/{{ project }}/keys /data/apps/{{ project }}/keys.old" + + - name: decomp file + shell: "cd /data/apps/{{ project }} && tar zxvf {{ dest_filename }} " \ No newline at end of file diff --git a/mp2ss.sh b/mp2ss.sh new file mode 100644 index 0000000..dfd9dae --- /dev/null +++ b/mp2ss.sh @@ -0,0 +1 @@ +curl "http://10.10.3.16:6500/webapp/index.php?c=Ops&a=mp2ss" diff --git a/myops/__init__.py b/myops/__init__.py new file mode 100644 index 0000000..7c68785 --- /dev/null +++ b/myops/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/ops/csv/1004_button_name.csv b/myops/csv/1004_button_name.csv similarity index 100% rename from ops/csv/1004_button_name.csv rename to myops/csv/1004_button_name.csv diff --git a/ops/csv/2001_b_n.csv b/myops/csv/2001_b_n.csv similarity index 100% rename from ops/csv/2001_b_n.csv rename to myops/csv/2001_b_n.csv diff --git a/ops/csv/2001_b_n.csv.old b/myops/csv/2001_b_n.csv.old similarity index 100% rename from ops/csv/2001_b_n.csv.old rename to myops/csv/2001_b_n.csv.old diff --git a/ops/csv/gameid.csv b/myops/csv/gameid.csv similarity index 100% rename from ops/csv/gameid.csv rename to myops/csv/gameid.csv diff --git a/ops/mansible.py b/myops/mansible.py similarity index 100% rename from ops/mansible.py rename to myops/mansible.py diff --git a/ops/mmysql.py b/myops/mmysql.py similarity index 100% rename from ops/mmysql.py rename to myops/mmysql.py diff --git a/ops/mylog.py b/myops/mylog.py similarity index 100% rename from ops/mylog.py rename to myops/mylog.py diff --git a/myops/ss_virtual_create.py b/myops/ss_virtual_create.py new file mode 100644 index 0000000..1c43f9d --- /dev/null +++ b/myops/ss_virtual_create.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +from urllib.parse import unquote, quote, urlencode + +def my_quote(data): + if isinstance(data, dict): + return urlencode(data) + elif isinstance(data, str): + return quote(data) + + +class SS_Virtual_command: + def __init__(self): + self.ss_virtual_url = "http://10.10.3.14:8993/v1/ta/meta/prop/virtual/dict/create" + self.method = "post" + self.loginName = "root" + self.password = "kingsome2016" + + def run_cmd(self, cmd): + import subprocess + msg = "Starting run: %s " % cmd + print("run_cmd {0}".format(msg)) + cmdref = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + output, error_info = cmdref.communicate() + if cmdref.returncode != 0: + if isinstance(error_info, list) or isinstance(error_info, tuple): + error_info = error_info[0] + msg = "RUN %s ERROR,error info: %s" % (cmd, error_info) + print(f"{msg},error output={output}") + return False + else: + print(f"Run Success,output was {output}") + return True + + + def upload_gameid(self): + filename = "/data/git/ops_interface/myops/csv/gameid.csv" + projectId = int(19) + createParam = {"commonHeader": {"projectId": 19}, + "mainColumn": {"property": {"columnName": "gameid", "tableType": 0}}, + "columns": [{"property": {"columnName": "gameid", "columnDesc": "主键ID", "selectType": "string"}}, + {"property": {"columnName": "gameid_china", "columnDesc": "游戏名称", "selectType": "string"}}]} + + body = {"projectId": projectId, "createParam": createParam, "loginName": self.loginName, + "password": self.password} + quote_body = my_quote(body) + + cmd = f"curl --header 'Accept: application/json' --form 'file=@{filename}' '{self.ss_virtual_url}?{quote_body}'" + print(cmd) + if self.run_cmd(cmd): + return True + else: + return False + + + def upload_2001_b_n(self): + filename = "/data/git/ops_interface/myops/csv/2001_b_n.csv" + projectId = int(22) + createParam = {"commonHeader": {"projectId": 22}, + "mainColumn": {"property": {"columnName": "#vp@b_n", "tableType": 0, "subTableType": "vprop_sql"}}, + "columns": [{"property": {"columnName": "b_n", "columnDesc": "跳转appid", "selectType": "string"}}, + {"property": {"columnName": "b_n_china", "columnDesc": "跳转中文名", "selectType": "string"}}]} + + body = {"projectId": projectId, "createParam": createParam, "loginName": self.loginName, + "password": self.password} + quote_body = my_quote(body) + + cmd = f"curl --header 'Accept: application/json' --form 'file=@{filename}' '{self.ss_virtual_url}?{quote_body}'" + print(cmd) + if self.run_cmd(cmd): + return True + else: + return False + + def upload_1004_button_name(self): + filename = "/data/git/ops_interface/myops/csv/1004_button_name.csv" + projectId = int(19) + createParam = {"commonHeader": {"projectId": 19}, + "mainColumn": {"property": {"columnName": "#vp@button_name", "tableType": 0, "subTableType": "vprop_sql"}}, + "columns": [{"property": {"columnName": "button_name", "columnDesc": "点击按钮属性", "selectType": "string"}}, + {"property": {"columnName": "button_name_cn", "columnDesc": "点击按钮属性(中文)", "selectType": "string"}}]} + + body = {"projectId": projectId, "createParam": createParam, "loginName": self.loginName, + "password": self.password} + quote_body = my_quote(body) + + cmd = f"curl --header 'Accept: application/json' --form 'file=@{filename}' '{self.ss_virtual_url}?{quote_body}'" + print(cmd) + if self.run_cmd(cmd): + return True + else: + return False + + + +def main(): + ss = SS_Virtual_command() + #ss.upload_gameid() + #ss.upload_2001_b_n() + ss.upload_1004_button_name() + +if __name__ == "__main__": + main() diff --git a/ops/__pycache__/__init__.cpython-37.pyc b/ops/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index 90290c1..0000000 Binary files a/ops/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/ops/__pycache__/mansible.cpython-37.pyc b/ops/__pycache__/mansible.cpython-37.pyc deleted file mode 100644 index cb5cf99..0000000 Binary files a/ops/__pycache__/mansible.cpython-37.pyc and /dev/null differ diff --git a/ops/__pycache__/mmysql.cpython-37.pyc b/ops/__pycache__/mmysql.cpython-37.pyc deleted file mode 100644 index cb3f66a..0000000 Binary files a/ops/__pycache__/mmysql.cpython-37.pyc and /dev/null differ diff --git a/ops/__pycache__/mylog.cpython-37.pyc b/ops/__pycache__/mylog.cpython-37.pyc deleted file mode 100644 index e0f0c58..0000000 Binary files a/ops/__pycache__/mylog.cpython-37.pyc and /dev/null differ diff --git a/ops/__pycache__/ss_virtual_create.cpython-37.pyc b/ops/__pycache__/ss_virtual_create.cpython-37.pyc deleted file mode 100644 index 67b2f4e..0000000 Binary files a/ops/__pycache__/ss_virtual_create.cpython-37.pyc and /dev/null differ diff --git a/web.py b/web.py index 0cf0f87..54a4635 100644 --- a/web.py +++ b/web.py @@ -1,20 +1,20 @@ # -*- coding: utf-8 -*- -# ops 通用接口 +# myops 通用接口 # python web.py --port=6013 import tornado.ioloop import tornado.web import tornado.options from tornado import gen -from ops.mylog import define_logger +from myops.mylog import define_logger import logging from config.config import * import pdb from data_channel.gamelog_external import Build_Gamelog_Config from data_channel.mp2shushu import Mp2shushu - -define_logger("/data/logs/ops/ops_interface.log") +from deploy.deploy_keys import DeployKeys +define_logger("/data/logs/myops/ops_interface.log") log = logging.getLogger(__name__) tornado.options.define("port", default=interface_port, type=int, help="run server on the given port.") @@ -27,13 +27,24 @@ class DispatchHandler(tornado.web.RequestHandler): self._selfgamelogexternal() elif self.get_query_argument('c') == 'Ops' and self.get_query_argument('a') == 'mp2ss': yield self._selfmp2ss() + elif self.get_query_argument('c') == 'Deploy' and self.get_query_argument('a') == 'keys': + yield self._selfdeploykeys() else: self.write("pls check args!") - @gen.coroutine - def post(self): - if self.get_query_argument('c') == 'Ops' and self.get_query_argument('a') == 'upJumpRecording': - self._selfupJumpRecording() + # @gen.coroutine + # def post(self): + # if self.get_query_argument('c') == 'Ops' and self.get_query_argument('a') == 'upJumpRecording': + # self._selfupJumpRecording() + + def _selfdeploykeys(self): + try: + project = self.get_query_argument('project') + dk = DeployKeys(project) + dk.run() + except Exception: + log.error(f"deploy keys with {project} failed", exc_info=True) + def _selfgamelogexternal(self): try: