From 2915c6301c50f8932571e57b823344ff2cdbfcc9 Mon Sep 17 00:00:00 2001 From: zhl Date: Thu, 17 Jun 2021 18:01:40 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0excel=E5=AF=BC=E5=85=A5?= =?UTF-8?q?=E6=97=B6=E7=9A=84=E5=AE=A1=E6=A0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 + .../controllers/shop_puzzle.controller.ts | 9 ++- src/models/AuditTask.ts | 36 ++++++++++ src/models/shop/ShopPuzzle.ts | 21 ++++-- src/services/AuditSvr.ts | 67 +++++++++++++++++++ yarn.lock | 7 ++ 6 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 src/models/AuditTask.ts create mode 100644 src/services/AuditSvr.ts diff --git a/package.json b/package.json index c67af76..ad74b01 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "node-html-to-image": "^3.1.0", "qrcode": "^1.4.4", "querystring": "^0.2.1", + "queue": "^6.0.2", "redis": "^3.1.2", "reflect-metadata": "^0.1.13", "svg-captcha": "^1.4.0", diff --git a/src/admin/controllers/shop_puzzle.controller.ts b/src/admin/controllers/shop_puzzle.controller.ts index 71326f2..ded203f 100644 --- a/src/admin/controllers/shop_puzzle.controller.ts +++ b/src/admin/controllers/shop_puzzle.controller.ts @@ -2,6 +2,8 @@ import BaseController from '../../common/base.controller' import { permission, router } from '../../decorators/router' import { ShopPuzzle } from '../../models/shop/ShopPuzzle' import { ZError } from '../../common/ZError' +import { AuditTask } from '../../models/AuditTask' +import { AuditSvr } from '../../services/AuditSvr' export default class ShopPuzzleController extends BaseController { @permission('shoppuzzle:read') @@ -111,7 +113,8 @@ export default class ShopPuzzleController extends BaseController { delete data.status } let result: any = {} - const initStatus = { status: 1, deleted: 0, is_hide: 0, quality: 1 } + const batchTag = Date.now() + const initStatus = { status: 0, deleted: 0, is_hide: 0, quality: 1, batchTag } try { datas.map(data => Object.assign(data, initStatus)) let results = await ShopPuzzle.bulkWrite( @@ -127,6 +130,10 @@ export default class ShopPuzzleController extends BaseController { result.insert = results.upsertedCount + results.insertedCount result.update = results.modifiedCount result.skip = 0 + let records = await ShopPuzzle.find({ batchTag }) + let ids = records.map(data => data._id + '') + let id = await AuditTask.addOneTask('shop_puzzle', ids) + new AuditSvr().addTask(id) } catch (err) { console.log(err) result.insert = err.upsertedCount + err.insertedCount diff --git a/src/models/AuditTask.ts b/src/models/AuditTask.ts new file mode 100644 index 0000000..bdb759f --- /dev/null +++ b/src/models/AuditTask.ts @@ -0,0 +1,36 @@ +import { dbconn } from '../decorators/dbconn' +import { getModelForClass, index, modelOptions, prop } from '@typegoose/typegoose' +import { Severity } from '@typegoose/typegoose/lib/internal/constants' +import { BaseModule } from './Base' + +@dbconn() +@index({ status: 1 }, { unique: false }) +@modelOptions({ + schemaOptions: { collection: 'audit_task', timestamps: true }, + options: { allowMixed: Severity.ALLOW }, +}) +export class AuditTaskClass extends BaseModule { + @prop() + public tableName: string + @prop({ type: () => [String] }) + public ids: string[] + + @prop({ default: 0 }) + public status: number + + @prop({ default: 0 }) + public processNum: number + + @prop({ default: 0 }) + public failNum: number + + public static async addOneTask(tableName: string, datas: string[]) { + let record = new AuditTask({}) + record.tableName = tableName + record.ids = datas + await record.save() + return record._id + } +} + +export const AuditTask = getModelForClass(AuditTaskClass, { existingConnection: AuditTaskClass.db }) diff --git a/src/models/shop/ShopPuzzle.ts b/src/models/shop/ShopPuzzle.ts index 0d6a8b2..87d7777 100644 --- a/src/models/shop/ShopPuzzle.ts +++ b/src/models/shop/ShopPuzzle.ts @@ -1,11 +1,10 @@ import { dbconn } from '../../decorators/dbconn' -import { getModelForClass, index, modelOptions, prop, ReturnModelType } from '@typegoose/typegoose' +import { getModelForClass, index, modelOptions, mongoose, prop, ReturnModelType } from '@typegoose/typegoose' import { BaseModule } from '../Base' import { Base, TimeStamps } from '@typegoose/typegoose/lib/defaultClasses' import { checkJson, noJson } from '../../decorators/nojson' -import { Puzzle, PuzzleClass } from '../content/Puzzle' -import { QCategoryCache } from '../../services/QCategoryCache' import { CompactPuzzleClass } from '../match/PuzzleSession' +import { Severity } from '@typegoose/typegoose/lib/internal/constants' const jsonExcludeKeys = ['updatedAt', '__v'] @@ -20,7 +19,10 @@ export interface ShopPuzzleClass extends Base, TimeStamps {} @index({ shop: 1, question: 1 }, { unique: true }) @index({ groups: 1 }, { unique: false }) @index({ shop: 1, tag: 1, sub_tag: 1 }, { unique: false }) -@modelOptions({ schemaOptions: { collection: 'shop_puzzle', timestamps: true } }) +@modelOptions({ + schemaOptions: { collection: 'shop_puzzle', timestamps: true }, + options: { allowMixed: Severity.ALLOW }, +}) export class ShopPuzzleClass extends BaseModule { @prop() public shop: string @@ -68,6 +70,9 @@ export class ShopPuzzleClass extends BaseModule { @prop({ type: () => [String] }) public groups: string[] + + @prop() + public batchTag: number /** * 是否删除 * @type {boolean} @@ -87,8 +92,11 @@ export class ShopPuzzleClass extends BaseModule { @prop({ default: 0 }) public status: number + @prop({ type: mongoose.Schema.Types.Mixed }) + public auditResult: {} + public static parseQueryParam(params) { - let { key, timeBegin, timeEnd, tag, sub_tag, groups, dp, quality, sort, length, shop } = params + let { key, timeBegin, timeEnd, tag, sub_tag, groups, dp, quality, sort, length, shop, status } = params let opt: any = { deleted: 0, is_hide: 0 } if (key) { opt.question = { $regex: key, $options: 'i' } @@ -125,6 +133,9 @@ export class ShopPuzzleClass extends BaseModule { if (length) { opt.length = { $gte: length } } + if (status !== undefined && status !== 9) { + opt.status = status + } let sortObj = { _id: 1 } if (sort) { sortObj = sort diff --git a/src/services/AuditSvr.ts b/src/services/AuditSvr.ts new file mode 100644 index 0000000..b4fec05 --- /dev/null +++ b/src/services/AuditSvr.ts @@ -0,0 +1,67 @@ +import axios, { AxiosRequestConfig } from 'axios' +import { singleton } from '../decorators/singleton' +import Queue from 'queue' +import { AuditTask } from '../models/AuditTask' +import { ShopPuzzle } from '../models/shop/ShopPuzzle' + +export async function auditTxt(txt: string) { + let data = { content: txt } + const url = 'http://127.0.0.1:8019/check_all' + let reqConfig: AxiosRequestConfig = { + method: 'post', + url, + headers: { + 'Cache-Control': 'no-cache', + 'Content-Type': 'application/json', + }, + data, + } + return axios(reqConfig) +} + +@singleton +export class AuditSvr { + private q + constructor() { + this.q = new Queue({ concurrency: 1, autostart: true }) + } + + addTask(taskId: string) { + this.q.push(async function (cb) { + let taskRecord = await AuditTask.findById(taskId) + if (!taskRecord) { + cb() + return + } + let failCount = 0 + let processNum = 0 + let riskNum = 0 + for (let id of taskRecord.ids) { + let record = await ShopPuzzle.findById(id) + if (record.status === 1) { + continue + } + processNum++ + let txt = `${record.question},${record.a1},${record.a2},${record.a3},${record.a4}` + let { data } = await auditTxt(txt) + record.auditResult = data + if (data.errcode) { + failCount++ + } else { + if (data.wx || data.bd || data.local) { + record.status = -1 + riskNum++ + } else { + record.status = 1 + } + } + await record.save() + } + taskRecord.processNum = processNum + taskRecord.failNum = failCount + await taskRecord.save() + console.log(`audit task finished: ${taskId}, found ${riskNum} risk`) + cb() + }) + } +} diff --git a/yarn.lock b/yarn.lock index cbae213..f46b8d1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2648,6 +2648,13 @@ queue-microtask@^1.1.2, queue-microtask@^1.2.2: resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== +queue@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/queue/-/queue-6.0.2.tgz#b91525283e2315c7553d2efa18d83e76432fed65" + integrity sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA== + dependencies: + inherits "~2.0.3" + quick-format-unescaped@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/quick-format-unescaped/-/quick-format-unescaped-4.0.3.tgz#6d6b66b8207aa2b35eef12be1421bb24c428f652"