add some script for data parser

package.json

@@ -18,7 +18,13 @@
     "checkredis": "ts-node -r tsconfig-paths/register src/checkredis.ts",
     "additem": "ts-node -r tsconfig-paths/register src/addboxdata.ts",
     "fixdata": "ts-node -r tsconfig-paths/register src/fixdata.ts",
+    "nochain": "ts-node -r tsconfig-paths/register src/removeNoChain.ts",
+    "out": "ts-node -r tsconfig-paths/register src/queryScoreList.ts",
+    "sybil": "ts-node -r tsconfig-paths/register src/removeSybil.ts",
+    "releation": "ts-node -r tsconfig-paths/register src/updateReleation.ts",
     "rankquery": "ts-node -r tsconfig-paths/register src/rankquery2.ts",
+    "mail": "ts-node -r tsconfig-paths/register src/batchMail.ts",
+    "eth": "ts-node -r tsconfig-paths/register src/queryEthTx.ts -s 0 -l 10000 -k TC7Y76AS4AMQ468XXNKKF2R3TI7V844HUC",
     "taskid": "ts-node -r tsconfig-paths/register src/generateTaskId.ts",
     "token": "ts-node -r tsconfig-paths/register src/generateToken.ts",
     "ingame": "ts-node -r tsconfig-paths/register src/fixIngame.ts",

src/fixdata.ts

@@ -16,9 +16,9 @@ import { randomIp, randomUserAgent } from 'common/Utils'
 import { ScoreRecord } from 'models/ScoreRecord'
 import { INVITE_REBATE, SCORE_INVITE_REBATE, SCORE_OPEN_CHEST } from 'common/Constants'
 import { ActivityChest } from 'models/ActivityChest'
-import { generateChestCfg } from 'services/game.svr'
+import { generateChestCfg, generateChestLevel } from 'services/game.svr'
 
-mongoose.set('debug', true)
+mongoose.set('debug', false)
 const sourceList = require('../configs/ingame_tasks.json')
 
 const dbMain = mongoose.createConnection(process.env.DB_MAIN)
@@ -32,20 +32,20 @@ const initEnv = async () => {
   activity = await ActivityInfo.findById(ACTIVITY)
 }
 
-const parseOneRecord = async (record: any) => {
-  if (!FixAddress.checkExist(record.from)) {
-    return
-  }
-  let addressEip55 = toEIP55(record.from)
-  console.log('parseOneRecord', addressEip55)
-  const user = await dbCtrl.collection('users_test').findOne({ address: addressEip55 })
-  if (!user) {
-    // 如果没有用户记录，插入用户记录, 并插入一条钱包登录记录(在插入用户记录之前)
-    const { uid, timestamp } = await inertUser(dbCtrl, addressEip55, record.blockTime)
-    await insertWalletLoginLog(dbCtrl, addressEip55, uid, timestamp - Math.floor(Math.random() * 2))
-  }
-  // 插入签到记录
-}
+// const parseOneRecord = async (record: any) => {
+//   if (!FixAddress.checkExist(record.from)) {
+//     return
+//   }
+//   let addressEip55 = toEIP55(record.from)
+//   console.log('parseOneRecord', addressEip55)
+//   const user = await dbCtrl.collection('users_test').findOne({ address: addressEip55 })
+//   if (!user) {
+//     // 如果没有用户记录，插入用户记录, 并插入一条钱包登录记录(在插入用户记录之前)
+//     const { uid, timestamp } = await inertUser(dbCtrl, addressEip55, record.blockTime)
+//     await insertWalletLoginLog(dbCtrl, addressEip55, uid, timestamp - Math.floor(Math.random() * 2))
+//   }
+//   // 插入签到记录
+// }
 // generate fake ObjectId from timestamp
 const generateObjectId = (oid: string, fixSeconds: number) => {
   let timestamp = parseInt(oid.slice(0, 8), 16) + fixSeconds
@@ -126,13 +126,15 @@ const updateTaskProgress = async (user: any) => {
       modifiedTasks.push({ id: task.id, task: task.task, status: TaskStatusEnum.NOT_START })
     }
   }
+  const loginRecord = await LoginRecord.findOne({ user: user.id }).sort({ _id: -1 })
   if (modifiedTasks.length > 0) {
     let result = await dbMain.collection('activity_user').updateOne(
       { _id: user._id },
       {
         $push: { taskProgress: { $each: modifiedTasks } },
         // TODO:: 根据实际情况更新updatedAt
-        $set: { updatedAt: new Date('2024-05-15T12:31:19.860Z') },
+        // @ts-ignore
+        $set: { updatedAt: loginRecord.createdAt },
         $inc: { __v: 1 },
       },
       {},
@@ -150,42 +152,108 @@ const updateTaskProgress = async (user: any) => {
  */
 const updateChestRecord = async (addressRecord: any) => {
   const records = await ScoreRecord.find({ user: addressRecord.user, activity: ACTIVITY, type: SCORE_OPEN_CHEST })
+  const user = await ActivityUser.findById(addressRecord.user)
+  let actionsBox = []
+  let actionsScore = []
   for (let record of records) {
     let { level, chestId } = record.data
+    if (level === 1) {
+      level = 3
+    } else if (level === 2) {
       level = 4
+    } else {
+      continue
+    }
     const { scoreInit, bounsCfg, maxBounsCount } = generateChestCfg(level)
-    await dbMain
-      .collection('activity_chest')
-      .updateOne({ _id: new Types.ObjectId(chestId) }, { $set: { level, scoreInit, bounsCfg, maxBounsCount } })
-    await dbMain
-      .collection('score_record')
-      .updateOne({ _id: record._id }, { $set: { score: scoreInit, 'data.level': level } })
-    const user = await ActivityUser.findById(addressRecord.user)
+    actionsBox.push({
+      updateOne: {
+        filter: { _id: new Types.ObjectId(chestId) },
+        update: { $set: { level, scoreInit, bounsCfg, maxBounsCount } },
+      },
+    })
+    actionsScore.push({
+      updateOne: {
+        filter: { _id: record._id },
+        update: { $set: { score: scoreInit, 'data.level': level } },
+      },
+    })
     if (user.inviteUser) {
       const fixSeconds = 0
       let oid = generateObjectId(record.id, fixSeconds)
-      let time = new Date(user.createdAt.getTime() + fixSeconds * 1000)
+      // @ts-ignore
+      let time = new Date(record.createdAt.getTime() + fixSeconds * 1000)
       const score1 = scoreInit * INVITE_REBATE
       let scoreParams = {
-        fromUser: user,
-        data: record.data.dateTag,
+        fromUser: user.id,
+        date: record.data.dateTag,
         chestId,
         level,
         items: record.data.items,
       }
-      await dbMain.collection('score_record').insertOne({
+      actionsScore.push({
+        insertOne: {
+          document: {
             _id: new Types.ObjectId(oid),
             user: user.inviteUser,
             score: score1,
             activity: record.activity,
             type: SCORE_INVITE_REBATE,
-        data: scoreParams,
+            data: JSON.parse(JSON.stringify(scoreParams)),
             createdAt: time,
             updatedAt: time,
             __v: 0.0,
+          },
+        },
       })
     }
   }
+  if (actionsBox.length > 0) {
+    await dbMain.collection('activity_box').bulkWrite(actionsBox)
+  }
+  if (actionsScore.length > 0) {
+    await dbMain.collection('score_record').bulkWrite(actionsScore)
+  }
+}
+/**
+ * 0607 删除sybil数据后, 调整买量用户的宝箱数据
+ * 所有4级改为2级
+ */
+
+const updateChest = async (addressRecord: any) => {
+  const records = await ScoreRecord.find({ user: addressRecord.user, activity: ACTIVITY, type: SCORE_OPEN_CHEST })
+  let actionsBox = []
+  let actionsScore = []
+  for (let record of records) {
+    let { level, chestId } = record.data
+    // if (level === 4) {
+    //   level = 2
+    // } else if (level === 3) {
+    //   level = 1
+    // } else {
+    //   continue
+    // }
+    level = generateChestLevel()
+    const { scoreInit, bounsCfg, maxBounsCount } = generateChestCfg(level)
+    actionsBox.push({
+      updateOne: {
+        filter: { _id: new Types.ObjectId(chestId) },
+        update: { $set: { level, scoreInit, bounsCfg, maxBounsCount } },
+      },
+    })
+    actionsScore.push({
+      updateOne: {
+        filter: { _id: record._id },
+        update: { $set: { score: scoreInit, 'data.level': level } },
+      },
+    })
+  }
+  if (actionsBox.length > 0) {
+    await dbMain.collection('activity_box').bulkWrite(actionsBox)
+  }
+  if (actionsScore.length > 0) {
+    await dbMain.collection('score_record').bulkWrite(actionsScore)
+  }
+  console.log('updateChest', addressRecord.user, actionsBox.length)
 }
 
 const fixTaskProgree = async (record: any) => {
@@ -209,10 +277,11 @@ const updateInGameRecords = async (record: any) => {
 ;(async () => {
   try {
     await initEnv()
-    await FixAddress.find().cursor().eachAsync(prepareUser)
-    await FixAddress.find().cursor().eachAsync(fixTaskProgree)
-    await FixAddress.find().cursor().eachAsync(updateChestRecord)
-    await FixAddress.find().cursor().eachAsync(updateInGameRecords)
+    // await FixAddress.find().cursor().eachAsync(prepareUser)
+    // await FixAddress.find().cursor().eachAsync(fixTaskProgree)
+    // await FixAddress.find().cursor().eachAsync(updateChestRecord)
+    await FixAddress.find().cursor().eachAsync(updateChest)
+    // await FixAddress.find().cursor().eachAsync(updateInGameRecords)
   } catch (e) {
     console.log(e)
   }

src/models/DeleteRecord.ts

@@ -8,7 +8,7 @@ import { BaseModule } from './Base'
 @dbconn()
 @index({ type: 1 }, { unique: false })
 @modelOptions({
-  schemaOptions: { collection: 'delete_record', timestamps: true },
+  schemaOptions: { collection: 'delete_record3', timestamps: true },
   options: { allowMixed: Severity.ALLOW },
 })
 export class DeleteRecordClass extends BaseModule {

src/models/EthTx.ts

@@ -0,0 +1,46 @@
+import { Severity, getModelForClass, index, modelOptions, mongoose, prop } from '@typegoose/typegoose'
+import { dbconn } from 'decorators/dbconn'
+import { BaseModule } from './Base'
+
+@dbconn()
+@index({ hash: 1, from: 1, to: 1 }, { unique: true })
+@index({ from: 1 }, { unique: false })
+@index({ to: 1 }, { unique: false })
+@modelOptions({
+  schemaOptions: { collection: 'eth_txs', timestamps: true },
+})
+export class EthTxClass extends BaseModule {
+  @prop({ required: true })
+  public from!: string
+  @prop({ required: true })
+  public to!: string
+  @prop({ required: true })
+  public value: string
+  @prop({ required: true })
+  public hash: string
+  @prop()
+  public blockNumber: number
+  @prop()
+  public blockHash: string
+  @prop()
+  public timeStamp: number
+  @prop()
+  public dateTag: string
+  @prop({ default: 0 })
+  public version: number
+
+  public static async saveEvent(event: any) {
+    let hash = event.hash
+    let data = {
+      $inc: { version: 1 },
+    }
+    data = Object.assign(event)
+
+    let record = await EthTx.insertOrUpdate({ hash }, data)
+    return record
+  }
+}
+
+export const EthTx = getModelForClass(EthTxClass, {
+  existingConnection: EthTxClass['db'],
+})

src/models/chain/GeneralEvent.ts

@@ -0,0 +1,63 @@
+import { Severity, getModelForClass, index, modelOptions, mongoose, prop } from '@typegoose/typegoose'
+import { dbconn } from 'decorators/dbconn'
+import { BaseModule } from '../Base'
+
+@dbconn('chain')
+@index({ chain: 1, hash: 1, logIndex: 1 }, { unique: true })
+@index({ chain: 1, address: 1, blockNumber: 1 }, { unique: false })
+@modelOptions({
+  schemaOptions: { collection: 'general_event', timestamps: true },
+  options: { allowMixed: Severity.ALLOW },
+})
+export class GeneralEventClass extends BaseModule {
+  @prop({ required: true })
+  public address!: string
+  @prop({ required: true })
+  public chain: string
+  @prop({ required: true })
+  public logIndex: number
+  @prop()
+  public event: string
+  // event hash
+  @prop({ required: true })
+  public hash: string
+  @prop()
+  public blockNumber: number
+  @prop()
+  public blockHash: string
+  @prop()
+  public dateTag: string
+  @prop()
+  public removed: boolean
+
+  @prop({ type: mongoose.Schema.Types.Mixed })
+  public decodedData: any
+
+  @prop()
+  public blockTime: number
+  @prop({ default: 0 })
+  public version: number
+
+  public static async saveEvent(event: any) {
+    const logIndex = parseInt(event.logIndex || '0')
+    const hash = event.hash || event.transactionHash
+    const data = {
+      address: event.address.toLowerCase(),
+      blockNumber: parseInt(event.blockNumber),
+      removed: event.removed,
+      decodedData: event.decodedData,
+      blockHash: event.blockHash,
+      blockTime: event.blockTime,
+      dateTag: event.dateTag,
+      event: event.event,
+      $inc: { version: 1 },
+    }
+
+    let record = await GeneralEvent.insertOrUpdate({ hash, logIndex, chain: event.chain }, data)
+    return record
+  }
+}
+
+export const GeneralEvent = getModelForClass(GeneralEventClass, {
+  existingConnection: GeneralEventClass['db'],
+})

src/models/ctrl/FixAddress.ts

@@ -2,7 +2,7 @@ import { getModelForClass, index, modelOptions, prop, Severity } from '@typegoos
 import { dbconn } from 'decorators/dbconn'
 import { BaseModule } from '../Base'
 
-@dbconn('ctrl')
+@dbconn()
 // case insensitive unique index
 @index({ address: 1 }, { unique: true, collation: { locale: 'en', strength: 2 } })
 @index({ user: 1 }, { unique: true, partialFilterExpression: { user: { $exists: true } } })

src/queryEthTx.ts

@@ -0,0 +1,135 @@
+import * as dotenv from 'dotenv'
+
+const envFile = process.env.NODE_ENV && process.env.NODE_ENV === 'production' ? `.env.production` : '.env.development'
+dotenv.config({ path: envFile })
+console.log(process.env.DB_MAIN)
+import mongoose from 'mongoose'
+import { EthTx } from 'models/EthTx'
+import { ActivityUser } from 'models/ActivityUser'
+let fs = require('fs')
+
+import yargs from 'yargs'
+/**
+ * 查询钱包native交易记录
+ */
+
+const cfgs = [
+  ['address_part1.txt', '66CEJ462RDC6C7J6KDBWVVEAFJ27XGFAVX'],
+  ['address_part2.txt', '18YV8FFRPIZAK1AVPN5ZSNJEYPZVNTW73R'],
+  ['address_part3.txt', 'TC7Y76AS4AMQ468XXNKKF2R3TI7V844HUC'],
+]
+
+const argv = yargs(process.argv.slice(2)).options({
+  i: { type: 'number', default: 0 },
+}).argv
+let count = 0
+mongoose.set('debug', false)
+let API_KEY = ''
+const NO_TX_ERR = 'No transactions found'
+const fetchNormalTx = async (address: string) => {
+  let res = await fetch(
+    `https://api-opbnb.bscscan.com/api?module=account&action=txlist&address=${address}&startblock=0&endblock=99999999&page=1&offset=10000&sort=asc&apikey=${API_KEY}`,
+    {
+      method: 'GET',
+    },
+  ).then(res => res.json())
+  if (res.status != '1') {
+    throw new Error(res.message)
+  }
+  return res.result
+}
+const fetchInternalTx = async (address: string) => {
+  const res = await fetch(
+    `https://api-opbnb.bscscan.com/api?module=account&action=txlistinternal&address=${address}&startblock=0&endblock=99999999&page=1&offset=10000&sort=asc&apikey=${API_KEY}`,
+    {
+      method: 'GET',
+    },
+  ).then(res => res.json())
+  if (res.status != '1') {
+    throw new Error(res.message)
+  }
+  return res.result
+}
+
+const parseOneUser = async (user: any) => {
+  let actions = []
+  let errNormal = false
+  let errInternal = false
+  try {
+    let res1 = await fetchNormalTx(user.address)
+    for (const sub of res1) {
+      if (sub.value == '0') {
+        continue
+      }
+      let data = {
+        $inc: { version: 1 },
+      }
+      data = Object.assign(sub)
+      actions.push({
+        updateOne: {
+          filter: { hash: sub.hash, from: sub.from, to: sub.to },
+          update: data,
+          upsert: true,
+        },
+      })
+      // await EthTx.saveEvent(sub)
+    }
+  } catch (err) {
+    if (err.message == NO_TX_ERR) {
+      errNormal = true
+    }
+  }
+  try {
+    let res2 = await fetchInternalTx(user.address)
+    for (const sub of res2) {
+      if (sub.value == '0') {
+        continue
+      }
+      let data: any = {
+        $inc: { version: 1 },
+      }
+      data = Object.assign(sub)
+      data.from = sub.from.toLowerCase()
+      data.to = sub.to.toLowerCase()
+      actions.push({
+        updateOne: {
+          filter: { hash: sub.hash, from: sub.from, to: sub.to },
+          update: data,
+          upsert: true,
+        },
+      })
+      // await EthTx.saveEvent(sub)
+    }
+  } catch (err) {
+    if (err.message == NO_TX_ERR) {
+      errInternal = true
+    }
+  }
+  if (errInternal && errNormal) {
+    console.log(user.address)
+  }
+  if (actions.length > 0) {
+    await EthTx.bulkWrite(actions)
+  }
+  count += 1
+}
+
+;(async () => {
+  try {
+    // @ts-ignore
+    const i = argv.i || 0
+    API_KEY = cfgs[i][1]
+    if (!API_KEY) {
+      throw new Error('invalid key')
+    }
+    const file = cfgs[i][0]
+    let list = fs.readFileSync(`configs/${file}`, 'utf-8').split('\n')
+    for (let address of list) {
+      await parseOneUser({ address: address.toLowerCase() })
+    }
+  } catch (e) {
+    console.log(e)
+  }
+  console.log('end')
+  process.exit(0)
+})()

src/queryReleation.ts

@@ -0,0 +1,104 @@
+import * as dotenv from 'dotenv'
+
+const envFile = process.env.NODE_ENV && process.env.NODE_ENV === 'production' ? `.env.production` : '.env.development'
+dotenv.config({ path: envFile })
+console.log(process.env.DB_MAIN)
+import mongoose from 'mongoose'
+import { ActivityUser } from 'models/ActivityUser'
+import { ScoreRecord } from 'models/ScoreRecord'
+import { ZRedisClient } from 'zutils'
+import { formatNumShow } from 'common/Utils'
+import { RANK_SCORE_SCALE } from 'common/Constants'
+let fs = require('fs')
+/**
+ * 生成积分明细
+ */
+
+mongoose.set('debug', false)
+const ACTIVITY = 'uaw_activity'
+
+const totalKey = 'uaw_activity:static:score'
+const minCount = 100
+
+const updateRedis2 = (key, vals) => {
+  return new Promise((resolve, reject) => {
+    new ZRedisClient().pub.zadd(key, vals, function (err, res) {
+      if (err) {
+        reject(err)
+      } else {
+        resolve(res)
+      }
+    })
+  })
+}
+
+const saveScoreToRedis = async (type: string) => {
+  const key = `uaw_score_${type}`
+  await new ZRedisClient().del(key)
+  let records = await ScoreRecord.aggregate([
+    { $match: { type } },
+    { $group: { _id: '$user', count: { $sum: '$score' } } },
+  ])
+  let vals = []
+
+  for (let i = 0, l = records.length; i < l; i++) {
+    if (records[i].count == 0) {
+      continue
+    }
+    vals.push((records[i].count * RANK_SCORE_SCALE) | 0)
+    vals.push(records[i]._id)
+    if (i % 1000 === 0) {
+      await updateRedis2(key, vals)
+      vals.length = 0
+    }
+  }
+  if (vals.length > 0) {
+    await updateRedis2(key, vals)
+  }
+}
+
+const queryList = async () => {
+  let records = await ActivityUser.aggregate([
+    { $match: { inviteUser: { $exists: true } } },
+    { $group: { _id: '$inviteUser', count: { $sum: 1 } } },
+    { $match: { count: { $gte: minCount } } },
+    { $sort: { count: -1 } },
+  ])
+  console.log(records.length)
+  let results = []
+  for (let i = 0, l = records.length; i < l; i++) {
+    const { _id, count } = records[i]
+    const user = await ActivityUser.findById(_id)
+    if (!user) {
+      continue
+    }
+    const totalScore = await new ZRedisClient().zscore(totalKey, _id)
+    const releationKey = `uaw_score_invite_rebate`
+    const releatScore = await new ZRedisClient().zscore(releationKey, _id)
+    const scoreShow = formatNumShow(totalScore ? parseInt(totalScore + '') / RANK_SCORE_SCALE : 0)
+    if (scoreShow == '0') {
+      continue
+    }
+    const releatScoreShow = formatNumShow(releatScore ? parseInt(releatScore + '') / RANK_SCORE_SCALE : 0)
+    results.push(`${_id}\t${user?.address || ''}\t${count}\t${scoreShow}\t${releatScoreShow}`)
+  }
+  fs.writeFileSync('outdatas/releation.txt', results.join('\n') + '\n', { flag: 'a+' })
+}
+
+;(async () => {
+  try {
+    let opts = { url: process.env.REDIS }
+    new ZRedisClient(opts)
+    console.time('update redis')
+    await saveScoreToRedis('invite_rebate')
+    console.timeEnd('update redis')
+    console.time('query')
+    fs.writeFileSync('outdatas/releation.txt', '')
+    await queryList()
+    console.timeEnd('query')
+  } catch (e) {
+    console.log(e)
+  }
+  console.log('end')
+  process.exit(0)
+})()

src/queryScoreList.ts

@@ -0,0 +1,244 @@
+import * as dotenv from 'dotenv'
+
+const envFile = process.env.NODE_ENV && process.env.NODE_ENV === 'production' ? `.env.production` : '.env.development'
+dotenv.config({ path: envFile })
+console.log(process.env.DB_MAIN)
+import mongoose from 'mongoose'
+import { ActivityUser } from 'models/ActivityUser'
+import { ScoreRecord } from 'models/ScoreRecord'
+import { ZRedisClient } from 'zutils'
+import { formatNumShow } from 'common/Utils'
+import { RANK_SCORE_SCALE } from 'common/Constants'
+import { InGameScoreRecord } from 'models/InGameScoreRecord'
+import { DeleteRecord } from 'models/DeleteRecord'
+import { Types } from 'mongoose'
+import { InGameStats } from 'models/InGameStats'
+import { NFTHolderRecord } from 'models/NFTHodlerRecord'
+let fs = require('fs')
+/**
+ * 生成积分明细
+ */
+
+mongoose.set('debug', false)
+const ACTIVITY = 'uaw_activity'
+
+const totalKey = 'uaw_activity:static:score'
+const gameTotalKey = 'uaw_activity:static:gamescore'
+const releationKey = `uaw_score_invite_rebate`
+const pageSize = 100
+
+const updateRedis2 = (key, vals) => {
+  return new Promise((resolve, reject) => {
+    new ZRedisClient().pub.zadd(key, vals, function (err, res) {
+      if (err) {
+        reject(err)
+      } else {
+        resolve(res)
+      }
+    })
+  })
+}
+
+const saveScoreToRedis = async () => {
+  let records = await ScoreRecord.aggregate([{ $group: { _id: '$user', count: { $sum: '$score' } } }])
+  let vals = []
+  for (let i = 0, l = records.length; i < l; i++) {
+    if (records[i].count == 0) {
+      continue
+    }
+    vals.push((records[i].count * RANK_SCORE_SCALE) | 0)
+    vals.push(records[i]._id)
+    if (i % 1000 === 0) {
+      await updateRedis2(totalKey, vals)
+      vals.length = 0
+    }
+  }
+  if (vals.length > 0) {
+    await updateRedis2(totalKey, vals)
+  }
+}
+
+const saveTypeScoreToRedis = async (type: string) => {
+  const key = `uaw_score_${type}`
+  await new ZRedisClient().del(key)
+  let records = await ScoreRecord.aggregate([
+    { $match: { type } },
+    { $group: { _id: '$user', count: { $sum: '$score' } } },
+  ])
+  let vals = []
+
+  for (let i = 0, l = records.length; i < l; i++) {
+    if (records[i].count == 0) {
+      continue
+    }
+    vals.push((records[i].count * RANK_SCORE_SCALE) | 0)
+    vals.push(records[i]._id)
+    if (i % 1000 === 0) {
+      await updateRedis2(key, vals)
+      vals.length = 0
+    }
+  }
+  if (vals.length > 0) {
+    await updateRedis2(key, vals)
+  }
+}
+
+const generateScoreList = async (start: number) => {
+  let end = start + pageSize
+  const records = await new ZRedisClient().zrevrange(totalKey, start, end)
+  let results = []
+  for (let i = 0; i < records.length; i += 2) {
+    const id = records[i]
+    let score = formatNumShow(parseInt(records[i + 1]) / RANK_SCORE_SCALE)
+    const releatScore = await new ZRedisClient().zscore(releationKey, id)
+    const releatScoreShow = formatNumShow(releatScore ? parseInt(releatScore + '') / RANK_SCORE_SCALE : 0)
+    const user = await ActivityUser.findById(id)
+    const rank = start + i / 2 + 1
+    results.push(
+      `${rank}\t${user?.address}\t${score}\t${releatScoreShow}\t${holderMap.get(user?.address.toLowerCase()) || 1}\t${founderSet.has(user.address) ? 1 : 0}\t${explorerSet.has(user.address) ? 1 : 0}\t${badgeSet.has(user.address) ? 1 : 0}\t${candySet.has(user.address) ? 1 : 0}`,
+    )
+  }
+  fs.writeFileSync('outdatas/scorelist.txt', results.join('\n') + '\n', { flag: 'a+' })
+}
+
+const saveGameScoreToRedis = async () => {
+  let records = await InGameScoreRecord.aggregate([{ $group: { _id: '$user', count: { $sum: '$score' } } }])
+  let vals = []
+  for (let i = 0, l = records.length; i < l; i++) {
+    if (records[i].count == 0) {
+      continue
+    }
+    vals.push((records[i].count * 100) | 0)
+    vals.push(records[i]._id)
+    if (i % 1000 === 0) {
+      await updateRedis2(gameTotalKey, vals)
+      vals.length = 0
+    }
+  }
+  if (vals.length > 0) {
+    await updateRedis2(gameTotalKey, vals)
+  }
+}
+
+const generateGameScoreList = async (start: number) => {
+  let end = start + pageSize
+  // const records = await new ZRedisClient().zrevrange(gameTotalKey, start, end)
+  const records = await InGameStats.find({ score: { $gt: 0 } })
+    .sort({ score: -1 })
+    .skip(start)
+    .limit(pageSize)
+  let results = []
+  for (let i = 0; i < records.length; i++) {
+    let score = records[i].score
+    if (score === 0) {
+      continue
+    }
+    let address = ''
+    const user = await ActivityUser.findById(records[i].user)
+    if (!user) {
+      const userDel = await DeleteRecord.findOne({ 'data._id': new Types.ObjectId(records[i].user) })
+      if (userDel) {
+        address = userDel.data.address
+      }
+    } else {
+      address = user.address
+    }
+    const rank = start + i + 1
+    results.push(`${rank}\t${address}\t${score}`)
+  }
+  fs.writeFileSync('outdatas/gameScorelist.txt', results.join('\n') + '\n', { flag: 'a+' })
+}
+
+const generateWhiteList = async () => {
+  let records = await ActivityUser.find({ whiteListNum: { $gt: 0 } })
+  let results = []
+  for (let i = 0; i < records.length; i++) {
+    results.push(`${records[i].address} \t ${records[i].whiteListNum}`)
+  }
+  let records2 = await DeleteRecord.find({ 'data.whiteListNum': { $gt: 0 } })
+  for (let i = 0; i < records2.length; i++) {
+    results.push(`${records2[i].data.address} \t ${records2[i].data.whiteListNum}`)
+  }
+  fs.writeFileSync('outdatas/whitelist.txt', results.join('\n') + '\n', { flag: 'a+' })
+}
+
+const holders = [
+  ['explorer', '0x0cee888fa25810ca648d697099bc17a2c9e1dfbf', 1.5],
+  ['candy', '0xefd4c863e73e7e9cc33d46fb30ce51510fcfdeb0', 1.5],
+  ['badge', '0xd728de3d9ebed90e84abe84539280cbc5b18e304', 2],
+  ['founder', '0xec23679653337d4c6390d0eeba682246a6067777', 3],
+]
+const holderMap = new Map()
+let explorerSet = new Set()
+let candySet = new Set()
+let badgeSet = new Set()
+let founderSet = new Set()
+const queryHolder = async () => {
+  for (let holder of holders) {
+    const [name, contract] = holder
+    let records = await NFTHolderRecord.find({ contract: contract + '' })
+    for (let i = 0; i < records.length; i++) {
+      const uid = records[i].user
+      let user = await ActivityUser.findById(uid)
+      if (!user) {
+        continue
+      }
+      let address = user.address.toLowerCase()
+      if (name === 'explorer') {
+        explorerSet.add(address)
+      } else if (name === 'candy') {
+        candySet.add(address)
+      } else if (name === 'badge') {
+        badgeSet.add(address)
+      } else if (name === 'founder') {
+        founderSet.add(address)
+      }
+      if (!holderMap.has(address)) {
+        holderMap.set(address, 1)
+      }
+      holderMap.set(address, holderMap.get(address) * parseFloat(holder[2] + ''))
+    }
+  }
+}
+
+;(async () => {
+  try {
+    let opts = { url: process.env.REDIS }
+    new ZRedisClient(opts)
+    await new ZRedisClient().del(totalKey)
+    // await new ZRedisClient().del(gameTotalKey)
+    console.time('update redis')
+    await saveScoreToRedis()
+    console.timeEnd('update redis')
+    console.time('query holder info')
+    await queryHolder()
+    console.timeEnd('query holder info')
+    console.time('update releation redis')
+    await saveTypeScoreToRedis('invite_rebate')
+    console.timeEnd('update releation redis')
+    console.time('generate score list')
+    const total = (await new ZRedisClient().zcard(totalKey)) as number
+    for (let i = 0; i < total; i += pageSize) {
+      await generateScoreList(i)
+    }
+    console.timeEnd('generate score list')
+
+    // console.time('update game redis')
+    // await saveGameScoreToRedis()
+    // console.timeEnd('update game redis')
+    // console.time('generate game score list')
+    // const totalGame = (await new ZRedisClient().zcard(gameTotalKey)) as number
+    // const totalGame = await InGameStats.countDocuments({ score: { $gt: 0 } })
+    // for (let i = 0; i < totalGame; i += pageSize) {
+    //   await generateGameScoreList(i)
+    // }
+    // console.timeEnd('generate game score list')
+    // console.time('generate white list')
+    // await generateWhiteList()
+    // console.timeEnd('generate white list')
+  } catch (e) {
+    console.log(e)
+  }
+  console.log('end')
+  process.exit(0)
+})()

src/removeNoChain.ts

@@ -0,0 +1,39 @@
+import * as dotenv from 'dotenv'
+
+const envFile = process.env.NODE_ENV && process.env.NODE_ENV === 'production' ? `.env.production` : '.env.development'
+dotenv.config({ path: envFile })
+console.log(process.env.DB_MAIN)
+import mongoose from 'mongoose'
+import { ActivityUser } from 'models/ActivityUser'
+import { GeneralEvent } from 'models/chain/GeneralEvent'
+import { DeleteRecord } from 'models/DeleteRecord'
+import { ScoreRecord } from 'models/ScoreRecord'
+
+mongoose.set('debug', true)
+const ACTIVITY = 'uaw_activity'
+
+const parseOneUser = async (record: any) => {
+  const address = record.address
+  let res = await GeneralEvent.findOne({ 'decodedData.user': address }).collation({ locale: 'en', strength: 2 })
+  if (!res) {
+    console.log(`no chain record: `, address)
+    let dRecord = new DeleteRecord({
+      type: 'user_no_chain',
+      data: record,
+    })
+    await ScoreRecord.deleteMany({ user: record.id, activity: ACTIVITY })
+    await ScoreRecord.deleteMany({ 'data.fromUser': record.id, type: 'invite_rebate' })
+    await dRecord.save()
+    await ActivityUser.deleteOne({ _id: record.id })
+  }
+}
+
+;(async () => {
+  try {
+    await ActivityUser.find().cursor().eachAsync(parseOneUser)
+  } catch (e) {
+    console.log(e)
+  }
+  console.log('end')
+  process.exit(0)
+})()

src/removeSybil.ts

@@ -0,0 +1,91 @@
+import * as dotenv from 'dotenv'
+
+const envFile = process.env.NODE_ENV && process.env.NODE_ENV === 'production' ? `.env.production` : '.env.development'
+dotenv.config({ path: envFile })
+console.log(process.env.DB_MAIN)
+import mongoose from 'mongoose'
+import { ActivityUser } from 'models/ActivityUser'
+import { DeleteRecord } from 'models/DeleteRecord'
+import { ScoreRecord } from 'models/ScoreRecord'
+import { FixAddress } from 'models/ctrl/FixAddress'
+
+mongoose.set('debug', false)
+const dbMain = mongoose.createConnection(process.env.DB_MAIN)
+const ACTIVITY = 'uaw_activity'
+let fs = require('fs')
+
+let sybilSet: Set<string> = new Set()
+let sybilArr: string[] = []
+
+// const loadSybils = async () => {
+//   let wallets = fs.readFileSync('configs/sybil.txt', 'utf-8').split('\n')
+//   // let fromSet = new Set(wallets)
+//   let records = await EthTx.find({ from: { $in: wallets } })
+//   for (let record of records) {
+//     sybilSet.add(record.to.toLowerCase())
+//   }
+// }
+const loadSybils = async (index: number) => {
+  sybilArr = fs.readFileSync(`data/address_list${(index + '').padStart(2, '0')}`, 'utf-8').split('\n')
+}
+
+const loadWhitelist = async () => {
+  let records = await FixAddress.find()
+  for (let record of records) {
+    if (sybilSet.has(record.address.toLowerCase())) {
+      sybilSet.delete(record.address.toLowerCase())
+    }
+  }
+}
+
+const deleteUser = async () => {
+  let userArr = []
+  let records = await ActivityUser.find({ address: { $in: sybilArr } })
+  for (let record of records) {
+    userArr.push(record.id)
+  }
+  await ActivityUser.aggregate([{ $match: { address: { $in: sybilArr } } }, { $out: 'delete_users2' }])
+  console.log('delete_users: ', userArr.length)
+  const res1 = await ActivityUser.deleteMany({ address: { $in: sybilArr } })
+  console.log(res1)
+  const res2 = await ScoreRecord.deleteMany({ user: { $in: userArr }, activity: ACTIVITY })
+  console.log(res2)
+  const res3 = await ScoreRecord.deleteMany({ 'data.fromUser': { $in: userArr }, type: 'invite_rebate' })
+  console.log(res3)
+}
+
+const parseOneUser = async (record: any) => {
+  const address = record.address
+  if (sybilSet.has(address)) {
+    console.log(`no chain record: `, address)
+    let dRecord = new DeleteRecord({
+      type: 'user_no_chain',
+      data: record,
+    })
+    await ScoreRecord.deleteMany({ user: record.id, activity: ACTIVITY })
+    await ScoreRecord.deleteMany({ 'data.fromUser': record.id, type: 'invite_rebate' })
+    await dRecord.save()
+    await ActivityUser.deleteOne({ _id: record.id })
+  }
+}
+
+;(async () => {
+  for (let i = 0; i < 33; i++) {
+    try {
+      await loadSybils(i)
+      console.log(i, 'sybilArr: ', sybilArr.length)
+      await deleteUser()
+    } catch (err) {
+      console.log(err)
+    }
+  }
+  // try {
+  //   // await loadWhitelist()
+  //   // console.log('sybilSet: ', sybilSet.size)
+  //   // await ActivityUser.find().cursor().eachAsync(parseOneUser)
+  // } catch (e) {
+  //   console.log(e)
+  // }
+  console.log('end')
+  process.exit(0)
+})()

src/services/email.svr.ts

@@ -62,14 +62,18 @@ const MAIL_SVR = process.env.EMAIL_SERVER
 
 @singleton
 export class EmailSvr {
-  public sendMail(msg: IMailData) {
+  public async sendMail(msg: IMailData) {
     let url = MAIL_SVR + '/mail/send'
     Object(DEFAULT_MSG_DATA).zssign(msg)
+    let headers: any = {
+      'Content-Type': 'application/json; charset=utf-8',
+    }
+    if (process.env.NODE_ENV !== 'production') {
+      headers['Authorization'] = 'Basic bWFpbHNlbmRlcjo3NjU0MzIxY2Yt'
+    }
     const options: any = {
       method: 'POST',
-      headers: {
-        'Content-Type': 'application/json; charset=utf-8',
-      },
+      headers,
       body: JSON.stringify({ message: msg }),
     }
 

src/updateReleation.ts

@@ -0,0 +1,31 @@
+import * as dotenv from 'dotenv'
+
+const envFile = process.env.NODE_ENV && process.env.NODE_ENV === 'production' ? `.env.production` : '.env.development'
+dotenv.config({ path: envFile })
+console.log(process.env.DB_MAIN)
+import mongoose from 'mongoose'
+import { FixAddress } from 'models/ctrl/FixAddress'
+
+mongoose.set('debug', false)
+const dbMain = mongoose.createConnection(process.env.DB_MAIN)
+
+;(async () => {
+  try {
+    let records = await FixAddress.find()
+    let users: string[] = records.map(r => r.user)
+    // let addressList: string[] = records.map(r => r.address)
+    console.log('users', users.length)
+    console.log(users[0])
+    // const res = await dbMain
+    //   .collection('activity_user')
+    //   .updateMany({ address: { $in: addressList } }, { $set: { star: 1 } })
+    const res = await dbMain.collection('score_record').updateMany({ user: { $in: users } }, { $set: { star: 1 } })
+    console.log(res)
+    const res1 = await dbMain.collection('activity_box').updateMany({ user: { $in: users } }, { $set: { star: 1 } })
+    console.log(res1)
+  } catch (e) {
+    console.log(e)
+  }
+  console.log('end')
+  process.exit(0)
+})()