diff --git a/.gitignore b/.gitignore index 2835c3a..bdf935c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ build dist .DS_Store .env -.env.development \ No newline at end of file +.env.development +.env.production \ No newline at end of file diff --git a/package.json b/package.json index 2b1fbb6..e02d991 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,9 @@ "dev:monitor": "NODE_ENV=development ts-node -r tsconfig-paths/register src/monitor.ts", "prod:monitor": "NODE_PATH=./dist node dist/monitor.js", "dev:scription": "ts-node -r tsconfig-paths/register src/scriptions.ts", - "prod:scription": "TZ='SG' NODE_PATH=./dist node dist/scriptions.js" + "prod:scription": "TZ='SG' NODE_PATH=./dist node dist/scriptions.js", + "dev:event": "ts-node -r tsconfig-paths/register src/events.ts", + "prod:event": "TZ='SG' NODE_PATH=./dist node dist/events.js" }, "author": "z", "license": "ISC", @@ -42,9 +44,10 @@ "fastify-xml-body-parser": "^2.2.0", "mongoose": "5.10.3", "mongoose-findorcreate": "^3.0.0", - "nodemailer": "^6.9.1", + "node-fetch": "2", "node-schedule": "^2.0.0", "node-xlsx": "^0.21.0", + "nodemailer": "^6.9.1", "redis": "^3.1.2", "tracer": "^1.1.6", "web3": "^1.7.4" @@ -52,8 +55,8 @@ "devDependencies": { "@types/dotenv": "^8.2.0", "@types/node": "^14.14.20", - "@types/nodemailer": "^6.4.7", "@types/node-schedule": "^2.1.0", + "@types/nodemailer": "^6.4.7", "@types/redis": "^2.8.28", "@typescript-eslint/eslint-plugin": "^4.25.0", "@typescript-eslint/parser": "^4.25.0", diff --git a/pm2_dev.sh b/pm2_dev.sh index 47382e0..224198c 100755 --- a/pm2_dev.sh +++ b/pm2_dev.sh @@ -1,2 +1,3 @@ pm2 start npm --name "chain-client" --log-date-format "YYYY-MM-DD HH:mm:ss" -- run "dev:api" -pm2 start npm --name "chain-monitor" --log-date-format "YYYY-MM-DD HH:mm:ss" -- run "dev:monitor" \ No newline at end of file +pm2 start npm --name "chain-scription" --log-date-format "YYYY-MM-DD HH:mm:ss" -- run "dev:scription" +pm2 start npm --name "chain-event" --log-date-format "YYYY-MM-DD HH:mm:ss" -- run "dev:event" \ No newline at end of file diff --git a/src/abis/Transfer.ts b/src/abis/Transfer.ts new file mode 100644 index 0000000..0c35393 --- /dev/null +++ b/src/abis/Transfer.ts @@ -0,0 +1,51 @@ +export const FT_TRANSFER_EVENT_ABI = { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "from", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "to", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "value", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" +} + +export const NFT_TRANSFER_EVENT_ABI = { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "from", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "to", + "type": "address" + }, + { + "indexed": true, + "internalType": "uint256", + "name": "tokenId", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" +} \ No newline at end of file diff --git a/src/chain/allchain.ts b/src/chain/allchain.ts index af1c5ec..ae1edb2 100644 --- a/src/chain/allchain.ts +++ b/src/chain/allchain.ts @@ -219,7 +219,7 @@ export const AllChains: IChain[] = [ { name: 'Arbitrum One', type: 'Mainnet', - rpc: 'https://arbitrum-mainnet.infura.io/v3/b6bf7d3508c941499b10025c0776eaf8', + rpc: 'https://arb-mainnet.g.alchemy.com/v2/2wVx68PmeMUCVgcMc9H-bKcnLDFBYlFS', id: 42161, network: 'ARBITRUM', symbol: 'ETH', @@ -277,7 +277,7 @@ export const AllChains: IChain[] = [ { name: 'Arbitrum Sepolia', type: 'Testnet', - rpc: 'https://sepolia-rollup.arbitrum.io/rpc|https://arbitrum-sepolia.infura.io/v3/b6bf7d3508c941499b10025c0776eaf8', + rpc: 'https://arb-sepolia.g.alchemy.com/v2/EKR1je8ZGia332kkemNc4mtXQuFskIq3', id: 421614, network: 'ARB_SEPOLIA', symbol: 'ETH', diff --git a/src/chain/chain.api.ts b/src/chain/chain.api.ts index 4569260..6c237c5 100644 --- a/src/chain/chain.api.ts +++ b/src/chain/chain.api.ts @@ -1,3 +1,4 @@ +import fetch from "node-fetch" const requestChain = async (rpc: string, method: string, params: any) => { const data = { @@ -47,4 +48,25 @@ export const batchEthBlocks = async (rpc: string, blockNumber: number, amount: n body: JSON.stringify(batch) }) .then((res) => res.json()) +} + +export const batchEthLogs = async (rpc: string, params: any) => { + // let batch = [] + // for (let i = 0; i < params.length; i++) { + // batch.push({ + // jsonrpc: "2.0", + // method: "eth_getLogs", + // params: [params[i]], + // id: ids[i] + // }) + // } + + return fetch(rpc, { + method: "POST", + headers: { + "Content-Type": "application/json; charset=utf-8" + }, + body: JSON.stringify(params) + }) + .then((res) => res.json()) } \ No newline at end of file diff --git a/src/config/events_cfg.ts b/src/config/events_cfg.ts new file mode 100644 index 0000000..bfacac2 --- /dev/null +++ b/src/config/events_cfg.ts @@ -0,0 +1,47 @@ +import { NFT_TRANSFER_EVENT_ABI } from 'abis/Transfer' +import { NftTransferEvent } from 'models/NftTransferEvent' + +export interface IEventCfg { + address: string, + event: string, + abi: any, + fromBlock: number, + eventProcesser: string, + chain: number, + topic?: string, +} + +export const EVENTS_CFG: IEventCfg[] = [ + { // hero + chain: 421614, + address: "0xCD4bb3402f1a444a1AF10F31946Ed37DaC0eaC4d", + event: "Transfer", + abi: NFT_TRANSFER_EVENT_ABI, + fromBlock: 3549077, + eventProcesser: NftTransferEvent, + }, + { // candy + chain: 421614, + address: "0x6a673D946a976776fd5F163d9d831b2fEB600015", + event: "Transfer", + abi: NFT_TRANSFER_EVENT_ABI, + fromBlock: 5814045, + eventProcesser: NftTransferEvent, + }, + { // Explorer + chain: 421614, + address: "0x7b6399DFbed8Bc46F6A498C6B1040E80c2B5C4bc", + event: "Transfer", + abi: NFT_TRANSFER_EVENT_ABI, + fromBlock: 5814491, + eventProcesser: NftTransferEvent, + }, + { // Gacha + chain: 421614, + address: "0xe2E4D5a4045fBFcbCBECAf5b8A94303712d2FA97", + event: "Transfer", + abi: NFT_TRANSFER_EVENT_ABI, + fromBlock: 3549613, + eventProcesser: NftTransferEvent, + }, +] \ No newline at end of file diff --git a/src/controllers/task.controllers.ts b/src/controllers/task.controllers.ts index 1f61d46..2fd3379 100644 --- a/src/controllers/task.controllers.ts +++ b/src/controllers/task.controllers.ts @@ -1,7 +1,9 @@ +import { ZERO_ADDRESS } from 'common/Constants' import { ZError } from 'common/ZError' import BaseController from 'common/base.controller' import { role, router } from 'decorators/router' import { CheckIn } from 'models/CheckIn' +import { NftTransferEvent } from 'models/NftTransferEvent' import { getMonthBegin, getNDayAgo } from 'utils/date.util' @@ -52,5 +54,28 @@ class TaskController extends BaseController { const record = await CheckIn.findOne({from: address.toLowerCase()}).sort({count: -1}) return record.toJson() } + + @role('anon') + @router('post /task/nft/checkburn') + async checkNFTBurn(req, res) { + let { address, chain, user } = req.params + if (!address || !chain || !user) { + throw new ZError(10, 'params mismatch') + } + address = address.toLowerCase() + user = user.toLowerCase() + + let records = await NftTransferEvent.find({address, chain, from: user, to: ZERO_ADDRESS}).sort({_id: -1}) + let result = [] + for (let record of records) { + result.push({ + address: record.address, + chain: record.chain, + user: record.from, + tokenId: record.tokenId, + }) + } + return result; + } } export default TaskController diff --git a/src/events.ts b/src/events.ts new file mode 100644 index 0000000..2096b4f --- /dev/null +++ b/src/events.ts @@ -0,0 +1,66 @@ +import * as dotenv from 'dotenv' +import logger from 'logger/logger' +import { RedisClient } from 'redis/RedisClient' +const envFile = process.env.NODE_ENV && process.env.NODE_ENV === 'production' ? `.env.production` : '.env.development' +dotenv.config({ path: envFile }) + + + +import 'common/Extend' +import { AllChains, IChain } from 'chain/allchain' +import { EVENTS_CFG, IEventCfg } from 'config/events_cfg' +import { EventBatchSvr } from 'service/event.batch.service' + +let svrs: any[] = [] +let lock = false + + +async function initEventSvrs() { + const cfgMap: Map = new Map(); + for (let cfg of EVENTS_CFG) { + cfg.address = cfg.address.toLowerCase() + const chainCfg = AllChains.find((chain) => chain.id === cfg.chain) + if (!chainCfg) { + logger.error('chainCfg not found: ' + cfg.chain) + process.exit(1) + } + if (!cfgMap.has(chainCfg)) { + cfgMap.set(chainCfg, []) + } + cfgMap.get(chainCfg)?.push(cfg) + } + for (let chainCfg of cfgMap.keys()) { + const svr = new EventBatchSvr(chainCfg, cfgMap.get(chainCfg)!) + svrs.push(svr) + } +} + +async function parseAllEvents() { + if (lock) { + logger.warn('sync in process, cancel.') + return + } + lock = true + logger.info('begin sync events with chains: ' + svrs.length) + for (let svr of svrs) { + try { + await svr.execute() + } catch (err) { + logger.info('sync events with error:: chain: ' + svr.chainCfg.id ) + logger.info(err) + } + } + lock = false +} + +;(async () => { + let opts = { url: process.env.REDIS } + new RedisClient(opts) + logger.info('REDIS Connected') + await initEventSvrs() + setInterval(function () { + parseAllEvents() + }, 10000) + parseAllEvents() +})(); + diff --git a/src/models/NftHolder.ts b/src/models/NftHolder.ts new file mode 100644 index 0000000..cc716d1 --- /dev/null +++ b/src/models/NftHolder.ts @@ -0,0 +1,54 @@ +import { getModelForClass, index, modelOptions, prop } from '@typegoose/typegoose' +import { dbconn } from 'decorators/dbconn' +import { BaseModule } from './Base' +import { ZERO_ADDRESS } from 'common/Constants' + +@dbconn() +@index({ chain: 1, address: 1, tokenId: 1 }, { unique: true }) +@index({ chain: 1, address: 1, user: 1 }, { unique: false }) +@modelOptions({ + schemaOptions: { collection: 'nft_holder', timestamps: true }, +}) +export class NftHolderClass extends BaseModule { + @prop({ required: true }) + public address!: string + @prop({ required: true }) + public chain: string + @prop({ required: true }) + public tokenId: string + @prop() + public blockNumber: number + @prop() + public user: string + @prop({default: false}) + public burn: boolean + + + public static async saveData(event: any) { + const address = event.address; + const chain = event.chain; + const tokenId = event.tokenId; + const blockNumer = event.blockNumber; + const burn = event.to === ZERO_ADDRESS + + let record = await NftHolder.findOne({ address, chain, tokenId }) + if (!record) { + record = new NftHolder({ address, chain, tokenId, blockNumber: blockNumer, user: event.to, burn }) + } else { + if (record.blockNumber < blockNumer) { + if (burn) { + record.burn = true + } else { + record.user = event.to + } + record.blockNumber = blockNumer + } + } + await record.save(); + + } +} + +export const NftHolder = getModelForClass(NftHolderClass, { + existingConnection: NftHolderClass['db'], +}) diff --git a/src/models/NftTransferEvent.ts b/src/models/NftTransferEvent.ts index 61ed427..511dbc4 100644 --- a/src/models/NftTransferEvent.ts +++ b/src/models/NftTransferEvent.ts @@ -1,20 +1,26 @@ import { getModelForClass, index, modelOptions, prop } from '@typegoose/typegoose' import { dbconn } from 'decorators/dbconn' import { BaseModule } from './Base' +import { NftHolder } from './NftHolder' @dbconn() -@index({ tokenId: 1 }, { unique: false }) -@index({ transactionHash: 1, tokenId: 1, from: 1, to: 1 }, { unique: true }) +@index({ chain: 1, address: 1, tokenId: 1 }, { unique: false }) +@index({ chain: 1, address: 1, from: 1, to: 1 }, { unique: false }) +@index({ chain: 1, hash: 1, logIndex: 1}, { unique: true }) @modelOptions({ schemaOptions: { collection: 'nft_transfer_event', timestamps: true }, }) export class NftTransferEventClass extends BaseModule { @prop({ required: true }) public address!: string + @prop({ required: true }) + public chain: string + @prop({ required: true }) + public logIndex: number @prop() public event: string @prop({ required: true }) - public transactionHash: string + public hash: string @prop() public blockNumber: number @prop() @@ -33,28 +39,27 @@ export class NftTransferEventClass extends BaseModule { public version: number public static async saveEvent(event: any) { - if (!event.success) { - return - } - const tokenId = event.tokenId + const tokenId = event.tokenId || event.value if (!tokenId) { return } - const from = event.source - const to = event.target + const logIndex = parseInt(event.logIndex || '0') + const from = event.from.toLowerCase() + const to = event.to.toLowerCase() + const hash = event.hash || event.transactionHash const data = { - address: event.tokenAddress, - blockNumber: event.blockHeight, + address: event.address.toLowerCase(), + blockNumber: parseInt(event.blockNumber), removed: event.removed, from, to, - transactionHash: event.hash, tokenId, - blockTime: new Date(event.time).getTime(), + // blockTime: new Date(event.time).getTime(), $inc: { version: 1 }, } - - return NftTransferEvent.insertOrUpdate({ transactionHash: event.hash, tokenId, from, to }, data) + + let record = NftTransferEvent.insertOrUpdate({ hash, logIndex, chain: event.chain }, data) + await NftHolder.saveData(record) } } diff --git a/src/models/ScheduleConfirmEvent.ts b/src/models/ScheduleConfirmEvent.ts deleted file mode 100644 index 8922f0c..0000000 --- a/src/models/ScheduleConfirmEvent.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { getModelForClass, index, modelOptions, prop } from '@typegoose/typegoose' -import { dbconn } from 'decorators/dbconn' -import logger from 'logger/logger' -import { TaskSvr } from 'service/task.service' -import { BaseModule } from './Base' - -@dbconn() -@index({ transactionHash: 1 }, { unique: true }) -@modelOptions({ - schemaOptions: { collection: 'schedule_confirm_event', timestamps: true }, -}) -export class ScheduleConfirmEventClass extends BaseModule { - @prop({ required: true }) - public address!: string - @prop() - public event: string - @prop({ required: true }) - public transactionHash: string - @prop() - public blockNumber: number - @prop() - public blockHash: string - @prop() - public removed: boolean - @prop() - public operater: string - @prop({ type: () => [String] }) - public scheduleIds: string[] - @prop() - public blockTime: number - @prop({ default: 0 }) - public version: number - - public static async saveEvent(event: any) { - logger.info(JSON.stringify(event)) - if (event.removed) { - return - } - - const data = { - address: event.address, - blockNumber: event.blockNumber, - removed: event.removed, - operater: event.returnValues.sender, - scheduleIds: event.returnValues.ids, - transactionHash: event.transactionHash, - blockTime: new Date(event.timestamp).getTime(), - $inc: { version: 1 }, - } - - let record = await ScheduleConfirmEvent.insertOrUpdate({ transactionHash: event.transactionHash }, data) - if (record.version === 1) { - logger.log('receive events: ' + JSON.stringify(record.scheduleIds)) - for (let id of record.scheduleIds) { - await new TaskSvr().parseOneSchedule(id) - } - } - return record - } -} - -export const ScheduleConfirmEvent = getModelForClass(ScheduleConfirmEventClass, { - existingConnection: ScheduleConfirmEventClass['db'], -}) diff --git a/src/models/ScheduleExecutedEvent.ts b/src/models/ScheduleExecutedEvent.ts deleted file mode 100644 index 71b2105..0000000 --- a/src/models/ScheduleExecutedEvent.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { getModelForClass, index, modelOptions, prop } from '@typegoose/typegoose' -import { dbconn } from 'decorators/dbconn' -import { BaseModule } from './Base' - -@dbconn() -@index({ transactionHash: 1, scheduleId: 1 }, { unique: true }) -@modelOptions({ - schemaOptions: { collection: 'schedule_executed_event', timestamps: true }, -}) -export class ScheduleExecutedEventClass extends BaseModule { - @prop({ required: true }) - public address!: string - @prop() - public event: string - @prop({ required: true }) - public transactionHash: string - @prop() - public blockNumber: number - @prop() - public blockHash: string - @prop() - public removed: boolean - @prop() - public operater: string - @prop() - public scheduleId: string - @prop() - public blockTime: number - @prop({ default: 0 }) - public version: number - - public static async saveEvent(event: any) { - if (event.removed) { - return - } - - const data = { - address: event.address, - blockNumber: event.blockNumber, - removed: event.removed, - operater: event.returnValues.sender, - transactionHash: event.transactionHash, - blockTime: new Date(event.timestamp).getTime(), - $inc: { version: 1 }, - } - - let record = await ScheduleExecutedEvent.insertOrUpdate( - { transactionHash: event.transactionHash, scheduleId: event.returnValues.id }, - data, - ) - if (record.version === 1) { - } - return record - } -} - -export const ScheduleExecutedEvent = getModelForClass(ScheduleExecutedEventClass, { - existingConnection: ScheduleExecutedEventClass['db'], -}) diff --git a/src/models/ScheduledAddedEvent.ts b/src/models/ScheduledAddedEvent.ts deleted file mode 100644 index bb684a3..0000000 --- a/src/models/ScheduledAddedEvent.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { getModelForClass, index, modelOptions, prop } from '@typegoose/typegoose' -import { dbconn } from 'decorators/dbconn' -import { BaseModule } from './Base' - -@dbconn() -@index({ transactionHash: 1, scheduleId: 1 }, { unique: true }) -@modelOptions({ - schemaOptions: { collection: 'schedule_added_event', timestamps: true }, -}) -export class ScheduledAddedEventClass extends BaseModule { - @prop({ required: true }) - public address!: string - @prop() - public event: string - @prop({ required: true }) - public transactionHash: string - @prop() - public blockNumber: number - @prop() - public blockHash: string - @prop() - public removed: boolean - @prop() - public operater: string - @prop() - public scheduleId: string - @prop() - public blockTime: number - @prop({ default: 0 }) - public version: number - - public static async saveEvent(event: any) { - if (event.removed) { - return - } - - const data = { - address: event.address, - blockNumber: event.blockNumber, - removed: event.removed, - operater: event.returnValues.sender, - transactionHash: event.transactionHash, - blockTime: new Date(event.timestamp).getTime(), - $inc: { version: 1 }, - } - - return ScheduledAddedEvent.insertOrUpdate( - { transactionHash: event.transactionHash, scheduleId: event.returnValues.id }, - data, - ) - } -} - -export const ScheduledAddedEvent = getModelForClass(ScheduledAddedEventClass, { - existingConnection: ScheduledAddedEventClass['db'], -}) diff --git a/src/monitor.ts b/src/monitor.ts index 226ad42..301df11 100644 --- a/src/monitor.ts +++ b/src/monitor.ts @@ -6,8 +6,6 @@ dotenv.config({ path: envFile }) import { EventSyncSvr } from 'service/event.sync.service' import { NftTransferEvent } from 'models/NftTransferEvent' import { FtTransferEvent } from 'models/FtTransferEvent' -import { ScheduleConfirmEvent } from 'models/ScheduleConfirmEvent' -import { ScheduleExecutedEvent } from 'models/ScheduleExecutedEvent' import 'common/Extend' @@ -17,14 +15,11 @@ let lock = false let eventProcessers = { NftTransferEvent: NftTransferEvent, FtTransferEvent: FtTransferEvent, - ScheduleConfirmEvent: ScheduleConfirmEvent, - ScheduleExecutedEvent: ScheduleExecutedEvent, } const events = require('config/events.json') async function initEventSvrs() { - // let nfts = [{ address: '0x37c30a2945799a53c5358636a721b442458fa691' }] for (let event of events) { let eventSvr = new EventSyncSvr({ address: event.address, diff --git a/src/service/block.sync.service.ts b/src/service/block.sync.service.ts index a1b0a9a..e7f609f 100644 --- a/src/service/block.sync.service.ts +++ b/src/service/block.sync.service.ts @@ -1,7 +1,6 @@ import { IChain } from "chain/allchain"; import { ethBlockNumber } from "chain/chain.api"; import { IScriptionCfg } from "config/scriptions_cfg"; -import { BlockData } from "models/BlockData"; import { RedisClient } from "redis/RedisClient"; import { getPastBlocksIter } from "utils/block.util"; import { formatDate } from "utils/date.util"; @@ -32,6 +31,7 @@ export class BlockSyncSvr { if (blockStr) { this.fromBlock = Math.max(parseInt(blockStr), this.fromBlock) } + //@ts-ignore const amount = parseInt(currentBlock.result, 16) - this.fromBlock let blocks = getPastBlocksIter({ chainId: this.chainCfg.id, @@ -49,7 +49,7 @@ export class BlockSyncSvr { for (const getPastBlockPromise of iterator) { const blocks = await getPastBlockPromise for (const block of blocks) { - await BlockData.saveBlock(block) + // await BlockData.saveBlock(block) if (!block.transactions || block.transactions.length === 0) { continue } diff --git a/src/service/event.batch.service.ts b/src/service/event.batch.service.ts new file mode 100644 index 0000000..955b523 --- /dev/null +++ b/src/service/event.batch.service.ts @@ -0,0 +1,103 @@ +import { IChain } from "chain/allchain"; +import { batchEthLogs, ethBlockNumber } from "chain/chain.api"; +import { IEventCfg } from "config/events_cfg"; +import logger from "logger/logger"; + +import { RedisClient } from "redis/RedisClient"; +import { getPastBlocksIter } from "utils/block.util"; +import { getTopics } from "utils/event.util"; +import web3abi from 'web3-eth-abi'; + + +export class EventBatchSvr { + chainCfg: IChain + eventCfgs: IEventCfg[] = [] + processer: Map = new Map() + fromBlock: number = Number.MAX_SAFE_INTEGER + redisKey = '' + rpc = ''; + constructor(_chainCfg: IChain, _eventCfgs: IEventCfg[]) { + this.chainCfg =_chainCfg + this.eventCfgs = _eventCfgs + this.rpc = _chainCfg.rpc.split('|')[0] + for (let cfg of this.eventCfgs) { + this.fromBlock = Math.min(this.fromBlock, cfg.fromBlock) + if (!cfg.topic) { + cfg.topic = getTopics(cfg) + } + this.processer.set(cfg.address+cfg.topic, cfg) + } + this.redisKey = `event_${this.chainCfg.id}` + } + + async execute() { + logger.info(`begin sync events with chain: ${this.chainCfg.id}`) + try { + // let currentBlock = await ethBlockNumber(this.rpc) + let params = [] + for (let cfg of this.eventCfgs) { + let param = await this.buildQueryParams(cfg) + params.push(param) + } + let results = await batchEthLogs(this.rpc, params) + for (let i = 0; i < results.length; i++) { + if (results[i].error) { + console.log(results[i].error) + continue + } + let events = results[i].result + await this.processEvents(events) + } + } catch (err) { + console.log(err) + } + } + + buildRedisKey(cfg: IEventCfg) { + return `event_${this.chainCfg.id}_${cfg.address}_${cfg.event}` + } + + async buildQueryParams(cfg: IEventCfg, toBlock?: string) { + const redisKey = this.buildRedisKey(cfg) + + const params: any = { + fromBlock: cfg.fromBlock, + toBlock: toBlock || 'latest', + address: cfg.address + } + let blockStr = await new RedisClient().get(redisKey) + if (blockStr) { + params.fromBlock = Math.max(parseInt(blockStr), cfg.fromBlock) + } + params.fromBlock = '0x' + params.fromBlock.toString(16) + params.topics = [getTopics(cfg)] + const result = { + jsonrpc: "2.0", + method: "eth_getLogs", + params: [params], + id: `${cfg.address}_${cfg.event}` + } + return result + } + + async processEvents(events: any[]) { + if (events.length === 0) { + return + } + const address = events[0].address + const topic = events[0].topics[0] + const cfg = this.processer.get(address+topic) + logger.info(`process events: ${cfg.chain} | ${address} | ${cfg.event} | ${events.length}`) + const abiInputs = cfg.abi.inputs; + for (const event of events) { + let result = web3abi.decodeLog(abiInputs, event.data, event.topics.slice(1)); + result.chain = this.chainCfg.id + '' + cfg.fromBlock = Math.max (parseInt(event.blockNumber, 16) + 1, cfg.fromBlock) + Object.assign(result, event) + // @ts-ignore + await cfg.eventProcesser.saveEvent(result) + } + const redisKey = this.buildRedisKey(cfg) + await new RedisClient().set(redisKey, cfg.fromBlock + '') + } +} \ No newline at end of file diff --git a/src/utils/event.util.ts b/src/utils/event.util.ts new file mode 100644 index 0000000..00ce6df --- /dev/null +++ b/src/utils/event.util.ts @@ -0,0 +1,13 @@ +import { IEventCfg } from "config/events_cfg"; +import { keccak256 } from "web3-utils"; + +export const getTopics = (cfg: IEventCfg) => { + let abi = cfg.abi + let topic = 'Transfer(' + for (let item of abi.inputs) { + topic += item.type + ',' + } + topic = topic.slice(0, -1) + topic += ')' + return keccak256(topic) +} \ No newline at end of file diff --git a/start_dev.json b/start_dev.json new file mode 100644 index 0000000..d31cafc --- /dev/null +++ b/start_dev.json @@ -0,0 +1,22 @@ +{ + "apps": [ + { + "name": "chain-client", + "script": "npm", + "args": "run prod:api", + "cwd": "/data/apps/web_chain_client", + "max_memory_restart": "1024M", + "log_date_format": "YYYY-MM-DD HH:mm Z", + "watch": false, + "ignore_watch": ["node_modules", "logs", "fixtures", "tasks"], + "instances": 1, + "exec_mode": "fork", + "env": { + "NODE_ENV": "production" + }, + "env_production": { + "NODE_ENV": "production" + } + } + ] +} diff --git a/yarn.lock b/yarn.lock index 1dbddf0..ebbb770 100644 --- a/yarn.lock +++ b/yarn.lock @@ -245,11 +245,6 @@ "@ethersproject/properties" "^5.7.0" "@ethersproject/strings" "^5.7.0" -"@fastify/accept-negotiator@^1.0.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@fastify/accept-negotiator/-/accept-negotiator-1.1.0.tgz#c1c66b3b771c09742a54dd5bc87c582f6b0630ff" - integrity sha512-OIHZrb2ImZ7XG85HXOONLcJWGosv7sIvM2ifAPQVhg9Lv7qdmMBNVaai4QTdyuaqbKM5eO6sLSQOYI7wEQeCJQ== - "@fastify/ajv-compiler@^3.5.0": version "3.5.0" resolved "https://registry.yarnpkg.com/@fastify/ajv-compiler/-/ajv-compiler-3.5.0.tgz#459bff00fefbf86c96ec30e62e933d2379e46670" @@ -311,38 +306,6 @@ fastify-plugin "^4.0.0" steed "^1.1.3" -"@fastify/send@^2.0.0": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@fastify/send/-/send-2.0.1.tgz#db10d1401883b4aef41669fcf2ddb4e1bb4630df" - integrity sha512-8jdouu0o5d0FMq1+zCKeKXc1tmOQ5tTGYdQP3MpyF9+WWrZT1KCBdh6hvoEYxOm3oJG/akdE9BpehLiJgYRvGw== - dependencies: - "@lukeed/ms" "^2.0.1" - escape-html "~1.0.3" - fast-decode-uri-component "^1.0.1" - http-errors "2.0.0" - mime "^3.0.0" - -"@fastify/static@^6.10.0": - version "6.10.0" - resolved "https://registry.yarnpkg.com/@fastify/static/-/static-6.10.0.tgz#cdb6a5ddcc3ea8691c79aad0c846bc986a4bc721" - integrity sha512-TGruNm6ZabkQz2oRNoarPnY2BvS9i9DNf8Nn1aDcZp+WjOQRPCq0Wy2ko78yGB5JHytdCWoHpprc128QtLl8hw== - dependencies: - "@fastify/accept-negotiator" "^1.0.0" - "@fastify/send" "^2.0.0" - content-disposition "^0.5.3" - fastify-plugin "^4.0.0" - glob "^8.0.1" - p-limit "^3.1.0" - readable-stream "^4.0.0" - -"@fastify/view@^7.4.1": - version "7.4.1" - resolved "https://registry.yarnpkg.com/@fastify/view/-/view-7.4.1.tgz#265daba48386a5d3f69dfc446af468d72e0a8757" - integrity sha512-ahmRmSbNVM8bIoz0BAFnY0jNigom+xbPQ9Q1ZjmNOtGVVT3nYXCxw2OMkTr9iXwrJ4Le3EtWDHlFkZ2fCQ2hJA== - dependencies: - fastify-plugin "^4.0.0" - hashlru "^2.3.0" - "@humanwhocodes/config-array@^0.5.0": version "0.5.0" resolved "https://registry.npmmirror.com/@humanwhocodes/config-array/-/config-array-0.5.0.tgz#1407967d4c6eecd7388f83acf1eaf4d0c6e58ef9" @@ -375,7 +338,7 @@ "@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/sourcemap-codec" "^1.4.10" -"@lukeed/ms@^2.0.0", "@lukeed/ms@^2.0.1": +"@lukeed/ms@^2.0.0": version "2.0.1" resolved "https://registry.yarnpkg.com/@lukeed/ms/-/ms-2.0.1.tgz#3c2bbc258affd9cc0e0cc7828477383c73afa6ee" integrity sha512-Xs/4RZltsAL7pkvaNStUQt7netTkyxrS0K+RILcVr3TRMS/ToOg4I6uNfhB9SlGsnWBym4U+EaXq0f0cEMNkHA== @@ -672,11 +635,6 @@ "@typescript-eslint/types" "4.33.0" eslint-visitor-keys "^2.0.0" -"@wecom/crypto@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@wecom/crypto/-/crypto-1.0.1.tgz#6918ed9829043b06075eaa8cff84de2475476a58" - integrity sha512-K4Ilkl1l64ceJDbj/kflx8ND/J88pcl8tKx4Ivp7IiCrshRJU+Uo5uWCjAa+PjUiLIdcQSZ4m4d0t1npMPCX5A== - abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" @@ -863,11 +821,6 @@ async-limiter@~1.0.0: resolved "https://registry.npmmirror.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd" integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== -async@^3.2.3: - version "3.2.4" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" - integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== - asynckit@^0.4.0: version "0.4.0" resolved "https://registry.npmmirror.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -1007,13 +960,6 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" -brace-expansion@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" - integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== - dependencies: - balanced-match "^1.0.0" - braces@^3.0.2, braces@~3.0.2: version "3.0.2" resolved "https://registry.npmmirror.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" @@ -1221,7 +1167,7 @@ chalk@^2.0.0, chalk@^2.3.0: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^4.0.0, chalk@^4.0.2: +chalk@^4.0.0: version "4.1.2" resolved "https://registry.npmmirror.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -1331,7 +1277,7 @@ concat-map@0.0.1: resolved "https://registry.npmmirror.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== -content-disposition@0.5.4, content-disposition@^0.5.3: +content-disposition@0.5.4: version "0.5.4" resolved "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== @@ -1638,13 +1584,6 @@ ee-first@1.1.1: resolved "https://registry.npmmirror.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== -ejs@^3.1.9: - version "3.1.9" - resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.9.tgz#03c9e8777fe12686a9effcef22303ca3d8eeb361" - integrity sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ== - dependencies: - jake "^10.8.5" - elliptic@6.5.4, elliptic@^6.4.0, elliptic@^6.5.2, elliptic@^6.5.3, elliptic@^6.5.4: version "6.5.4" resolved "https://registry.npmmirror.com/elliptic/-/elliptic-6.5.4.tgz#da37cebd31e79a1367e941b592ed1fbebd58abbb" @@ -2295,13 +2234,6 @@ file-entry-cache@^6.0.1: dependencies: flat-cache "^3.0.4" -filelist@^1.0.1: - version "1.0.4" - resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.4.tgz#f78978a1e944775ff9e62e744424f215e58352b5" - integrity sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q== - dependencies: - minimatch "^5.0.1" - fill-range@^7.0.1: version "7.0.1" resolved "https://registry.npmmirror.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" @@ -2496,17 +2428,6 @@ glob@^7.1.1, glob@^7.1.3: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^8.0.1: - version "8.1.0" - resolved "https://registry.yarnpkg.com/glob/-/glob-8.1.0.tgz#d388f656593ef708ee3e34640fdfb99a9fd1c33e" - integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ== - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^5.0.1" - once "^1.3.0" - global@~4.4.0: version "4.4.0" resolved "https://registry.npmmirror.com/global/-/global-4.4.0.tgz#3e7b105179006a323ed71aafca3e9c57a5cc6406" @@ -2646,11 +2567,6 @@ hash.js@1.1.7, hash.js@^1.0.0, hash.js@^1.0.3, hash.js@^1.1.7: inherits "^2.0.3" minimalistic-assert "^1.0.1" -hashlru@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/hashlru/-/hashlru-2.3.0.tgz#5dc15928b3f6961a2056416bb3a4910216fdfb51" - integrity sha512-0cMsjjIC8I+D3M44pOQdsy0OHXGLVz6Z0beRuufhKa0KfaD2wGwAev6jILzXsd3/vpnNQJmWyZtIILqM1N+n5A== - helmet@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/helmet/-/helmet-6.0.1.tgz#52ec353638b2e87f14fe079d142b368ac11e79a4" @@ -2955,16 +2871,6 @@ isstream@~0.1.2: resolved "https://registry.npmmirror.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" integrity sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g== -jake@^10.8.5: - version "10.8.5" - resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.5.tgz#f2183d2c59382cb274226034543b9c03b8164c46" - integrity sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw== - dependencies: - async "^3.2.3" - chalk "^4.0.2" - filelist "^1.0.1" - minimatch "^3.0.4" - js-sha3@0.8.0, js-sha3@^0.8.0: version "0.8.0" resolved "https://registry.npmmirror.com/js-sha3/-/js-sha3-0.8.0.tgz#b9b7a5da73afad7dedd0f8c463954cbde6818840" @@ -3213,11 +3119,6 @@ mime@1.6.0: resolved "https://registry.npmmirror.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== -mime@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/mime/-/mime-3.0.0.tgz#b374550dca3a0c18443b0c950a6a58f1931cf7a7" - integrity sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A== - mimic-response@^1.0.0: version "1.0.1" resolved "https://registry.npmmirror.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" @@ -3252,13 +3153,6 @@ minimatch@^3.0.4, minimatch@^3.1.1: dependencies: brace-expansion "^1.1.7" -minimatch@^5.0.1: - version "5.1.6" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.6.tgz#1cfcb8cf5522ea69952cd2af95ae09477f122a96" - integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g== - dependencies: - brace-expansion "^2.0.1" - minimist@^1.2.0, minimist@^1.2.6: version "1.2.6" resolved "https://registry.npmmirror.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" @@ -3446,6 +3340,13 @@ node-addon-api@^2.0.0: resolved "https://registry.npmmirror.com/node-addon-api/-/node-addon-api-2.0.2.tgz#432cfa82962ce494b132e9d72a15b29f71ff5d32" integrity sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA== +node-fetch@2: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-fetch@2.6.7: version "2.6.7" resolved "https://registry.npmmirror.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" @@ -3580,13 +3481,6 @@ p-cancelable@^3.0.0: resolved "https://registry.npmmirror.com/p-cancelable/-/p-cancelable-3.0.0.tgz#63826694b54d61ca1c20ebcb6d3ecf5e14cd8050" integrity sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw== -p-limit@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" - integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== - dependencies: - yocto-queue "^0.1.0" - parent-module@^1.0.0: version "1.0.1" resolved "https://registry.npmmirror.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -5204,8 +5098,3 @@ yn@3.1.1: version "3.1.1" resolved "https://registry.npmmirror.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== - -yocto-queue@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" - integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==