From 5343bf4456a73ebdce19ca7374501a685341fcdf Mon Sep 17 00:00:00 2001 From: aozhiwei Date: Mon, 10 Jul 2023 10:31:09 +0800 Subject: [PATCH] 1 --- database/bcevent.sql | 22 ++ server/common/constant.js | 2 + .../web3bcspider/services/erc721_refresher.js | 231 ++++++++++++++++++ 3 files changed, 255 insertions(+) diff --git a/database/bcevent.sql b/database/bcevent.sql index d4ed891..0a3b767 100644 --- a/database/bcevent.sql +++ b/database/bcevent.sql @@ -135,6 +135,28 @@ CREATE TABLE `t_dbprocess_last_idx` ( ) ENGINE=InnoDB AUTO_INCREMENT=10001 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; /*!40101 SET character_set_client = @saved_cs_client */; +-- +-- Table structure for table `t_erc721_refresh` +-- + +DROP TABLE IF EXISTS `t_erc721_refresh`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t_erc721_refresh` ( + `idx` bigint NOT NULL AUTO_INCREMENT COMMENT '自增id', + `net_id` bigint NOT NULL DEFAULT '0' COMMENT 'net_id', + `contract_address` varchar(60) NOT NULL DEFAULT '' COMMENT 'contract_address', + `contract_name` varchar(60) NOT NULL DEFAULT '' COMMENT 'contract_name', + `token_id` varchar(60) NOT NULL DEFAULT '' COMMENT 'token_id', + `state` int(11) NOT NULL DEFAULT '0' COMMENT 'state', + `createtime` int(11) NOT NULL DEFAULT '0' COMMENT '创建时间', + `modifytime` int(11) NOT NULL DEFAULT '0' COMMENT '修改时间', + PRIMARY KEY (`idx`), + UNIQUE KEY `unikey` (`net_id`, `contract_address`, `token_id`), + KEY `net_id_contract_address_state` (`net_id`, `contract_address`, `state`) +) ENGINE=InnoDB AUTO_INCREMENT=10001 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; +/*!40101 SET character_set_client = @saved_cs_client */; + -- -- Table structure for table `t_721nft_transfer` -- diff --git a/server/common/constant.js b/server/common/constant.js index e6cc926..71bc01d 100644 --- a/server/common/constant.js +++ b/server/common/constant.js @@ -2,7 +2,9 @@ const EVENTDB_STATE_PENDING = 0; const EVENTDB_STATE_HANDLED = 1; const GAMEDB_NAME = 'GameDb0'; +const BCEVENTDB_NAME = 'BcEventDb0'; exports.EVENTDB_STATE_PENDING = EVENTDB_STATE_PENDING; exports.EVENTDB_STATE_HANDLED = EVENTDB_STATE_HANDLED; exports.GAMEDB_NAME = GAMEDB_NAME; +exports.BCEVENTDB_NAME = BCEVENTDB_NAME; diff --git a/server/web3bcspider/services/erc721_refresher.js b/server/web3bcspider/services/erc721_refresher.js index e69de29..bc1da7e 100644 --- a/server/web3bcspider/services/erc721_refresher.js +++ b/server/web3bcspider/services/erc721_refresher.js @@ -0,0 +1,231 @@ +const app = require('j7/app'); +const utils = require('j7/utils'); +const bcutils = require('j7/bcutils'); +const log = require('j7/log'); +const constant = require('common/constant'); +const BaseService = require('./baseservice'); + +class Erc721Refresher extends BaseService { + + async init(bc, net, event) { + const {err, conn} = await app.getDbConn(constant.BCEVENTDB_NAME); + this.conn = conn; + this.lastBlockNumber = 0; + this.bc = bc; + this.net = net; + this.event = event; + this.eventConf = this.event['eventConf']; + this.progInfo = this.event['progressInfo']; + await this.start(); + } + + async start() { + while (true) { + await this.pullEvent(); + await utils.sleep(8000 + utils.randRange(500, 1500)); + } + } + + async pullEvent() { + const logHead = this.getInstanceName() + ' pullEvent: '; + while (true) { + await this.bc.lockQuery(); + try { + const fromBlock = await this.getFromBlock(); + const toBlock = await this.calcToBlock(fromBlock); + if (toBlock > fromBlock) { + const events = await this.bc.getPastEvents( + this.getContractName(), + this.getEventName(), + { + fromBlock: fromBlock, + toBlock: toBlock, + }, + ); + await this.processEvents(events, fromBlock, toBlock); + await this.saveLastBlockNumber(toBlock); + } + ++this.progInfo['pullCount']; + return; + } catch (err) { + log.error(logHead + err); + await utils.sleep(1000 + utils.randRange(10, 2000)); + } finally { + await this.bc.unlockQuery(); + } + } + } + + async processEvents(events, fromBlock, toBlock) { + this.progInfo['fromBlock'] = fromBlock; + this.progInfo['toBlock'] = toBlock; + this.progInfo['currBlock'] = this.bc.getCurrBlockNumber(); + this.progInfo['eventCount'] += events.length; + if (events.length <= 0) { + return; + } + console.log(events); + utils.serial + (events, + async (event) => { + while (true) { + try { + await this.saveToDb(event); + return; + } catch (err) { + log.error(err); + } + await utils.sleep(8000 + utils.randRange(500, 1500)); + } + }); + } + + async getFromBlock() { + const logHead = this.getInstanceName() + ' getFromBlock: '; + const firstBlockNumber = this.getInitBlock(); + while (this.lastBlockNumber < 1) { + try { + const {err, row} = await this.conn.ormSelectOne( + 't_last_block', + [ + ['net_id', this.getNetId()], + ['contract_address', this.getContractAddress()], + ['event_name', this.getEventName()], + ] + ); + if (!err) { + if (row) { + this.lastBlockNumber = Number(row['block_number']); + } else { + this.lastBlockNumber = firstBlockNumber; + } + } + console.log(logHead, this.lastBlockNumber, this.bc.getCurrBlockNumber()); + while (this.lastBlockNumber + 8 > this.bc.getCurrBlockNumber()) { + await utils.sleep(1000 + utils.randRange(500, 1500)); + } + continue; + } catch (err) { + log.error(err); + } + await utils.sleep(5000 + utils.randRange(500, 1500)); + } + return this.lastBlockNumber + 1; + } + + async calcToBlock(fromBlock) { + const currBlockNumber = this.bc.getCurrBlockNumber(); + const distanceBlock = currBlockNumber - fromBlock - 8; + const batchBlockNum = 888; + if (distanceBlock > 0) { + if (distanceBlock > batchBlockNum) { + return fromBlock + batchBlockNum; + } else { + return fromBlock + distanceBlock; + } + } + return fromBlock; + } + + async saveLastBlockNumber(blockNumber) { + const logHead = this.getInstanceName() + ' event_process.saveLastBlockNumber: '; + while (true) { + const {err} = await this.conn.upsert( + 't_last_block', + [ + ['net_id', this.getNetId()], + ['contract_address', this.getContractAddress()], + ['event_name', this.getEventName()], + ], + [ + ['block_number', blockNumber], + ['modifytime', utils.getUtcTime()], + ], + [ + ['net_id', this.getNetId()], + ['contract_address', this.getContractAddress()], + ['event_name', this.getEventName()], + ['block_number', blockNumber], + ['contract_name', this.getContractName()], + ['createtime', utils.getUtcTime()], + ['modifytime', utils.getUtcTime()], + ] + ); + if (!err) { + break; + } + log.error(logHead + err); + await utils.sleep(5000 + utils.randRange(500, 1500)); + } + this.lastBlockNumber = blockNumber; + } + + async saveToDb(event) { + const logHead = this.getInstanceName() + ' event_process.saveToDb: '; + while (true) { + const nowTime = utils.getUtcTime(); + const returnValues = event['returnValues']; + const hashCode = ''; + const {err} = await this.conn.upsert( + 't_blockchain_event', + [ + ['txhash', event['transactionHash']], + ['hash_code', hashCode], + ['log_index', event['logIndex']], + ['net_id', this.bc.getNetId()], + ['event_name', this.getEventName()], + ['contract_address', this.getContractAddress()], + ], + [ + ], + [ + ['txhash', event['transactionHash']], + ['hash_code', hashCode], + ['log_index', event['logIndex']], + ['net_id', this.bc.getNetId()], + ['event_name', this.getEventName()], + ['contract_address', this.getContractAddress()], + ['contract_name', this.getContractName()], + ['block_number', event['blockNumber']], + ['raw_data', utils.jsonEncode(event)], + ['return_values', utils.jsonEncode(returnValues)], + ['createtime', utils.getUtcTime()], + ['modifytime', utils.getUtcTime()], + ] + ); + if (!err) { + break; + } + log.error(logHead + err); + await utils.sleep(5000 + utils.randRange(500, 1500)); + } + } + + getNetId() { + return this.net['net_id']; + } + + getEventName() { + return this.eventConf['event_name']; + } + + getInitBlock() { + return this.eventConf['init_block']; + } + + getContractAddress() { + return this.bc.getContractAddressByName(this.getContractName()); + } + + getContractName() { + return this.eventConf['contract_name']; + } + + getInstanceName() { + const instName = this.getNetId() + '.' + this.getContractName() + '.' + this.getEventName(); + return instName; + } + +} + +module.exports = Erc721Refresher;