import { batchEthBlocks } from "chain/chain.api"; import { IScriptionCfg } from "interface/IScriptionCfg"; import logger from "logger/logger"; import { RedisClient } from "redis/RedisClient"; import { utf8ToHex } from "./string.util"; const MAX_BATCH_AMOUNT = +process.env.MAX_BLOCK_BATCH_AMOUNT const REQUEST_INTERVAL = 0.5 * 1000 export async function divQueryPassBlocks({chainId, rpc, fromBlock, amount} : {chainId: number, rpc: string, fromBlock: number, amount: number}) { const middleBlock = fromBlock + Math.floor(amount / 2) const firstBlocks = await getPastBlocks({chainId, rpc, fromBlock, amount: middleBlock - fromBlock}) const secondBlocks = await getPastBlocks({chainId, rpc, fromBlock: middleBlock, amount: amount - (middleBlock - fromBlock)}) return [...firstBlocks, ...secondBlocks] } export async function getPastBlocks({chainId, rpc, fromBlock, amount} : {chainId: number, rpc: string, fromBlock: number, amount: number}) { let blocks = [] logger.info(`getPastBlocks: ${chainId} from: ${fromBlock} amount: ${amount}`) let blockNumber = fromBlock const redisKey = `blocknum_${chainId}` let retryCount = 0; const parseBlocksAndRetry = async (blockNums: number[]) => { let records = await batchEthBlocks(rpc, blockNums) if (records.error) { throw new Error(records.error.message) } let realAmount = 0 let retryNums: number[] = [] let maxBlockNumber = 0 for (let i = 0; i < records.length; i++) { const block = records[i].result; if (block?.hash) { blocks.push(block) realAmount++ maxBlockNumber = Math.max(maxBlockNumber, blockNumber + i) } else { if (block) { logger.warn(`block ${blockNumber + i}: ${block}`) } else { logger.warn(`block ${blockNumber + i} is null`) retryNums.push(blockNumber + i) } } } if (retryNums.length > 0 && ++retryCount < 3) { logger.info(`${retryCount} retry ${retryNums.length} blocks`) const retryBlocks = await parseBlocksAndRetry(retryNums) realAmount += retryBlocks } return realAmount } try { const numsArr = Array.from({length: amount}, (v, k) => k + blockNumber) const realAmount = await parseBlocksAndRetry(numsArr) if (retryCount > 0) { blocks.sort((a, b) => parseInt(a.number) - parseInt(b.number)) } await new RedisClient().set(redisKey, blockNumber + amount + '') await new Promise(resolve => setTimeout(resolve, REQUEST_INTERVAL)) } catch (e) { logger.log(e.message || e) if (e.message && /Too Many Requests/.test(e.message) && amount > 1) { blocks = await divQueryPassBlocks({chainId, rpc, fromBlock, amount}) } else if (e.message && /Public RPC Rate Limit Hit, limit will reset in \d+ seconds/.test(e.message)) { const match = e.message.match(/Public RPC Rate Limit Hit, limit will reset in (\d+) seconds/) const seconds = parseInt(match[1]) await new Promise(resolve => setTimeout(resolve, seconds * 1000)) blocks = await getPastBlocks({chainId, rpc, fromBlock, amount}) }else { throw e } } return blocks } export function* getPastBlocksIter({chainId, rpc, fromBlock, amount} : {chainId: number, rpc: string, fromBlock: number, amount: number}) { logger.info(`*getPastBlocksIter: ${chainId} from: ${fromBlock} amount: ${amount}`) let remain = amount while (remain > 0) { yield getPastBlocks({chainId, rpc, fromBlock, amount: Math.min(MAX_BATCH_AMOUNT, remain)}) fromBlock += MAX_BATCH_AMOUNT remain -= MAX_BATCH_AMOUNT } } export const buildScriptionFilters = (cfg: IScriptionCfg) => { if (cfg.filter) { return cfg.filter } if (cfg.filters) { let body = '' for (let i = 0; i < cfg.filters.length; i++) { if (i > 0) { body += ' && ' } let filter = cfg.filters[i] let value: any = filter.value let op = '' switch (filter.op) { case 'eq': op = '===' break case 'ne': op = '!==' break case 'gt': op = '>' break case 'gte': op = '>=' break case 'lt': op = '<' break case 'lte': op = '<=' break case 'in': op = 'in' break case 'nin': op = 'nin' break case 'like': op = 'like' break case 'nlike': op = 'nlike' break case 'isNull': body += `!event.${filter.key}` break case 'isNotNull': body += `!!event.${filter.key}` break } if (filter.type === 'address') { value = `'${value.toLowerCase()}'` } else if (filter.type === 'utf8_data') { value = `'0x${utf8ToHex(value)}'` } else if (filter.type === 'hex_data') { value = `'${value.indexOf('0x') === 0 ? value : '0x'+value}'` } else if (filter.type === 'number') { value = parseInt(value) } else if (filter.type === 'boolean') { value = !!value } else { value = `'${value}'` } if (op) { if (op === 'in') { body += `event.${filter.key}.indexOf(${value}) >= 0` } else if (op === 'nin') { body += `!(event.${filter.key}.indexOf(${value}) >= 0)` } else if (op === 'nlike') { body += `!new RegExp(${value}).test(event.${filter.key})` } else if (op === 'like') { body += new RegExp(value).test(`event.${filter.key}`) } else { body += `event.${filter.key}${op}${value}` } } } return new Function('event', `return ${body}`) } }