site-activity-chain/src/utils/block.util.ts

import { batchEthBlocks } from "chain/chain.api";
import { IScriptionCfg } from "interface/IScriptionCfg";
import logger from "logger/logger";
import { RedisClient } from "redis/RedisClient";
import { utf8ToHex } from "./string.util";

const MAX_BATCH_AMOUNT = +process.env.MAX_BLOCK_BATCH_AMOUNT
const REQUEST_INTERVAL = 0.5 * 1000

export async function divQueryPassBlocks({chainId, rpc, fromBlock, amount}
  : {chainId: number, rpc: string, fromBlock: number, amount: number}) {
  const middleBlock = fromBlock + Math.floor(amount / 2)
  const firstBlocks = await getPastBlocks({chainId, rpc, fromBlock, amount: middleBlock - fromBlock})
  const secondBlocks = await getPastBlocks({chainId, rpc, fromBlock: middleBlock, amount: amount - (middleBlock - fromBlock)})
  return [...firstBlocks, ...secondBlocks]
}

export async function getPastBlocks({chainId, rpc, fromBlock, amount}
  : {chainId: number, rpc: string, fromBlock: number, amount: number}) {
  let blocks = []
  logger.info(`getPastBlocks: ${chainId} from: ${fromBlock} amount: ${amount}`)
  let blockNumber = fromBlock
  const redisKey = `blocknum_${chainId}`
  let retryCount = 0;
  const parseBlocksAndRetry = async (blockNums: number[]) => {
    let records = await batchEthBlocks(rpc, blockNums)
    if (records.error) {
      throw new Error(records.error.message)
    }
    let realAmount = 0
    let retryNums: number[] = []
    let maxBlockNumber = 0
    for (let i = 0; i < records.length; i++) {
      const block = records[i].result;
      if (block?.hash) {
        blocks.push(block)
        realAmount++
        maxBlockNumber = Math.max(maxBlockNumber, blockNumber + i)
      } else {
        if (block) {
          logger.warn(`block ${blockNumber + i}: ${block}`)
        } else {
          logger.warn(`block ${blockNumber + i} is null`)
          retryNums.push(blockNumber + i)
        }
      }
    }
    if (retryNums.length > 0 && ++retryCount < 3) {
      logger.info(`${retryCount} retry ${retryNums.length} blocks`)
      const retryBlocks = await parseBlocksAndRetry(retryNums)
      realAmount += retryBlocks
    }
    return realAmount
  }
  try {
    const numsArr = Array.from({length: amount}, (v, k) => k + blockNumber)
    const realAmount = await parseBlocksAndRetry(numsArr)
    if (retryCount > 0) {
      blocks.sort((a, b) => parseInt(a.number) - parseInt(b.number))
    }
    await new RedisClient().set(redisKey, blockNumber + amount + '')
    await new Promise(resolve => setTimeout(resolve, REQUEST_INTERVAL))
  } catch (e) {
    logger.log(e.message || e)
    if (e.message && /Too Many Requests/.test(e.message) && amount > 1) {
      blocks = await divQueryPassBlocks({chainId, rpc, fromBlock, amount})
    } else if (e.message && /Public RPC Rate Limit Hit, limit will reset in \d+ seconds/.test(e.message)) {
      const match = e.message.match(/Public RPC Rate Limit Hit, limit will reset in (\d+) seconds/)
      const seconds = parseInt(match[1])
      await new Promise(resolve => setTimeout(resolve, seconds * 1000))
      blocks = await getPastBlocks({chainId, rpc, fromBlock, amount})
    }else {
      throw e
    }
  }
  return blocks
}

export function* getPastBlocksIter({chainId, rpc, fromBlock, amount}
  : {chainId: number, rpc: string, fromBlock: number, amount: number}) {
  logger.info(`*getPastBlocksIter: ${chainId} from: ${fromBlock} amount: ${amount}`)
  let remain = amount
  while (remain > 0) {
    yield getPastBlocks({chainId, rpc, fromBlock, amount: Math.min(MAX_BATCH_AMOUNT, remain)})
    fromBlock += MAX_BATCH_AMOUNT
    remain -= MAX_BATCH_AMOUNT
  }
}


export const buildScriptionFilters = (cfg: IScriptionCfg) => {
  if (cfg.filter) {
    return cfg.filter
  }
  if (cfg.filters) {
    let body = ''
    for (let i = 0; i < cfg.filters.length; i++) {
      if (i > 0) {
        body += ' && '
      }
      let filter = cfg.filters[i]
      let value: any = filter.value
      let op = ''
      switch (filter.op) {
        case 'eq':
          op = '==='
          break
        case 'ne':
          op = '!=='
          break
        case 'gt':
          op = '>'
          break
        case 'gte':
          op = '>='
          break
        case 'lt':
          op = '<'
          break
        case 'lte':
          op = '<='
          break
        case 'in':
          op = 'in'
          break
        case 'nin':
          op = 'nin'
          break
        case 'like':
          op = 'like'
          break
        case 'nlike':
          op = 'nlike'
          break
        case 'isNull':
          body += `!event.${filter.key}`
          break
        case 'isNotNull':
          body += `!!event.${filter.key}`
          break
      }
      if (filter.type === 'address') {
        value = `'${value.toLowerCase()}'`
      } else if (filter.type === 'utf8_data') {
        value = `'0x${utf8ToHex(value)}'`
      } else if (filter.type === 'hex_data') {
        value = `'${value.indexOf('0x') === 0 ? value : '0x'+value}'`
      } else if (filter.type === 'number') {
        value = parseInt(value)
      } else if (filter.type === 'boolean') {
        value = !!value
      } else {
        value = `'${value}'`
      }
      if (op) {
        if (op === 'in') {
          body += `event.${filter.key}.indexOf(${value}) >= 0`
        } else if (op === 'nin') {
          body += `!(event.${filter.key}.indexOf(${value}) >= 0)`
        } else if (op === 'nlike') {
          body += `!new RegExp(${value}).test(event.${filter.key})`
        } else if (op === 'like') {
          body += new RegExp(value).test(`event.${filter.key}`)
        } else {
          body += `event.${filter.key}${op}${value}` 
        }
      }
    }
    return new Function('event', `return ${body}`)
  }
}