1

.gitignore

@@ -1,4 +1,5 @@
 node_modules/
 *#*
 dist/
-res/
+res/
+nohup.out

server/web3dbspider/app.js

@@ -0,0 +1,22 @@
+const app = require('j7/app');
+const event = require('j7/event');
+const config = require('j7/config');
+const metaFactory = require('./metadata/factory');
+
+process.on('unhandledRejection', (reason, promise) => {
+  console.log('Unhandled Rejection at:', promise, 'reason:', reason);
+  throw reason;
+});
+
+event.addListener(event.APP_INITIALIZED_EVENT, async () => {
+  await require('./metadata/factory').init();
+  await require('./middlewares/factory').init();
+  await require('./controllers/factory').init();
+  await require('./models/factory').init();
+  await require('./services/factory').init();
+  await require('./tasks/factory').init();
+  app.injectionSession(require('./session'));
+  console.log(metaFactory.getWeb3BcSpiderConf()['listen_port']);
+  app.listen(metaFactory.getWeb3BcSpiderConf()['listen_port']);
+});
+app.init();

server/web3dbspider/blockchain.js

@@ -0,0 +1,139 @@
+const util = require('util');
+const Web3 = require('web3');
+const utils = require('j7/utils');
+const bcutils = require('j7/bcutils');
+const event = require('j7/event');
+const sync = require("j7/sync");
+const log = require("j7/log");
+const contract = require('common/contract');
+const bcconst = require('common/bcconst');
+const metaFactory = require('./metadata/factory');
+
+class BlockChain {
+
+  constructor(netId) {
+    this.actived = false;
+    this.netId = netId;
+    this.lastQueryTime = utils.getUtcTime();
+    this.queryLockTimes = 0;
+    this.currBlockNumber = 0;
+    this.refreshCond = new sync.Cond();
+    this.lastRefreshTime = utils.getUtcTime();
+    setTimeout(this.refreshBlockNumber.bind(this), 1000 * 0.01);
+  }
+
+  async initInstance(user, address, jsonUrl) {
+    const json = utils.readJsonFromFile(jsonUrl);
+    return new this.web3.eth.Contract(
+      json.abi,
+      address,
+      { from: user }
+    );
+  }
+
+  async init() {
+    this.web3Conf = metaFactory.getWeb3Conf(this.netId);
+    this.contractsConf = metaFactory.getContractsConf(this.netId);
+    this.netDir = metaFactory.getNetDir(this.netId);
+
+    this.web3 = new Web3(this.getRpcUrl());
+    this.web3.eth.handleRevert = true;
+    this.web3.eth.accounts.wallet.add(this.getPrivateKey());
+    for (const data of this.contractsConf) {
+      this[`${data.name}Instance`] = await this.initInstance
+      (this.getUserAddress(), data.address, this.netDir + data.json);
+    }
+    const chainId = await this.web3.eth.getChainId();
+    if (chainId != this.netId) {
+      log.warning(util.format('net id error %s %s',
+                              chainId,
+                              this.netId
+                             ));
+    }
+    log.info(util.format('local.net_id:%s remote_net_id:%s',
+                         this.netId,
+                         chainId
+                        ));
+    {
+      await this.mustBeActive();
+      const netId = this.getNetId();
+      console.log('net_id:', netId, ' blockNumber:', this.getCurrBlockNumber(),
+                  ' handleRevert:', this.web3.eth.handleRevert, ' isOnlineEnv:', utils.isOnlineEnv());
+    }
+  }
+
+  async mustBeActive() {
+    while (!this.actived) {
+      await utils.sleep(1000);
+    }
+  }
+
+  getNetId() {
+    return this.netId;
+  }
+
+  getRpcUrl() {
+    return this.web3Conf['block_server'];
+  }
+
+  getUserAddress() {
+    return this.web3Conf['user_address'];
+  }
+
+  getPrivateKey() {
+    return this.web3Conf['private_key'];
+  }
+
+  getContractByName(name) {
+    let contract = null;
+    this.contractsConf.forEach((item) => {
+      if (item['name'] == name) {
+        contract = item;
+      }
+    });
+    return contract;
+  }
+
+  getCurrBlockNumber() {
+    return this.currBlockNumber;
+  }
+
+  async lockQuery() {
+    while (this.queryLockTimes > 3) {
+      await utils.sleep(100 + utils.randRange(10, 100));
+    }
+    ++this.queryLockTimes;
+    this.lastQueryTime = utils.getUtcTime();
+  }
+
+  async unlockQuery() {
+    await utils.sleep(10 + utils.randRange(10, 50));
+    --this.queryLockTimes;
+  }
+
+  isAddress(address) {
+    return this.web3.utils.isAddress(address);
+  }
+
+  async refreshBlockNumber() {
+    const logHead = ' refreshBlockNumber:';
+    while (true) {
+      try {
+        this.currBlockNumber = await this.web3.eth.getBlockNumber();
+        this.actived = true;
+        this.lastRefreshTime = utils.getUtcTime();
+        console.log('currBlockNumber', this.currBlockNumber);
+      } catch (e) {
+        this.actived = false;
+        log.warning(util.format('%s err:%s',
+                                logHead,
+                                e
+                               ));
+      }
+      await this.refreshCond.wait(1000 * 3);
+    }
+  }
+
+}
+
+module.exports = BlockChain;

server/web3dbspider/controllers/factory.js

@@ -0,0 +1,11 @@
+const controllers = {};
+
+function add(name) {
+  controllers[name] = require(`./${name}`);
+  controllers[name].init();
+}
+
+async function init() {
+}
+
+exports.init = init;

server/web3dbspider/metadata/BcEventDb.js

@@ -0,0 +1,8 @@
+const utils = require('j7/utils');
+const basewrap = require('./basewrap');
+
+class BcEventDb extends basewrap.BaseWrap {
+
+}
+
+module.exports = BcEventDb;

server/web3dbspider/metadata/BcNftDb.js

@@ -0,0 +1,8 @@
+const utils = require('j7/utils');
+const basewrap = require('./basewrap');
+
+class BcNftDb extends basewrap.BaseWrap {
+
+}
+
+module.exports = BcNftDb;

server/web3dbspider/metadata/Config.js

@@ -0,0 +1,8 @@
+const utils = require('j7/utils');
+const basewrap = require('./basewrap');
+
+class Config extends basewrap.BaseWrap {
+
+}
+
+module.exports = Config;

server/web3dbspider/metadata/Web3BcSpider.js

@@ -0,0 +1,8 @@
+const utils = require('j7/utils');
+const basewrap = require('./basewrap');
+
+class Web3BcSpider extends basewrap.BaseWrap {
+
+}
+
+module.exports = Web3BcSpider;

server/web3dbspider/metadata/basewrap.js

@@ -0,0 +1,57 @@
+const utils = require('j7/utils');
+
+class BaseWrap {
+
+  #writeLock = false;
+
+  constructor(json, metaClass) {
+    this._json = json;
+    //this._metaClass = metaClass;
+    this._metaName = metaClass['wrapClassName'];
+  }
+
+  lock() {
+    //this.#writeLock = true;
+  }
+
+  _getHandler() {
+    return {
+      get: (obj, prop) => {
+        if (prop in obj._json) {
+          let val = obj._json[prop];
+          if (utils.isArray(val)) {
+            return new Proxy(val, {
+              set: () => {
+                console.log(33333);
+              }
+            });
+          } else if (utils.isObject(val)) {
+            return new Proxy(val, {
+              set: () => {
+                console.log(33333);
+              }
+            });
+          } else {
+            return val;
+          }
+        }
+        return prop in obj ? obj[prop] : null;
+      },
+      set: (obj, prop, val) => {
+        if (this.#writeLock) {
+          console.log(111111);
+        } else {
+          Reflect.set(obj, prop, val);
+        }
+        return true;
+      }
+    };
+  }
+
+  getMetaName() {
+    return this._metaName;
+  }
+
+}
+
+exports.BaseWrap = BaseWrap;

server/web3dbspider/metadata/factory.js

@@ -0,0 +1,211 @@
+const app = require('j7/app');
+const utils = require('j7/utils');
+const log = require('j7/log');
+
+const metaClassList = [];
+const metaClasses = {};
+const web3ConfHash = {};
+let web3BcSpiderConf = null;
+let configDir = './config/';
+
+function registerMetaClass(fileName, primKey, wrapClass) {
+  const metaClass = {
+    'fileName' : fileName,
+    'primKey' : primKey,
+    'wrapClassName': wrapClass,
+    'wrapClass': require('./' + wrapClass),
+    'rawList' : [],
+    'rawHash' : {},
+    'wrapList' : [],
+    'wrapHash' : {},
+  };
+  metaClassList.push(metaClass);
+  metaClasses[wrapClass] = metaClass;
+}
+
+function load() {
+  metaClassList.forEach((metaClass) => {
+    const json = utils.readJsonFromFile(metaClass['fileName']);
+    if (!json) {
+      throw new Error('读取配置' + metaClass['fileName'] + '失败');
+    }
+    if (Array.isArray(json)) {
+      metaClass['rawList'] = json;
+
+    } else {
+      metaClass['rawList'].push(json);
+    }
+    let idx = 0;
+    metaClass['rawList'].forEach(function (item) {
+      const wrapOjb = new metaClass['wrapClass'](item, metaClass);
+      const wrapProxy = new Proxy(wrapOjb, wrapOjb._getHandler());
+      metaClass['wrapList'].push(wrapProxy);
+      if (metaClass['primKey'] == '') {
+        metaClass['rawHash'][idx] = item;
+        metaClass['wrapHash'][idx] = wrapProxy;
+      } else {
+        metaClass['rawHash'][item[metaClass['primKey']]] = item;
+        metaClass['wrapHash'][item[metaClass['primKey']]] = wrapProxy;
+      }
+      ++idx;
+    });
+    //log.debug(utils.jsonEncode(metaClass));
+  });
+  {
+    for (let i = 0; i < 3; ++i) {
+      metaClassList.forEach((metaClass) => {
+        metaClass['wrapList'].forEach((item) => {
+          if (item['_init' + i]) {
+            item['_init' + i](exports);
+          }
+        });
+      });
+    }
+    metaClassList.forEach((metaClass) => {
+      metaClass['wrapList'].forEach((item) => {
+        item.lock();
+      });
+    });
+  }
+}
+
+async function init() {
+  configDir = './config/';
+  if (utils.isOnlineEnv()) {
+    configDir = '../config/';
+  } else if (utils.getArgv('env') == 'dev'){
+    configDir = './config_dev/';
+  }
+  console.log(configDir, utils.getArgv('env'));
+
+  let resDir = './res/';
+  if (utils.isOnlineEnv()) {
+    resDir = '../res/';
+  } else if (utils.getArgv('env') == 'dev'){
+    resDir = './res_dev/';
+  }
+  console.log(resDir);
+
+  registerMetaClass(configDir + 'config.json',
+                    '',
+                    'Config'
+                   );
+  registerMetaClass(configDir + 'bcevent_mysql.json',
+                    '',
+                    'BcEventDb'
+                   );
+  registerMetaClass(configDir + 'bcnft_mysql.json',
+                    '',
+                    'BcNftDb'
+                   );
+  registerMetaClass(configDir + 'web3bcspider.cluster.json',
+                    '',
+                    'Web3BcSpider'
+                   );
+  load();
+  {
+    traverseMetaList('BcEventDb', (dbConf, idx) => {
+      app.registerDb('BcEventDb' + idx, dbConf);
+    });
+    traverseMetaList('BcNftDb', (dbConf, idx) => {
+      app.registerDb('BcNftDb' + idx, dbConf);
+    });
+    traverseMetaList('Web3BcSpider', (item, idx) => {
+      if (item['instance_id'] == app.getInstanceId()) {
+        web3BcSpiderConf = item;
+        item['nets'].forEach((net) => {
+          const netId = net['net_id'];
+          const netDir = configDir + 'nets/' + netId + '/';
+          web3ConfHash[netId] = {
+            'net_id': netId,
+            'netId': netId,
+            'Web3': utils.readJsonFromFile(netDir + 'web3.json'),
+            'Contract': utils.readJsonFromFile(netDir + 'contract.json'),
+            'Events': net['events'],
+          };
+        });
+      }
+    });
+  }
+}
+
+function getMetaClass(name) {
+  return utils.hasKey(metaClasses, name) ? metaClasses[name] : null;
+}
+
+function getMetaByKey(name, key) {
+  const metaClass = getMetaClass(name);
+  return metaClass && key in metaClass['wrapHash'] ? metaClass['wrapHash'][key] : null;
+}
+
+function getMetaList(name) {
+  const metaClass = getMetaClass(name);
+  return metaClass ? metaClass['wrapList'] : null;
+}
+
+function callMetaStatic(name, method, ...args) {
+  const metaClass = getMetaClass(name);
+  return metaClass['wrapClass'][method](...args);
+}
+
+function traverseMetaList(name, cb) {
+  const metaList = getMetaList(name);
+  if (metaList) {
+    for (let i = 0; i < metaList.length; ++i) {
+      if (!cb(metaList[i], i, metaList.length)) {
+        break;
+      }
+    }
+  }
+}
+
+function getConf(netId) {
+  const keys = Object.keys(web3ConfHash);
+  for (let i = 0; i < keys.length; ++i) {
+    const conf = web3ConfHash[keys[i]];
+    if (conf['net_id'] == netId) {
+      return conf;
+    }
+  }
+  return null;
+}
+
+function getWeb3Conf(netId) {
+  const conf = getConf(netId);
+  return conf ? conf['Web3'] : null;
+}
+
+function getWeb3BcSpiderConf() {
+  return web3BcSpiderConf;
+}
+
+function getContractsConf(netId) {
+  const conf = getConf(netId);
+  return conf ? conf['Contract'] : null;
+}
+
+function getNetList() {
+  const netList = [];
+  const keys = Object.keys(web3ConfHash);
+  for (let i = 0; i < keys.length; ++i) {
+    netList.push(web3ConfHash[keys[i]]);
+  }
+  return netList;
+}
+
+function getNetDir(netId) {
+  const netDir = configDir + 'nets/' + netId + '/';
+  return netDir;
+}
+
+exports.init = init;
+
+exports.getMetaByKey = getMetaByKey;
+exports.traverseMetaList = traverseMetaList;
+exports.callMetaStatic = callMetaStatic;
+
+exports.getWeb3Conf = getWeb3Conf;
+exports.getWeb3BcSpiderConf = getWeb3BcSpiderConf;
+exports.getContractsConf = getContractsConf;
+exports.getNetList = getNetList;
+exports.getNetDir = getNetDir;

server/web3dbspider/middlewares/factory.js

@@ -0,0 +1,5 @@
+async function init() {
+
+}
+
+exports.init = init;

server/web3dbspider/models/factory.js

@@ -0,0 +1,5 @@
+async function init() {
+
+}
+
+exports.init = init;

server/web3dbspider/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "web3dbspider",
+  "version": "1.0.0",
+  "description": "",
+  "private": true,
+  "scripts": {},
+  "dependencies": {
+    "j7": "file:../../third_party/j7",
+    "common": "file:../common",
+    "@metamask/eth-sig-util": "^4.0.0",
+    "express": "^4.17.2",
+    "log4js": "~6.3.0",
+    "mysql": "~2.18.1",
+    "web3": "^1.6.1"
+  }
+}

server/web3dbspider/services/baseservice.js

@@ -0,0 +1,13 @@
+class BaseService {
+
+  constructor(session) {
+    this.session = session;
+  }
+
+  getSession() {
+    return this.session;
+  }
+
+}
+
+module.exports = BaseService;

server/web3dbspider/services/blockchain.js

@@ -0,0 +1,68 @@
+const app = require('j7/app');
+const utils = require('j7/utils');
+const bcutils = require('j7/bcutils');
+const sync = require("j7/sync");
+const log = require("j7/log");
+const metaFactory = require('../metadata/factory');
+const bcClass = require('../blockchain');
+const BaseService = require('./baseservice');
+
+const netIdHash = {};
+
+function getBc(netId) {
+  return utils.hasKey(netIdHash, netId) ? netIdHash[netId] : null;
+}
+
+class BlockChain extends BaseService {
+
+  #bc = null;
+
+  static async staticInit() {
+    metaFactory.getNetList().forEach(async (net) => {
+      const bc = new bcClass(net['netId']);
+      netIdHash[net['netId']] = bc;
+      await bc.init();
+    });
+  }
+
+  init(netId) {
+    this.#bc = getBc(netId);
+    return this.#bc != null;
+  }
+
+  async mustBeActive() {
+    await this.#bc.mustBeActive();
+  }
+
+  getNetId() {
+    return this.#bc.getNetId();
+  }
+
+  getCurrBlockNumber() {
+    return this.#bc.currBlockNumber;
+  }
+
+  async lockQuery() {
+    await this.#bc.lockQuery();
+  }
+
+  async unlockQuery() {
+    await this.#bc.unlockQuery();
+  }
+
+  isAddress(address) {
+    return this.#bc.isAddress();
+  }
+
+  getContractAddressByName(name) {
+    const contract = this.#bc.getContractByName(name);
+    return contract ? bcutils.toNormalAddress(contract['address']) : '';
+  }
+
+  async getPastEvents(contractName, eventName, ...args) {
+    return this.#bc[contractName + 'Instance'].getPastEvents(eventName, ...args);
+  }
+
+}
+
+module.exports = BlockChain;

server/web3dbspider/services/factory.js

@@ -0,0 +1,35 @@
+const services = {};
+
+async function internalAdd(clsName, modName, isSingle) {
+  const modClass = require('./' + modName);
+  services[clsName] = {
+    'clsName': clsName,
+    'modName': modName,
+    'class': modClass,
+    'isSingle': isSingle
+  };
+  if (modClass.staticInit) {
+    await modClass.staticInit();
+  }
+}
+
+async function add(clsName, modName) {
+  await internalAdd(clsName, modName, false);
+}
+
+async function addSingle(clsName, modName) {
+  await internalAdd(clsName, modName, true);
+}
+
+async function init() {
+  await add("BlockChain", 'blockchain');
+  add(['PullBcEvent'], 'pull_bcevent');
+}
+
+function create(name, session = null) {
+  const service = services[name];
+  return new service['class'](session);
+}
+
+exports.init = init;
+exports.create = create;

server/web3dbspider/services/pull_bcevent.js

@@ -0,0 +1,230 @@
+const app = require('j7/app');
+const utils = require('j7/utils');
+const bcutils = require('j7/bcutils');
+const log = require('j7/log');
+const BaseService = require('./baseservice');
+
+class PullBcEvent extends BaseService {
+
+  async init(bc, net, event) {
+    const {err, conn} = await app.getDbConn('BcEventDb0');
+    this.conn = conn;
+    this.lastBlockNumber = 0;
+    this.bc = bc;
+    this.net = net;
+    this.event = event;
+    this.eventConf = this.event['eventConf'];
+    this.progInfo = this.event['progressInfo'];
+    await this.start();
+  }
+
+  async start() {
+    while (true) {
+      await this.pullEvent();
+      await utils.sleep(8000 + utils.randRange(500, 1500));
+    }
+  }
+
+  async pullEvent() {
+    const logHead = this.getInstanceName() + ' pullEvent: ';
+    while (true) {
+      await this.bc.lockQuery();
+      try {
+        const fromBlock = await this.getFromBlock();
+        const toBlock = await this.calcToBlock(fromBlock);
+        if (toBlock > fromBlock) {
+          const events = await this.bc.getPastEvents(
+            this.getContractName(),
+            this.getEventName(),
+            {
+              fromBlock: fromBlock,
+              toBlock: toBlock,
+            },
+          );
+          await this.processEvents(events, fromBlock, toBlock);
+          await this.saveLastBlockNumber(toBlock);
+        }
+        ++this.progInfo['pullCount'];
+        return;
+      } catch (err) {
+        log.error(logHead + err);
+        await utils.sleep(1000 + utils.randRange(10, 2000));
+      } finally {
+        await this.bc.unlockQuery();
+      }
+    }
+  }
+
+  async processEvents(events, fromBlock, toBlock) {
+    this.progInfo['fromBlock'] = fromBlock;
+    this.progInfo['toBlock'] = toBlock;
+    this.progInfo['currBlock'] = this.bc.getCurrBlockNumber();
+    this.progInfo['eventCount'] += events.length;
+    if (events.length <= 0) {
+      return;
+    }
+    console.log(events);
+    utils.serial
+    (events,
+     async (event) => {
+      while (true) {
+        try {
+          await this.saveToDb(event);
+          return;
+        } catch (err) {
+          log.error(err);
+        }
+        await utils.sleep(8000 + utils.randRange(500, 1500));
+      }
+     });
+  }
+
+  async getFromBlock() {
+    const logHead = this.getInstanceName() + ' getFromBlock: ';
+    const firstBlockNumber  = this.getInitBlock();
+    while (this.lastBlockNumber < 1) {
+      try {
+        const {err, row} = await this.conn.ormSelectOne(
+          't_last_block',
+          [
+            ['net_id', this.getNetId()],
+            ['contract_address', this.getContractAddress()],
+            ['event_name', this.getEventName()],
+          ]
+        );
+        if (!err) {
+          if (row) {
+            this.lastBlockNumber = Number(row['block_number']);
+          } else {
+            this.lastBlockNumber = firstBlockNumber;
+          }
+        }
+        console.log(logHead, this.lastBlockNumber, this.bc.getCurrBlockNumber());
+        while (this.lastBlockNumber + 8 > this.bc.getCurrBlockNumber()) {
+          await utils.sleep(1000 + utils.randRange(500, 1500));
+        }
+        continue;
+      } catch (err) {
+        log.error(err);
+      }
+      await utils.sleep(5000 + utils.randRange(500, 1500));
+    }
+    return this.lastBlockNumber + 1;
+  }
+
+  async calcToBlock(fromBlock) {
+    const currBlockNumber = this.bc.getCurrBlockNumber();
+    const distanceBlock = currBlockNumber - fromBlock - 8;
+    const batchBlockNum = 888;
+    if (distanceBlock > 0) {
+      if (distanceBlock > batchBlockNum) {
+        return fromBlock + batchBlockNum;
+      } else {
+        return fromBlock + distanceBlock;
+      }
+    }
+    return fromBlock;
+  }
+
+  async saveLastBlockNumber(blockNumber) {
+    const logHead = this.getInstanceName() + ' event_process.saveLastBlockNumber: ';
+    while (true) {
+      const {err} = await this.conn.upsert(
+        't_last_block',
+        [
+          ['net_id', this.getNetId()],
+          ['contract_address', this.getContractAddress()],
+          ['event_name', this.getEventName()],
+        ],
+        [
+          ['block_number', blockNumber],
+          ['modifytime', utils.getUtcTime()],
+        ],
+        [
+          ['net_id', this.getNetId()],
+          ['contract_address', this.getContractAddress()],
+          ['event_name', this.getEventName()],
+          ['block_number', blockNumber],
+          ['contract_name', this.getContractName()],
+          ['createtime', utils.getUtcTime()],
+          ['modifytime', utils.getUtcTime()],
+        ]
+      );
+      if (!err) {
+        break;
+      }
+      log.error(logHead + err);
+      await utils.sleep(5000 + utils.randRange(500, 1500));
+    }
+    this.lastBlockNumber = blockNumber;
+  }
+
+  async saveToDb(event) {
+    const logHead = this.getInstanceName() + ' event_process.saveToDb: ';
+    while (true) {
+      const nowTime = utils.getUtcTime();
+      const returnValues = event['returnValues'];
+      const hashCode = '';
+      const {err} = await this.conn.upsert(
+        't_blockchain_event',
+        [
+          ['txhash', event['transactionHash']],
+          ['hash_code', hashCode],
+          ['log_index', event['logIndex']],
+          ['net_id', this.bc.getNetId()],
+          ['event_name', this.getEventName()],
+          ['contract_address', this.getContractAddress()],
+        ],
+        [
+        ],
+        [
+          ['txhash', event['transactionHash']],
+          ['hash_code', hashCode],
+          ['log_index', event['logIndex']],
+          ['net_id', this.bc.getNetId()],
+          ['event_name', this.getEventName()],
+          ['contract_address', this.getContractAddress()],
+          ['contract_name', this.getContractName()],
+          ['block_number', event['blockNumber']],
+          ['raw_data', utils.jsonEncode(event)],
+          ['return_values', utils.jsonEncode(returnValues)],
+          ['createtime', utils.getUtcTime()],
+          ['modifytime', utils.getUtcTime()],
+        ]
+      );
+      if (!err) {
+        break;
+      }
+      log.error(logHead + err);
+      await utils.sleep(5000 + utils.randRange(500, 1500));
+    }
+  }
+
+  getNetId() {
+    return this.net['net_id'];
+  }
+
+  getEventName() {
+    return this.eventConf['event_name'];
+  }
+
+  getInitBlock() {
+    return this.eventConf['init_block'];
+  }
+
+  getContractAddress() {
+    return this.bc.getContractAddressByName(this.getContractName());
+  }
+
+  getContractName() {
+    return this.eventConf['contract_name'];
+  }
+
+  getInstanceName() {
+    const instName = this.getNetId() + '.' + this.getContractName() + '.' + this.getEventName();
+    return instName;
+  }
+
+}
+
+module.exports = PullBcEvent;

server/web3dbspider/session.js

@@ -0,0 +1,128 @@
+const utils = require('j7/utils');
+const app = require('j7/app');
+const error = require('j7/error');
+const modelsFactory = require('./models/factory');
+const serviceFactory = require('./services/factory');
+const metaFactory = require('./metadata/factory');
+
+class Session {
+
+  constructor(req, rsp) {
+    this.req = req;
+    this.rsp = rsp;
+    this.nowTime = utils.getUtcTime();
+    this.useConns = {};
+  }
+
+  async destory() {
+    if (this.user) {
+      await this.user.destory();
+    }
+    for (let key in this.useConns) {
+      this.useConns[key].release();
+    }
+    //console.log(new Error().stack);
+    this.useConns = null;
+  }
+
+  getNowTime() {
+    return this.nowTime;
+  }
+
+  getNowDaySeconds() {
+    return this.getNowDaySeconds(this.getNowTime());
+  }
+
+  getDaySeconds(utcTime) {
+
+  }
+
+  rspErr(errCode, errMsg) {
+    utils.rspErr(this.rsp, errCode, errMsg);
+  }
+
+  rspOk() {
+    utils.rspOk(this.rsp);
+  }
+
+  rspData(data) {
+    utils.rspData(this.rsp, data);
+  }
+
+  dieErr(errCode, errMsg) {
+    this.rspErr(errCode, errMsg);
+  }
+
+  throwError(errCode, errMsg) {
+    throw new error.InternalError(errCode, errMsg);
+  }
+
+  request(name, defVal = null) {
+    return name in this.req.query ? this.req.query[name] : defVal;
+  }
+
+  requestToJson() {
+    return utils.jsonEncode(this.req.query);
+  }
+
+  async getMarketDb() {
+    const idx = 0;
+    const dbKey = 'MarketDb' + idx;
+    if (this.useConns[dbKey]) {
+      return this.useConns[dbKey];
+    }
+    const {err, conn} = await app.getDbConn(dbKey);
+    if (err) {
+      console.log(err);
+    }
+    //console.log(new Error().stack);
+    if (!err && conn) {
+      this.useConns[dbKey] = conn;
+    }
+    return conn;
+  }
+
+  createModel(name) {
+    return modelsFactory.create(name, this);
+  }
+
+  createService(name) {
+    return serviceFactory.create(name, this);
+  }
+
+  getMeta(name, key) {
+    return metaFactory.getMetaByKey(name, key);
+  }
+
+  callMetaStatic(name, method, ...args) {
+    return metaFactory.callMetaStatic(name, method, this, ...args);
+  }
+
+  traverseMetaList(name, cb) {
+    return metaFactory.traverseMetaList(name, cb);
+  }
+
+  callMetaFactory(name, ...args) {
+    return metaFactory[name](this, ...args);
+  }
+
+  async marketConn(method, ...args) {
+    const conn = await this.getMarketDb();
+    const ret = await conn[method](...args);
+    if (ret.err){
+      this.throwError(500, 'internal error');
+      log.error(ret.err);
+      return;
+    }
+    if (utils.hasKey(ret, 'row')) {
+      return ret['row'];
+    } else if (utils.hasKey(ret, 'rows')) {
+      return ret['rows'];
+    } else {
+      return null;
+    }
+  }
+
+}
+
+module.exports = Session;

server/web3dbspider/tasks/basetask.js

@@ -0,0 +1,4 @@
+class BaseTask {
+}
+
+module.exports = BaseTask;

server/web3dbspider/tasks/bcspider.js

@@ -0,0 +1,78 @@
+const app = require('j7/app');
+const utils = require('j7/utils');
+const bcutils = require('j7/bcutils');
+const log = require('j7/log');
+const BaseTask = require('./basetask');
+const factory = require('./factory');
+const serviceFactory = require('../services/factory');
+const metaFactory = require('../metadata/factory');
+
+class BcSpider extends BaseTask {
+
+  async init() {
+    const netList = metaFactory.getNetList();
+    netList.forEach
+    (
+      (net) => {
+       this.initNet(net);
+     });
+  }
+
+  async initNet(net) {
+    const events = [];
+    net['Events'].forEach
+    (
+      (eventConf) => {
+        const event = {
+            'eventConf': eventConf,
+            'progressInfo': {
+              'pullCount': 0,
+              'eventCount': 0,
+              'fromBlock': 0,
+              'toBlock': 0,
+              'currBlock': 0
+            }
+        };
+        events.push(event);
+        this.createPullBcEventService(net, event);
+      });
+    this.outputProgressInfo(net, events);
+  }
+
+  createPullBcEventService(net, event) {
+    const bc = serviceFactory.create('BlockChain');
+    bc.init(net['net_id']);
+    const pullBcEventService = serviceFactory.create('PullBcEvent');
+    event['pullBcEventService'] = pullBcEventService;
+    pullBcEventService.init(bc, net, event);
+    return pullBcEventService;
+  }
+
+  async outputProgressInfo(net, events) {
+    let count = 0;
+    while (true) {
+      log.info('----------------------------------------------------------');
+      events.forEach
+      (
+        (event) => {
+          const eventConf = event['eventConf'];
+          const progInfo = event['progressInfo'];
+          const logObj = 'net_id: ' + net['net_id'] + ' ' +
+                eventConf['contract_name'] + '.' +
+                eventConf['event_name'] + ' ' +
+                ' pullCount:' + progInfo['pullCount'] +
+                ' eventCount:' + progInfo['eventCount'] +
+                ' fromBlock:' + progInfo['fromBlock'] +
+                ' toBlock:' + progInfo['toBlock'] +
+                ' currBlock:' + progInfo['currBlock']
+                ;
+          log.info(logObj);
+        }
+      );
+      await utils.sleep(1000 * 10);
+    }
+  }
+
+}
+
+module.exports = BcSpider;

server/web3dbspider/tasks/factory.js

@@ -0,0 +1,37 @@
+const utils = require('j7/utils');
+
+const tasks = {};
+
+function add(clsNames, modName) {
+  const modClass = require('./' + modName);
+  clsNames.forEach((clsName) => {
+    tasks[clsName] = {
+      'clsName': clsName,
+      'modName': modName,
+      'class': modClass
+    };
+  });
+}
+
+async function init() {
+  add(['BcSpider'], 'bcspider');
+  const initTasks = [
+    'BcSpider'
+  ];
+  await utils.serial(
+    [
+      'BcSpider'
+    ],
+    async (name) =>
+      {
+        const task = create(name);
+        await task.init();
+      });
+}
+
+function create(name) {
+  const module = tasks[name];
+  return new module['class']();
+}
+
+exports.init = init;

server/web3dbspider/user.js

@@ -0,0 +1,41 @@
+const utils = require('j7/utils');
+const app = require('j7/app');
+const db = require('j7/db');
+const log = require('j7/log');
+
+class User {
+
+  constructor(session) {
+    this.session = session;
+    this.accountId = session.request('account_id');
+    this.sessionId = session.request('session_id');
+    this.useConns = {};
+    this.selfDb = null;
+  }
+
+  async destory() {
+    for (let key in this.useConns) {
+      this.useConns[key].release();
+    }
+    this.useConns = null;
+  }
+
+  getAccountId() {
+    return this.accountId;
+  }
+
+  getChannel() {
+    return utils.extractChannel(this.getAccountId());
+  }
+
+  getSessionId() {
+    return this.sessionId;
+  }
+
+  getRegisterTime() {
+    return utils.extractRegisterTime(this.getAccountId(), this.getSessionId());
+  }
+
+}
+
+exports.User = User;