Colm3na · Bigomby · Mar 14, 2020 · Mar 14, 2020 · Mar 14, 2020 · Mar 15, 2020
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -0,0 +1,32 @@
+module.exports = {
+  parserOptions: {
+    ecmaVersion: 2017,
+  },
+
+  env: {
+    es2017: true,
+    node: true,
+  },
+
+  extends: ['eslint:recommended', 'prettier'],
+
+  rules: {
+    // ERROR
+    'no-var': 'error',
+    'lines-between-class-members': ['error', 'always'],
+    'padding-line-between-statements': [
+      'error',
+      { blankLine: 'always', prev: 'block-like', next: '*' },
+      { blankLine: 'always', prev: 'function', next: 'function' },
+    ],
+    'newline-before-return': 'error',
+    'object-shorthand': 'error',
+    'space-before-blocks': 'error',
+    'no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
+    'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off',
+
+    // WARNING
+    'guard-for-in': 'warn',
+    'no-console': 'warn',
+  },
+};
diff --git a/.prettierrc b/.prettierrc
@@ -0,0 +1,4 @@
+{
+    "singleQuote": true,
+    "trailingComma": "all"
+}
diff --git a/backend.config.js b/backend.config.js
@@ -1,20 +1,42 @@
+// Also wss://kusama-rpc.polkadot.io
+const DEFAULT_WS_PROVIDER_URL = 'ws://substrate-node:9944';
+
 module.exports = {
-  // Local Polkadot Kusama node
-  wsProviderUrl: 'ws://substrate-node:9944',
-  // Postgres database connection params
+  wsProviderUrl: process.env.WS_PROVIDER_URL || DEFAULT_WS_PROVIDER_URL,
+
   postgresConnParams: {
     user: 'polkastats',
     host: 'postgres',
     database: 'polkastats',
     password: 'polkastats',
     port: 5432,
   },
-  BLOCK_HARVESTER_POLLING_TIME: 1 * 60 * 1000, // Run every 10 min
-  ACTIVE_ACCOUNTS_POLLING_TIME: 1 * 60 * 1000, // Run every 60 min
-  PHRAGMEN_POLLING_TIME: 5 * 60 * 1000, // Run every 5 min
+
+  staking: {
+    enabled: true,
+  },
+
+  rewards: {
+    enabled: true,
+  },
+
+  blockListener: {
+    enabled: true,
+  },
+
+  blockHarvester: {
+    enabled: true,
+    pollingTime: 1 * 60 * 1000,
+  },
+
+  accounts: {
+    enabled: true,
+    pollingTime: 1 * 60 * 1000,
+  },
 
   phragmen: {
-    wsProviderUrl: 'ws://substrate-node:9944',
+    enabled: true,
+    pollingTime: 5 * 60 * 1000,
     phragmenOutputDir: '/tmp/phragmen',
     offlinePhragmenPath: '/usr/app/polkastats-backend-v3/offline-phragmen',
   },

diff --git a/index.js b/index.js
@@ -1,9 +1,9 @@
-
 const config = require('./backend.config.js');
 const BackendV3 = require('./lib/BackendV3.js');
 
 async function main () {
   const backendV3 = new BackendV3(config); 
+  backendV3.runCrawlers();
 }
 
 main().catch((error) => {

diff --git a/lib/BackendV3.js b/lib/BackendV3.js
@@ -1,5 +1,3 @@
-// @ts-check
-
 const { ApiPromise, WsProvider } = require('@polkadot/api');
 const { Pool } = require('pg');
 const { blockListener } = require('./crawlers/blockListener.js');
@@ -14,49 +12,52 @@ class BackendV3 {
   constructor(config) {
     this.config = config;
     this.nodeisSyncing = true;
-    this.runCrawlers();
   }
+
   async runCrawlers() {
-    let api;
-    api = await this.getPolkadotAPI();
+    const pool = await this.getPool();
+
+    let api = await this.getPolkadotAPI();
     while (!api) {
       await wait(10000);
       api = await this.getPolkadotAPI();
     }
-    const pool = await this.getPool();
-
-    console.log(`[PolkaStats backend v3] - \x1b[32m${JSON.stringify(this.config, null, 2)}\x1b[0m`);
 
     console.log(`[PolkaStats backend v3] - \x1b[32mRunning crawlers\x1b[0m`);
-    // Listen to new blocks and system events and add them to database
-    console.log(`[PolkaStats backend v3] - \x1b[32mStarting block listener...\x1b[0m`);
-    blockListener(api, pool);
-    // Get rewards
-    console.log(`[PolkaStats backend v3] - \x1b[32mStarting rewards crawler...\x1b[0m`);
-    rewards(api, pool);
-    // Fill the gaps in block and event tables
-    setTimeout(() => {
-      console.log(`[PolkaStats backend v3] - \x1b[32mStarting block harvester...\x1b[0m`);
-      blockHarvester(api, pool, this.config);
-    }, this.config.BLOCK_HARVESTER_POLLING_TIME);
-    // Get information about active accounts
-    setTimeout(() => {
-      console.log(`[PolkaStats backend v3] - \x1b[32mStarting active accounts crawler...\x1b[0m`);
-      activeAccounts(api, pool, this.config);
-    }, this.config.ACTIVE_ACCOUNTS_POLLING_TIME);
-    // Get offline phragmen output
-    setInterval(
-      () => phragmen(api, pool, this.config.phragmen),
-      this.config.PHRAGMEN_POLLING_TIME,
-    );
-    // Get staking info
-    setTimeout(() => {
-      console.log(`[PolkaStats backend v3] - \x1b[32mStarting staking crawler...\x1b[0m`);
-      staking(api, pool);
-    }, this.config.STAKING_POLLING_TIME);
+
+    if (this.config.blockListener && this.config.blockListener.enabled) {
+      blockListener(api, pool, this.config.blockListener);
+    }
+
+    if (this.config.rewards && this.config.rewards.enabled) {
+      rewards(api, pool, this.config.rewards);
+    }
+
+    if (this.config.blockHarvester && this.config.blockHarvester.enabled) {
+      blockHarvester(api, pool, this.config.blockHarvester);
+    }
+
+    if (this.config.accounts && this.config.accounts.enabled) {
+      activeAccounts(api, pool, this.config.accounts);
+    }
+
+    if (this.config.staking && this.config.staking.enabled) {
+      staking(api, pool, this.config.staking);
+    }
+
+    if (this.config.phragmen && this.config.phragmen.enabled) {
+      phragmen(api, pool, {
+        ...this.config.phragmen,
+        wsProviderUrl: this.config.wsProviderUrl,
+      });
+    }
   }
+
   async getPolkadotAPI() {
-    console.log(`[PolkaStats backend v3] - \x1b[32mConnecting to ${this.config.wsProviderUrl}\x1b[0m`);
+    console.log(
+      `[PolkaStats backend v3] - \x1b[32mConnecting to ${this.config.crawlerRunner.wsProviderUrl}\x1b[0m`,
+    );
+
     const provider = new WsProvider(this.config.wsProviderUrl);
     const api = await ApiPromise.create({ provider });
     await api.isReady;
@@ -66,30 +67,41 @@ class BackendV3 {
     let node;
     try {
       node = await api.rpc.system.health();
-    } catch {
-      console.log(`[PolkaStats backend v3] - \x1b[31mCan't connect to node! Waiting 10s...\x1b[0m`);
+    } catch (_err) {
+      console.log(
+        `[PolkaStats backend v3] - \x1b[31mCan't connect to node! Waiting 10s...\x1b[0m`,
+      );
       api.disconnect();
       await wait(10000);
+
       return false;
     }
 
-    console.log(`[PolkaStats backend v3] - \x1b[32mNode: ${JSON.stringify(node)}\x1b[0m`);
+    console.log(
+      `[PolkaStats backend v3] - \x1b[32mNode: ${JSON.stringify(node)}\x1b[0m`,
+    );
 
     if (node && node.isSyncing.eq(false)) {
       // Node is synced!
       console.log(`[PolkaStats backend v3] - \x1b[32mNode is synced!\x1b[0m`);
       this.nodeisSyncing = false;
+
       return api;
     } else {
-      console.log(`[PolkaStats backend v3] - \x1b[33mNode is not synced! Waiting 10s...\x1b[0m`);
+      console.log(
+        `[PolkaStats backend v3] - \x1b[33mNode is not synced! Waiting 10s...\x1b[0m`,
+      );
       api.disconnect();
       await wait(10000);
     }
+
     return false;
   }
+
   async getPool() {
     const pool = new Pool(this.config.postgresConnParams);
     await pool.connect();
+
     return pool;
   }
 }

diff --git a/lib/crawlers/activeAccounts.js b/lib/crawlers/activeAccounts.js
@@ -1,33 +1,37 @@
-// @ts-check
 module.exports = {
-  activeAccounts: async function (api, pool, config) {
-
-    const POLLING_TIME = config.ACTIVE_ACCOUNTS_POLLING_TIME;
-
+  async activeAccounts(api, pool, config) {
+    console.log(
+      `[PolkaStats backend v3] - \x1b[32mStarting active accounts crawler...\x1b[0m`,
+    );
+
     // Fetch active accounts
     const accounts = await api.derive.accounts.indexes();
 
     let accountsInfo = [];
 
-    for (var key in accounts ) {
+    for (var key in accounts) {
       let accountId = key;
-      let accountIndex = accounts[key]
+      let accountIndex = accounts[key];
       let accountInfo = await api.derive.accounts.info(accountId);
-      let identity = accountInfo.identity.display ? JSON.stringify(accountInfo.identity) : '';
+      let identity = accountInfo.identity.display
+        ? JSON.stringify(accountInfo.identity)
+        : '';
       let nickname = accountInfo.nickname ? accountInfo.nickname : '';
       let balances = await api.derive.balances.all(accountId);
       accountsInfo[accountId] = {
         accountId,
         accountIndex,
         identity,
         nickname,
-        balances
-      }
-      console.log(`[PolkaStats backend v3] - Active Accounts - \x1b[32mProcessing account ${accountId}\x1b[0m`);
+        balances,
+      };
+      console.log(
+        `[PolkaStats backend v3] - Active Accounts - \x1b[32mProcessing account ${accountId}\x1b[0m`,
+      );
     }
 
     // Main loop
-    for (var key in accountsInfo ) {
+    for (var key in accountsInfo) {
       if (accountsInfo.hasOwnProperty(key)) {
         // console.log(key + " -> " + accounts[key]);
         let sql = `SELECT account_id FROM account WHERE account_id = '${key}'`;
@@ -36,25 +40,51 @@ module.exports = {
         const resBlockHeight = await pool.query(sqlBlockHeight);
         if (res.rows.length > 0) {
           const timestamp = new Date().getTime();
-          sql = `UPDATE account SET account_index = '${accountsInfo[key].accountIndex}', nickname = '${accountsInfo[key].nickname}', identity = '${accountsInfo[key].identity}', balances = '${JSON.stringify(accountsInfo[key].balances)}', timestamp = '${timestamp}', block_height = '${resBlockHeight.rows[0].block_number}' WHERE account_id = '${key}'`;
+          sql = `UPDATE account SET account_index = '${
+            accountsInfo[key].accountIndex
+          }', nickname = '${accountsInfo[key].nickname}', identity = '${
+            accountsInfo[key].identity
+          }', balances = '${JSON.stringify(
+            accountsInfo[key].balances,
+          )}', timestamp = '${timestamp}', block_height = '${
+            resBlockHeight.rows[0].block_number
+          }' WHERE account_id = '${key}'`;
           try {
-            console.log(`[PolkaStats backend v3] - Active Accounts - \x1b[32mUpdating account ${accountsInfo[key].accountIndex} [${key}]\x1b[0m`);
+            console.log(
+              `[PolkaStats backend v3] - Active Accounts - \x1b[32mUpdating account ${accountsInfo[key].accountIndex} [${key}]\x1b[0m`,
+            );
             await pool.query(sql);
           } catch (error) {
-            console.log(`[PolkaStats backend v3] - Active Accounts - \x1b[31mError updating account ${key}\x1b[0m`);
+            console.log(
+              `[PolkaStats backend v3] - Active Accounts - \x1b[31mError updating account ${key}\x1b[0m`,
+            );
           }
         } else {
           const timestamp = new Date().getTime();
-          sql = `INSERT INTO account (account_id, account_index, nickname, identity, balances, timestamp, block_height) VALUES ('${key}', '${accountsInfo[key].accountIndex}', '${accountsInfo[key].nickname}', '${accountsInfo[key].idenity}', '${JSON.stringify(accountsInfo[key].balances)}', '${timestamp}', '${resBlockHeight.rows[0].block_number}');`;
+          sql = `INSERT INTO account (account_id, account_index, nickname, identity, balances, timestamp, block_height) VALUES ('${key}', '${
+            accountsInfo[key].accountIndex
+          }', '${accountsInfo[key].nickname}', '${
+            accountsInfo[key].idenity
+          }', '${JSON.stringify(
+            accountsInfo[key].balances,
+          )}', '${timestamp}', '${resBlockHeight.rows[0].block_number}');`;
           try {
-            console.log(`[PolkaStats backend v3] - Active Accounts - \x1b[32mAdding account ${accountsInfo[key].accountIndex} [${key}]\x1b[0m`);
+            console.log(
+              `[PolkaStats backend v3] - Active Accounts - \x1b[32mAdding account ${accountsInfo[key].accountIndex} [${key}]\x1b[0m`,
+            );
             await pool.query(sql);
           } catch (error) {
-            console.log(`[PolkaStats backend v3] - Active Accounts - \x1b[31mError adding new account ${key}\x1b[0m`);
+            console.log(
+              `[PolkaStats backend v3] - Active Accounts - \x1b[31mError adding new account ${key}\x1b[0m`,
+            );
           }
-        } 
+        }
       }
     }
-    setTimeout(module.exports.activeAccounts(api, pool, config), POLLING_TIME);
-  }
-}
+
+    setTimeout(
+      () => module.exports.activeAccounts(api, pool, config),
+      config.pollingTime,
+    );
+  },
+};