Merge branch 'release/3.0.0-milestone2-1'

.dockerignore

@@ -0,0 +1 @@
+.env

backend.config.js

@@ -55,7 +55,7 @@ module.exports = {
 
     {
       enabled: !process.env.CRAWLER_REWARDS_DISABLE,
-      module: require('./lib/crawlers/rewards'),
+      module: require('./lib/crawlers/rewardsSlashes'),
     },
 
     {

docker/polkastats-backend/backend/Dockerfile

@@ -8,7 +8,7 @@
 #
 #RUN cargo build --release
 
-FROM node
+FROM node:erbium
 
 WORKDIR /usr/app/polkastats-backend-v3
 

docker/polkastats-backend/docker-compose.yml

@@ -14,7 +14,7 @@ services:
       - '30333:30333'
       - '9933:9933'
       - '9944:9944'
-    command: -d /data --rpc-external --ws-external --rpc-cors=all --pruning=archive --name 'Polkastats 🤖 v3'
+    command: -d /data --unsafe-rpc-external --unsafe-rpc-expose --unsafe-ws-external --rpc-cors all --pruning=archive --name 'Polkastats 🤖 v3'
     restart: on-failure
   #
   # SQL data base

docker/polkastats-backend/sql/polkastats.sql

@@ -57,6 +57,24 @@ CREATE TABLE IF NOT EXISTS rewards  (
   PRIMARY KEY ( block_number, era_index, stash_id )  
 );
 
+CREATE TABLE IF NOT EXISTS validator_slashes_era  (
+  block_number BIGINT NOT NULL,
+  era_index INT NOT NULL,
+  account_id VARCHAR(50),
+  amount BIGINT NOT NULL,
+  timestamp BIGINT NOT NULL,
+  PRIMARY KEY ( block_number, era_index, account_id )  
+);
+
+CREATE TABLE IF NOT EXISTS nominator_slashes_era  (
+  block_number BIGINT NOT NULL,
+  era_index INT NOT NULL,
+  account_id VARCHAR(50),
+  amount BIGINT NOT NULL,
+  timestamp BIGINT NOT NULL,
+  PRIMARY KEY ( block_number, era_index, account_id )  
+);
+
 CREATE TABLE IF NOT EXISTS validator_staking  (  
   block_number BIGINT NOT NULL,
   session_index INT NOT NULL,

docker/polkastats-backend/substrate-client/Dockerfile

@@ -2,7 +2,7 @@ FROM phusion/baseimage:0.11
 LABEL maintainer "@ColmenaLabs_svq"
 LABEL description="Small image with the Substrate binary."
 
-ARG VERSION=v0.7.30
+ARG VERSION=v0.7.32
 
 RUN apt-get update && apt-get install wget curl jq -y
 

lib/crawlers/blockHarvester.js

@@ -130,7 +130,7 @@ module.exports = {
       const genesisSlot = new BigNumber(await api.query.babe.genesisSlot.at(blockHash));
       const currentEra = new BigNumber(await api.query.staking.currentEra.at(blockHash));
 
-      // This only works for the last HISTORY_DEPTH eras (api.query.staking.historyDepth)
+      // CAUTION: This only works for the last HISTORY_DEPTH eras (api.query.staking.historyDepth)
       const erasStartSessionIndex = await api.query.staking.erasStartSessionIndex(currentEra.toString());
       const currentEraStartSessionIndex = new BigNumber(erasStartSessionIndex);
 
@@ -161,8 +161,8 @@ module.exports = {
         // Get runtime spec name and version
         const runtimeVersion = await api.rpc.state.getRuntimeVersion(blockHash);
 
-        // We can't get the timestamp for old blocks so we put the harvest timestap
-        const timestamp = new Date().getTime();
+        const timestampMs = await api.query.timestamp.now.at(blockHash);
+        const timestamp = Math.floor(timestampMs / 1000);
 
         // Total events
         const totalEvents = blockEvents.length || 0;

lib/crawlers/blockListener.js

@@ -1,5 +1,6 @@
 // @ts-check
 const { shortHash } = require('../utils.js');
+const {BigNumber} = require('bignumber.js');
 
 module.exports = {
   start: async function (api, pool, _config) {
@@ -40,7 +41,23 @@ module.exports = {
       const runtimeVersion = await api.rpc.state.getRuntimeVersion(blockHash);
 
       // Get session info
-      const session = await api.derive.session.info();
+      const isEpoch = true;
+      const validatorCount = await api.query.staking.validatorCount.at(blockHash);
+      const currentIndex = new BigNumber(await api.query.session.currentIndex.at(blockHash));
+      const currentEra = new BigNumber(await api.query.staking.currentEra.at(blockHash));
+      const erasStartSessionIndex = new BigNumber(await api.query.staking.erasStartSessionIndex(currentEra.toString()));
+      const currentSlot = new BigNumber(await api.query.babe.currentSlot.at(blockHash));
+      const epochIndex = new BigNumber(await api.query.babe.epochIndex.at(blockHash));
+      const genesisSlot = new BigNumber(await api.query.babe.genesisSlot.at(blockHash));
+      const epochDuration = new BigNumber(api.consts.babe.epochDuration);
+      const sessionsPerEra = new BigNumber(api.consts.staking.sessionsPerEra);
+      const eraLength = epochDuration.multipliedBy(sessionsPerEra);
+      const epochStartSlot = new BigNumber(epochIndex).multipliedBy(epochDuration).plus(genesisSlot); 
+      const sessionProgress = currentSlot.minus(epochStartSlot);
+      const eraProgress = new BigNumber(currentIndex)
+        .minus(erasStartSessionIndex)
+        .multipliedBy(epochDuration)
+        .plus(sessionProgress)
 
       // Handle chain reorganizations
       const sqlSelect = `SELECT block_number FROM block WHERE block_number = '${blockNumber}'`;
@@ -82,7 +99,10 @@ module.exports = {
 
         // Store new block
         console.log(`[PolkaStats backend v3] - Block listener - \x1b[32mAdding block #${blockNumber} (${shortHash(blockHash.toString())})\x1b[0m`);
-        const timestamp = await api.query.timestamp.now();
+
+        const timestampMs = await api.query.timestamp.now.at(blockHash);
+        const timestamp = Math.floor(timestampMs / 1000);
+
         const sqlInsert =
           `INSERT INTO block (
             block_number,
@@ -117,15 +137,15 @@ module.exports = {
             '${parentHash}',
             '${extrinsicsRoot}',
             '${stateRoot}',
-            '${session.currentEra}',
-            '${session.currentIndex}',
-            '${session.eraLength}',
-            '${session.eraProgress}',
-            '${session.isEpoch}',
-            '${session.sessionLength}',
-            '${session.sessionsPerEra}',
-            '${session.sessionProgress}',
-            '${session.validatorCount}',
+            '${currentEra.toString()}',
+            '${currentIndex.toString()}',
+            '${eraLength.toString()}',
+            '${eraProgress.toString()}',
+            '${isEpoch}',
+            '${epochDuration.toString()}',
+            '${sessionsPerEra.toString()}',
+            '${sessionProgress.toString()}',
+            '${validatorCount}',
             '${runtimeVersion.specName}',
             '${runtimeVersion.specVersion}',
             '${totalEvents}',

lib/crawlers/rewards.js → lib/crawlers/rewardsSlashes.js

@@ -23,7 +23,7 @@ module.exports = {
       currentDBEraIndex = currentEraIndex;
       const block = await api.rpc.chain.getBlock()
       const blockNumber = block.block.header.number.toNumber()
-      await module.exports.storeRewardsInfo(api, pool, blockNumber, currentEraIndex);
+      await module.exports.storeEraRewardsSlashes(api, pool, blockNumber, currentEraIndex);
     }
 
     // Subscribe to new blocks
@@ -40,21 +40,51 @@ module.exports = {
 
       if (currentEraIndex > currentDBEraIndex) {
         currentDBEraIndex = currentEraIndex;
-        await module.exports.storeRewardsInfo(api, pool, blockNumber, currentEraIndex);
+        await module.exports.storeEraRewardsSlashes(api, pool, blockNumber, currentEraIndex);
       }
     });
   },
-  storeRewardsInfo: async function (api, pool, blockNumber, currentEraIndex) {
+  storeEraRewardsSlashes: async function (api, pool, blockNumber, currentEraIndex) {
 
     //
-    // We want the rewards for the LAST era
+    // We want the rewards and slashes for the LAST era
     //
     const eraIndex = currentEraIndex - 1;
     console.log(`[PolkaStats backend v3] - Rewards crawler - \x1b[32mStoring rewards at block #${blockNumber} for era #${eraIndex}\x1b[0m`);
 
-    // Get last era rewards
-    let eraRewards = await api.query.staking.erasValidatorReward(eraIndex);
-    // console.log(`[PolkaStats backend v3] - Rewards crawler - \x1b[32mRewards for era #${eraIndex}: ${JSON.stringify(eraRewards, null, 2)}\x1b[0m`);
+    // Get Validator rewards for last era
+    const eraRewards = await api.query.staking.erasValidatorReward(eraIndex);
+
+    // Fetch and store Validator and Nominator slashes for last era
+    const eraSlahes = await api.derive.staking.eraSlashes(eraIndex);
+    const eraValidatorSlashes = eraSlahes.validators;
+    const eraNominatorSlashes = eraSlahes.nominators;
+
+    for (const validator in eraValidatorSlashes) {
+      const amount = eraValidatorSlashes[validator].toString();
+      const sqlInsert = `INSERT INTO validator_slashes_era (block_number, era_index, account_id, amount, timestamp) 
+      VALUES ('${blockNumber}', '${eraIndex}', '${validator}', '${amount}', extract(epoch from now()));`;
+      try {
+        console.log(`[PolkaStats backend v3] - Rewards crawler - \x1b[32m=> Adding validator slash for ${validator} in era ${eraIndex}: ${new BigNumber(amount).dividedBy(1e12).toNumber().toFixed(3)} KSM\x1b[0m`);
+        const res = await pool.query(sqlInsert);
+      } catch (error) {
+        console.log(`[PolkaStats backend v3] - Rewards crawler - \x1b[31mSQL: ${sqlInsert}\x1b[0m`);
+        console.log(`[PolkaStats backend v3] - Rewards crawler - \x1b[31mERROR: ${JSON.stringify(error)}\x1b[0m`);
+      }
+    }
+
+    for (const nominator in eraNominatorSlashes) {
+      const amount = eraNominatorSlashes[nominator].toString();
+      const sqlInsert = `INSERT INTO nominator_slashes_era (block_number, era_index, account_id, amount, timestamp) 
+      VALUES ('${blockNumber}', '${eraIndex}', '${nominator}', '${amount}', extract(epoch from now()));`;
+      try {
+        console.log(`[PolkaStats backend v3] - Rewards crawler - \x1b[32m=> Adding nominator slash for ${nominator} in era ${eraIndex}: ${new BigNumber(amount).dividedBy(1e12).toNumber().toFixed(3)} KSM\x1b[0m`);
+        const res = await pool.query(sqlInsert);
+      } catch (error) {
+        console.log(`[PolkaStats backend v3] - Rewards crawler - \x1b[31mSQL: ${sqlInsert}\x1b[0m`);
+        console.log(`[PolkaStats backend v3] - Rewards crawler - \x1b[31mERROR: ${JSON.stringify(error)}\x1b[0m`);
+      }
+    }
 
     // Get last era points
     let eraPoints = await api.query.staking.erasRewardPoints(eraIndex);

lib/crawlers/staking.js

@@ -92,7 +92,7 @@ module.exports = {
     //
     const { nextElected } = JSON.parse(JSON.stringify(stakingOverview));
     validatorStaking.forEach(function (validator) {
-      if (nextElected.includes(validator)) {
+      if (nextElected.includes(validator.accountId.toString())) {
         validator.currentElected = true;
       } else {
         validator.currentElected = false;