feat: more canonical validators list

feat: upgrade to new getVoteAccounts() API, integrate dashboard model
fix: add uniq key prop
feat: style updates for api code
fix: yarn lint

api/api.js

@@ -29,8 +29,8 @@ import config from './config';
 const FULLNODE_URL = 'http://localhost:8899';
 
 const GLOBAL_STATS_BROADCAST_INTERVAL_MS = 2000;
-const CLUSTER_INFO_BROADCAST_INTERVAL_MS = 16000;
-const CLUSTER_INFO_CACHE_TIME_SECS = 35000;
+const CLUSTER_INFO_BROADCAST_INTERVAL_MS = 5000;
+const CLUSTER_INFO_CACHE_TIME_SECS = 4500;
 const CONFIG_PROGRAM_ID = 'Config1111111111111111111111111111111111111';
 const MAX_KEYBASE_USER_LOOKUP = 50;
 
@@ -67,6 +67,10 @@ function randomOffset(seedString) {
   return (x - Math.floor(x)) / 10 - 0.05;
 }
 
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
 function getClient() {
   let props = config.redis.path
     ? {path: config.redis.path}
@@ -498,65 +502,175 @@ const DEFAULT_LNG = 165.3768;
 
 async function getClusterInfo() {
   const connection = new solanaWeb3.Connection(FULLNODE_URL);
+  const nodeConnectionCache = {};
   let ts = new Date().toISOString();
   let [, feeCalculator] = await connection.getRecentBlockhash();
   let currentSlot = await getAsync('!blk-last-slot');
   let networkInflationRate = getNetworkInflationRate(currentSlot);
   let supply = await connection.getTotalSupply();
-  let cluster = await connection.getClusterNodes();
-  let identities = await fetchValidatorIdentities(cluster.map(c => c.pubkey));
-  let votingNow = await connection.getEpochVoteAccounts();
-  let votingAll = await connection.getProgramAccounts(
+  let clusterNodes = await connection.getClusterNodes();
+  const leader = await connection.getSlotLeader();
+  let identities = await fetchValidatorIdentities(
+    clusterNodes.map(c => c.pubkey),
+  );
+  let voteAccounts = await connection.getVoteAccounts();
+  let allVoteAccounts = await connection.getProgramAccounts(
     solanaWeb3.VOTE_ACCOUNT_KEY,
   );
   let uptimeJson = await getAsync('!uptime');
   let uptime = uptimeJson && JSON.parse(uptimeJson);
 
   let totalStaked = _.reduce(
-    votingNow,
+    (voteAccounts.current || []).concat(voteAccounts.delinquent || []),
     (a, v) => {
-      a += v.stake || 0;
+      a += v.activatedStake || 0;
 
       return a;
     },
     0,
   );
 
-  votingAll = _.map(votingAll, e => {
-    const [, v] = e;
+  const network = {};
 
-    let voteAccount = solanaWeb3.VoteAccount.fromAccountData(v.data);
-    voteAccount.nodePubkey = voteAccount.nodePubkey.toString();
-    voteAccount.authorizedVoterPubkey = voteAccount.authorizedVoterPubkey.toString();
+  for (const clusterNode of clusterNodes) {
+    const {pubkey, rpc, tpu, gossip} = clusterNode;
 
-    return voteAccount;
-  });
+    if (!tpu) {
+      continue;
+    }
 
-  cluster = _.map(cluster, c => {
-    let ip = c.gossip.split(':')[0];
+    let ip = tpu.split(':')[0];
     const geoip = geoipLookup(ip);
     let ll = geoip ? geoip.ll : null;
-    let newc = _.clone(c, true);
 
     // compute different but deterministic offsets
     let offsetLat = randomOffset(ip);
-    let offsetLng = randomOffset(c.gossip);
+    let offsetLng = randomOffset(tpu);
+
+    let lat = ((ll && ll[0]) || DEFAULT_LAT) + offsetLat;
+    let lng = ((ll && ll[1]) || DEFAULT_LNG) + offsetLng;
+
+    network[pubkey] = Object.assign(network[pubkey] || {}, {
+      online: true,
+      gossip,
+      rpc,
+      tpu,
+      lat,
+      lng,
+      coordinates: [lng, lat],
+    });
+  }
 
-    newc.lat = ((ll && ll[0]) || DEFAULT_LAT) + offsetLat;
-    newc.lng = ((ll && ll[1]) || DEFAULT_LNG) + offsetLng;
+  for (let [votePubkey, voteAccountInfo] of allVoteAccounts) {
+    voteAccountInfo.owner =
+      voteAccountInfo.owner && voteAccountInfo.owner.toString();
 
-    return newc;
-  });
+    let voteAccount = solanaWeb3.VoteAccount.fromAccountData(
+      voteAccountInfo.data,
+    );
+    voteAccount.authorizedVoterPubkey =
+      voteAccount.authorizedVoterPubkey &&
+      voteAccount.authorizedVoterPubkey.toString();
+    voteAccount.nodePubkey =
+      voteAccount.nodePubkey && voteAccount.nodePubkey.toString();
+
+    const nodePubkey = voteAccount.nodePubkey.toString();
+    const node = network[nodePubkey];
+    if (!node) {
+      continue;
+    }
+    if (node.votePubkey && node.votePubkey != votePubkey) {
+      if (node.warning && node.warning.hasMultipleVoteAccounts) {
+        node.warning.hasMultipleVoteAccounts.push(
+          voteAccount.authorizedVoterPubkey,
+        );
+      } else {
+        node.warning |= {};
+        node.warning.hasMultipleVoteAccounts = [
+          node.votePubkey,
+          voteAccount.authorizedVoterPubkey,
+        ];
+      }
+      continue;
+    }
+    node.nodePubkey = nodePubkey;
+    node.voteAccount = voteAccount;
+    node.votePubkey = votePubkey;
+    node.identity = identities.find(x => {
+      return x.pubkey === nodePubkey;
+    });
+    node.uptime =
+      uptime &&
+      uptime.find(x => {
+        return x.nodePubkey === nodePubkey;
+      });
+
+    node.voteStatus =
+      voteAccounts.current.find(x => {
+        return x.nodePubkey === nodePubkey;
+      }) ||
+      voteAccounts.delinquent.find(x => {
+        return x.nodePubkey === nodePubkey;
+      });
+    node.activatedStake = node.voteStatus && node.voteStatus.activatedStake;
+    node.commission = node.voteStatus && node.voteStatus.commission;
+  }
+
+  for (const node of Object.keys(network).sort()) {
+    const {online, rpc, tpu} = network[node];
+    if (!online && !tpu) {
+      continue;
+    }
+
+    const balanceLamports = await connection.getBalance(
+      new solanaWeb3.PublicKey(node),
+    );
+    let currentSlot = null;
+    if (rpc) {
+      try {
+        let nodeConnection = nodeConnectionCache[rpc];
+        if (nodeConnection === undefined) {
+          nodeConnectionCache[rpc] = nodeConnection = new solanaWeb3.Connection(
+            `http://${rpc}`,
+          );
+        }
+        currentSlot = await Promise.race([
+          nodeConnection.getSlot(),
+          sleep(1000),
+        ]);
+        if (currentSlot === undefined) {
+          currentSlot = 'timeout';
+        }
+      } catch (err) {
+        currentSlot = 'error';
+      }
+    }
+
+    let what;
+    if (!tpu && online) {
+      what = 'Spy';
+    } else {
+      what = 'Validator';
+    }
+
+    let newNode = network[node] || {};
+    newNode.leader = leader === node;
+    newNode.what = what;
+    newNode.balanceLamports = balanceLamports;
+    newNode.currentSlot = currentSlot;
+    network[node] = newNode;
+  }
 
   let rest = {
     feeCalculator,
     supply,
     networkInflationRate,
     totalStaked,
-    cluster,
+    network,
+    clusterNodes,
     identities,
-    votingAll,
-    votingNow,
+    voteAccounts,
+    allVoteAccounts,
     uptime,
     ts,
   };
@@ -566,6 +680,7 @@ async function getClusterInfo() {
     CLUSTER_INFO_CACHE_TIME_SECS,
     JSON.stringify(rest),
   );
+
   return rest;
 }
 

api/uptime-crawler.js

@@ -31,47 +31,50 @@ function getVoteAccountUptime(x) {
   const t1 = new Date().getTime();
 
   const p = new Promise((resolve, reject) => {
-    exec(
-      `solana-wallet -u ${FULLNODE_URL} show-vote-account ${x.votePubkey}`,
-      (err, stdout, stderr) => {
-        const t2 = new Date().getTime();
-
-        if (err) {
-          // node couldn't execute the command
-          console.log('err', err, stderr);
-          reject(err);
-          return;
-        }
-
-        const result = YAML.parse(stdout);
-
-        const uptime = _.reduce(
-          result['epoch voting history'],
-          (a, v) => {
-            a.unshift({
-              epoch: v.epoch,
-              credits_earned: v['credits earned'],
-              slots_in_epoch: v['slots in epoch'],
-              percentage: (
-                (v['credits earned'] * 1.0) /
-                (v['slots in epoch'] * 1.0)
-              ).toFixed(6),
-            });
-            return a;
-          },
-          [],
-        );
-
-        const uptimeValue = {
-          votePubkey: x.votePubkey,
-          uptime: uptime,
-          lat: t2 - t1,
-          ts: t1,
-        };
-
-        resolve(uptimeValue);
-      },
-    );
+    if (!x.votePubkey || x.votePubkey.length !== 44) {
+      reject(`invalid pubkey: ${x}`);
+      return;
+    }
+    let command = `solana-wallet -u ${FULLNODE_URL} show-vote-account ${x.votePubkey}`;
+    exec(command, (err, stdout, stderr) => {
+      const t2 = new Date().getTime();
+
+      if (err) {
+        // node couldn't execute the command
+        console.log('err', err, stderr);
+        reject(err);
+        return;
+      }
+
+      const result = YAML.parse(stdout);
+
+      const uptime = _.reduce(
+        result['epoch voting history'],
+        (a, v) => {
+          a.unshift({
+            epoch: v.epoch,
+            credits_earned: v['credits earned'],
+            slots_in_epoch: v['slots in epoch'],
+            percentage: (
+              (v['credits earned'] * 1.0) /
+              (v['slots in epoch'] * 1.0)
+            ).toFixed(6),
+          });
+          return a;
+        },
+        [],
+      );
+
+      const uptimeValue = {
+        nodePubkey: result['node id'],
+        authorizedVoterPubkey: result['authorized voter pubkey'],
+        uptime: uptime,
+        lat: t2 - t1,
+        ts: t1,
+      };
+
+      resolve(uptimeValue);
+    });
   });
 
   return p;
@@ -80,9 +83,9 @@ function getVoteAccountUptime(x) {
 async function refreshUptime() {
   console.log('uptime updater: updating...');
   const connection = new solanaWeb3.Connection(FULLNODE_URL);
-  let voting = await connection.getEpochVoteAccounts();
+  let voting = await connection.getVoteAccounts();
 
-  const allTasks = _.map(voting, v => {
+  const allTasks = _.map(voting.current.concat(...voting.delinquent), v => {
     return getVoteAccountUptime(v);
   });
 

src/v2/components/Dashboard/NetworkOverview/StatCards/index.jsx

@@ -12,17 +12,18 @@ import Socket from 'v2/stores/socket';
 
 import Loader from '../../Loader';
 import useStyles from './styles';
+import {size} from 'lodash';
 
 const StatCards = () => {
   const {globalStats} = OverviewStore;
-  const {cluster} = NodesStore;
+  const {network} = NodesStore;
   const {isLoading} = Socket;
   const classes = useStyles();
 
   const cards = [
     {
       title: 'Node Count',
-      value: cluster.nodes.length,
+      value: size(network),
     },
     {
       title: 'Block Height',

src/v2/components/TourDeSol/Cards/index.jsx

@@ -14,10 +14,12 @@ const Cards = ({
 }) => {
   const classes = useStyles();
   const {
-    cluster,
+    network,
     validators,
     inactiveValidators,
-    totalStakedTokens,
+    supply,
+    totalStaked,
+    networkInflationRate,
   } = NodesStore;
 
   const cards = [
@@ -41,19 +43,19 @@ const Cards = ({
     },
     {
       title: 'Total SOL In Circulation',
-      value: (cluster.supply / Math.pow(2, 34)).toFixed(2),
+      value: (supply / Math.pow(2, 34)).toFixed(2),
       changes: '',
       period: 'since yesterday',
     },
     {
-      title: 'Total Staked Tokens',
-      value: totalStakedTokens,
+      title: 'Total Staked SOL',
+      value: totalStaked,
       changes: '',
       period: 'since yesterday',
     },
     {
       title: 'Current Network Inflation Rate',
-      value: (cluster.networkInflationRate * 100.0).toFixed(3) + '%',
+      value: (networkInflationRate * 100.0).toFixed(3) + '%',
       changes: '',
       period: 'since yesterday',
     },