feat: initial implementation of uptime crawler

api/api.js

@@ -102,6 +102,7 @@ let handleRedis = type => (channel, message) => {
 const client = getClient();
 
 const setexAsync = promisify(client.setex).bind(client);
+const getAsync = promisify(client.get).bind(client);
 const mgetAsync = promisify(client.mget).bind(client);
 const existsAsync = promisify(client.exists).bind(client);
 const lrangeAsync = promisify(client.lrange).bind(client);
@@ -502,6 +503,8 @@ async function getClusterInfo() {
   let cluster = await connection.getClusterNodes();
   let identities = await fetchValidatorIdentities(cluster.map(c => c.pubkey));
   let voting = await connection.getEpochVoteAccounts();
+  let uptime = await getAsync('!uptime');
+
   let totalStaked = _.reduce(
     voting,
     (a, v) => {
@@ -535,8 +538,10 @@ async function getClusterInfo() {
     cluster,
     identities,
     voting,
+    uptime,
     ts,
   };
+
   await setexAsync(
     '!clusterInfo',
     CLUSTER_INFO_CACHE_TIME_SECS,

api/uptime-crawler.js

@@ -0,0 +1,92 @@
+import * as solanaWeb3 from '@solana/web3.js';
+import _ from 'lodash';
+import YAML from 'yaml';
+import redis from 'redis';
+import {promisify} from 'util';
+import {exec} from 'child_process';
+
+import config from './config';
+
+const SOLANA_WALLET_PATH =
+  process.env.SOLANA_WALLET_PATH || '../solana/target/debug';
+const FULLNODE_URL = process.env.FULLNODE_URL || 'http://localhost:8899';
+
+const REFRESH_INTERVAL = 10 * 60 * 1000; // 10min
+
+function getClient() {
+  let props = config.redis.path
+    ? {path: config.redis.path}
+    : {host: config.redis.host, port: config.redis.port};
+
+  return redis.createClient(props);
+}
+
+const client = getClient();
+const setAsync = promisify(client.set).bind(client);
+
+function getVoteAccountUptime(x) {
+  const t1 = new Date().getTime();
+
+  const p = new Promise((resolve, reject) => {
+    exec(
+      `${SOLANA_WALLET_PATH}/solana-wallet -u ${FULLNODE_URL} show-vote-account ${x.votePubkey}`,
+      (err, stdout, stderr) => {
+        const t2 = new Date().getTime();
+
+        if (err) {
+          // node couldn't execute the command
+          console.log('err', err, stderr);
+          reject(err);
+          return;
+        }
+
+        const result = YAML.parse(stdout);
+
+        const uptime = _.reduce(
+          result['epoch voting history'],
+          (a, v) => {
+            a.unshift({
+              epoch: v.epoch,
+              credits_earned: v['credits earned'],
+              slots_in_epoch: v['slots in epoch'],
+              percentage: (
+                (v['credits earned'] * 1.0) /
+                (v['slots in epoch'] * 1.0)
+              ).toFixed(6),
+            });
+            return a;
+          },
+          [],
+        );
+
+        const uptimeValue = {
+          votePubkey: x.votePubkey,
+          uptime: uptime,
+          lat: t2 - t1,
+          ts: t1,
+        };
+
+        resolve(uptimeValue);
+      },
+    );
+  });
+
+  return p;
+}
+
+async function refreshUptime() {
+  const connection = new solanaWeb3.Connection(FULLNODE_URL);
+  let voting = await connection.getEpochVoteAccounts();
+
+  const allTasks = _.map(voting, v => {
+    return getVoteAccountUptime(v);
+  });
+
+  Promise.all(allTasks).then(async results => {
+    await setAsync('!uptime', JSON.stringify(results));
+  });
+}
+
+console.log('uptime updater process running...');
+refreshUptime();
+setInterval(refreshUptime, REFRESH_INTERVAL);

package.json

@@ -99,6 +99,7 @@
     "prepack": "set -ex; npm run lint; npm run build",
     "pretty": "prettier --write '{,{api,proxy,src}/**/}*.js{,x}'",
     "re": "semantic-release --repository-url [email protected]:solana-labs/blockexplorer.git",
+    "start:uptime-crawler": "set -ex; redis-cli ping; babel-node --presets env api/uptime-crawler.js",
     "start:api": "set -ex; redis-cli ping; babel-node --presets env api/api.js",
     "start:proxy": "babel-node --presets env proxy",
     "start:ui": "react-app-rewired start",