From 46b325eb8d6fdfaf538b0ee2fab5593822c186ee Mon Sep 17 00:00:00 2001 From: Sunny Gleason Date: Fri, 16 Aug 2019 12:08:36 -0400 Subject: [PATCH] feat: more canonical validators list feat: upgrade to new getVoteAccounts() API, integrate dashboard model fix: add uniq key prop feat: style updates for api code fix: yarn lint --- api/api.js | 167 +++++++++++++++--- api/uptime-crawler.js | 89 +++++----- .../NetworkOverview/StatCards/index.jsx | 5 +- src/v2/components/TourDeSol/Cards/index.jsx | 14 +- src/v2/components/TourDeSol/Table/index.jsx | 8 +- src/v2/components/TourDeSol/index.jsx | 6 +- src/v2/components/Validators/All/index.jsx | 6 +- src/v2/components/Validators/Detail/index.jsx | 12 +- src/v2/components/Validators/Table/index.jsx | 16 +- src/v2/components/Validators/index.jsx | 8 +- src/v2/stores/nodes.js | 104 +++++------ src/v2/utils/parseMessage.js | 21 +-- 12 files changed, 279 insertions(+), 177 deletions(-) diff --git a/api/api.js b/api/api.js index 02045373..a70d76da 100644 --- a/api/api.js +++ b/api/api.js @@ -29,8 +29,8 @@ import config from './config'; const FULLNODE_URL = 'http://localhost:8899'; const GLOBAL_STATS_BROADCAST_INTERVAL_MS = 2000; -const CLUSTER_INFO_BROADCAST_INTERVAL_MS = 16000; -const CLUSTER_INFO_CACHE_TIME_SECS = 35000; +const CLUSTER_INFO_BROADCAST_INTERVAL_MS = 5000; +const CLUSTER_INFO_CACHE_TIME_SECS = 4500; const CONFIG_PROGRAM_ID = 'Config1111111111111111111111111111111111111'; const MAX_KEYBASE_USER_LOOKUP = 50; @@ -67,6 +67,10 @@ function randomOffset(seedString) { return (x - Math.floor(x)) / 10 - 0.05; } +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + function getClient() { let props = config.redis.path ? {path: config.redis.path} @@ -498,65 +502,175 @@ const DEFAULT_LNG = 165.3768; async function getClusterInfo() { const connection = new solanaWeb3.Connection(FULLNODE_URL); + const nodeConnectionCache = {}; let ts = new Date().toISOString(); let [, feeCalculator] = await connection.getRecentBlockhash(); let currentSlot = await getAsync('!blk-last-slot'); let networkInflationRate = getNetworkInflationRate(currentSlot); let supply = await connection.getTotalSupply(); - let cluster = await connection.getClusterNodes(); - let identities = await fetchValidatorIdentities(cluster.map(c => c.pubkey)); - let votingNow = await connection.getEpochVoteAccounts(); - let votingAll = await connection.getProgramAccounts( + let clusterNodes = await connection.getClusterNodes(); + const leader = await connection.getSlotLeader(); + let identities = await fetchValidatorIdentities( + clusterNodes.map(c => c.pubkey), + ); + let voteAccounts = await connection.getVoteAccounts(); + let allVoteAccounts = await connection.getProgramAccounts( solanaWeb3.VOTE_ACCOUNT_KEY, ); let uptimeJson = await getAsync('!uptime'); let uptime = uptimeJson && JSON.parse(uptimeJson); let totalStaked = _.reduce( - votingNow, + (voteAccounts.current || []).concat(voteAccounts.delinquent || []), (a, v) => { - a += v.stake || 0; + a += v.activatedStake || 0; return a; }, 0, ); - votingAll = _.map(votingAll, e => { - const [, v] = e; + const network = {}; - let voteAccount = solanaWeb3.VoteAccount.fromAccountData(v.data); - voteAccount.nodePubkey = voteAccount.nodePubkey.toString(); - voteAccount.authorizedVoterPubkey = voteAccount.authorizedVoterPubkey.toString(); + for (const clusterNode of clusterNodes) { + const {pubkey, rpc, tpu, gossip} = clusterNode; - return voteAccount; - }); + if (!tpu) { + continue; + } - cluster = _.map(cluster, c => { - let ip = c.gossip.split(':')[0]; + let ip = tpu.split(':')[0]; const geoip = geoipLookup(ip); let ll = geoip ? geoip.ll : null; - let newc = _.clone(c, true); // compute different but deterministic offsets let offsetLat = randomOffset(ip); - let offsetLng = randomOffset(c.gossip); + let offsetLng = randomOffset(tpu); + + let lat = ((ll && ll[0]) || DEFAULT_LAT) + offsetLat; + let lng = ((ll && ll[1]) || DEFAULT_LNG) + offsetLng; + + network[pubkey] = Object.assign(network[pubkey] || {}, { + online: true, + gossip, + rpc, + tpu, + lat, + lng, + coordinates: [lng, lat], + }); + } - newc.lat = ((ll && ll[0]) || DEFAULT_LAT) + offsetLat; - newc.lng = ((ll && ll[1]) || DEFAULT_LNG) + offsetLng; + for (let [votePubkey, voteAccountInfo] of allVoteAccounts) { + voteAccountInfo.owner = + voteAccountInfo.owner && voteAccountInfo.owner.toString(); - return newc; - }); + let voteAccount = solanaWeb3.VoteAccount.fromAccountData( + voteAccountInfo.data, + ); + voteAccount.authorizedVoterPubkey = + voteAccount.authorizedVoterPubkey && + voteAccount.authorizedVoterPubkey.toString(); + voteAccount.nodePubkey = + voteAccount.nodePubkey && voteAccount.nodePubkey.toString(); + + const nodePubkey = voteAccount.nodePubkey.toString(); + const node = network[nodePubkey]; + if (!node) { + continue; + } + if (node.votePubkey && node.votePubkey != votePubkey) { + if (node.warning && node.warning.hasMultipleVoteAccounts) { + node.warning.hasMultipleVoteAccounts.push( + voteAccount.authorizedVoterPubkey, + ); + } else { + node.warning |= {}; + node.warning.hasMultipleVoteAccounts = [ + node.votePubkey, + voteAccount.authorizedVoterPubkey, + ]; + } + continue; + } + node.nodePubkey = nodePubkey; + node.voteAccount = voteAccount; + node.votePubkey = votePubkey; + node.identity = identities.find(x => { + return x.pubkey === nodePubkey; + }); + node.uptime = + uptime && + uptime.find(x => { + return x.nodePubkey === nodePubkey; + }); + + node.voteStatus = + voteAccounts.current.find(x => { + return x.nodePubkey === nodePubkey; + }) || + voteAccounts.delinquent.find(x => { + return x.nodePubkey === nodePubkey; + }); + node.activatedStake = node.voteStatus && node.voteStatus.activatedStake; + node.commission = node.voteStatus && node.voteStatus.commission; + } + + for (const node of Object.keys(network).sort()) { + const {online, rpc, tpu} = network[node]; + if (!online && !tpu) { + continue; + } + + const balanceLamports = await connection.getBalance( + new solanaWeb3.PublicKey(node), + ); + let currentSlot = null; + if (rpc) { + try { + let nodeConnection = nodeConnectionCache[rpc]; + if (nodeConnection === undefined) { + nodeConnectionCache[rpc] = nodeConnection = new solanaWeb3.Connection( + `http://${rpc}`, + ); + } + currentSlot = await Promise.race([ + nodeConnection.getSlot(), + sleep(1000), + ]); + if (currentSlot === undefined) { + currentSlot = 'timeout'; + } + } catch (err) { + currentSlot = 'error'; + } + } + + let what; + if (!tpu && online) { + what = 'Spy'; + } else { + what = 'Validator'; + } + + let newNode = network[node] || {}; + newNode.leader = leader === node; + newNode.what = what; + newNode.balanceLamports = balanceLamports; + newNode.currentSlot = currentSlot; + network[node] = newNode; + } let rest = { feeCalculator, supply, networkInflationRate, totalStaked, - cluster, + network, + clusterNodes, identities, - votingAll, - votingNow, + voteAccounts, + allVoteAccounts, uptime, ts, }; @@ -566,6 +680,7 @@ async function getClusterInfo() { CLUSTER_INFO_CACHE_TIME_SECS, JSON.stringify(rest), ); + return rest; } diff --git a/api/uptime-crawler.js b/api/uptime-crawler.js index 479e1c63..3007ac87 100644 --- a/api/uptime-crawler.js +++ b/api/uptime-crawler.js @@ -31,47 +31,50 @@ function getVoteAccountUptime(x) { const t1 = new Date().getTime(); const p = new Promise((resolve, reject) => { - exec( - `solana-wallet -u ${FULLNODE_URL} show-vote-account ${x.votePubkey}`, - (err, stdout, stderr) => { - const t2 = new Date().getTime(); - - if (err) { - // node couldn't execute the command - console.log('err', err, stderr); - reject(err); - return; - } - - const result = YAML.parse(stdout); - - const uptime = _.reduce( - result['epoch voting history'], - (a, v) => { - a.unshift({ - epoch: v.epoch, - credits_earned: v['credits earned'], - slots_in_epoch: v['slots in epoch'], - percentage: ( - (v['credits earned'] * 1.0) / - (v['slots in epoch'] * 1.0) - ).toFixed(6), - }); - return a; - }, - [], - ); - - const uptimeValue = { - votePubkey: x.votePubkey, - uptime: uptime, - lat: t2 - t1, - ts: t1, - }; - - resolve(uptimeValue); - }, - ); + if (!x.votePubkey || x.votePubkey.length !== 44) { + reject(`invalid pubkey: ${x}`); + return; + } + let command = `solana-wallet -u ${FULLNODE_URL} show-vote-account ${x.votePubkey}`; + exec(command, (err, stdout, stderr) => { + const t2 = new Date().getTime(); + + if (err) { + // node couldn't execute the command + console.log('err', err, stderr); + reject(err); + return; + } + + const result = YAML.parse(stdout); + + const uptime = _.reduce( + result['epoch voting history'], + (a, v) => { + a.unshift({ + epoch: v.epoch, + credits_earned: v['credits earned'], + slots_in_epoch: v['slots in epoch'], + percentage: ( + (v['credits earned'] * 1.0) / + (v['slots in epoch'] * 1.0) + ).toFixed(6), + }); + return a; + }, + [], + ); + + const uptimeValue = { + nodePubkey: result['node id'], + authorizedVoterPubkey: result['authorized voter pubkey'], + uptime: uptime, + lat: t2 - t1, + ts: t1, + }; + + resolve(uptimeValue); + }); }); return p; @@ -80,9 +83,9 @@ function getVoteAccountUptime(x) { async function refreshUptime() { console.log('uptime updater: updating...'); const connection = new solanaWeb3.Connection(FULLNODE_URL); - let voting = await connection.getEpochVoteAccounts(); + let voting = await connection.getVoteAccounts(); - const allTasks = _.map(voting, v => { + const allTasks = _.map(voting.current.concat(...voting.delinquent), v => { return getVoteAccountUptime(v); }); diff --git a/src/v2/components/Dashboard/NetworkOverview/StatCards/index.jsx b/src/v2/components/Dashboard/NetworkOverview/StatCards/index.jsx index f05e2592..f35e1346 100644 --- a/src/v2/components/Dashboard/NetworkOverview/StatCards/index.jsx +++ b/src/v2/components/Dashboard/NetworkOverview/StatCards/index.jsx @@ -12,17 +12,18 @@ import Socket from 'v2/stores/socket'; import Loader from '../../Loader'; import useStyles from './styles'; +import {size} from 'lodash'; const StatCards = () => { const {globalStats} = OverviewStore; - const {cluster} = NodesStore; + const {network} = NodesStore; const {isLoading} = Socket; const classes = useStyles(); const cards = [ { title: 'Node Count', - value: cluster.nodes.length, + value: size(network), }, { title: 'Block Height', diff --git a/src/v2/components/TourDeSol/Cards/index.jsx b/src/v2/components/TourDeSol/Cards/index.jsx index 64c5df45..07573d51 100644 --- a/src/v2/components/TourDeSol/Cards/index.jsx +++ b/src/v2/components/TourDeSol/Cards/index.jsx @@ -14,10 +14,12 @@ const Cards = ({ }) => { const classes = useStyles(); const { - cluster, + network, validators, inactiveValidators, - totalStakedTokens, + supply, + totalStaked, + networkInflationRate, } = NodesStore; const cards = [ @@ -41,19 +43,19 @@ const Cards = ({ }, { title: 'Total SOL In Circulation', - value: (cluster.supply / Math.pow(2, 34)).toFixed(2), + value: (supply / Math.pow(2, 34)).toFixed(2), changes: '', period: 'since yesterday', }, { - title: 'Total Staked Tokens', - value: totalStakedTokens, + title: 'Total Staked SOL', + value: totalStaked, changes: '', period: 'since yesterday', }, { title: 'Current Network Inflation Rate', - value: (cluster.networkInflationRate * 100.0).toFixed(3) + '%', + value: (networkInflationRate * 100.0).toFixed(3) + '%', changes: '', period: 'since yesterday', }, diff --git a/src/v2/components/TourDeSol/Table/index.jsx b/src/v2/components/TourDeSol/Table/index.jsx index 514eec18..a2ecc399 100644 --- a/src/v2/components/TourDeSol/Table/index.jsx +++ b/src/v2/components/TourDeSol/Table/index.jsx @@ -32,7 +32,7 @@ const ValidatorsTable = ({separate}: {separate: boolean}) => { const renderRow = row => { const uptime = getUptime(row); - const {identity = {}, nodePubkey, stake} = row; + const {identity = {}, nodePubkey, activatedStake} = row; return ( 1 @@ -42,14 +42,14 @@ const ValidatorsTable = ({separate}: {separate: boolean}) => {
{identity.name || nodePubkey}
- {(stake * LAMPORT_SOL_RATIO).toFixed(8)} + {(activatedStake * LAMPORT_SOL_RATIO).toFixed(8)} {uptime}% ); }; const renderCard = card => { const uptime = getUptime(card); - const {identity = {}, nodePubkey, stake} = card; + const {identity = {}, nodePubkey, activatedStake} = card; return (
{
Stake
-
{(stake * LAMPORT_SOL_RATIO).toFixed(4)}
+
{(activatedStake * LAMPORT_SOL_RATIO).toFixed(4)}
Uptime
diff --git a/src/v2/components/TourDeSol/index.jsx b/src/v2/components/TourDeSol/index.jsx index cad70e09..74d4771f 100644 --- a/src/v2/components/TourDeSol/index.jsx +++ b/src/v2/components/TourDeSol/index.jsx @@ -64,7 +64,7 @@ const TourDeSol = () => { if (isActive) { return ( -
  • +
  • {stage.title} (LIVE!) @@ -73,7 +73,7 @@ const TourDeSol = () => { ); } else if (isFinished) { return ( -
  • +
  • {stage.title}
    @@ -83,7 +83,7 @@ const TourDeSol = () => { ); } else { return ( -
  • +
  • {stage.title}
    diff --git a/src/v2/components/Validators/All/index.jsx b/src/v2/components/Validators/All/index.jsx index da520f87..131408dd 100644 --- a/src/v2/components/Validators/All/index.jsx +++ b/src/v2/components/Validators/All/index.jsx @@ -8,12 +8,14 @@ import ValidatorsTable from '../Table'; import useStyles from './styles'; const ValidatorsAll = () => { - const {validators} = NodesStore; + const {validators, inactiveValidators} = NodesStore; const classes = useStyles(); return ( -
    {validators.length}
    +
    + {validators.length + inactiveValidators.length} +
    diff --git a/src/v2/components/Validators/Detail/index.jsx b/src/v2/components/Validators/Detail/index.jsx index 5b5e40e3..b0e8e7d0 100644 --- a/src/v2/components/Validators/Detail/index.jsx +++ b/src/v2/components/Validators/Detail/index.jsx @@ -35,7 +35,7 @@ const mapStyles = { }; const ValidatorsDetail = ({match}: {match: Match}) => { - const {validators} = NodesStore; + const {validators, inactiveValidators} = NodesStore; const {globalStats} = OverviewStore; const classes = useStyles(); @@ -47,13 +47,15 @@ const ValidatorsDetail = ({match}: {match: Match}) => { Mixpanel.track(`Clicked Validator ${params.id}`); }, [params.id]); - const node = find({nodePubkey: params.id})(validators); + let node = + find({nodePubkey: params.id})(validators) || + find({nodePubkey: params.id})(inactiveValidators); if (!node) { return
    Loading...
    ; } - const {nodePubkey, gossip, stake, commission, identity = {}} = node; + const {nodePubkey, gossip, activatedStake, commission, identity = {}} = node; const renderMarker = () => ( { { label: 'Voting power', hint: '', - value: (stake * LAMPORT_SOL_RATIO).toFixed(8), + value: (activatedStake * LAMPORT_SOL_RATIO).toFixed(8), }, { label: 'Website', @@ -231,7 +233,7 @@ const ValidatorsDetail = ({match}: {match: Match}) => { )) } - {renderMarker()} + {node.coordinates && renderMarker()}
    diff --git a/src/v2/components/Validators/Table/index.jsx b/src/v2/components/Validators/Table/index.jsx index cb3b410b..762c5ff4 100644 --- a/src/v2/components/Validators/Table/index.jsx +++ b/src/v2/components/Validators/Table/index.jsx @@ -30,7 +30,7 @@ const ValidatorsTable = ({separate}: {separate: boolean}) => { const {validators, inactiveValidators} = NodesStore; const renderRow = row => { const uptime = row.uptime && getUptime(row); - const {identity = {}, nodePubkey, stake, commission} = row; + const {identity = {}, nodePubkey, activatedStake, commission} = row; return ( @@ -40,7 +40,9 @@ const ValidatorsTable = ({separate}: {separate: boolean}) => { - {(stake && (stake * LAMPORT_SOL_RATIO).toFixed(8)) || 'N/A'} + {(activatedStake && + (activatedStake * LAMPORT_SOL_RATIO).toFixed(8)) || + 'N/A'} {commission || 'N/A'} {(uptime && uptime + '%') || 'Unavailable'} @@ -49,7 +51,7 @@ const ValidatorsTable = ({separate}: {separate: boolean}) => { }; const renderCard = card => { const uptime = card.uptime && getUptime(card); - const {identity = {}, nodePubkey, stake, commission} = card; + const {identity = {}, nodePubkey, activatedStake, commission} = card; return (
    {
    Stake
    - {(stake && (stake * LAMPORT_SOL_RATIO).toFixed(4)) || 'N/A'} + {(activatedStake && + (activatedStake * LAMPORT_SOL_RATIO).toFixed(4)) || + 'N/A'}
    @@ -83,7 +87,9 @@ const ValidatorsTable = ({separate}: {separate: boolean}) => { {!separate && (
    Validators - {validators.length} + + {validators.length + inactiveValidators.length} + See all > diff --git a/src/v2/components/Validators/index.jsx b/src/v2/components/Validators/index.jsx index 9d746a94..e5f39bd6 100644 --- a/src/v2/components/Validators/index.jsx +++ b/src/v2/components/Validators/index.jsx @@ -15,7 +15,7 @@ import useStyles from './styles'; const Validators = () => { const classes = useStyles(); - const {cluster, validators, fetchClusterInfo, totalStakedTokens} = NodesStore; + const {supply, validators, fetchClusterInfo, totalStaked} = NodesStore; useEffect(() => { fetchClusterInfo(); // eslint-disable-next-line react-hooks/exhaustive-deps @@ -23,13 +23,13 @@ const Validators = () => { const cards = [ { title: 'Total Circulating SOL', - value: (cluster.supply / Math.pow(2, 34)).toFixed(2), + value: (supply / Math.pow(2, 34)).toFixed(2), changes: '', period: 'since yesterday', }, { - title: 'Total Staked Tokens', - value: totalStakedTokens, + title: 'Total Staked SOL', + value: totalStaked, changes: '', period: 'since yesterday', }, diff --git a/src/v2/stores/nodes.js b/src/v2/stores/nodes.js index 284bba43..2565693a 100644 --- a/src/v2/stores/nodes.js +++ b/src/v2/stores/nodes.js @@ -1,96 +1,80 @@ -import { - sumBy, - eq, - get, - compose, - keys, - filter, - map, - mapValues, - pick, - merge, - find, -} from 'lodash/fp'; -import {action, computed, decorate, observable, observe, flow} from 'mobx'; -import {parseClusterInfo} from 'v2/utils/parseMessage'; +import {filter, map} from 'lodash/fp'; +import {action, computed, decorate, observable, flow} from 'mobx'; import * as API from 'v2/api/stats'; -import calcChanges from 'v2/utils/calcChanges'; class Store { cluster = { - nodes: [], - votingNow: [], - votingAll: [], + network: {}, }; clusterChanges = {}; constructor() { - observe(this, 'cluster', ({oldValue, newValue}) => { - if (!keys(oldValue).length) { - return; - } - this.clusterChanges = compose( - mapValues.convert({cap: false})((value, key) => { - if (eq('nodes', key)) { - return calcChanges(oldValue[key].length, value.length); - } - return calcChanges(oldValue[key], value); - }), - pick(['nodes', 'supply']), - )(newValue); - }); + // observe(this, 'network', ({oldValue, newValue}) => { + // if (!keys(oldValue).length) { + // return; + // } + // this.clusterChanges = compose( + // mapValues.convert({cap: false})((value, key) => { + // if (eq('nodes', key)) { + // return calcChanges(oldValue[key].length, value.length); + // } + // return calcChanges(oldValue[key], value); + // }), + // pick(['nodes', 'supply']), + // )(newValue); + // }); } updateClusterInfo = data => { - this.cluster = merge(this.cluster, parseClusterInfo(data)); + data = JSON.parse(data); + this.network = data.network || {}; + this.totalStaked = data.totalStaked; + this.supply = data.supply; + this.networkInflationRate = data.networkInflationRate; }; fetchClusterInfo = flow(function*() { const res = yield API.getClusterInfo(); - this.cluster = merge(this.cluster, res.data); + const data = res.data; + this.network = data.network; + this.totalStaked = data.totalStaked; + this.supply = data.supply; + this.networkInflationRate = data.networkInflationRate; }); get mapMarkers() { - return map(({pubkey: name, gossip, lat, lng}) => ({ + let validators = filter(node => node.what === 'Validator')(this.network); + + validators = map(({nodePubkey: name, tpu: gossip, coordinates}) => ({ name, gossip, - coordinates: [lng, lat], - }))(this.cluster.nodes); + coordinates, + }))(validators); + + return validators; } get validators() { - return map(vote => { - const cluster = - find({pubkey: vote.nodePubkey})(this.cluster.cluster) || {}; - const {lng = 0, lat = 0, gossip} = cluster; - return { - ...vote, - coordinates: [lng, lat], - gossip, - uptime: find({votePubkey: vote.votePubkey})(this.cluster.uptime), - identity: find({pubkey: vote.nodePubkey})(this.cluster.identities), - }; - })(this.cluster.votingNow); + let active = filter( + node => node.what === 'Validator' && node.activatedStake, + )(this.network); + + return active; } get inactiveValidators() { let inactive = filter( - vote => !find({pubkey: vote.nodePubkey})(this.cluster.cluster), - )(this.cluster.votingAll); + node => node.what === 'Validator' && !node.activatedStake, + )(this.network); return inactive; } - - get totalStakedTokens() { - return compose( - sumBy('stake'), - get('votingNow'), - )(this.cluster); - } } decorate(Store, { - cluster: observable, + network: observable, + supply: observable, + stakedTokens: observable, updateClusterInfo: action.bound, mapMarkers: computed, validators: computed, diff --git a/src/v2/utils/parseMessage.js b/src/v2/utils/parseMessage.js index 149d837b..08f79370 100644 --- a/src/v2/utils/parseMessage.js +++ b/src/v2/utils/parseMessage.js @@ -1,4 +1,4 @@ -import {find, compose, split, map} from 'lodash/fp'; +import {compose, split, map} from 'lodash/fp'; export function parseTransaction(message) { const [h, l, s, dt, entry_id, id, inst] = split('#')(message); @@ -42,25 +42,12 @@ export function parseBlock(message) { } export function parseClusterInfo(data) { - const { - votingNow, - votingAll, - cluster: gossip, - supply, - feeCalculator, - identities, - } = JSON.parse(data); - - const nodes = map(g => ({ - ...g, - voteAccount: find({nodePubkey: g.pubKey})(votingNow), - }))(gossip); + const {network, supply, totalStaked, feeCalculator} = JSON.parse(data); return { - nodes, + network, supply, + totalStaked, feeCalculator, - identities, - votingAll, }; }