Skip to content
This repository has been archived by the owner on Nov 15, 2022. It is now read-only.

Mario/staking refactor #191

Open
wants to merge 21 commits into
base: develop
Choose a base branch
from
Open

Mario/staking refactor #191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
45c94df
staking crawler quick refactor
mariopino Nov 29, 2020
3077619
change polling times
mariopino Nov 29, 2020
6225536
fix staking
mariopino Nov 29, 2020
03b744d
fix
mariopino Nov 29, 2020
3343696
multiple rows insert in validator table
mariopino Nov 29, 2020
9dd7dae
multiple inserts
mariopino Nov 29, 2020
c4c3adf
revert multiple inserts
mariopino Nov 29, 2020
9842f90
fix sessionId in intentions
mariopino Nov 30, 2020
2f4437d
fix intention sessionIds
mariopino Nov 30, 2020
19864f3
fix
mariopino Nov 30, 2020
f46d097
fix
mariopino Nov 30, 2020
297302f
fix
mariopino Nov 30, 2020
38029b9
parallelize bonds query
mariopino Nov 30, 2020
f943c50
polkadot node
mariopino Feb 4, 2021
4351ca2
typo
mariopino Feb 4, 2021
02a554d
@polkadot/api
mariopino Feb 4, 2021
000da9e
upgrade parity polkadot node
Feb 22, 2021
b62d4cd
bump @polkadot/api
Feb 22, 2021
d6df56b
3.9.3
Feb 22, 2021
e6ec200
debug
Feb 22, 2021
bf847c9
fix mess
Feb 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions backend.config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ module.exports = {
config: {
pollingTime:
parseInt(process.env.CRAWLER_BLOCK_LISTENER_POLLING_TIME_MS) ||
1 * 60 * 1000,
60 * 60 * 1000,
},
},

Expand All @@ -45,7 +45,7 @@ module.exports = {
config: {
pollingTime:
parseInt(process.env.CRAWLER_ACTIVE_ACCOUNTS_POLLING_TIME_MS) ||
60 * 60 * 1000,
30 * 60 * 1000,
},
},

Expand Down
2 changes: 1 addition & 1 deletion docker/polkastats-backend/substrate-client/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ FROM phusion/baseimage:0.11
LABEL maintainer "@ColmenaLabs_svq"
LABEL description="Small image with the Substrate binary."

ARG VERSION=v0.8.25
ARG VERSION=v0.8.28-1

RUN apt-get update && apt-get install wget curl jq -y

Expand Down
194 changes: 89 additions & 105 deletions lib/crawlers/staking.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,12 @@ module.exports = {
logger.info(loggerOptions, `First execution, no session index found in DB!`);

const sessionInfo = await api.derive.session.info();
const currentEraIndex = sessionInfo.activeEra.toNumber();
const currentSessionIndex = sessionInfo.currentIndex.toNumber();
currentKnownSessionIndex = currentSessionIndex;

const block = await api.rpc.chain.getBlock();
const blockNumber = block.block.header.number.toNumber();
await module.exports.storeSessionStakingInfo(api, pool, blockNumber, sessionInfo, currentEraIndex);
await module.exports.storeSessionStakingInfo(api, pool, blockNumber, sessionInfo);
}

// Subscribe to new blocks
Expand All @@ -39,16 +38,15 @@ module.exports = {
const blockNumber = header.number.toNumber();

const sessionInfo = await api.derive.session.info();
const currentEraIndex = sessionInfo.activeEra.toNumber();
const currentSessionIndex = sessionInfo.currentIndex.toNumber();
logger.info(loggerOptions, `currentSessionIndex: ${currentSessionIndex}, currentKnownSessionIndex: ${currentKnownSessionIndex}`);
if (currentSessionIndex > currentKnownSessionIndex) {
currentKnownSessionIndex = currentSessionIndex;
await module.exports.storeSessionStakingInfo(api, pool, blockNumber, sessionInfo, currentEraIndex);
await module.exports.storeSessionStakingInfo(api, pool, blockNumber, sessionInfo);
}
});
},
storeSessionStakingInfo: async function (api, pool, blockNumber, sessionInfo, currentEraIndex) {
storeSessionStakingInfo: async function (api, pool, blockNumber, sessionInfo) {

// Start execution
const startTime = new Date().getTime();
Expand All @@ -57,71 +55,56 @@ module.exports = {
logger.info(loggerOptions, `Storing validators staking info for session #${currentIndex} (block #${blockNumber})`);

//
// Get active validators, next elected and nominators
// Get active validators, next elected, nominators and waitingInfo
//
logger.info(loggerOptions, `Get active validators, next elected and nominators`);
const [validatorAddresses, stakingOverview, nominators] = await Promise.all([
logger.info(loggerOptions, `Get active validators, next elected, nominators and waitingInfo`);
const [validatorAddresses, { nextElected }, nominators, { info }] = await Promise.all([
api.query.session.validators(),
api.derive.staking.overview(),
api.query.staking.nominators.entries()
api.query.staking.nominators.entries(),
api.derive.staking.waitingInfo()
]);

//
// Map validator authorityId to staking info object
// Validator staking info
//
logger.info(loggerOptions, `Map validator authorityId to staking info object`);
const validatorStaking = await Promise.all(
validatorAddresses.map(authorityId => api.derive.staking.account(authorityId))
logger.info(loggerOptions, `Getting validator staking info`);
let validatorStaking = [];
validatorStaking = await Promise.all(
validatorAddresses
.map(authorityId => api.derive.staking.account(authorityId)
.then((validator) => {
return {
...validator,
sessionIdHex: validator.sessionIds.length !== 0 ? validator.sessionIds.toHex() : ``,
nextSessionIdHex: validator.nextSessionIds.length !== 0 ? validator.nextSessionIds.toHex() : ``,
nextElected: nextElected.includes(validator.accountId),
}
}
)
)
);

//
// Add hex representation of sessionId[] and nextSessionId[]
//
logger.info(loggerOptions, `Add hex representation of sessionId[] and nextSessionId[]`);
validatorStaking.forEach(validator => {
validator.sessionIdHex = validator.sessionIds.length !== 0 ? validator.sessionIds.toHex() : ``;
validator.nextSessionIdHex = validator.nextSessionIds.length !== 0 ? validator.nextSessionIds.toHex() : ``;
})

//
// Add next elected property
//
logger.info(loggerOptions, `Add next elected property`);
const { nextElected } = JSON.parse(JSON.stringify(stakingOverview));
validatorStaking.forEach(function (validator) {
if (nextElected.includes(validator.accountId.toString())) {
validator.nextElected = true;
} else {
validator.nextElected = false;
}
});

//
// Add identity
//
logger.info(loggerOptions, `Add identity`);
for(let i = 0; i < validatorStaking.length; i++) {
let validator = validatorStaking[i];
const { identity } = await api.derive.accounts.info(validator.accountId);
validator.identity = identity;
validator.displayName = getDisplayName(identity);
}

//
// Add next elected property
//
logger.info(loggerOptions, `Add next elected property`);
validatorStaking.forEach(function (validator) {
if (nextElected.includes(validator.accountId.toString())) {
validator.nextElected = true;
} else {
validator.nextElected = false;
}
});
logger.info(loggerOptions, `Getting validator identities`);
validatorStaking = await Promise.all(
validatorStaking.map((validator) =>
api.derive.accounts.info(validator.accountId).then(({ identity }) => {
return {
...validator,
identity,
displayName: getDisplayName(identity),
}
})
)
)

//
// Get nominations
//
logger.info(loggerOptions, `Getting nominations`);
const nominations = nominators.map(([key, nominations]) => {
const nominator = key.toHuman()[0];
const targets = nominations.toHuman().targets;
Expand All @@ -130,20 +113,15 @@ module.exports = {
targets
}
});

//
// Stakers
//

// Get all nominator addresses
const allNominatorAddresses = nominations.map(nomination => nomination.nominator);

// Get all nominator identities
logger.info(loggerOptions, `Get all nominator identities`);
// Get nominator identities
logger.info(loggerOptions, `Getting nominator identities`);
const allNominatorIdentities = await Promise.all(
allNominatorAddresses.map(accountId => api.derive.accounts.info(accountId))
);


logger.info(loggerOptions, `Getting validator stakers`);
for(let i = 0; i < validatorStaking.length; i++) {
let validator = validatorStaking[i];
let stakers = [];
Expand Down Expand Up @@ -178,6 +156,7 @@ module.exports = {
//
// Sort by stake / add rank
//
logger.info(loggerOptions, `Sorting validators by stake / add rank`);
validatorStaking.sort((a, b) => {
const A = new BigNumber(a.exposure.total);
const B = new BigNumber(b.exposure.total);
Expand All @@ -193,6 +172,7 @@ module.exports = {
//
// Populate validator table
//
logger.info(loggerOptions, `Populating validators table`);
for(let i = 0; i < validatorStaking.length; i++) {
const validator = validatorStaking[i];
const sql = `
Expand Down Expand Up @@ -270,57 +250,43 @@ module.exports = {
//
// Get intention staking info
//
const { info } = await api.derive.staking.waitingInfo();
const intentionStaking = info;

//
// Add hex representation of sessionId[] and nextSessionId[]
//
for(let i = 0; i < intentionStaking.length; i++) {
let intention = intentionStaking[i];
if (intention.sessionIds) {
intention.sessionIdHex = intention.sessionIds.toHex();
}
if (intention.nextSessionIds) {
intention.nextSessionIdHex = intention.nextSessionIds.toHex();
logger.info(loggerOptions, `Getting intention staking info`);
let intentionStaking = []
intentionStaking = info.map(intention => {
return {
...intention,
nextSessionIds: "[]",
sessionIds: "[]",
sessionIdHex: '',
nextSessionIdHex: '',
nextElected: nextElected.includes(intention.accountId),
stakers: nominations
.filter(nomination => nomination.targets.some(target => target === intention.accountId.toString()))
.map(nomination => nomination.nominator),
}
}

//
// Add intention stakers
//
});

for(let i = 0; i < intentionStaking.length; i++) {
let intention = intentionStaking[i];
intention.stakers = nominations
.filter(nomination => nomination.targets.some(target => target === intention.accountId.toString()))
.map(nomination => nomination.nominator);
}

//
// Add identity
//
for(let i = 0; i < intentionStaking.length; i++) {
let intention = intentionStaking[i];
const { identity } = await api.derive.accounts.info(intention.accountId);
intention.identity = identity;
intention.displayName = getDisplayName(identity);
}

//
// Add next elected property
//
intentionStaking.forEach(function (intention) {
if (nextElected.includes(intention.accountId.toString())) {
intention.nextElected = true;
} else {
intention.nextElected = false;
}
});
logger.info(loggerOptions, `Getting intention identities`);
intentionStaking = await Promise.all(
intentionStaking.map((intention) =>
api.derive.accounts.info(intention.accountId).then(({ identity }) => {
return {
...intention,
identity,
displayName: getDisplayName(identity),
}
})
)
)

//
// Sort by stake / add rank
//
logger.info(loggerOptions, `Sorting intentions by stake / add rank`);
intentionStaking.sort((a, b) => {
const A = new BigNumber(a.stakingLedger.total);
const B = new BigNumber(b.stakingLedger.total);
Expand All @@ -336,6 +302,7 @@ module.exports = {
//
// Populate intention table
//
logger.info(loggerOptions, `Populating intentions table`);
for(let i = 0; i < intentionStaking.length; i++) {
const intention = intentionStaking[i];
const sql = `
Expand Down Expand Up @@ -397,13 +364,28 @@ module.exports = {
logger.info(loggerOptions, `Storing nominator info for session #${currentIndex} (block #${blockNumber}`);

// Get nominator balances
logger.info(loggerOptions, `Getting nominator balances`);
const allNominatorBalances = await Promise.all(
allNominatorAddresses.map(accountId => api.derive.balances.all(accountId))
);

// Get nominator bonds
logger.info(loggerOptions, `Getting nominator bonds`);
const allNominatorBonds = await Promise.all(
allNominatorAddresses.map(accountId => api.query.staking.bonded(accountId).then(
(bond) => {
return {
accountId,
bond,
}
})
)
);

//
// Populate nominator table
//
logger.info(loggerOptions, `Populating nominators table`);
let nominatorStaking = [];
for (let i = 0; i < validatorStaking.length; i++) {
let validator = validatorStaking[i];
Expand All @@ -428,7 +410,7 @@ module.exports = {
});
} else {
let totalStakedBN = new BigNumber(exposure.value.toString());
const bonded = await api.query.staking.bonded(exposure.who);
const bonded = allNominatorBonds.find(bond => bond.accountId.toString() === exposure.who.toString());
let balances = allNominatorBalances.find(balance => balance.accountId.toString() === exposure.who.toString());
let displayName = ``;
const identity = allNominatorIdentities.find(identity => identity.accountId.toString() === exposure.who.toString());
Expand Down Expand Up @@ -474,12 +456,14 @@ module.exports = {
}

// Sort by total staked
logger.info(loggerOptions, `Sorting nominators by total staked`);
nominatorStaking.sort((a, b) => {
const A = new BigNumber(a.totalStaked);
const B = new BigNumber(b.totalStaked);
return A.lt(B) ? 1 : -1;
});

logger.info(loggerOptions, `Populating nominators table`);
rank = 1;
for(let i = 0; i < nominatorStaking.length; i++) {
const nominator = nominatorStaking[i];
Expand Down
Loading