feat(fetch): store missing epoch even if they dont exist

this is happening because on one testnet restart the genesis was the same as before but the data was not keep, it was coming from a snapshot

packages/nimiq-validators-score/src/fetcher.ts

@@ -11,8 +11,11 @@ export async function fetchActivity(client: NimiqRPCClient, epochIndex: number)
 
   const electionBlock = genesisBlockNumber + (epochIndex * blocksPerEpoch)
   const { data: block, error } = await client.blockchain.getBlockByNumber(electionBlock, { includeTransactions: true })
-  if (error || !block)
-    throw new Error(JSON.stringify({ epochIndex, error, block }))
+  if (error || !block) {
+    // throw new Error(JSON.stringify({ epochIndex, error, block }))
+    console.error(JSON.stringify({ epochIndex, error, block }))
+    return {}
+  }
   if (!('isElectionBlock' in block))
     throw new Error(JSON.stringify({ message: 'Block is not election block', epochIndex, block }))
 
@@ -113,6 +116,9 @@ export async function fetchActivity(client: NimiqRPCClient, epochIndex: number)
 export async function* fetchEpochs(client: NimiqRPCClient, epochsIndexes: number[]) {
   for (const epochIndex of epochsIndexes) {
     const validatorActivities = await fetchActivity(client, epochIndex)
+    // If validatorActivities is empty, it means that the epoch cannot be fetched
+    if (Object.keys(validatorActivities).length === 0)
+      yield { epochIndex, address: '', activity: null }
     for (const [address, activity] of Object.entries(validatorActivities)) {
       yield { address, epochIndex, activity }
     }

server/lib/fetch.ts

@@ -3,7 +3,7 @@ import { consola } from 'consola'
 import type { EpochsActivities } from 'nimiq-validators-score'
 import { fetchCurrentEpoch, fetchEpochs, getRange } from 'nimiq-validators-score'
 import { findMissingValidators, storeValidator } from '../utils/validators'
-import { findMissingEpochs, storeActivities } from '../utils/activities'
+import { findMissingEpochs, storeActivities, storeSingleActivity } from '../utils/activities'
 
 const EPOCHS_IN_PARALLEL = 3
 
@@ -74,6 +74,10 @@ async function fetchMissingEpochs(client: NimiqRPCClient) {
       const { value: pair, done } = await epochGenerator.next()
       if (done || !pair)
         break
+      if (pair.activity === null) {
+        await storeSingleActivity({ address: '', activity: null, epochNumber: pair.epochIndex })
+        continue
+      }
       if (!epochsActivities[`${pair.epochIndex}`])
         epochsActivities[`${pair.epochIndex}`] = {}
       const epoch = epochsActivities[`${pair.epochIndex}`]
@@ -87,7 +91,7 @@ async function fetchMissingEpochs(client: NimiqRPCClient) {
     const percentage = Math.round((fetchedEpochs.length / missingEpochs.length) * 100).toFixed(2)
     consola.info(`Fetched ${newestEpoch} epochs. ${percentage}%`)
 
-    if (epochs.length === 0)
+    if ((await findMissingEpochs(range)).length === 0)
       break
     await storeActivities(epochsActivities)
   }

server/utils/activities.ts

@@ -38,11 +38,13 @@ export async function storeActivities(epochs: EpochsActivities) {
 
 interface StoreActivityParams {
   address: string
-  activity: Activity
+  activity: Activity | null
   epochNumber: number
 }
 
-async function storeSingleActivity({ address, activity, epochNumber }: StoreActivityParams) {
+const defaultActivity: Activity = { likelihood: -1, missed: -1, rewarded: -1, sizeRatio: 0, sizeRatioViaSlots: false }
+
+export async function storeSingleActivity({ address, activity, epochNumber }: StoreActivityParams) {
   const validatorId = await storeValidator(address)
   if (!validatorId)
     return
@@ -56,7 +58,7 @@ async function storeSingleActivity({ address, activity, epochNumber }: StoreActi
       eq(tables.activity.validatorId, validatorId),
     ))
 
-  const { likelihood, rewarded, missed, sizeRatio: _sizeRatio, sizeRatioViaSlots: _sizeRatioViaSlots } = activity
+  const { likelihood, rewarded, missed, sizeRatio: _sizeRatio, sizeRatioViaSlots: _sizeRatioViaSlots } = activity || defaultActivity
 
   // We always want to update db except the columns `sizeRatio` and `sizeRatioViaSlots`.
   // If we have `sizeRatioViaSlots` as false and `sizeRatio` > 0, then we won't update only those columns