From 42d26fd27b8c5aff74020b759be8998046f459e6 Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Fri, 22 Nov 2024 17:25:31 +0100 Subject: [PATCH] Added recovery mechanism for missing block result --- .../src/services/prisma/PrismaBlockStorage.ts | 32 ++++++++++++++----- .../sequencing/BlockProducerModule.ts | 29 +++++++++++++++-- .../storage/inmemory/InMemoryBlockStorage.ts | 29 +++++++++++++++-- packages/sequencer/src/storage/model/Block.ts | 5 +++ .../src/storage/repositories/BlockStorage.ts | 9 ++++-- 5 files changed, 88 insertions(+), 16 deletions(-) diff --git a/packages/persistance/src/services/prisma/PrismaBlockStorage.ts b/packages/persistance/src/services/prisma/PrismaBlockStorage.ts index 53be8654..52a06346 100644 --- a/packages/persistance/src/services/prisma/PrismaBlockStorage.ts +++ b/packages/persistance/src/services/prisma/PrismaBlockStorage.ts @@ -8,6 +8,7 @@ import { BlockStorage, BlockWithResult, BlockWithPreviousResult, + BlockWithMaybeResult, } from "@proto-kit/sequencer"; import { filterNonNull, log } from "@proto-kit/common"; import { @@ -39,7 +40,7 @@ export class PrismaBlockStorage private async getBlockByQuery( where: { height: number } | { hash: string } - ): Promise { + ): Promise { const dbResult = await this.connection.prismaClient.block.findFirst({ where, include: { @@ -57,18 +58,15 @@ export class PrismaBlockStorage const transactions = dbResult.transactions.map( (txresult) => this.transactionResultMapper.mapIn([txresult, txresult.tx]) ); - if (dbResult.result === undefined || dbResult.result === null) { - throw new Error( - `No Metadata has been set for block ${JSON.stringify(where)} yet` - ); - } return { block: { ...this.blockMapper.mapIn(dbResult), transactions, }, - result: this.blockResultMapper.mapIn(dbResult.result), + result: dbResult.result + ? this.blockResultMapper.mapIn(dbResult.result) + : undefined, }; } @@ -169,7 +167,9 @@ export class PrismaBlockStorage return (result?._max.height ?? -1) + 1; } - public async getLatestBlock(): Promise { + public async getLatestBlockAndResult(): Promise< + BlockWithMaybeResult | undefined + > { const latestBlock = await this.connection.prismaClient.$queryRaw< { hash: string }[] >`SELECT b1."hash" FROM "Block" b1 @@ -185,6 +185,22 @@ export class PrismaBlockStorage }); } + public async getLatestBlock(): Promise { + const result = await this.getLatestBlockAndResult(); + if (result !== undefined) { + if (result.result === undefined) { + throw new Error( + `Block result for block ${result.block.height.toString()} not found` + ); + } + return { + block: result.block, + result: result.result, + }; + } + return result; + } + public async getNewBlocks(): Promise { const blocks = await this.connection.prismaClient.block.findMany({ where: { diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts index ec71fb1f..e98447d4 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts @@ -158,19 +158,31 @@ export class BlockProducerModule extends SequencerModule { }> { const txs = await this.mempool.getTxs(this.maximumBlockSize()); - const parentBlock = await this.blockQueue.getLatestBlock(); + const parentBlock = await this.blockQueue.getLatestBlockAndResult(); + + let metadata: BlockWithResult; if (parentBlock === undefined) { log.debug( "No block metadata given, assuming first block, generating genesis metadata" ); + metadata = BlockWithResult.createEmpty(); + } else if (parentBlock.result === undefined) { + throw new Error( + `Metadata for block at height ${parentBlock.block.height.toString()} not available` + ); + } else { + metadata = { + block: parentBlock.block, + // By reconstructing this object, typescript correctly infers the result to be defined + result: parentBlock.result, + }; } const messages = await this.messageStorage.getMessages( parentBlock?.block.toMessagesHash.toString() ?? ACTIONS_EMPTY_HASH.toString() ); - const metadata = parentBlock ?? BlockWithResult.createEmpty(); log.debug( `Block collected, ${txs.length} txs, ${messages.length} messages` @@ -215,6 +227,17 @@ export class BlockProducerModule extends SequencerModule { } public async start() { - noop(); + // Check if metadata height is behind block production. + // This can happen when the sequencer crashes after a block has been produced + // but before the metadata generation has finished + const latestBlock = await this.blockQueue.getLatestBlockAndResult(); + // eslint-disable-next-line sonarjs/no-collapsible-if + if (latestBlock !== undefined) { + if (latestBlock.result === undefined) { + await this.generateMetadata(latestBlock.block); + } + // Here, the metadata has been computed already + } + // If we reach here, its a genesis startup, no blocks exist yet } } diff --git a/packages/sequencer/src/storage/inmemory/InMemoryBlockStorage.ts b/packages/sequencer/src/storage/inmemory/InMemoryBlockStorage.ts index a12f846e..cf332bb4 100644 --- a/packages/sequencer/src/storage/inmemory/InMemoryBlockStorage.ts +++ b/packages/sequencer/src/storage/inmemory/InMemoryBlockStorage.ts @@ -5,7 +5,12 @@ import { BlockQueue, BlockStorage, } from "../repositories/BlockStorage"; -import type { Block, BlockResult, BlockWithResult } from "../model/Block"; +import type { + Block, + BlockResult, + BlockWithMaybeResult, + BlockWithResult, +} from "../model/Block"; import { BlockWithPreviousResult } from "../../protocol/production/BatchProducerModule"; import { BatchStorage } from "../repositories/BatchStorage"; @@ -29,10 +34,12 @@ export class InMemoryBlockStorage return this.blocks.length; } - public async getLatestBlock(): Promise { + public async getLatestBlockAndResult(): Promise< + BlockWithMaybeResult | undefined + > { const currentHeight = await this.getCurrentBlockHeight(); const block = await this.getBlockAt(currentHeight - 1); - const result = this.results[currentHeight - 1]; + const result: BlockResult | undefined = this.results[currentHeight - 1]; if (block === undefined) { return undefined; } @@ -42,6 +49,22 @@ export class InMemoryBlockStorage }; } + public async getLatestBlock(): Promise { + const result = await this.getLatestBlockAndResult(); + if (result !== undefined) { + if (result.result === undefined) { + throw new Error( + `Block result for block ${result.block.height.toString()} not found` + ); + } + return { + block: result.block, + result: result.result, + }; + } + return result; + } + public async getNewBlocks(): Promise { const latestBatch = await this.batchStorage.getLatestBatch(); diff --git a/packages/sequencer/src/storage/model/Block.ts b/packages/sequencer/src/storage/model/Block.ts index 5aa87c4a..ef7c0994 100644 --- a/packages/sequencer/src/storage/model/Block.ts +++ b/packages/sequencer/src/storage/model/Block.ts @@ -61,6 +61,11 @@ export interface BlockWithResult { result: BlockResult; } +export interface BlockWithMaybeResult { + block: Block; + result?: BlockResult; +} + // eslint-disable-next-line @typescript-eslint/no-redeclare export const BlockWithResult = { createEmpty: () => diff --git a/packages/sequencer/src/storage/repositories/BlockStorage.ts b/packages/sequencer/src/storage/repositories/BlockStorage.ts index 741332ea..2347162f 100644 --- a/packages/sequencer/src/storage/repositories/BlockStorage.ts +++ b/packages/sequencer/src/storage/repositories/BlockStorage.ts @@ -1,11 +1,16 @@ import { BlockWithPreviousResult } from "../../protocol/production/BatchProducerModule"; -import type { Block, BlockResult, BlockWithResult } from "../model/Block"; +import { + Block, + BlockResult, + BlockWithMaybeResult, + BlockWithResult, +} from "../model/Block"; export interface BlockQueue { pushBlock: (block: Block) => Promise; pushResult: (result: BlockResult) => Promise; getNewBlocks: () => Promise; - getLatestBlock: () => Promise; + getLatestBlockAndResult: () => Promise; } export interface BlockStorage {