diff --git a/apps/processing/src/index.ts b/apps/processing/src/index.ts index 97d77ad..9705da5 100644 --- a/apps/processing/src/index.ts +++ b/apps/processing/src/index.ts @@ -6,7 +6,7 @@ import { ProcessingService } from "./services/processing.service.js"; let processor: ProcessingService; const main = async (): Promise => { - processor = new ProcessingService(environment); + processor = await ProcessingService.initialize(environment); await processor.start(); }; diff --git a/apps/processing/src/services/processing.service.ts b/apps/processing/src/services/processing.service.ts index 285cd8f..cf75f58 100644 --- a/apps/processing/src/services/processing.service.ts +++ b/apps/processing/src/services/processing.service.ts @@ -23,10 +23,9 @@ export class ProcessingService { private readonly logger = new Logger({ className: "ProcessingService" }); private readonly kyselyDatabase: SharedDependencies["kyselyDatabase"]; - constructor(private readonly env: Environment) { + private constructor(env: Environment, sharedDependencies: SharedDependencies) { const { CHAINS: chains } = env; - const { core, registries, indexerClient, kyselyDatabase } = - SharedDependenciesService.initialize(env); + const { core, registries, indexerClient, kyselyDatabase } = sharedDependencies; this.kyselyDatabase = kyselyDatabase; for (const chain of chains) { @@ -49,6 +48,11 @@ export class ProcessingService { } } + static async initialize(env: Environment): Promise { + const sharedDependencies = await SharedDependenciesService.initialize(env); + return new ProcessingService(env, sharedDependencies); + } + /** * Start the processor service * diff --git a/apps/processing/src/services/sharedDependencies.service.ts b/apps/processing/src/services/sharedDependencies.service.ts index 6e1bc25..b2e05ae 100644 --- a/apps/processing/src/services/sharedDependencies.service.ts +++ b/apps/processing/src/services/sharedDependencies.service.ts @@ -1,7 +1,10 @@ import { CoreDependencies, + DatabaseStrategyRegistry, + IEventsRegistry, + InMemoryCachedStrategyRegistry, InMemoryEventsRegistry, - InMemoryStrategyRegistry, + IStrategyRegistry, } from "@grants-stack-indexer/data-flow"; import { EnvioIndexerClient } from "@grants-stack-indexer/indexer-client"; import { IpfsProvider } from "@grants-stack-indexer/metadata"; @@ -13,6 +16,7 @@ import { KyselyDonationRepository, KyselyProjectRepository, KyselyRoundRepository, + KyselyStrategyRegistryRepository, } from "@grants-stack-indexer/repository"; import { Logger } from "@grants-stack-indexer/shared"; @@ -21,8 +25,8 @@ import { Environment } from "../config/index.js"; export type SharedDependencies = { core: Omit; registries: { - eventsRegistry: InMemoryEventsRegistry; - strategyRegistry: InMemoryStrategyRegistry; + eventsRegistry: IEventsRegistry; + strategyRegistry: IStrategyRegistry; }; indexerClient: EnvioIndexerClient; kyselyDatabase: ReturnType; @@ -35,7 +39,7 @@ export type SharedDependencies = { * - Initializes indexer client */ export class SharedDependenciesService { - static initialize(env: Environment): SharedDependencies { + static async initialize(env: Environment): Promise { // Initialize repositories const kyselyDatabase = createKyselyDatabase({ connectionString: env.DATABASE_URL, @@ -68,8 +72,16 @@ export class SharedDependenciesService { const eventsRegistry = new InMemoryEventsRegistry( new Logger({ className: "InMemoryEventsRegistry" }), ); - const strategyRegistry = new InMemoryStrategyRegistry( - new Logger({ className: "InMemoryStrategyRegistry" }), + const strategyRepository = new KyselyStrategyRegistryRepository( + kyselyDatabase, + env.DATABASE_SCHEMA, + ); + const strategyRegistry = await InMemoryCachedStrategyRegistry.initialize( + new Logger({ className: "InMemoryCachedStrategyRegistry" }), + new DatabaseStrategyRegistry( + new Logger({ className: "DatabaseStrategyRegistry" }), + strategyRepository, + ), ); // Initialize indexer client diff --git a/apps/processing/test/unit/processing.service.spec.ts b/apps/processing/test/unit/processing.service.spec.ts index 802ed72..6623fde 100644 --- a/apps/processing/test/unit/processing.service.spec.ts +++ b/apps/processing/test/unit/processing.service.spec.ts @@ -52,9 +52,9 @@ describe("ProcessingService", () => { DATABASE_SCHEMA: "public", }; - beforeEach(() => { + beforeEach(async () => { vi.clearAllMocks(); - processingService = new ProcessingService(mockEnv as Environment); + processingService = await ProcessingService.initialize(mockEnv as Environment); }); afterEach(() => { diff --git a/apps/processing/test/unit/sharedDependencies.service.spec.ts b/apps/processing/test/unit/sharedDependencies.service.spec.ts index fc633dd..9d309b9 100644 --- a/apps/processing/test/unit/sharedDependencies.service.spec.ts +++ b/apps/processing/test/unit/sharedDependencies.service.spec.ts @@ -17,6 +17,11 @@ vi.mock("@grants-stack-indexer/repository", () => ({ KyselyApplicationRepository: vi.fn(), KyselyDonationRepository: vi.fn(), KyselyApplicationPayoutRepository: vi.fn(), + KyselyStrategyRegistryRepository: vi.fn().mockImplementation(() => ({ + getStrategies: vi.fn().mockResolvedValue([]), + getStrategyId: vi.fn(), + saveStrategyId: vi.fn(), + })), })); vi.mock("@grants-stack-indexer/pricing", () => ({ @@ -33,6 +38,26 @@ vi.mock("@grants-stack-indexer/indexer-client", () => ({ EnvioIndexerClient: vi.fn(), })); +vi.mock("@grants-stack-indexer/data-flow", () => { + const mockStrategyRegistry = { + getStrategies: vi.fn(), + getStrategyId: vi.fn(), + saveStrategyId: vi.fn(), + }; + + return { + InMemoryEventsRegistry: vi.fn(), + InMemoryCachedStrategyRegistry: { + initialize: vi.fn().mockResolvedValue(mockStrategyRegistry), + }, + DatabaseStrategyRegistry: vi.fn().mockImplementation(() => ({ + getStrategies: vi.fn(), + getStrategyId: vi.fn(), + saveStrategyId: vi.fn(), + })), + }; +}); + describe("SharedDependenciesService", () => { const mockEnv: Pick< Environment, @@ -51,8 +76,8 @@ describe("SharedDependenciesService", () => { PRICING_SOURCE: "dummy", }; - it("initializes all dependencies correctly", () => { - const dependencies = SharedDependenciesService.initialize(mockEnv as Environment); + it("initializes all dependencies correctly", async () => { + const dependencies = await SharedDependenciesService.initialize(mockEnv as Environment); // Verify database initialization expect(createKyselyDatabase).toHaveBeenCalledWith({ @@ -89,5 +114,8 @@ describe("SharedDependenciesService", () => { // Verify registries expect(dependencies.registries).toHaveProperty("eventsRegistry"); expect(dependencies.registries).toHaveProperty("strategyRegistry"); + + // Verify InMemoryCachedStrategyRegistry initialization + expect(dependencies.registries.strategyRegistry).toBeDefined(); }); }); diff --git a/package.json b/package.json index 0ec53b2..68e8a44 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "build": "turbo run build", "check-types": "turbo run check-types", "clean": "turbo run clean", + "db:migrate": "pnpm run --filter @grants-stack-indexer/migrations db:migrate", + "db:reset": "pnpm run --filter @grants-stack-indexer/migrations db:reset", "dev": "turbo run dev", "format": "turbo run format", "format:fix": "turbo run format:fix", @@ -17,8 +19,6 @@ "lint": "turbo run lint", "lint:fix": "turbo run lint:fix", "prepare": "husky", - "script:db:migrate": "pnpm run --filter @grants-stack-indexer/migrations script:db:migrate", - "script:db:reset": "pnpm run --filter @grants-stack-indexer/migrations script:db:reset", "start": "turbo run start", "test": "turbo run test", "test:cov": "turbo run test:cov", diff --git a/packages/data-flow/README.md b/packages/data-flow/README.md index 0ab5fe7..d6ab601 100644 --- a/packages/data-flow/README.md +++ b/packages/data-flow/README.md @@ -90,9 +90,13 @@ The `EventsProcessor` class is responsible for processing events in the processi The `EventsFetcher` class is responsible for fetching events from the blockchain. -### [StrategyRegistry](./src/strategyRegistry.ts) +### [StrategyRegistry](./src/registries/) The `StrategyRegistry` stores strategy IDs to populate strategy events with them given the Strategy address. +There are 3 implementations: + +- `DatabaseStrategyRegistry`: persists data to database using IStrategyRepository +- `InMemoryCachedStrategyRegistry`: stores map in-memory as cache and persists to database ### [DataLoader](./src/data-loader/dataLoader.ts) diff --git a/packages/data-flow/src/external.ts b/packages/data-flow/src/external.ts index 28cb58e..4a22f36 100644 --- a/packages/data-flow/src/external.ts +++ b/packages/data-flow/src/external.ts @@ -1,7 +1,8 @@ export { DataLoader, InMemoryEventsRegistry, - InMemoryStrategyRegistry, + InMemoryCachedStrategyRegistry, + DatabaseStrategyRegistry, Orchestrator, } from "./internal.js"; diff --git a/packages/data-flow/src/interfaces/strategyRegistry.interface.ts b/packages/data-flow/src/interfaces/strategyRegistry.interface.ts index 8d3aa44..965ad44 100644 --- a/packages/data-flow/src/interfaces/strategyRegistry.interface.ts +++ b/packages/data-flow/src/interfaces/strategyRegistry.interface.ts @@ -1,22 +1,38 @@ import { Address, Hex } from "viem"; +import { Strategy } from "@grants-stack-indexer/repository"; +import { ChainId } from "@grants-stack-indexer/shared"; + /** * The strategy registry saves the mapping between the strategy address and the strategy id. Serves as a Cache * to avoid having to read from the chain to get the strategy id every time. */ -//TODO: implement a mechanism to record Strategy that we still don't have a corresponding handler -// we need to store and mark that this strategy is not handled yet, so when it's supported we can process all of the pending events for it export interface IStrategyRegistry { /** - * Get the strategy id by the strategy address + * Get the strategy id by the strategy address and chain id + * + * @param chainId - The chain id * @param strategyAddress - The strategy address - * @returns The strategy id or undefined if the strategy address is not registered + * @returns The strategy or undefined if the strategy address is not registered */ - getStrategyId(strategyAddress: Address): Promise; + getStrategyId(chainId: ChainId, strategyAddress: Address): Promise; /** - * Save the strategy id by the strategy address + * Save the strategy id by the strategy address and chain id + * @param chainId - The chain id * @param strategyAddress - The strategy address * @param strategyId - The strategy id + * @param handled - Whether the strategy is handled + */ + saveStrategyId( + chainId: ChainId, + strategyAddress: Address, + strategyId: Hex, + handled: boolean, + ): Promise; + + /** + * Get all the strategies + * @returns The strategies */ - saveStrategyId(strategyAddress: Address, strategyId: Hex): Promise; + getStrategies(filters?: { handled?: boolean; chainId?: ChainId }): Promise; } diff --git a/packages/data-flow/src/internal.ts b/packages/data-flow/src/internal.ts index accf61a..cb9a1a9 100644 --- a/packages/data-flow/src/internal.ts +++ b/packages/data-flow/src/internal.ts @@ -5,7 +5,7 @@ export * from "./abis/index.js"; export * from "./utils/index.js"; export * from "./data-loader/index.js"; export * from "./eventsFetcher.js"; -export * from "./strategyRegistry.js"; +export * from "./registries/index.js"; export * from "./eventsRegistry.js"; export * from "./eventsProcessor.js"; export * from "./orchestrator.js"; diff --git a/packages/data-flow/src/orchestrator.ts b/packages/data-flow/src/orchestrator.ts index ede93c6..7b41b3c 100644 --- a/packages/data-flow/src/orchestrator.ts +++ b/packages/data-flow/src/orchestrator.ts @@ -113,15 +113,17 @@ export class Orchestrator { await this.eventsRegistry.saveLastProcessedEvent(this.chainId, event); event = await this.enhanceStrategyId(event); - if (event.contractName === "Strategy" && "strategyId" in event) { + if (this.isPoolCreated(event)) { + const handleable = existsHandler(event.strategyId); + await this.strategyRegistry.saveStrategyId( + this.chainId, + event.srcAddress, + event.strategyId, + handleable, + ); + } else if (event.contractName === "Strategy" && "strategyId" in event) { if (!existsHandler(event.strategyId)) { - //TODO: save to registry as unsupported strategy, so when the strategy is handled it will be backwards compatible and process all of the events - //TODO: decide if we want to log this - // this.logger.info( - // `No handler found for strategyId: ${event.strategyId}. Event: ${stringify( - // event, - // )}`, - // ); + // we skip the event if the strategy id is not handled yet continue; } } @@ -216,9 +218,12 @@ export class Orchestrator { * @returns The strategy id */ private async getOrFetchStrategyId(strategyAddress: Address): Promise { - const existingId = await this.strategyRegistry.getStrategyId(strategyAddress); - if (existingId) { - return existingId; + const cachedStrategy = await this.strategyRegistry.getStrategyId( + this.chainId, + strategyAddress, + ); + if (cachedStrategy) { + return cachedStrategy.id; } const strategyId = await this.dependencies.evmProvider.readContract( @@ -227,11 +232,20 @@ export class Orchestrator { "getStrategyId", ); - await this.strategyRegistry.saveStrategyId(strategyAddress, strategyId); - return strategyId; } + /** + * Check if the event is a PoolCreated event from Allo contract + * @param event - The event + * @returns True if the event is a PoolCreated event from Allo contract, false otherwise + */ + private isPoolCreated( + event: ProcessorEvent, + ): event is ProcessorEvent<"Allo", "PoolCreated"> { + return isAlloEvent(event) && event.eventName === "PoolCreated"; + } + /** * Check if the event requires a strategy id * @param event - The event @@ -240,6 +254,6 @@ export class Orchestrator { private requiresStrategyId( event: ProcessorEvent, ): event is ProcessorEvent<"Allo", "PoolCreated"> | ProcessorEvent<"Strategy", StrategyEvent> { - return (isAlloEvent(event) && event.eventName === "PoolCreated") || isStrategyEvent(event); + return this.isPoolCreated(event) || isStrategyEvent(event); } } diff --git a/packages/data-flow/src/registries/index.ts b/packages/data-flow/src/registries/index.ts new file mode 100644 index 0000000..c6319bf --- /dev/null +++ b/packages/data-flow/src/registries/index.ts @@ -0,0 +1 @@ +export * from "./strategy/index.js"; diff --git a/packages/data-flow/src/registries/strategy/cachedStrategyRegistry.ts b/packages/data-flow/src/registries/strategy/cachedStrategyRegistry.ts new file mode 100644 index 0000000..16899cc --- /dev/null +++ b/packages/data-flow/src/registries/strategy/cachedStrategyRegistry.ts @@ -0,0 +1,89 @@ +import { Strategy } from "@grants-stack-indexer/repository"; +import { Address, ChainId, Hex, ILogger } from "@grants-stack-indexer/shared"; + +import { IStrategyRegistry } from "../../internal.js"; + +/** + * Proxy class to cache the strategy ids in memory or fallback to another strategy registry + */ +export class InMemoryCachedStrategyRegistry implements IStrategyRegistry { + private cache: Map>; + + private constructor( + private readonly logger: ILogger, + private readonly strategyRegistry: IStrategyRegistry, + cache: Map>, + ) { + this.cache = structuredClone(cache); + } + + /** @inheritdoc */ + async getStrategies(filters?: { handled?: boolean; chainId?: ChainId }): Promise { + return this.strategyRegistry.getStrategies(filters); + } + + /** + * Creates a new cached strategy registry instance. It will load the strategies into memory and cache them and + * fallback to the strategy registry if the strategy is not found in the cache. + * + * @param logger - The logger instance + * @param strategyRegistry - The strategy registry instance + * @returns The initialized cached strategy registry + */ + static async initialize( + logger: ILogger, + strategyRegistry: IStrategyRegistry, + ): Promise { + const strategies = await strategyRegistry.getStrategies(); + const cache = new Map>(); + + logger.debug(`Loading strategies into memory...`); + + for (const strategy of strategies) { + if (!cache.has(strategy.chainId)) { + cache.set(strategy.chainId, new Map()); + } + cache.get(strategy.chainId)?.set(strategy.address, strategy); + } + + return new InMemoryCachedStrategyRegistry(logger, strategyRegistry, cache); + } + + /** @inheritdoc */ + async getStrategyId(chainId: ChainId, strategyAddress: Address): Promise { + const cache = this.cache.get(chainId)?.get(strategyAddress); + if (cache) { + return cache; + } + + const strategy = await this.strategyRegistry.getStrategyId(chainId, strategyAddress); + if (strategy) { + this.cache.get(chainId)?.set(strategyAddress, strategy); + } + return strategy; + } + + /** @inheritdoc */ + async saveStrategyId( + chainId: ChainId, + strategyAddress: Address, + strategyId: Hex, + handled: boolean, + ): Promise { + if (this.cache.get(chainId)?.get(strategyAddress)?.handled === handled) { + return; + } + + this.logger.debug( + `Saving strategy id ${strategyId} for address ${strategyAddress} and chainId ${chainId}`, + ); + await this.strategyRegistry.saveStrategyId(chainId, strategyAddress, strategyId, handled); + + this.cache.get(chainId)?.set(strategyAddress, { + address: strategyAddress, + id: strategyId, + chainId, + handled, + }); + } +} diff --git a/packages/data-flow/src/registries/strategy/dbStrategyRegistry.ts b/packages/data-flow/src/registries/strategy/dbStrategyRegistry.ts new file mode 100644 index 0000000..e29fa3f --- /dev/null +++ b/packages/data-flow/src/registries/strategy/dbStrategyRegistry.ts @@ -0,0 +1,44 @@ +import type { Address, Hex } from "viem"; + +import { IStrategyRegistryRepository, Strategy } from "@grants-stack-indexer/repository"; +import { ChainId, ILogger } from "@grants-stack-indexer/shared"; + +import { IStrategyRegistry } from "../../internal.js"; + +/** + * Class to store strategy ids in Database + */ +export class DatabaseStrategyRegistry implements IStrategyRegistry { + constructor( + private logger: ILogger, + private strategyRepository: IStrategyRegistryRepository, + ) {} + + /** @inheritdoc */ + async getStrategies(filters?: { handled?: boolean; chainId?: ChainId }): Promise { + return this.strategyRepository.getStrategies(filters); + } + + /** @inheritdoc */ + async getStrategyId(chainId: ChainId, strategyAddress: Address): Promise { + return this.strategyRepository.getStrategyByChainIdAndAddress(chainId, strategyAddress); + } + + /** @inheritdoc */ + async saveStrategyId( + chainId: ChainId, + strategyAddress: Address, + strategyId: Hex, + handled: boolean, + ): Promise { + this.logger.debug( + `Saving strategy id ${strategyId} for address ${strategyAddress} and chainId ${chainId} in Database`, + ); + await this.strategyRepository.saveStrategy({ + chainId, + address: strategyAddress, + id: strategyId, + handled, + }); + } +} diff --git a/packages/data-flow/src/registries/strategy/index.ts b/packages/data-flow/src/registries/strategy/index.ts new file mode 100644 index 0000000..8ab6469 --- /dev/null +++ b/packages/data-flow/src/registries/strategy/index.ts @@ -0,0 +1,2 @@ +export * from "./cachedStrategyRegistry.js"; +export * from "./dbStrategyRegistry.js"; diff --git a/packages/data-flow/src/strategyRegistry.ts b/packages/data-flow/src/strategyRegistry.ts deleted file mode 100644 index d32fd59..0000000 --- a/packages/data-flow/src/strategyRegistry.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { Address, Hex } from "viem"; - -import { ILogger } from "@grants-stack-indexer/shared"; - -import type { IStrategyRegistry } from "./internal.js"; - -/** - * Class to store strategy ids in memory - */ -//TODO: Implement storage to persist strategies. since we're using address, do we need ChainId? -export class InMemoryStrategyRegistry implements IStrategyRegistry { - private strategiesMap: Map = new Map(); - constructor(private logger: ILogger) {} - - /** @inheritdoc */ - async getStrategyId(strategyAddress: Address): Promise { - return this.strategiesMap.get(strategyAddress); - } - - /** @inheritdoc */ - async saveStrategyId(strategyAddress: Address, strategyId: Hex): Promise { - this.logger.debug(`Saving strategy id ${strategyId} for address ${strategyAddress}`); - this.strategiesMap.set(strategyAddress, strategyId); - } -} diff --git a/packages/data-flow/test/registries/cachedStrategyRegistry.spec.ts b/packages/data-flow/test/registries/cachedStrategyRegistry.spec.ts new file mode 100644 index 0000000..d3edecb --- /dev/null +++ b/packages/data-flow/test/registries/cachedStrategyRegistry.spec.ts @@ -0,0 +1,140 @@ +import { Address, Hex } from "viem"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { Strategy } from "@grants-stack-indexer/repository"; +import { ChainId, ILogger } from "@grants-stack-indexer/shared"; + +import { IStrategyRegistry } from "../../src/internal.js"; +import { InMemoryCachedStrategyRegistry } from "../../src/registries/strategy/cachedStrategyRegistry.js"; + +describe("InMemoryCachedStrategyRegistry", () => { + const logger: ILogger = { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + }; + + const mockStrategyRegistry: IStrategyRegistry = { + getStrategies: vi.fn(), + getStrategyId: vi.fn(), + saveStrategyId: vi.fn(), + }; + + const chainId = 1 as ChainId; + const strategyAddress = "0x123" as Address; + const strategyId = "0xabc" as Hex; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("initialize with existing strategies", async () => { + const strategies: Strategy[] = [ + { + id: strategyId, + address: strategyAddress, + chainId, + handled: true, + }, + ]; + + vi.mocked(mockStrategyRegistry.getStrategies).mockResolvedValue(strategies); + + const registry = await InMemoryCachedStrategyRegistry.initialize( + logger, + mockStrategyRegistry, + ); + const cached = await registry.getStrategyId(chainId, strategyAddress); + + expect(cached).toEqual(strategies[0]); + expect(mockStrategyRegistry.getStrategyId).not.toHaveBeenCalled(); + }); + + it("fetch from underlying registry when not in cache", async () => { + const strategy: Strategy = { + id: strategyId, + address: strategyAddress, + chainId, + handled: true, + }; + + vi.mocked(mockStrategyRegistry.getStrategies).mockResolvedValue([]); + vi.mocked(mockStrategyRegistry.getStrategyId).mockResolvedValue(strategy); + + const registry = await InMemoryCachedStrategyRegistry.initialize( + logger, + mockStrategyRegistry, + ); + const result = await registry.getStrategyId(chainId, strategyAddress); + + expect(result).toEqual(strategy); + expect(mockStrategyRegistry.getStrategyId).toHaveBeenCalledWith(chainId, strategyAddress); + }); + + it("save strategy and update cache", async () => { + vi.mocked(mockStrategyRegistry.getStrategies).mockResolvedValue([]); + + const registry = await InMemoryCachedStrategyRegistry.initialize( + logger, + mockStrategyRegistry, + ); + await registry.saveStrategyId(chainId, strategyAddress, strategyId, true); + + const cached = await registry.getStrategyId(chainId, strategyAddress); + expect(cached).toEqual({ + id: strategyId, + address: strategyAddress, + chainId, + handled: true, + }); + expect(mockStrategyRegistry.saveStrategyId).toHaveBeenCalledWith( + chainId, + strategyAddress, + strategyId, + true, + ); + }); + + it("don't save if strategy already exists with same handled status", async () => { + const strategy: Strategy = { + id: strategyId, + address: strategyAddress, + chainId, + handled: true, + }; + + vi.mocked(mockStrategyRegistry.getStrategies).mockResolvedValue([strategy]); + + const registry = await InMemoryCachedStrategyRegistry.initialize( + logger, + mockStrategyRegistry, + ); + await registry.saveStrategyId(chainId, strategyAddress, strategyId, true); + + expect(mockStrategyRegistry.saveStrategyId).not.toHaveBeenCalled(); + }); + + it("delegate getStrategies to underlying registry", async () => { + const strategies: Strategy[] = [ + { + id: strategyId, + address: strategyAddress, + chainId, + handled: true, + }, + ]; + + vi.mocked(mockStrategyRegistry.getStrategies).mockResolvedValue(strategies); + + const registry = await InMemoryCachedStrategyRegistry.initialize( + logger, + mockStrategyRegistry, + ); + const params = { handled: true, chainId }; + const result = await registry.getStrategies(params); + + expect(result).toEqual(strategies); + expect(mockStrategyRegistry.getStrategies).toHaveBeenCalledWith(params); + }); +}); diff --git a/packages/data-flow/test/registries/dbStrategyRegistry.spec.ts b/packages/data-flow/test/registries/dbStrategyRegistry.spec.ts new file mode 100644 index 0000000..1974cdb --- /dev/null +++ b/packages/data-flow/test/registries/dbStrategyRegistry.spec.ts @@ -0,0 +1,94 @@ +import { Address, Hex } from "viem"; +import { describe, expect, it, vi } from "vitest"; + +import { IStrategyRegistryRepository, Strategy } from "@grants-stack-indexer/repository"; +import { ChainId, ILogger } from "@grants-stack-indexer/shared"; + +import { DatabaseStrategyRegistry } from "../../src/registries/strategy/dbStrategyRegistry.js"; + +describe("DatabaseStrategyRegistry", () => { + const logger: ILogger = { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + }; + + const mockStrategyRepository: IStrategyRegistryRepository = { + getStrategies: vi.fn(), + getStrategyByChainIdAndAddress: vi.fn(), + saveStrategy: vi.fn(), + }; + + const chainId = 1 as ChainId; + + it("return undefined for non-existent strategy address", async () => { + const registry = new DatabaseStrategyRegistry(logger, mockStrategyRepository); + const strategyAddress = "0x123" as Address; + + vi.mocked(mockStrategyRepository.getStrategyByChainIdAndAddress).mockResolvedValue( + undefined, + ); + + const strategy = await registry.getStrategyId(chainId, strategyAddress); + expect(strategy).toBeUndefined(); + expect(mockStrategyRepository.getStrategyByChainIdAndAddress).toHaveBeenCalledWith( + chainId, + strategyAddress, + ); + }); + + it("save and retrieve strategy id", async () => { + const registry = new DatabaseStrategyRegistry(logger, mockStrategyRepository); + const strategyAddress = "0x123" as Address; + const strategyId = "0xabc" as Hex; + const strategy: Strategy = { + id: strategyId, + address: strategyAddress, + chainId, + handled: true, + }; + + vi.mocked(mockStrategyRepository.saveStrategy).mockResolvedValue(); + vi.mocked(mockStrategyRepository.getStrategyByChainIdAndAddress).mockResolvedValue( + strategy, + ); + + await registry.saveStrategyId(chainId, strategyAddress, strategyId, true); + const retrieved = await registry.getStrategyId(chainId, strategyAddress); + + expect(mockStrategyRepository.saveStrategy).toHaveBeenCalledWith(strategy); + expect(retrieved).toEqual(strategy); + }); + + it("get all strategies", async () => { + const registry = new DatabaseStrategyRegistry(logger, mockStrategyRepository); + const strategies: Strategy[] = [ + { + id: "0xabc" as Hex, + address: "0x123" as Address, + chainId: 1 as ChainId, + handled: true, + }, + { + id: "0xdef" as Hex, + address: "0x456" as Address, + chainId: 1 as ChainId, + handled: false, + }, + ]; + + vi.mocked(mockStrategyRepository.getStrategies).mockResolvedValue(strategies); + + const result = await registry.getStrategies(); + expect(result).toEqual(strategies); + }); + + it("get strategies with filters", async () => { + const registry = new DatabaseStrategyRegistry(logger, mockStrategyRepository); + const params = { handled: true, chainId: 1 as ChainId }; + + await registry.getStrategies(params); + expect(mockStrategyRepository.getStrategies).toHaveBeenCalledWith(params); + }); +}); diff --git a/packages/data-flow/test/unit/orchestrator.spec.ts b/packages/data-flow/test/unit/orchestrator.spec.ts index 6943c78..0412e5a 100644 --- a/packages/data-flow/test/unit/orchestrator.spec.ts +++ b/packages/data-flow/test/unit/orchestrator.spec.ts @@ -83,6 +83,7 @@ describe("Orchestrator", { sequential: true }, () => { mockStrategyRegistry = { getStrategyId: vi.fn(), saveStrategyId: vi.fn(), + getStrategies: vi.fn(), }; mockEvmProvider = { @@ -140,6 +141,12 @@ describe("Orchestrator", { sequential: true }, () => { const eventsProcessorSpy = vi.spyOn(orchestrator["eventsProcessor"], "processEvent"); vi.spyOn(mockEventsRegistry, "getLastProcessedEvent").mockResolvedValue(undefined); + vi.spyOn(mockStrategyRegistry, "getStrategyId").mockResolvedValue({ + id: "0x6f9291df02b2664139cec5703c124e4ebce32879c74b6297faa1468aa5ff9ebf", + address: "0x123", + chainId, + handled: false, + }); vi.spyOn(mockIndexerClient, "getEventsAfterBlockNumberAndLogIndex") .mockResolvedValueOnce(mockEvents) .mockResolvedValue([]); @@ -154,6 +161,9 @@ describe("Orchestrator", { sequential: true }, () => { vi.spyOn(mockEventsRegistry, "saveLastProcessedEvent").mockImplementation(() => { return Promise.resolve(); }); + vi.spyOn(mockStrategyRegistry, "saveStrategyId").mockImplementation(() => { + return Promise.resolve(); + }); runPromise = orchestrator.run(abortController.signal); @@ -258,10 +268,6 @@ describe("Orchestrator", { sequential: true }, () => { }); expect(orchestrator["eventsProcessor"].processEvent).toHaveBeenCalledTimes(1); - expect(mockStrategyRegistry.saveStrategyId).toHaveBeenCalledWith( - strategyAddress, - strategyId, - ); expect(orchestrator["eventsProcessor"].processEvent).toHaveBeenCalledWith({ ...mockEvent, strategyId, @@ -274,6 +280,53 @@ describe("Orchestrator", { sequential: true }, () => { ); }); + it("save strategyId to registry on PoolCreated event", async () => { + const strategyAddress = "0x123" as Address; + const strategyId = "0xunknown" as Hex; + const existingStrategyId = + "0x6f9291df02b2664139cec5703c124e4ebce32879c74b6297faa1468aa5ff9ebf" as Hex; + + const mockEvent = createMockEvent("Allo", "PoolCreated", 1, { + strategy: strategyAddress, + poolId: "1", + profileId: "0x123", + token: "0x123", + amount: "100", + metadata: ["1", "1"], + }); + + const eventsProcessorSpy = vi.spyOn(orchestrator["eventsProcessor"], "processEvent"); + vi.spyOn(mockStrategyRegistry, "getStrategyId").mockResolvedValue(undefined); + vi.spyOn(mockEvmProvider, "readContract") + .mockResolvedValueOnce(strategyId) + .mockResolvedValueOnce(existingStrategyId); + vi.spyOn(mockEventsRegistry, "getLastProcessedEvent").mockResolvedValue(undefined); + vi.spyOn(mockIndexerClient, "getEventsAfterBlockNumberAndLogIndex") + .mockResolvedValueOnce([mockEvent, mockEvent]) + .mockResolvedValue([]); + + runPromise = orchestrator.run(abortController.signal); + + await vi.waitFor(() => { + if (eventsProcessorSpy.mock.calls.length < 1) throw new Error("Not yet called"); + }); + + expect(mockStrategyRegistry.saveStrategyId).toHaveBeenNthCalledWith( + 1, + chainId, + strategyAddress, + strategyId, + false, + ); + expect(mockStrategyRegistry.saveStrategyId).toHaveBeenNthCalledWith( + 2, + chainId, + strategyAddress, + existingStrategyId, + true, + ); + }); + const strategyEvents: Record = { RegisteredWithSender: "", DistributedWithRecipientAddress: "", @@ -321,7 +374,12 @@ describe("Orchestrator", { sequential: true }, () => { .mockResolvedValueOnce([mockEvent]) .mockResolvedValue([]); - vi.spyOn(mockStrategyRegistry, "getStrategyId").mockResolvedValue(strategyId); + vi.spyOn(mockStrategyRegistry, "getStrategyId").mockResolvedValue({ + id: strategyId, + address: strategyAddress, + chainId, + handled: true, + }); eventsProcessorSpy.mockResolvedValue(changesets); vi.spyOn(mockEventsRegistry, "saveLastProcessedEvent").mockImplementation(() => { return Promise.resolve(); @@ -345,7 +403,10 @@ describe("Orchestrator", { sequential: true }, () => { ...mockEvent, strategyId, }); - expect(mockStrategyRegistry.getStrategyId).toHaveBeenCalledWith(strategyAddress); + expect(mockStrategyRegistry.getStrategyId).toHaveBeenCalledWith( + chainId, + strategyAddress, + ); expect(orchestrator["dataLoader"].applyChanges).toHaveBeenCalledTimes(1); expect(orchestrator["dataLoader"].applyChanges).toHaveBeenCalledWith(changesets); expect(mockEventsRegistry.saveLastProcessedEvent).toHaveBeenCalledWith( @@ -375,7 +436,12 @@ describe("Orchestrator", { sequential: true }, () => { .mockResolvedValueOnce([mockEvent]) .mockResolvedValue([]); - vi.spyOn(mockStrategyRegistry, "getStrategyId").mockResolvedValue(unhandledStrategyId); + vi.spyOn(mockStrategyRegistry, "getStrategyId").mockResolvedValue({ + id: unhandledStrategyId, + address: strategyAddress, + chainId, + handled: false, + }); vi.spyOn(mockEvmProvider, "readContract").mockResolvedValue(unhandledStrategyId); runPromise = orchestrator.run(abortController.signal); @@ -388,6 +454,7 @@ describe("Orchestrator", { sequential: true }, () => { expect(orchestrator["eventsProcessor"].processEvent).not.toHaveBeenCalled(); expect(orchestrator["dataLoader"].applyChanges).not.toHaveBeenCalled(); expect(mockEventsRegistry.saveLastProcessedEvent).not.toHaveBeenCalled(); + expect(mockStrategyRegistry.saveStrategyId).not.toHaveBeenCalled(); }); it("uses cached strategy ID from registry", async () => { @@ -418,12 +485,16 @@ describe("Orchestrator", { sequential: true }, () => { vi.spyOn(mockStrategyRegistry, "getStrategyId") .mockResolvedValueOnce(undefined) - .mockResolvedValue(strategyId); + .mockResolvedValue({ + id: strategyId, + address: strategyAddress, + chainId, + handled: true, + }); + vi.spyOn(mockEvmProvider, "readContract").mockResolvedValue(strategyId); vi.spyOn(mockIndexerClient, "getEventsAfterBlockNumberAndLogIndex") - .mockResolvedValueOnce([poolCreatedEvent]) - .mockResolvedValueOnce([]) - .mockResolvedValueOnce([registeredEvent]) + .mockResolvedValueOnce([poolCreatedEvent, registeredEvent]) .mockResolvedValue([]); eventsProcessorSpy.mockResolvedValue([]); @@ -437,6 +508,9 @@ describe("Orchestrator", { sequential: true }, () => { vi.spyOn(mockEventsRegistry, "saveLastProcessedEvent").mockImplementation(() => { return Promise.resolve(); }); + vi.spyOn(mockStrategyRegistry, "saveStrategyId").mockImplementation(() => { + return Promise.resolve(); + }); runPromise = orchestrator.run(abortController.signal); @@ -451,7 +525,10 @@ describe("Orchestrator", { sequential: true }, () => { ); expect(mockEvmProvider.readContract).toHaveBeenCalledTimes(1); - expect(mockStrategyRegistry.getStrategyId).toHaveBeenLastCalledWith(strategyAddress); + expect(mockStrategyRegistry.getStrategyId).toHaveBeenLastCalledWith( + chainId, + strategyAddress, + ); expect(eventsProcessorSpy).toHaveBeenLastCalledWith({ ...registeredEvent, strategyId, @@ -580,7 +657,12 @@ describe("Orchestrator", { sequential: true }, () => { ); const error = new UnsupportedStrategy(strategyId); - vi.spyOn(mockStrategyRegistry, "getStrategyId").mockResolvedValue(strategyId); + vi.spyOn(mockStrategyRegistry, "getStrategyId").mockResolvedValue({ + id: strategyId, + address: mockEvent.srcAddress, + chainId, + handled: true, + }); vi.spyOn(mockIndexerClient, "getEventsAfterBlockNumberAndLogIndex") .mockResolvedValueOnce([mockEvent]) .mockResolvedValue([]); diff --git a/packages/data-flow/test/unit/strategyRegistry.spec.ts b/packages/data-flow/test/unit/strategyRegistry.spec.ts deleted file mode 100644 index 01e4806..0000000 --- a/packages/data-flow/test/unit/strategyRegistry.spec.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { Address, Hex } from "viem"; -import { describe, expect, it, vi } from "vitest"; - -import { ILogger } from "@grants-stack-indexer/shared"; - -import { InMemoryStrategyRegistry } from "../../src/strategyRegistry.js"; - -describe("InMemoryStrategyRegistry", () => { - const logger: ILogger = { - debug: vi.fn(), - error: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - }; - it("return undefined for non-existent strategy address", async () => { - const registry = new InMemoryStrategyRegistry(logger); - const strategyAddress = "0x123" as Address; - - const strategyId = await registry.getStrategyId(strategyAddress); - expect(strategyId).toBeUndefined(); - }); - - it("save and retrieve strategy id", async () => { - const registry = new InMemoryStrategyRegistry(logger); - const strategyAddress = "0x123" as Address; - const strategyId = "0xabc" as Hex; - - await registry.saveStrategyId(strategyAddress, strategyId); - const retrievedId = await registry.getStrategyId(strategyAddress); - - expect(retrievedId).toBe(strategyId); - }); - - it("handle multiple strategy addresses independently", async () => { - const registry = new InMemoryStrategyRegistry(logger); - const firstAddress = "0x123" as Address; - const secondAddress = "0x456" as Address; - const firstStrategyId = "0xabc" as Hex; - const secondStrategyId = "0xdef" as Hex; - - await registry.saveStrategyId(firstAddress, firstStrategyId); - await registry.saveStrategyId(secondAddress, secondStrategyId); - - const retrievedFirstId = await registry.getStrategyId(firstAddress); - const retrievedSecondId = await registry.getStrategyId(secondAddress); - - expect(retrievedFirstId).toBe(firstStrategyId); - expect(retrievedSecondId).toBe(secondStrategyId); - }); -}); diff --git a/packages/repository/src/db/connection.ts b/packages/repository/src/db/connection.ts index 8b0a352..d9e5568 100644 --- a/packages/repository/src/db/connection.ts +++ b/packages/repository/src/db/connection.ts @@ -20,6 +20,7 @@ import { Round, RoundRole as RoundRoleTable, StatusSnapshot, + Strategy as StrategyTable, } from "../internal.js"; const { Pool } = pg; @@ -59,6 +60,7 @@ export interface Database { applications: ApplicationTable; donations: DonationTable; applicationsPayouts: ApplicationPayoutTable; + strategies: StrategyTable; } /** diff --git a/packages/repository/src/external.ts b/packages/repository/src/external.ts index 8d10c3c..fec981b 100644 --- a/packages/repository/src/external.ts +++ b/packages/repository/src/external.ts @@ -8,6 +8,7 @@ export type { IApplicationReadRepository, IDonationRepository, IApplicationPayoutRepository, + IStrategyRegistryRepository, DatabaseConfig, } from "./internal.js"; @@ -35,6 +36,8 @@ export type { Donation, NewDonation } from "./types/index.js"; export type { NewApplicationPayout, ApplicationPayout } from "./types/index.js"; +export type { Strategy, NewStrategy } from "./types/index.js"; + export type { Changeset, ProjectChangeset, @@ -50,6 +53,7 @@ export { KyselyApplicationRepository, KyselyDonationRepository, KyselyApplicationPayoutRepository, + KyselyStrategyRegistryRepository, } from "./repositories/kysely/index.js"; export { diff --git a/packages/repository/src/interfaces/index.ts b/packages/repository/src/interfaces/index.ts index 26a47f1..3562351 100644 --- a/packages/repository/src/interfaces/index.ts +++ b/packages/repository/src/interfaces/index.ts @@ -3,3 +3,4 @@ export * from "./roundRepository.interface.js"; export * from "./applicationRepository.interface.js"; export * from "./donationRepository.interface.js"; export * from "./applicationPayoutRepository.interface.js"; +export * from "./strategyRepository.interface.js"; diff --git a/packages/repository/src/interfaces/strategyRepository.interface.ts b/packages/repository/src/interfaces/strategyRepository.interface.ts new file mode 100644 index 0000000..c58d344 --- /dev/null +++ b/packages/repository/src/interfaces/strategyRepository.interface.ts @@ -0,0 +1,31 @@ +import { Address, ChainId } from "@grants-stack-indexer/shared"; + +import { Strategy } from "../internal.js"; + +export interface IStrategyRegistryRepository { + /** + * Retrieves a strategy by its chain ID and address. + * @param chainId - The chain ID of the strategy. + * @param strategyAddress - The address of the strategy. + * @returns A promise that resolves to the strategy object or undefined if not found. + */ + getStrategyByChainIdAndAddress( + chainId: ChainId, + strategyAddress: Address, + ): Promise; + + /** + * Saves a strategy to the repository. + * @param strategy - The strategy to save. + */ + saveStrategy(strategy: Strategy): Promise; + + /** + * Retrieves all strategies from the repository. + * @param filters - The parameters to filter the strategies. + * @param filters.handled - Whether to include handled strategies. + * @param filters.chainId - The chain ID to filter the strategies. + * @returns A promise that resolves to an array of strategies. + */ + getStrategies(filters?: { handled?: boolean; chainId?: ChainId }): Promise; +} diff --git a/packages/repository/src/repositories/kysely/index.ts b/packages/repository/src/repositories/kysely/index.ts index 6b5d977..ae4143a 100644 --- a/packages/repository/src/repositories/kysely/index.ts +++ b/packages/repository/src/repositories/kysely/index.ts @@ -3,3 +3,4 @@ export * from "./round.repository.js"; export * from "./application.repository.js"; export * from "./donation.repository.js"; export * from "./applicationPayout.repository.js"; +export * from "./strategyRegistry.repository.js"; diff --git a/packages/repository/src/repositories/kysely/strategyRegistry.repository.ts b/packages/repository/src/repositories/kysely/strategyRegistry.repository.ts new file mode 100644 index 0000000..a251f42 --- /dev/null +++ b/packages/repository/src/repositories/kysely/strategyRegistry.repository.ts @@ -0,0 +1,52 @@ +import { Kysely } from "kysely"; + +import { Address, ChainId } from "@grants-stack-indexer/shared"; + +import { IStrategyRegistryRepository } from "../../interfaces/index.js"; +import { Database, Strategy } from "../../internal.js"; + +export class KyselyStrategyRegistryRepository implements IStrategyRegistryRepository { + constructor( + private readonly db: Kysely, + private readonly schemaName: string, + ) {} + + /** @inheritdoc */ + async getStrategyByChainIdAndAddress( + chainId: ChainId, + strategyAddress: Address, + ): Promise { + return this.db + .withSchema(this.schemaName) + .selectFrom("strategies") + .where("chainId", "=", chainId) + .where("address", "=", strategyAddress) + .selectAll() + .executeTakeFirst(); + } + + /** @inheritdoc */ + async saveStrategy(strategy: Strategy): Promise { + await this.db + .withSchema(this.schemaName) + .insertInto("strategies") + .values(strategy) + .onConflict((oc) => oc.columns(["chainId", "address"]).doUpdateSet(strategy)) + .execute(); + } + + /** @inheritdoc */ + async getStrategies(filters?: { handled?: boolean; chainId?: ChainId }): Promise { + const query = this.db.withSchema(this.schemaName).selectFrom("strategies"); + + if (filters?.chainId) { + query.where("chainId", "=", filters.chainId); + } + + if (filters?.handled) { + query.where("handled", "=", filters.handled); + } + + return query.selectAll().execute(); + } +} diff --git a/packages/repository/src/types/index.ts b/packages/repository/src/types/index.ts index f573b25..c34de7d 100644 --- a/packages/repository/src/types/index.ts +++ b/packages/repository/src/types/index.ts @@ -4,3 +4,4 @@ export * from "./application.types.js"; export * from "./changeset.types.js"; export * from "./donation.types.js"; export * from "./applicationPayout.types.js"; +export * from "./strategy.types.js"; diff --git a/packages/repository/src/types/strategy.types.ts b/packages/repository/src/types/strategy.types.ts new file mode 100644 index 0000000..8bee12f --- /dev/null +++ b/packages/repository/src/types/strategy.types.ts @@ -0,0 +1,10 @@ +import { Address, ChainId, Hex } from "@grants-stack-indexer/shared"; + +export type Strategy = { + address: Address; + id: Hex; + chainId: ChainId; + handled: boolean; +}; + +export type NewStrategy = Strategy; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eeca826..92721dd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -279,10 +279,16 @@ importers: kysely: specifier: 0.27.4 version: 0.27.4 + yargs: + specifier: 17.7.2 + version: 17.7.2 zod: specifier: 3.23.8 version: 3.23.8 devDependencies: + "@types/yargs": + specifier: 17.0.33 + version: 17.0.33 tsx: specifier: 4.19.2 version: 4.19.2 @@ -1527,6 +1533,18 @@ packages: integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==, } + "@types/yargs-parser@21.0.3": + resolution: + { + integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==, + } + + "@types/yargs@17.0.33": + resolution: + { + integrity: sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==, + } + "@typescript-eslint/eslint-plugin@5.62.0": resolution: { @@ -5770,6 +5788,12 @@ snapshots: "@types/triple-beam@1.3.5": {} + "@types/yargs-parser@21.0.3": {} + + "@types/yargs@17.0.33": + dependencies: + "@types/yargs-parser": 21.0.3 + "@typescript-eslint/eslint-plugin@5.62.0(@typescript-eslint/parser@5.62.0(eslint@8.56.0)(typescript@5.5.4))(eslint@8.56.0)(typescript@5.5.4)": dependencies: "@eslint-community/regexpp": 4.12.1 diff --git a/scripts/migrations/README.md b/scripts/migrations/README.md index fb442af..1d9736b 100644 --- a/scripts/migrations/README.md +++ b/scripts/migrations/README.md @@ -4,10 +4,10 @@ This package contains scripts for managing the database schema and migrations. ## Available Scripts -| Script | Description | -| ------------------- | --------------------------------------- | -| `script:db:migrate` | Runs all pending database migrations | -| `script:db:reset` | Drops and recreates the database schema | +| Script | Description | +| ------------ | --------------------------------------- | +| `db:migrate` | Runs all pending database migrations | +| `db:reset` | Drops and recreates the database schema | ## Environment Setup @@ -16,17 +16,13 @@ This package contains scripts for managing the database schema and migrations. ```env # Database connection URL DATABASE_URL=postgresql://user:password@localhost:5432/mydb - -# Schema name to manage -DATABASE_SCHEMA=grants_stack ``` ### Environment Variables -| Variable | Description | Example | -| ----------------- | ------------------------- | ------------------------------------------------ | -| `DATABASE_URL` | PostgreSQL connection URL | `postgresql://user:password@localhost:5432/mydb` | -| `DATABASE_SCHEMA` | Database schema name | `grants_stack` | +| Variable | Description | Example | +| -------------- | ------------------------- | ------------------------------------------------ | +| `DATABASE_URL` | PostgreSQL connection URL | `postgresql://user:password@localhost:5432/mydb` | ## Usage @@ -41,9 +37,13 @@ pnpm install To apply all pending migrations: ```bash -pnpm script:db:migrate +pnpm db:migrate --schema=schema_name ``` +Optional arguments: + +- `--schema` or `-s`: Database schema name where migrations are applied. Defaults to `public`. + This will: 1. Load environment variables @@ -57,7 +57,7 @@ This will: To completely reset the database schema: ```bash -pnpm script:db:reset +pnpm db:reset --schema=schema_name ``` **Warning**: This will: @@ -70,10 +70,10 @@ pnpm script:db:reset ### Adding New Migrations -1. Create a new migration file in [`packages/repository/src/migrations`](../../packages//repository//migrations) +1. Create a new migration file in [`packages/repository/src/migrations`](../../packages//repository/migrations) 2. Name it using the format: `YYYYMMDDTHHmmss_description.ts` 3. Implement the `up` and `down` functions -4. Run `pnpm script:db:migrate` to apply the new migration +4. Run `pnpm db:migrate` to apply the new migration Example migration file: diff --git a/scripts/migrations/package.json b/scripts/migrations/package.json index 737e796..c4c3180 100644 --- a/scripts/migrations/package.json +++ b/scripts/migrations/package.json @@ -16,12 +16,12 @@ "build": "tsc -p tsconfig.build.json", "check-types": "tsc --noEmit -p ./tsconfig.json", "clean": "rm -rf dist/", + "db:migrate": "tsx src/migrateDb.script.ts", + "db:reset": "tsx src/resetDb.script.ts", "format": "prettier --check \"{src,test}/**/*.{js,ts,json}\"", "format:fix": "prettier --write \"{src,test}/**/*.{js,ts,json}\"", "lint": "eslint \"{src,test}/**/*.{js,ts,json}\"", "lint:fix": "pnpm lint --fix", - "script:db:migrate": "tsx src/migrateDb.script.ts", - "script:db:reset": "tsx src/resetDb.script.ts", "test": "vitest run --config vitest.config.ts --passWithNoTests", "test:cov": "vitest run --config vitest.config.ts --coverage --passWithNoTests" }, @@ -29,9 +29,11 @@ "@grants-stack-indexer/repository": "workspace:*", "dotenv": "16.4.5", "kysely": "0.27.4", + "yargs": "17.7.2", "zod": "3.23.8" }, "devDependencies": { + "@types/yargs": "17.0.33", "tsx": "4.19.2" } } diff --git a/scripts/migrations/src/migrateDb.script.ts b/scripts/migrations/src/migrateDb.script.ts index 3d7245b..bef2362 100644 --- a/scripts/migrations/src/migrateDb.script.ts +++ b/scripts/migrations/src/migrateDb.script.ts @@ -1,10 +1,9 @@ -import path from "path"; import { configDotenv } from "dotenv"; import { createKyselyDatabase } from "@grants-stack-indexer/repository"; import { getDatabaseConfigFromEnv } from "./schemas/index.js"; -import { migrateToLatest } from "./utils/index.js"; +import { getMigrationsFolder, migrateToLatest, parseArguments } from "./utils/index.js"; configDotenv(); @@ -21,7 +20,9 @@ configDotenv(); * * Environment variables required: * - DATABASE_URL: PostgreSQL connection string - * - DATABASE_SCHEMA: Schema name to migrate (e.g. "grants_stack") + * + * Script arguments: + * - schema: Database schema name where migrations are applied * * The script will: * - Create the schema if it doesn't exist @@ -31,22 +32,20 @@ configDotenv(); */ export const main = async (): Promise => { - const { DATABASE_URL, DATABASE_SCHEMA } = getDatabaseConfigFromEnv(); + const { DATABASE_URL } = getDatabaseConfigFromEnv(); + const { schema } = parseArguments(); const db = createKyselyDatabase({ connectionString: DATABASE_URL, - withSchema: DATABASE_SCHEMA, + withSchema: schema, }); - console.log(`Migrating database schema '${DATABASE_SCHEMA}'...`); + console.log(`Migrating database schema '${schema}'...`); const migrationResults = await migrateToLatest({ db, - schema: DATABASE_SCHEMA, - migrationsFolder: path.join( - path.dirname(new URL(import.meta.url).pathname), - "./migrations", - ), + schema, + migrationsFolder: getMigrationsFolder(), }); if (migrationResults && migrationResults?.length > 0) { diff --git a/scripts/migrations/src/migrations/20241210T175001_strategy_registry.ts b/scripts/migrations/src/migrations/20241210T175001_strategy_registry.ts new file mode 100644 index 0000000..45a5936 --- /dev/null +++ b/scripts/migrations/src/migrations/20241210T175001_strategy_registry.ts @@ -0,0 +1,27 @@ +import { Kysely } from "kysely"; + +/** + * The up function is called when you update your database schema to the next version and down when you go back to previous version. + * The only argument for the functions is an instance of Kysely. It's important to use Kysely and not Kysely. + * ref: https://kysely.dev/docs/migrations#migration-files + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function up(db: Kysely): Promise { + const ADDRESS_TYPE = "text"; + const CHAIN_ID_TYPE = "integer"; + + await db.schema + .createTable("strategies") + .addColumn("address", ADDRESS_TYPE) + .addColumn("id", "text") + .addColumn("chainId", CHAIN_ID_TYPE) + .addColumn("handled", "boolean") + .addPrimaryKeyConstraint("strategies_pkey", ["address", "chainId"]) + .execute(); +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function down(db: Kysely): Promise { + // Drop everything in reverse order + await db.schema.dropTable("strategies").execute(); +} diff --git a/scripts/migrations/src/resetDb.script.ts b/scripts/migrations/src/resetDb.script.ts index 905c9eb..d5ca249 100644 --- a/scripts/migrations/src/resetDb.script.ts +++ b/scripts/migrations/src/resetDb.script.ts @@ -1,10 +1,9 @@ -import path from "path"; import { configDotenv } from "dotenv"; import { createKyselyDatabase } from "@grants-stack-indexer/repository"; import { getDatabaseConfigFromEnv } from "./schemas/index.js"; -import { resetDatabase } from "./utils/index.js"; +import { getMigrationsFolder, parseArguments, resetDatabase } from "./utils/index.js"; configDotenv(); @@ -21,7 +20,9 @@ configDotenv(); * * Environment variables required: * - DATABASE_URL: PostgreSQL connection string - * - DATABASE_SCHEMA: Schema name to reset (e.g. "grants_stack") + * + * Script arguments: + * - schema: Database schema name where migrations are applied * * The script will: * - Drop the schema if it exists @@ -34,22 +35,20 @@ configDotenv(); */ const main = async (): Promise => { - const { DATABASE_URL, DATABASE_SCHEMA } = getDatabaseConfigFromEnv(); + const { DATABASE_URL } = getDatabaseConfigFromEnv(); + const { schema } = parseArguments(); const db = createKyselyDatabase({ connectionString: DATABASE_URL, - withSchema: DATABASE_SCHEMA, + withSchema: schema, }); - console.log(`Resetting database schema '${DATABASE_SCHEMA}'...`); + console.log(`Resetting database schema '${schema}'...`); const resetResults = await resetDatabase({ db, - schema: DATABASE_SCHEMA, - migrationsFolder: path.join( - path.dirname(new URL(import.meta.url).pathname), - "./migrations", - ), + schema, + migrationsFolder: getMigrationsFolder(), }); if (resetResults && resetResults?.length > 0) { diff --git a/scripts/migrations/src/schemas/index.ts b/scripts/migrations/src/schemas/index.ts index a18141a..5b5ec4e 100644 --- a/scripts/migrations/src/schemas/index.ts +++ b/scripts/migrations/src/schemas/index.ts @@ -2,7 +2,6 @@ import { z } from "zod"; const dbEnvSchema = z.object({ DATABASE_URL: z.string().url(), - DATABASE_SCHEMA: z.string().min(1), }); export type DbEnvConfig = z.infer; diff --git a/scripts/migrations/src/utils/index.ts b/scripts/migrations/src/utils/index.ts index 7b157b3..e720a9c 100644 --- a/scripts/migrations/src/utils/index.ts +++ b/scripts/migrations/src/utils/index.ts @@ -1 +1,2 @@ export * from "./kysely.js"; +export * from "./parsing.js"; diff --git a/scripts/migrations/src/utils/parsing.ts b/scripts/migrations/src/utils/parsing.ts new file mode 100644 index 0000000..664bed0 --- /dev/null +++ b/scripts/migrations/src/utils/parsing.ts @@ -0,0 +1,43 @@ +import { existsSync } from "fs"; +import path from "path"; +import yargs from "yargs"; +import { hideBin } from "yargs/helpers"; +import { z } from "zod"; + +const DEFAULT_SCHEMA = "public"; + +const zodSchema = z.object({ + schema: z + .string() + .default(DEFAULT_SCHEMA) + .describe("Database schema name where migrations are applied"), +}); + +export const parseArguments = (): z.infer => { + return yargs(hideBin(process.argv)) + .option("schema", { + alias: "s", + type: "string", + demandOption: true, + description: "Database schema name where migrations are applied", + default: DEFAULT_SCHEMA, + }) + .check((argv) => { + zodSchema.parse(argv); + return true; + }) + .parseSync(); +}; + +export const getMigrationsFolder = (): string => { + const migrationsFolder = path.join( + path.dirname(new URL(import.meta.url).pathname), + `../migrations`, + ); + + if (!existsSync(migrationsFolder)) { + throw new Error(`Migrations folder not found`); + } + + return migrationsFolder; +};