[6] Implement dynamic contract registry as entity (#340)

* Implement entity history tables for each entity

* Wip implement per table insert history item function with tests

* Add entity history schema creator

* Add eval function for inserting entity history

* Fix spelling of variable

* Move EntityHistory to its own file

* Add EntityHistoryRowAction enum

* Handle delete entities with enum flag

* Use new entity history tables for saving entity history

* Add serial to entity history

* Make InMemomoryStore static

* Move InMemomoryStore to static templates dir

* Add new rollback diffing queries

* Implement delete rolled back entity history

* Implement multichain rollback filters

* Implement unordered and ordered multichain versions of rollbacks

* Make tests compile

* Fix broken magic in memory table function

* Fix double quoted dynamic table name in query

* Add new test expectations

* Refactor simpler in memory table entity history state

* Fix diff and delete entity history functions

* Fix tests

* Fix inverted rollback check

* Implement entity history pruning

* Remove all code related to previous entity_history table

* Remove union schemas for float and int

* Fix tests

* Implement dynamic contract registry as entity

* Fix test to allow for duplicates in raw events table

* Add back condition for validating evm contract address

* Remove unused dynamic contract registry key

codegenerator/cli/templates/dynamic/codegen/src/ContextEnv.res.hbs

@@ -61,20 +61,27 @@ let make = (~eventBatchQueueItem: Types.eventBatchQueueItem, ~logger) => {
 let getAddedDynamicContractRegistrations = (contextEnv: t) =>
   contextEnv.addedDynamicContractRegistrations
 
-let makeDynamicContractRegisterFn = (~contextEnv: t, ~contractName, ~inMemoryStore) => (
-  contractAddress: Address.t,
-) => {
+let makeDynamicContractId = (~chainId, ~contractAddress) => {
+  chainId->Belt.Int.toString ++ "-" ++ contractAddress->Address.toString
+}
+let makeDynamicContractRegisterFn = (
+  ~contextEnv: t,
+  ~contractName,
+  ~inMemoryStore,
+  ~shouldSaveHistory,
+) => (contractAddress: Address.t) => {
   {{#if is_evm_ecosystem}} {{!-- TODO: Add validation for Fuel --}}
   // Even though it's the Address.t type on ReScript side, for TS side it's a string.
   // So we need to ensure that it's a valid checksummed address.
   let contractAddress = contractAddress->Address.Evm.fromAddressOrThrow
-
   {{/if}}
+
   let {eventBatchQueueItem, addedDynamicContractRegistrations} = contextEnv
   let {chain, timestamp, blockNumber, logIndex} = eventBatchQueueItem
 
   let chainId = chain->ChainMap.Chain.toChainId
   let dynamicContractRegistration: TablesStatic.DynamicContractRegistry.t = {
+    id: makeDynamicContractId(~chainId, ~contractAddress),
     chainId,
     registeringEventBlockNumber: blockNumber,
     registeringEventLogIndex: logIndex,
@@ -88,9 +95,21 @@ let makeDynamicContractRegisterFn = (~contextEnv: t, ~contractName, ~inMemorySto
 
   addedDynamicContractRegistrations->Js.Array2.push(dynamicContractRegistration)->ignore
 
-  inMemoryStore.InMemoryStore.dynamicContractRegistry->InMemoryTable.set(
-    {chainId, contractAddress},
-    dynamicContractRegistration,
+  let eventIdentifier: Types.eventIdentifier = {
+    chainId,
+    blockTimestamp: timestamp,
+    blockNumber,
+    logIndex,
+  }
+
+  inMemoryStore.InMemoryStore.entities
+  ->InMemoryStore.EntityTables.get(module(TablesStatic.DynamicContractRegistry))
+  ->InMemoryTable.Entity.set(
+    Set(dynamicContractRegistration)->Types.mkEntityUpdate(
+      ~eventIdentifier,
+      ~entityId=dynamicContractRegistration.id,
+    ),
+    ~shouldSaveHistory,
   )
 }
 
@@ -139,10 +158,10 @@ let makeEntityHandlerContext = (
   }
 }
 
-let getContractRegisterContext = (contextEnv, ~inMemoryStore) => {
+let getContractRegisterContext = (contextEnv, ~inMemoryStore, ~shouldSaveHistory) => {
   //TODO only add contracts we've registered for the event in the config
   {{#each codegen_contracts as |contract| }}
-  add{{contract.name.capitalized}}:  makeDynamicContractRegisterFn(~contextEnv, ~inMemoryStore, ~contractName={{contract.name.capitalized}}),
+  add{{contract.name.capitalized}}:  makeDynamicContractRegisterFn(~contextEnv, ~inMemoryStore, ~contractName={{contract.name.capitalized}}, ~shouldSaveHistory),
   {{/each}}
 }
 
@@ -198,9 +217,9 @@ let getHandlerContext = (
   }
 }
 
-let getContractRegisterArgs = (contextEnv, ~inMemoryStore) => {
+let getContractRegisterArgs = (contextEnv, ~inMemoryStore, ~shouldSaveHistory) => {
   Types.HandlerTypes.event: contextEnv.eventBatchQueueItem.event,
-  context: contextEnv->getContractRegisterContext(~inMemoryStore),
+  context: contextEnv->getContractRegisterContext(~inMemoryStore, ~shouldSaveHistory),
 }
 
 let getLoaderArgs = (contextEnv, ~inMemoryStore, ~loadLayer) => {

codegenerator/cli/templates/dynamic/codegen/src/IO.res.hbs

@@ -167,32 +167,21 @@ let executeBatch = async (
     ~items=inMemoryStore.rawEvents->InMemoryTable.values,
   )
 
-  let setDynamicContracts = executeSet(
-    _,
-    ~dbFunction=DbFunctions.DynamicContractRegistry.batchSet,
-    ~items=inMemoryStore.dynamicContractRegistry->InMemoryTable.values,
-  )
-
-  {{#each entities as | entity |}}
-  let set{{entity.name.capitalized}}s = entityDbExecutionComposer(
-    _,
-    ~entityMod=module(Entities.{{entity.name.capitalized}}),
-    ~inMemoryStore,
-  )
+  let setEntities = Entities.allEntities->Belt.Array.map(entityMod => {
+    entityDbExecutionComposer(_, ~entityMod, ~inMemoryStore)
+  })
 
-  {{/each}}
   //In the event of a rollback, rollback all meta tables based on the given
   //valid event identifier, where all rows created after this eventIdentifier should
   //be deleted
   let rollbackTables = switch inMemoryStore.rollBackEventIdentifier {
-  | Some(eventIdentifier) =>
-    [
+  | Some(eventIdentifier) => [
       DbFunctions.EntityHistory.deleteAllEntityHistoryAfterEventIdentifier(
+        _,
         ~isUnorderedMultichainMode=config.isUnorderedMultichainMode,
+        ~eventIdentifier,
       ),
-      DbFunctions.RawEvents.deleteAllRawEventsAfterEventIdentifier,
-      DbFunctions.DynamicContractRegistry.deleteAllDynamicContractRegistrationsAfterEventIdentifier,
-    ]->Belt.Array.map(fn => fn(_, ~eventIdentifier))
+    ]
   | None => []
   }
 
@@ -218,14 +207,8 @@ let executeBatch = async (
     Belt.Array.concatMany([
       //Rollback tables need to happen first in the traction
       rollbackTables,
-      [
-        setEventSyncState,
-        setRawEvents,
-        setDynamicContracts,
-        {{#each entities as | entity |}}
-        set{{entity.name.capitalized}}s,
-        {{/each}}
-      ],
+      [setEventSyncState, setRawEvents],
+      setEntities,
       //History pruning needs to happen last in the transaction
       //It deletes all unneeded history rows outside of the reorg threshold
       pruneEntityHistory,

codegenerator/cli/templates/dynamic/codegen/src/TestHelpers.res.hbs

@@ -133,6 +133,7 @@ module EventFunctions = {
             false,
           ~dynamicContractRegistrations=None,
           ~inMemoryStore,
+          ~shouldSaveHistory=false,
         ) {
         | Ok(_) => ()
         | Error(e) => e->ErrorHandling.logAndRaise

codegenerator/cli/templates/dynamic/codegen/src/TestHelpers_MockDb.res.hbs

@@ -54,10 +54,7 @@ type rec t = {
   entities: entities,
   rawEvents: storeOperations<InMemoryStore.rawEventsKey, TablesStatic.RawEvents.t>,
   eventSyncState: storeOperations<Types.chainId, TablesStatic.EventSyncState.t>,
-  dynamicContractRegistry: storeOperations<
-    InMemoryStore.dynamicContractRegistryKey,
-    TablesStatic.DynamicContractRegistry.t,
-  >,
+  dynamicContractRegistry: entityStoreOperations<TablesStatic.DynamicContractRegistry.t>,
 }
 
 // Each user defined entity will be in this record with all the store or "mockdb" operators
@@ -224,11 +221,13 @@ let rec makeWithInMemoryStore: InMemoryStore.t => t = (inMemoryStore: InMemorySt
     ~getKey=({chainId}) => chainId,
   )
 
-  let dynamicContractRegistry = makeStoreOperatorMeta(
+  let dynamicContractRegistry = makeStoreOperatorEntity(
     ~inMemoryStore,
-    ~getStore=db => db.dynamicContractRegistry,
+    ~getStore=db =>
+      db.entities->InMemoryStore.EntityTables.get(module(TablesStatic.DynamicContractRegistry)),
     ~makeMockDb=makeWithInMemoryStore,
-    ~getKey=({chainId, contractAddress}) => {chainId, contractAddress},
+    ~getKey=({chainId, contractAddress}) =>
+      ContextEnv.makeDynamicContractId(~chainId, ~contractAddress),
   )
 
   let entities = {
@@ -273,7 +272,10 @@ let cloneMockDb = (self: t) => {
 
 let getEntityOperations = (mockDb: t, ~entityMod): entityStoreOperations<Entities.internalEntity> => {
   let module(Entity: Entities.InternalEntity) = entityMod
-  (mockDb.entities->Utils.magic)->Utils.Dict.dangerouslyGetNonOption(Entity.key)->Utils.Option.getExn("Mocked operations for entity " ++ Entity.key ++ " not found")
+  mockDb.entities
+  ->Utils.magic
+  ->Utils.Dict.dangerouslyGetNonOption((Entity.name :> string))
+  ->Utils.Option.getExn("Mocked operations for entity " ++ (Entity.name :> string) ++ " not found")
 }
 
 let makeLoadEntitiesByIds = (mockDb: t) => {
@@ -368,13 +370,10 @@ let writeFromMemoryStore = (mockDb: t, ~inMemoryStore: InMemoryStore.t) => {
     ~getKey=entity => entity.chainId,
   )
 
-  mockDb->executeRowsMeta(
+  mockDb->executeRowsEntity(
     ~inMemoryStore,
-    ~getInMemTable=inMemStore => {inMemStore.dynamicContractRegistry},
-    ~getKey=(entity): InMemoryStore.dynamicContractRegistryKey => {
-      chainId: entity.chainId,
-      contractAddress: entity.contractAddress,
-    },
+    ~entityMod=module(TablesStatic.DynamicContractRegistry),
+    ~getKey=entity => entity.id,
   )
 
 //ENTITY EXECUTION

codegenerator/cli/templates/dynamic/codegen/src/db/Entities.res.hbs

@@ -5,7 +5,6 @@ type id = string
 type internalEntity
 module type Entity = {
   type t
-  let key: string
   let name: Enums.EntityType.t
   let schema: S.schema<t>
   let rowsSchema: S.schema<array<t>>
@@ -26,7 +25,6 @@ type whereOperations<'entity, 'fieldType> = {eq: 'fieldType => promise<array<'en
 
 {{#each entities as |entity|}}
 module {{entity.name.capitalized}} = {
-  let key = "{{entity.name.original}}"
   let name = {{entity.name.capitalized}}
   @genType
   type t = {
@@ -93,26 +91,12 @@ module {{entity.name.capitalized}} = {
  
 {{/each}}
 
-type entity = 
-{{#each entities as | entity |}}
-  | {{entity.name.capitalized}}({{entity.name.capitalized}}.t)
-{{/each}}
-
-let makeGetter = (schema, accessor) => json => json->S.parseWith(schema)->Belt.Result.map(accessor)
-
-let getEntityParamsDecoder = (entityName: Enums.EntityType.t) =>
-  switch entityName {
-{{#each entities as | entity |}}
-  | {{entity.name.capitalized}} => makeGetter({{entity.name.capitalized}}.schema, e => {{entity.name.capitalized}}(e))
-{{/each}}
-  }
-
 let allEntities: array<module(InternalEntity)> =
   [
-
 {{#each entities as |entity|}}
     module({{entity.name.capitalized}}),
 {{/each}}
+    module(TablesStatic.DynamicContractRegistry),
   ]->(Utils.magic: array<module(Entity)> => array<module(InternalEntity)>)
 
 let allTables: array<table> = allEntities->Belt.Array.map(entityMod => {

codegenerator/cli/templates/dynamic/codegen/src/db/Enums.res.hbs

@@ -48,14 +48,16 @@ module EntityType = {
   @genType
   type t = 
   {{#each entities as | entity |}}
-    | @as("{{entity.name.capitalized}}") {{entity.name.capitalized}}
+    | @as("{{entity.name.original}}") {{entity.name.capitalized}}
   {{/each}}
+    | @as("dynamic_contract_registry") DynamicContractRegistry
 
   let name = "ENTITY_TYPE"
   let variants = [
   {{#each entities as | entity |}}
     {{entity.name.capitalized}},
   {{/each}}
+    DynamicContractRegistry,
   ]
 
   let enum = mkEnum(~name, ~variants)

codegenerator/cli/templates/static/codegen/src/EventProcessing.res

@@ -121,12 +121,15 @@ let runEventContractRegister = (
   ~dynamicContractRegistrations: option<dynamicContractRegistrations>,
   ~inMemoryStore,
   ~preRegisterLatestProcessedBlocks=?,
+  ~shouldSaveHistory,
 ) => {
   let {chain, event, blockNumber} = eventBatchQueueItem
 
   let contextEnv = ContextEnv.make(~eventBatchQueueItem, ~logger)
 
-  switch contractRegister(contextEnv->ContextEnv.getContractRegisterArgs(~inMemoryStore)) {
+  switch contractRegister(
+    contextEnv->ContextEnv.getContractRegisterArgs(~inMemoryStore, ~shouldSaveHistory),
+  ) {
   | exception exn =>
     exn
     ->ErrorHandling.make(
@@ -358,6 +361,7 @@ let rec registerDynamicContracts = (
   ~dynamicContractRegistrations: option<dynamicContractRegistrations>=None,
   ~inMemoryStore,
   ~preRegisterLatestProcessedBlocks=?,
+  ~shouldSaveHistory,
 ) => {
   switch eventBatch[index] {
   | None => (eventsBeforeDynamicRegistrations, dynamicContractRegistrations)->Ok
@@ -382,6 +386,7 @@ let rec registerDynamicContracts = (
           ~dynamicContractRegistrations,
           ~inMemoryStore,
           ~preRegisterLatestProcessedBlocks?,
+          ~shouldSaveHistory,
         )
       | None =>
         dynamicContractRegistrations
@@ -406,6 +411,7 @@ let rec registerDynamicContracts = (
         ~dynamicContractRegistrations,
         ~inMemoryStore,
         ~preRegisterLatestProcessedBlocks?,
+        ~shouldSaveHistory,
       )
     | Error(e) => Error(e)
     }
@@ -527,6 +533,7 @@ let getDynamicContractRegistrations = (
         ~logger,
         ~inMemoryStore,
         ~preRegisterLatestProcessedBlocks,
+        ~shouldSaveHistory=false,
       )
       ->propogate
 
@@ -578,7 +585,12 @@ let processEventBatch = (
       dynamicContractRegistrations,
     ) =
       eventBatch
-      ->registerDynamicContracts(~checkContractIsRegistered, ~logger, ~inMemoryStore)
+      ->registerDynamicContracts(
+        ~checkContractIsRegistered,
+        ~logger,
+        ~inMemoryStore,
+        ~shouldSaveHistory=config->Config.shouldSaveHistory(~isInReorgThreshold),
+      )
       ->propogate
 
     let elapsedAfterContractRegister =

codegenerator/cli/templates/static/codegen/src/InMemoryStore.res

@@ -7,30 +7,21 @@ type rawEventsKey = {
 let hashRawEventsKey = (key: rawEventsKey) =>
   EventUtils.getEventIdKeyString(~chainId=key.chainId, ~eventId=key.eventId)
 
-@genType
-type dynamicContractRegistryKey = {
-  chainId: int,
-  contractAddress: Address.t,
-}
-
-let hashDynamicContractRegistryKey = ({chainId, contractAddress}) =>
-  EventUtils.getContractAddressKeyString(~chainId, ~contractAddress)
-
 module EntityTables = {
   type t = dict<InMemoryTable.Entity.t<Entities.internalEntity>>
-  exception UndefinedEntity(string)
+  exception UndefinedEntity(Enums.EntityType.t)
   let make = (entities: array<module(Entities.InternalEntity)>): t => {
     let init = Js.Dict.empty()
     entities->Belt.Array.forEach(entity => {
       let module(Entity) = entity
-      init->Js.Dict.set(Entity.key, InMemoryTable.Entity.make())
+      init->Js.Dict.set((Entity.name :> string), InMemoryTable.Entity.make())
     })
     init
   }
 
   let get = (type entity, self: t, entityMod: module(Entities.Entity with type t = entity)) => {
     let module(Entity) = entityMod
-    switch self->Utils.Dict.dangerouslyGetNonOption(Entity.key) {
+    switch self->Utils.Dict.dangerouslyGetNonOption((Entity.name :> string)) {
     | Some(table) =>
       table->(
         Utils.magic: InMemoryTable.Entity.t<Entities.internalEntity> => InMemoryTable.Entity.t<
@@ -39,7 +30,7 @@ module EntityTables = {
       )
 
     | None =>
-      UndefinedEntity(Entity.key)->ErrorHandling.mkLogAndRaise(
+      UndefinedEntity(Entity.name)->ErrorHandling.mkLogAndRaise(
         ~msg="Unexpected, entity InMemoryTable is undefined",
       )
     }
@@ -56,10 +47,6 @@ module EntityTables = {
 type t = {
   eventSyncState: InMemoryTable.t<int, TablesStatic.EventSyncState.t>,
   rawEvents: InMemoryTable.t<rawEventsKey, TablesStatic.RawEvents.t>,
-  dynamicContractRegistry: InMemoryTable.t<
-    dynamicContractRegistryKey,
-    TablesStatic.DynamicContractRegistry.t,
-  >,
   entities: Js.Dict.t<InMemoryTable.Entity.t<Entities.internalEntity>>,
   rollBackEventIdentifier: option<Types.eventIdentifier>,
 }
@@ -70,15 +57,13 @@ let make = (
 ): t => {
   eventSyncState: InMemoryTable.make(~hash=v => v->Belt.Int.toString),
   rawEvents: InMemoryTable.make(~hash=hashRawEventsKey),
-  dynamicContractRegistry: InMemoryTable.make(~hash=hashDynamicContractRegistryKey),
   entities: EntityTables.make(entities),
   rollBackEventIdentifier,
 }
 
 let clone = (self: t) => {
   eventSyncState: self.eventSyncState->InMemoryTable.clone,
   rawEvents: self.rawEvents->InMemoryTable.clone,
-  dynamicContractRegistry: self.dynamicContractRegistry->InMemoryTable.clone,
   entities: self.entities->EntityTables.clone,
   rollBackEventIdentifier: self.rollBackEventIdentifier->InMemoryTable.structuredClone,
 }