Fix for fork handling issues (#1083)

* fixing sorting i count distinct aggregation * more debugging info for fork handling * yet another possible fix or sth... * more possible fixes * another possible fix * another approach to fixing forks * another change in conseil.sql * fix ofr the fork handling issue - invalidating all of the "leader" forks * reverting previous fork change * change fork-handling to be off by default
Cryptonomic · Jun 29, 2022 · 48c6beb · 48c6beb
1 parent c4694bf
commit 48c6beb
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 20 deletions.
diff --git a/...lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala b/...lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala
@@ -2,7 +2,6 @@ package tech.cryptonomic.conseil.indexer.tezos
 
 import java.sql.Timestamp
 import java.time.{Instant, ZoneOffset}
-
 import cats.effect.Sync
 import cats.implicits._
 import scribe._
@@ -12,14 +11,8 @@ import tech.cryptonomic.conseil.common.io.Logging.ConseilLogSupport
 import tech.cryptonomic.conseil.common.config.ChainEvent.AccountIdPattern
 import tech.cryptonomic.conseil.common.generic.chain.DataTypes.{Query => _}
 import tech.cryptonomic.conseil.common.sql.CustomProfileExtension
-import tech.cryptonomic.conseil.common.tezos.Tables
-import tech.cryptonomic.conseil.common.tezos.Tables.{
-  AccountsRow,
-  GovernanceRow,
-  OperationsRow,
-  OriginatedAccountMapsRow,
-  RegisteredTokensRow
-}
+import tech.cryptonomic.conseil.common.tezos.{Fork, Tables}
+import tech.cryptonomic.conseil.common.tezos.Tables.{AccountsRow, GovernanceRow, OperationsRow, OriginatedAccountMapsRow, RegisteredTokensRow}
 import tech.cryptonomic.conseil.common.tezos.TezosTypes.Fee.AverageFees
 import tech.cryptonomic.conseil.common.tezos.TezosTypes._
 import tech.cryptonomic.conseil.common.util.ConfigUtil
@@ -35,7 +28,6 @@ import scala.concurrent.{ExecutionContext, Future}
 import scala.math
 import scala.util.{Failure, Success}
 import java.util.UUID
-
 import slick.dbio.DBIOAction
 import tech.cryptonomic.conseil.indexer.tezos.RegisteredTokensFetcher.RegisteredToken
 
@@ -997,7 +989,7 @@ object TezosDatabaseOperations extends ConseilLogSupport {
         assert(fromLevel > 0, message = "Invalidation due to fork can be performed from positive block level only")
         val asOfTimestamp = Timestamp.from(asOf)
         query
-          .filter(levelColumn(_) >= fromLevel)
+          .filter(x => levelColumn(x) >= fromLevel && forkIdColumn(x) === Fork.mainForkId)
           .map(e => (invalidationTimeColumn(e), forkIdColumn(e)))
           .update(asOfTimestamp.some, forkId)
       }

diff --git a/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala b/conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala
@@ -128,7 +128,8 @@ class TezosIndexer private (
       unhandled <- accountsResetHandler.applyUnhandledAccountsResets(
         reloadedAccountEvents.orElse(lastReloadedAccountEvents).getOrElse(accountResetEvents)
       )
-      _ <- processTezosBlocks(maxLevel)
+      maxLevelNew <- indexedData.fetchMaxLevel
+      _ <- processTezosBlocks(maxLevelNew)
       _ <- feeOperations
         .processTezosAverageFees(lorreConf.feesAverageTimeWindow)
         .whenA(iteration % lorreConf.feeUpdateInterval == 0)

diff --git a/...rc/main/scala/tech/cryptonomic/conseil/indexer/tezos/forks/BacktracingForkProcessor.scala b/...rc/main/scala/tech/cryptonomic/conseil/indexer/tezos/forks/BacktracingForkProcessor.scala
@@ -84,6 +84,7 @@ class BacktracingForkProcessor(
         val forkLevel = xs.min
         for {
           forkBlockId <- indexerSearch.searchForLevel(forkLevel)
+          _ = logger.info(s"Found forked block hash $forkBlockId")
           amendment <- amender.amendFork(forkLevel, forkBlockId, currentHeadLevel, Instant.now())
         } yield amendment.some
     }

diff --git a/...ain/scala/tech/cryptonomic/conseil/indexer/tezos/forks/TezosForkInvalidatingAmender.scala b/...ain/scala/tech/cryptonomic/conseil/indexer/tezos/forks/TezosForkInvalidatingAmender.scala
@@ -3,12 +3,14 @@ package tech.cryptonomic.conseil.indexer.tezos.forks
 import tech.cryptonomic.conseil.common.tezos.TezosTypes.{BlockLevel, TezosBlockHash}
 import tech.cryptonomic.conseil.indexer.tezos.{TezosDatabaseOperations => DBOps}
 import tech.cryptonomic.conseil.indexer.forks.ForkAmender
+
 import java.time.Instant
 import scala.concurrent.{ExecutionContext, Future}
 import slick.dbio.DBIO
 import slick.jdbc.PostgresProfile.api._
 import cats.implicits._
 import slickeffect.implicits._
+import tech.cryptonomic.conseil.common.io.Logging.ConseilLogSupport
 
 /** Provides static utilities */
 object TezosForkInvalidatingAmender {
@@ -24,7 +26,7 @@ object TezosForkInvalidatingAmender {
   * E.g. removing processed chain events from the db registry.
   */
 class TezosForkInvalidatingAmender(db: Database)(implicit ec: ExecutionContext)
-    extends ForkAmender[Future, TezosBlockHash] {
+    extends ForkAmender[Future, TezosBlockHash]  with ConseilLogSupport {
 
   /* Note that we need to defer constraint checks manually with postgres
    * policies on consistency levels.
@@ -46,8 +48,11 @@ class TezosForkInvalidatingAmender(db: Database)(implicit ec: ExecutionContext)
         indexedHeadLevel,
         detectionTime
       )
+      _ = logger.info(s"Wrote fork with ID $forkId")
       _ <- DBOps.deferConstraints()
+      _ = logger.info(s"Defered constraints")
       invalidated <- invalidateData(forkLevel, detectionTime, forkId)
+      _ = logger.info(s"Invalidated data $invalidated")
     } yield (forkId, invalidated)
 
     db.run(forkAndInvalidateAction.transactionally)

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -15,6 +15,8 @@ services:
     image: postgres:11.9
     ports:
       - 5432:5432
+    # uncomment if you want to see SQL queries in the logs
+    # command: ["postgres", "-c", "log_statement=all"]
     environment:
       POSTGRES_USER: "conseiluser"
       POSTGRES_PASSWORD: "p@ssw0rd"

diff --git a/sql/conseil.sql b/sql/conseil.sql
@@ -779,7 +779,7 @@ ALTER TABLE ONLY tezos.accounts
     ADD CONSTRAINT accounts_block_id_fkey
     FOREIGN KEY (block_id, fork_id)
     REFERENCES tezos.blocks(hash, fork_id)
-    DEFERRABLE INITIALLY IMMEDIATE;
+    DEFERRABLE INITIALLY DEFERRED;
 
 
 --
@@ -790,7 +790,7 @@ ALTER TABLE ONLY tezos.operation_groups
     ADD CONSTRAINT block
     FOREIGN KEY (block_id, fork_id)
     REFERENCES tezos.blocks(hash, fork_id)
-    DEFERRABLE INITIALLY IMMEDIATE;
+    DEFERRABLE INITIALLY DEFERRED;
 
 --
 -- Name: delegates delegates_block_id_fkey; Type: FK CONSTRAINT; Schema: tezos; Owner: -
@@ -800,7 +800,7 @@ ALTER TABLE ONLY tezos.bakers
     ADD CONSTRAINT bakers_block_id_fkey
     FOREIGN KEY (block_id, fork_id)
     REFERENCES tezos.blocks(hash, fork_id)
-    DEFERRABLE INITIALLY IMMEDIATE;
+    DEFERRABLE INITIALLY DEFERRED;
 
 
 --
@@ -812,7 +812,7 @@ ALTER TABLE ONLY tezos.baking_rights
     FOREIGN KEY (block_hash, fork_id)
     REFERENCES tezos.blocks(hash, fork_id)
     NOT VALID
-    DEFERRABLE INITIALLY IMMEDIATE;
+    DEFERRABLE INITIALLY DEFERRED;
 
 
 --
@@ -824,7 +824,7 @@ ALTER TABLE ONLY tezos.endorsing_rights
     FOREIGN KEY (block_hash, fork_id)
     REFERENCES tezos.blocks(hash, fork_id)
     NOT VALID
-    DEFERRABLE INITIALLY IMMEDIATE;
+    DEFERRABLE INITIALLY DEFERRED;
 
 
 --
@@ -835,7 +835,7 @@ ALTER TABLE ONLY tezos.operations
     ADD CONSTRAINT fk_blockhashes
     FOREIGN KEY (block_hash, fork_id)
     REFERENCES tezos.blocks(hash, fork_id)
-    DEFERRABLE INITIALLY IMMEDIATE;
+    DEFERRABLE INITIALLY DEFERRED;
 
 
 --
@@ -846,7 +846,7 @@ ALTER TABLE ONLY tezos.operations
     ADD CONSTRAINT fk_opgroups
     FOREIGN KEY (operation_group_hash, block_hash, fork_id)
     REFERENCES tezos.operation_groups(hash, block_id, fork_id)
-    DEFERRABLE INITIALLY IMMEDIATE;
+    DEFERRABLE INITIALLY DEFERRED;
 
 --
 -- PostgreSQL database dump complete