Merge branch 'release/1.1.2-rc.3-metricsfix' into release/txtracing/1…

….1.2-rc.2

cmd/opera/launcher/chaincmd.go

@@ -8,9 +8,18 @@ import (
 var (
 	EvmExportMode = cli.StringFlag{
 		Name:  "export.evm.mode",
-		Usage: `EVM export mode ("full" or "ext-mpt" or "mpt" or "none")`,
+		Usage: `EVM export mode ("full" or "ext-mpt" or "mpt")`,
 		Value: "mpt",
 	}
+	EvmExportExclude = cli.StringFlag{
+		Name:  "export.evm.exclude",
+		Usage: `DB of EVM keys to exclude from genesis`,
+	}
+	GenesisExportSections = cli.StringFlag{
+		Name:  "export.sections",
+		Usage: `Genesis sections to export separated by comma (e.g. "brs-1" or "ers" or "evm-2")`,
+		Value: "brs,ers,evm",
+	}
 	importCommand = cli.Command{
 		Name:      "import",
 		Usage:     "Import a blockchain file",
@@ -132,11 +141,13 @@ be gzipped
 			{
 				Name:      "genesis",
 				Usage:     "Export current state into a genesis file",
-				ArgsUsage: "<filename> [<epochFrom> <epochTo>] [--export.evm.mode=none]",
+				ArgsUsage: "<filename or dry-run> [<epochFrom> <epochTo>] [--export.evm.mode=MODE --export.evm.exclude=DB_PATH --export.sections=A,B,C]",
 				Action:    utils.MigrateFlags(exportGenesis),
 				Flags: []cli.Flag{
 					DataDirFlag,
 					EvmExportMode,
+					EvmExportExclude,
+					GenesisExportSections,
 				},
 				Description: `
     opera export genesis
@@ -146,6 +157,20 @@ Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
 last epoch to write.
 EVM export mode is configured with --export.evm.mode.
+`,
+			},
+			{
+				Name:      "evm-keys",
+				Usage:     "Export EVM node keys",
+				ArgsUsage: "<directory>",
+				Action:    utils.MigrateFlags(exportEvmKeys),
+				Flags: []cli.Flag{
+					DataDirFlag,
+				},
+				Description: `
+    opera export evm-keys
+
+Requires a first argument of the DB directory to write to.
 `,
 			},
 		},

cmd/opera/launcher/check.go

@@ -27,12 +27,14 @@ func checkEvm(ctx *cli.Context) error {
 	start, reported := time.Now(), time.Now()
 
 	var prevPoint idx.Block
+	var prevIndex idx.Block
 	checkBlocks := func(stateOK func(root common.Hash) (bool, error)) {
 		var (
 			lastIdx            = gdb.GetLatestBlockIndex()
 			prevPointRootExist bool
 		)
 		gdb.ForEachBlock(func(index idx.Block, block *inter.Block) {
+			prevIndex = index
 			found, err := stateOK(common.Hash(block.Root))
 			if found != prevPointRootExist {
 				if index > 0 && found {
@@ -57,9 +59,9 @@ func checkEvm(ctx *cli.Context) error {
 		})
 	}
 
-	if err := evms.CheckEvm(checkBlocks); err != nil {
+	if err := evms.CheckEvm(checkBlocks, true); err != nil {
 		return err
 	}
-	log.Info("EVM storage is verified", "last", prevPoint, "elapsed", common.PrettyDuration(time.Since(start)))
+	log.Info("EVM storage is verified", "last", prevIndex, "elapsed", common.PrettyDuration(time.Since(start)))
 	return nil
 }

cmd/opera/launcher/config.go

@@ -90,6 +90,11 @@ var (
 		Usage: "Sets a cap on transaction fee (in FTM) that can be sent via the RPC APIs (0 = no cap)",
 		Value: gossip.DefaultConfig(cachescale.Identity).RPCTxFeeCap,
 	}
+	RPCGlobalTimeoutFlag = cli.DurationFlag{
+		Name:  "rpc.timeout",
+		Usage: "Time limit for RPC calls execution",
+		Value: gossip.DefaultConfig(cachescale.Identity).RPCTimeout,
+	}
 
 	SyncModeFlag = cli.StringFlag{
 		Name:  "syncmode",
@@ -328,6 +333,9 @@ func gossipConfigWithFlags(ctx *cli.Context, src gossip.Config) (gossip.Config,
 	if ctx.GlobalIsSet(RPCGlobalTxFeeCapFlag.Name) {
 		cfg.RPCTxFeeCap = ctx.GlobalFloat64(RPCGlobalTxFeeCapFlag.Name)
 	}
+	if ctx.GlobalIsSet(RPCGlobalTimeoutFlag.Name) {
+		cfg.RPCTimeout = ctx.GlobalDuration(RPCGlobalTimeoutFlag.Name)
+	}
 	if ctx.GlobalIsSet(SyncModeFlag.Name) {
 		if syncmode := ctx.GlobalString(SyncModeFlag.Name); syncmode != "full" && syncmode != "snap" {
 			utils.Fatalf("--%s must be either 'full' or 'snap'", SyncModeFlag.Name)

cmd/opera/launcher/db-transform.go

@@ -16,6 +16,7 @@ import (
 	"gopkg.in/urfave/cli.v1"
 
 	"github.com/Fantom-foundation/go-opera/integration"
+	"github.com/Fantom-foundation/go-opera/utils/compactdb"
 )
 
 func dbTransform(ctx *cli.Context) error {
@@ -255,16 +256,17 @@ func transformComponent(datadir string, dbTypes, tmpDbTypes map[multidb.TypeName
 				}
 				oldDB = batched.Wrap(oldDB)
 				defer oldDB.Close()
+				oldHumanName := path.Join(string(e.Old.Type), e.Old.Name)
 				newDB, err := tmpDbTypes[e.New.Type].OpenDB(e.New.Name)
 				if err != nil {
 					return err
 				}
 				toMove[dbLocatorOf(e.New)] = true
-				newDbName := "tmp/" + e.New.Name
 				newDB = batched.Wrap(newDB)
 				defer newDB.Close()
-				log.Info("Copying DB table", "req", e.Req, "old_db_type", e.Old.Type, "old_db_name", e.Old.Name, "old_table", e.Old.Table,
-					"new_db_type", e.New.Type, "new_db_name", newDbName, "new_table", e.New.Table)
+				newHumanName := path.Join("tmp", string(e.New.Type), e.New.Name)
+				log.Info("Copying DB table", "req", e.Req, "old_db", oldHumanName, "old_table", e.Old.Table,
+					"new_db", newHumanName, "new_table", e.New.Table)
 				oldTable := table.New(oldDB, []byte(e.Old.Table))
 				newTable := table.New(newDB, []byte(e.New.Table))
 				it := oldTable.NewIterator(nil, nil)
@@ -288,6 +290,11 @@ func transformComponent(datadir string, dbTypes, tmpDbTypes map[multidb.TypeName
 					keys = keys[:0]
 					values = values[:0]
 				}
+				err = compactdb.Compact(newTable, newHumanName)
+				if err != nil {
+					log.Error("Database compaction failed", "err", err)
+					return err
+				}
 				return nil
 			}()
 			if err != nil {

cmd/opera/launcher/dbcmd.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/Fantom-foundation/go-opera/gossip"
 	"github.com/Fantom-foundation/go-opera/integration"
+	"github.com/Fantom-foundation/go-opera/utils/compactdb"
 )
 
 var (
@@ -127,31 +128,36 @@ func compact(ctx *cli.Context) error {
 	producers := makeCheckedDBsProducers(cfg)
 	for typ, p := range producers {
 		for _, name := range p.Names() {
-			humanName := path.Join(string(typ), name)
-			db, err := p.OpenDB(name)
-			defer db.Close()
-			if err != nil {
-				log.Error("Cannot open db or db does not exists", "db", humanName)
+			if err := compactDB(typ, name, p); err != nil {
 				return err
 			}
+		}
+	}
 
-			log.Info("Stats before compaction", "db", humanName)
-			showDbStats(db)
+	return nil
+}
 
-			log.Info("Triggering compaction", "db", humanName)
-			for b := byte(0); b < 255; b++ {
-				log.Trace("Compacting chain database", "db", humanName, "range", fmt.Sprintf("0x%0.2X-0x%0.2X", b, b+1))
-				if err := db.Compact([]byte{b}, []byte{b + 1}); err != nil {
-					log.Error("Database compaction failed", "err", err)
-					return err
-				}
-			}
+func compactDB(typ multidb.TypeName, name string, producer kvdb.DBProducer) error {
+	humanName := path.Join(string(typ), name)
+	db, err := producer.OpenDB(name)
+	defer db.Close()
+	if err != nil {
+		log.Error("Cannot open db or db does not exists", "db", humanName)
+		return err
+	}
 
-			log.Info("Stats after compaction", "db", humanName)
-			showDbStats(db)
-		}
+	log.Info("Stats before compaction", "db", humanName)
+	showDbStats(db)
+
+	err = compactdb.Compact(db, humanName)
+	if err != nil {
+		log.Error("Database compaction failed", "err", err)
+		return err
 	}
 
+	log.Info("Stats after compaction", "db", humanName)
+	showDbStats(db)
+
 	return nil
 }
 

cmd/opera/launcher/export.go

@@ -10,11 +10,14 @@ import (
 
 	"github.com/Fantom-foundation/lachesis-base/hash"
 	"github.com/Fantom-foundation/lachesis-base/inter/idx"
+	"github.com/Fantom-foundation/lachesis-base/kvdb/batched"
+	"github.com/Fantom-foundation/lachesis-base/kvdb/pebble"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/status-im/keycard-go/hexutils"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 	"gopkg.in/urfave/cli.v1"
 
 	"github.com/Fantom-foundation/go-opera/gossip"
@@ -114,3 +117,38 @@ func exportTo(w io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) {
 
 	return
 }
+
+func exportEvmKeys(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires an argument.")
+	}
+
+	cfg := makeAllConfigs(ctx)
+
+	rawDbs := makeDirectDBsProducer(cfg)
+	gdb := makeGossipStore(rawDbs, cfg)
+	defer gdb.Close()
+
+	fn := ctx.Args().First()
+
+	keysDB_, err := pebble.New(fn, 1024*opt.MiB, utils.MakeDatabaseHandles()/2, nil, nil)
+	if err != nil {
+		return err
+	}
+	keysDB := batched.Wrap(keysDB_)
+	defer keysDB.Close()
+
+	it := gdb.EvmStore().EvmDb.NewIterator(nil, nil)
+	// iterate only over MPT data
+	it = mptAndPreimageIterator{it}
+	defer it.Release()
+
+	log.Info("Exporting EVM keys", "dir", fn)
+	for it.Next() {
+		if err := keysDB.Put(it.Key(), []byte{0}); err != nil {
+			return err
+		}
+	}
+	log.Info("Exported EVM keys", "dir", fn)
+	return nil
+}