Merge pull request #3900 from TrueBlocks/feature/chifra-scrape-touch

Feature/chifra scrape touch

src/apps/chifra/internal/names/handle_autoname_integration_test.go

@@ -31,7 +31,7 @@ func TestNamesOptions_autoname(t *testing.T) {
 	if err != nil {
 		t.Errorf("Error making backup = %+v", err)
 	}
-	defer backup.Restore() // put the file back
+	defer backup.Restore() // put the file back if it still exists (i.e., an error occurred)
 
 	type fields struct {
 		Autoname string

src/apps/chifra/internal/scrape/handle_touch.go

@@ -5,15 +5,113 @@
 package scrapePkg
 
 import (
+	"context"
+	"encoding/binary"
 	"fmt"
+	"os"
 
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/colors"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/index"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/output"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/sigintTrap"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/tslib"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
 )
 
 func (opts *ScrapeOptions) HandleTouch(rCtx *output.RenderCtx) error {
+	chain := opts.Globals.Chain
 	fetchData := func(modelChan chan types.Modeler, errorChan chan error) {
-		errorChan <- fmt.Errorf("chifra scrape --touch is not yet implemented")
+		meta, _ := opts.Conn.GetMetaData(false)
+		rng := base.FileRange{
+			First: meta.Finalized,
+			Last:  opts.Touch,
+		}
+		indexPath := index.ToIndexPath(rng.RangeToFilename(chain))
+		stagePath := index.ToStagingPath(rng.RangeToFilename(chain))
+		var chunk index.Chunk
+		var bm = BlazeManager{}
+		_ = file.StringToAsciiFile(stagePath, base.SkippedSender.Hex()+"\t0\t0\n")
+		appMap, _, nAppearances := bm.AsciiFileToAppearanceMap(stagePath)
+		if report, err := chunk.Write(chain, base.ZeroAddr, indexPath, appMap, nAppearances); err != nil {
+			errorChan <- err
+		} else {
+			modelChan <- &types.Message{Msg: report.Report()}
+		}
+		if reportStr, err := opts.backfillTimestamps(); err != nil {
+			errorChan <- err
+		} else {
+			modelChan <- &types.Message{Msg: reportStr}
+		}
 	}
 	return output.StreamMany(rCtx, fetchData, opts.Globals.OutputOpts())
 }
+
+func (opts *ScrapeOptions) backfillTimestamps() (string, error) {
+	chain := opts.Globals.Chain
+	tsPath := config.PathToTimestamps(chain)
+	logger.Info(tsPath)
+
+	sigintCtx, cancel := context.WithCancel(context.Background())
+	cleanOnQuit := func() {
+		// We only print a warning here, as the scrape.pid file will be
+		// removed by the deferred function
+		logger.Warn(sigintTrap.TrapMessage)
+	}
+	trapChannel := sigintTrap.Enable(sigintCtx, cancel, cleanOnQuit)
+	defer sigintTrap.Disable(trapChannel)
+
+	tmpPath := tsPath + ".backup"
+	backup, err := file.MakeBackup(tmpPath, tsPath)
+	if err != nil {
+		return "", fmt.Errorf("error making backup = %+v", err)
+	}
+	defer backup.Restore() // put the file back if it still exists (i.e., an error occurred)
+
+	file.Remove(tsPath)
+	fp, err := os.OpenFile(tsPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		tslib.ClearCache(chain)
+		fp.Close()
+	}()
+
+	if nTimestamps, err := tslib.NTimestamps(chain); err != nil {
+		return "", err
+	} else {
+		logger.Info("nTimestamps:", nTimestamps)
+		if nTimestamps > opts.Touch {
+			return "", fmt.Errorf("nTimestamps > opts.Touch - should not happen")
+		}
+
+		firstTs := uint32(0)
+		for block := uint32(0); block <= uint32(opts.Touch); block++ {
+			if sigintCtx.Err() != nil {
+				// This means the context got cancelled, i.e. we got a SIGINT.
+				// The backup will be restored by the deferred function
+				break
+			}
+			if block == 0 {
+				firstTs = uint32(opts.Conn.GetBlockTimestamp(base.Value(block)))
+			}
+			ts := tslib.TimestampRecord{
+				Bn: block,
+				Ts: uint32(block + firstTs),
+			}
+			msg := fmt.Sprintf("Backfilling timestamps at block %d", block)
+			logger.Progress(true, msg)
+			if err := binary.Write(fp, binary.LittleEndian, &ts); err != nil {
+				return "", err
+			}
+		}
+		msg := fmt.Sprintf("{%s} were backfilled to block {%d}", "Timestamps", opts.Touch)
+		ret := colors.ColoredWith(msg, colors.BrightBlue)
+		os.Remove(tmpPath)
+		return ret, nil
+	}
+}

src/apps/chifra/internal/scrape/save_timestamps.go

@@ -42,7 +42,7 @@ func (bm *BlazeManager) WriteTimestamps(ctx context.Context, blocks []base.Blknu
 		// we need to catch up (for example, the user truncated the timestamps file while debugging)
 		// don't get more than maxBlocks at a time
 		cnt := 0
-		maxBlocks := 1000
+		maxBlocks := 2000
 		for block := nTimestamps; block < blocks[0] && cnt < maxBlocks; block++ {
 			if ctx.Err() != nil {
 				// This means the context got cancelled, i.e. we got a SIGINT.

src/apps/chifra/internal/scrape/scrape_consolidate.go

@@ -106,7 +106,7 @@ func (bm *BlazeManager) Consolidate(ctx context.Context, blocks []base.Blknum) e
 			} else {
 				report.Snapped = isSnap
 				report.FileSize = file.FileSize(chunkPath)
-				report.Report()
+				logger.Info(report.Report())
 			}
 			if err = bm.opts.NotifyChunkWritten(chunk, chunkPath); err != nil {
 				return err

src/apps/chifra/internal/scrape/scrape_prepare.go

@@ -65,7 +65,7 @@ func (opts *ScrapeOptions) Prepare() (ok bool, err error) {
 	} else {
 		report.Snapped = true // assumes block zero is a snap to grid (which it is in a sense)
 		report.FileSize = file.FileSize(indexPath)
-		report.Report()
+		logger.Info(report.Report())
 	}
 	if err = opts.NotifyChunkWritten(chunk, indexPath); err != nil {
 		return false, err

src/apps/chifra/internal/scrape/validate.go

@@ -81,6 +81,10 @@ func (opts *ScrapeOptions) validateScrape() error {
 		fmt.Println(validate.Usage("The index ({0}) is ahead of the chain ({1}).", fmt.Sprintf("%d", m), fmt.Sprintf("%d", meta.Latest)))
 	}
 
+	if opts.Touch > 0 && opts.Touch < meta.Finalized {
+		return validate.Usage("The index is already finalized ({0}) past the touch block ({1}).", fmt.Sprintf("%d", meta.Finalized), fmt.Sprintf("%d", opts.Touch))
+	}
+
 	if len(opts.Publisher) > 0 {
 		err := validate.ValidateExactlyOneAddr([]string{opts.Publisher})
 		if err != nil {

src/apps/chifra/pkg/base/baddresses.go

@@ -8,4 +8,5 @@ var (
 	BlockRewardSender = HexToAddress("0x0000000000000000000000000000004d696e6572") // The word "Miner" in hex
 	UncleRewardSender = HexToAddress("0x000000000000000000000000000000556e636c65") // The word "Uncle" in hex
 	WithdrawalSender  = HexToAddress("0x000000000000000000005769746864726177616C") // The word "Withdrawal" in hex
+	SkippedSender     = HexToAddress("0x00000000000000000000000000536B6970706564") // The word "Skipped" in hex
 )

src/apps/chifra/pkg/index/chunk_write.go

@@ -12,7 +12,6 @@ import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/colors"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
-	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
 )
 
@@ -24,13 +23,13 @@ type writeReport struct {
 	Snapped      bool
 }
 
-func (c *writeReport) Report() {
+func (c *writeReport) Report() string {
 	report := `Wrote {%d} address and {%d} appearance records to {$INDEX/%s.bin}`
 	if c.Snapped {
 		report += ` @(snapped to grid)}`
 	}
 	report += " (size: {%d} , span: {%d})"
-	logger.Info(colors.ColoredWith(fmt.Sprintf(report, c.nAddresses, c.nAppearances, c.Range, c.FileSize, c.Range.Span()), colors.BrightBlue))
+	return colors.ColoredWith(fmt.Sprintf(report, c.nAddresses, c.nAppearances, c.Range, c.FileSize, c.Range.Span()), colors.BrightBlue)
 }
 
 func (chunk *Chunk) Write(chain string, publisher base.Address, fileName string, addrAppearanceMap map[string][]types.AppRecord, nApps int) (*writeReport, error) {

src/apps/chifra/pkg/manifest/remove_chunk.go

@@ -38,14 +38,19 @@ func RemoveChunk(chain string, publisher base.Address, bloomFn, indexFn string)
 		}
 	}()
 
-	var man *Manifest
-	man, err = LoadManifest(chain, publisher, LocalCache)
-	if err != nil {
-		return err
-	}
+	exists := file.FileExists(manifestFn)
 
-	if _, err = file.Copy(manifestBackup, manifestFn); err != nil {
-		return err
+	var man *Manifest
+	if exists {
+		man, err = LoadManifest(chain, publisher, LocalCache)
+		if err != nil {
+			return err
+		}
+		if _, err = file.Copy(manifestBackup, manifestFn); err != nil {
+			return err
+		}
+	} else {
+		man = &Manifest{}
 	}
 
 	if _, err = file.Copy(indexBackup, indexFn); err != nil {
@@ -64,28 +69,31 @@ func RemoveChunk(chain string, publisher base.Address, bloomFn, indexFn string)
 		return err
 	}
 
-	removedRange, err1 := base.RangeFromFilenameE(bloomFn)
-	if err1 != nil {
-		err = err1
-		return err
-	}
+	if exists {
+		removedRange, err1 := base.RangeFromFilenameE(bloomFn)
+		if err1 != nil {
+			err = err1
+			return err
+		}
 
-	newChunks := []types.ChunkRecord{}
-	for _, chunk := range man.Chunks {
-		chunkRange := base.RangeFromRangeString(chunk.Range)
-		if chunkRange.EarlierThan(removedRange) {
-			newChunks = append(newChunks, chunk)
-			// 	fmt.Println(colors.Green, "Keeping", chunk.Range, colors.Off)
-			// } else {
-			// 	fmt.Println(colors.Red, "Removing", chunk.Range, colors.Off)
+		newChunks := []types.ChunkRecord{}
+		for _, chunk := range man.Chunks {
+			chunkRange := base.RangeFromRangeString(chunk.Range)
+			if chunkRange.EarlierThan(removedRange) {
+				newChunks = append(newChunks, chunk)
+				// 	fmt.Println(colors.Green, "Keeping", chunk.Range, colors.Off)
+				// } else {
+				// 	fmt.Println(colors.Red, "Removing", chunk.Range, colors.Off)
+			}
 		}
-	}
-	man.Chunks = newChunks
-	if err = man.SaveManifest(chain, config.PathToManifest(chain)); err != nil {
-		return err
+		man.Chunks = newChunks
+		if err = man.SaveManifest(chain, config.PathToManifest(chain)); err != nil {
+			return err
+		}
+
+		os.Remove(manifestBackup)
 	}
 
-	os.Remove(manifestBackup)
 	os.Remove(indexBackup)
 	os.Remove(bloomBackup)
 

src/apps/chifra/pkg/rpc/is_node.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/prefunds"
 )
 
@@ -21,6 +23,10 @@ import (
 // false, when in fact the node may be an archive node.
 func (conn *Connection) IsNodeArchive() bool {
 	thePath := filepath.Join(config.MustGetPathToChainConfig(conn.Chain), "allocs.csv")
+	if !file.FileExists(thePath) {
+		logger.Warn("No pre-allocation file found at", thePath, "assuming an archive node")
+		return true
+	}
 	largest, err := prefunds.GetLargestPrefund(conn.Chain, thePath)
 	if err != nil {
 		return false

src/other/install/init_configs.cmake

@@ -89,23 +89,24 @@ PrintLine("Copying files and folder to config...")
 
 # ---------------------------------------------------------------
 # Function        InFolder                                 InFile                     OutFolder
-CopyNotPresent    (${INSTALL_SOURCE}                   "trueBlocks.toml"          ${INSTALL_DEST})
+CopyNotPresent    (${INSTALL_SOURCE}                    "trueBlocks.toml"         ${INSTALL_DEST})
 
-CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/mainnet "allocs.csv"               ${INSTALL_DEST}/config/mainnet/)
-CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/gnosis  "allocs.csv"               ${INSTALL_DEST}/config/gnosis/)
-CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/sepolia "allocs.csv"               ${INSTALL_DEST}/config/sepolia/)
-CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/goerli  "allocs.csv"               ${INSTALL_DEST}/config/goerli/)
-CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/polygon "allocs.csv"               ${INSTALL_DEST}/config/polygon/)
+CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/mainnet  "allocs.csv"              ${INSTALL_DEST}/config/mainnet/)
+CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/gnosis   "allocs.csv"              ${INSTALL_DEST}/config/gnosis/)
+CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/sepolia  "allocs.csv"              ${INSTALL_DEST}/config/sepolia/)
+CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/goerli   "allocs.csv"              ${INSTALL_DEST}/config/goerli/)
+CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/polygon  "allocs.csv"              ${INSTALL_DEST}/config/polygon/)
 
-CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/    "names.tab"                ${INSTALL_DEST}/config/mainnet/)
-CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/    "names.tab"                ${INSTALL_DEST}/config/gnosis/)
-CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/    "names.tab"                ${INSTALL_DEST}/config/sepolia/)
+CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/     "names.tab"               ${INSTALL_DEST}/config/mainnet/)
+CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/     "names.tab"               ${INSTALL_DEST}/config/gnosis/)
+CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/     "names.tab"               ${INSTALL_DEST}/config/sepolia/)
 
-CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/mainnet "specials.csv"             ${INSTALL_DEST}/config/mainnet/)
-CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/gnosis  "specials.csv"             ${INSTALL_DEST}/config/gnosis/)
-CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/sepolia "specials.csv"             ${INSTALL_DEST}/config/sepolia/)
+CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/mainnet  "specials.csv"            ${INSTALL_DEST}/config/mainnet/)
+CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/gnosis   "specials.csv"            ${INSTALL_DEST}/config/gnosis/)
+CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/sepolia  "specials.csv"            ${INSTALL_DEST}/config/sepolia/)
+CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/optimism "specials.csv"            ${INSTALL_DEST}/config/optimism/)
 
-CopyNotPresent    (${INSTALL_SOURCE}/names/            "names_custom.tab"         ${INSTALL_DEST}/config/mainnet/)
+CopyNotPresent    (${INSTALL_SOURCE}/names/             "names_custom.tab"        ${INSTALL_DEST}/config/mainnet/)
 
 CopyFolder        (${INSTALL_SOURCE}/abis/known-000/                              ${INSTALL_DEST}/abis/known-000/)
 CopyFolder        (${INSTALL_SOURCE}/abis/known-005/                              ${INSTALL_DEST}/abis/known-005/)

src/other/install/per-chain/optimism/specials.csv

@@ -0,0 +1,3 @@
+component,block/epoch,name,timestamp,date,description
+execution,0,first,1636665386,2021-11-11T21:16:26,The chain's first block
+execution,105235063,Bedrock,1686068903,2023-06-06T16:28:23,The Bedrock hard fork block