Removes --pin and --remote from chifra scrape. Use chifra chunks --pi…

…n instead.

docs/content/api/openapi.yaml

@@ -1960,22 +1960,6 @@ paths:
           schema:
             type: number
             format: uint64
-        - name: pin
-          description: pin new chunks (requires locally-running IPFS daemon or --remote)
-          required: false
-          style: form
-          in: query
-          explode: true
-          schema:
-            type: boolean
-        - name: remote
-          description: pin new chunks to the gateway (requires pinning service keys)
-          required: false
-          style: form
-          in: query
-          explode: true
-          schema:
-            type: boolean
         - name: sleep
           description: seconds to sleep between scraper passes
           required: false

docs/content/chifra/admin.md

@@ -191,8 +191,6 @@ Usage:
 
 Flags:
   -n, --block_cnt uint     maximum number of blocks to process per pass (default 2000)
-  -i, --pin                pin new chunks (requires locally-running IPFS daemon or --remote)
-  -r, --remote             pin new chunks to the gateway (requires pinning service keys)
   -s, --sleep float        seconds to sleep between scraper passes (default 14)
   -l, --start_block uint   first block to visit when scraping (snapped back to most recent snap_to_grid mark)
   -v, --verbose            enable verbose output

docs/readmes/admin-scrape.md

@@ -28,8 +28,6 @@ Usage:
 
 Flags:
   -n, --block_cnt uint     maximum number of blocks to process per pass (default 2000)
-  -i, --pin                pin new chunks (requires locally-running IPFS daemon or --remote)
-  -r, --remote             pin new chunks to the gateway (requires pinning service keys)
   -s, --sleep float        seconds to sleep between scraper passes (default 14)
   -l, --start_block uint   first block to visit when scraping (snapped back to most recent snap_to_grid mark)
   -v, --verbose            enable verbose output

sdk/python/src/_scrape.py

@@ -8,8 +8,6 @@
 scrapeFmt = "json"
 scrapeOpts = {
     "blockCnt": {"hotkey": "-n", "type": "flag"},
-    "pin": {"hotkey": "-i", "type": "switch"},
-    "remote": {"hotkey": "-r", "type": "switch"},
     "sleep": {"hotkey": "-s", "type": "flag"},
     "startBlock": {"hotkey": "-l", "type": "flag"},
     "fmt": {"hotkey": "-x", "type": "flag"},

sdk/typescript/src/paths/scrape.ts

@@ -9,8 +9,6 @@ import { ChunkRecord, double, Manifest, uint64 } from '../types';
 export function getScrape(
   parameters?: {
     blockCnt?: uint64,
-    pin?: boolean,
-    remote?: boolean,
     sleep?: double,
     startBlock?: uint64,
     chain: string,

src/apps/chifra/cmd/scrape.go

@@ -58,8 +58,6 @@ func init() {
 	scrapeCmd.Flags().SortFlags = false
 
 	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().BlockCnt, "block_cnt", "n", 2000, "maximum number of blocks to process per pass")
-	scrapeCmd.Flags().BoolVarP(&scrapePkg.GetOptions().Pin, "pin", "i", false, "pin new chunks (requires locally-running IPFS daemon or --remote)")
-	scrapeCmd.Flags().BoolVarP(&scrapePkg.GetOptions().Remote, "remote", "r", false, "pin new chunks to the gateway (requires pinning service keys)")
 	scrapeCmd.Flags().Float64VarP(&scrapePkg.GetOptions().Sleep, "sleep", "s", 14, "seconds to sleep between scraper passes")
 	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().StartBlock, "start_block", "l", 0, "first block to visit when scraping (snapped back to most recent snap_to_grid mark)")
 	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().RunCount, "run_count", "u", 0, "run the scraper this many times, then quit (hidden)")

src/apps/chifra/internal/chunks/handle_pin.go

@@ -34,14 +34,14 @@ func (opts *ChunksOptions) HandlePin(blockNums []uint64) error {
 		if len(blockNums) == 0 {
 			var err error
 			tsPath := config.PathToTimestamps(chain)
-			if report.TsHash, err = pinning.PinOneFile(chain, "timestamps", tsPath, config.IpfsRunning(), opts.Remote); err != nil {
+			if report.TsHash, _, err = pinning.PinOneFile(chain, "timestamps", tsPath, config.IpfsRunning(), opts.Remote); err != nil {
 				errorChan <- err
 				cancel()
 				return
 			}
 
 			manPath := config.PathToManifest(chain)
-			if report.ManifestHash, err = pinning.PinOneFile(chain, "manifest", manPath, config.IpfsRunning(), opts.Remote); err != nil {
+			if report.ManifestHash, _, err = pinning.PinOneFile(chain, "manifest", manPath, config.IpfsRunning(), opts.Remote); err != nil {
 				errorChan <- err
 				cancel()
 				return
@@ -63,37 +63,27 @@ func (opts *ChunksOptions) HandlePin(blockNums []uint64) error {
 					return false, fmt.Errorf("should not happen in pinChunk")
 				}
 
-				result, err := pinning.PinOneChunk(chain, index.ToBloomPath(path), index.ToIndexPath(path), config.IpfsRunning(), opts.Remote)
+				local, remote, err := pinning.PinOneChunk(chain, index.ToBloomPath(path), index.ToIndexPath(path), config.IpfsRunning(), opts.Remote)
 				if err != nil {
 					errorChan <- err
 					cancel() // keep going...
 					return true, nil
 				}
 
-				if config.IpfsRunning() {
-					report.Pinned = append(report.Pinned, result.Local.BloomHash)
-					report.Pinned = append(report.Pinned, result.Local.IndexHash)
-				}
-
-				if opts.Remote {
-					report.Pinned = append(report.Pinned, result.Remote.BloomHash)
-					report.Pinned = append(report.Pinned, result.Remote.IndexHash)
-				}
-
-				if !result.Matches() {
+				if !pinning.Matches(&local, &remote) {
 					logger.Warn("Local and remote pins do not match")
-					logger.Warn(colors.Yellow+result.Local.BloomHash.String(), "-", result.Local.IndexHash, colors.Off)
-					logger.Warn(colors.Yellow+result.Remote.BloomHash.String(), "-", result.Remote.IndexHash, colors.Off)
+					logger.Warn(colors.Yellow+local.BloomHash.String(), "-", local.IndexHash, colors.Off)
+					logger.Warn(colors.Yellow+remote.BloomHash.String(), "-", remote.IndexHash, colors.Off)
 					logger.Fatal("IPFS hashes between local and remote do not match")
 				} else if opts.Remote && config.IpfsRunning() {
-					logger.Info(colors.BrightGreen+"Matches: "+result.Remote.BloomHash.String(), "-", result.Remote.IndexHash, colors.Off)
+					logger.Info(colors.BrightGreen+"Matches: "+remote.BloomHash.String(), "-", remote.IndexHash, colors.Off)
 				}
 
 				if opts.Globals.Verbose {
 					if opts.Remote {
-						fmt.Println("result.Remote:", result.Remote.String())
+						fmt.Println("result.Remote:", remote.String())
 					} else {
-						fmt.Println("result.Local:", result.Local.String())
+						fmt.Println("result.Local:", local.String())
 					}
 				}
 

src/apps/chifra/internal/chunks/types_chunkpinreport.go

@@ -23,7 +23,6 @@ type simpleChunkPinReport struct {
 	TsHash        base.IpfsHash `json:"tsHash"`
 	Version       string        `json:"version"`
 	// EXISTING_CODE
-	Pinned []base.IpfsHash `json:"-"`
 	// EXISTING_CODE
 }
 
@@ -42,15 +41,13 @@ func (s *simpleChunkPinReport) Model(chain, format string, verbose bool, extraOp
 		"specification": s.Specification,
 		"tsHash":        s.TsHash,
 		"version":       s.Version,
-		"nPinned":       len(s.Pinned),
 	}
 	order = []string{
 		"chain",
 		"manifestHash",
 		"specification",
 		"tsHash",
 		"version",
-		"nPinned",
 	}
 	// EXISTING_CODE
 

src/apps/chifra/internal/scrape/README.md

@@ -28,8 +28,6 @@ Usage:
 
 Flags:
   -n, --block_cnt uint     maximum number of blocks to process per pass (default 2000)
-  -i, --pin                pin new chunks (requires locally-running IPFS daemon or --remote)
-  -r, --remote             pin new chunks to the gateway (requires pinning service keys)
   -s, --sleep float        seconds to sleep between scraper passes (default 14)
   -l, --start_block uint   first block to visit when scraping (snapped back to most recent snap_to_grid mark)
   -v, --verbose            enable verbose output

src/apps/chifra/internal/scrape/options.go

@@ -23,8 +23,6 @@ import (
 // ScrapeOptions provides all command options for the chifra scrape command.
 type ScrapeOptions struct {
 	BlockCnt     uint64                `json:"blockCnt,omitempty"`     // Maximum number of blocks to process per pass
-	Pin          bool                  `json:"pin,omitempty"`          // Pin new chunks (requires locally-running IPFS daemon or --remote)
-	Remote       bool                  `json:"remote,omitempty"`       // Pin new chunks to the gateway (requires pinning service keys)
 	Sleep        float64               `json:"sleep,omitempty"`        // Seconds to sleep between scraper passes
 	StartBlock   uint64                `json:"startBlock,omitempty"`   // First block to visit when scraping (snapped back to most recent snap_to_grid mark)
 	RunCount     uint64                `json:"runCount,omitempty"`     // Run the scraper this many times, then quit
@@ -47,8 +45,6 @@ var defaultScrapeOptions = ScrapeOptions{
 // testLog is used only during testing to export the options for this test case.
 func (opts *ScrapeOptions) testLog() {
 	logger.TestLog(opts.BlockCnt != 2000, "BlockCnt: ", opts.BlockCnt)
-	logger.TestLog(opts.Pin, "Pin: ", opts.Pin)
-	logger.TestLog(opts.Remote, "Remote: ", opts.Remote)
 	logger.TestLog(opts.Sleep != float64(14), "Sleep: ", opts.Sleep)
 	logger.TestLog(opts.StartBlock != 0, "StartBlock: ", opts.StartBlock)
 	logger.TestLog(opts.RunCount != 0, "RunCount: ", opts.RunCount)
@@ -83,10 +79,6 @@ func scrapeFinishParseApi(w http.ResponseWriter, r *http.Request) *ScrapeOptions
 		switch key {
 		case "blockCnt":
 			opts.BlockCnt = globals.ToUint64(value[0])
-		case "pin":
-			opts.Pin = true
-		case "remote":
-			opts.Remote = true
 		case "sleep":
 			opts.Sleep = globals.ToFloat64(value[0])
 		case "startBlock":

src/apps/chifra/internal/scrape/scrape_consolidate.go

@@ -8,7 +8,6 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
-	"time"
 
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
@@ -104,7 +103,7 @@ func (bm *BlazeManager) Consolidate(blocks []base.Blknum) (error, bool) {
 			// Make a chunk - i.e., consolidate
 			chunkPath := indexPath + "finalized/" + chunkRange.String() + ".bin"
 			publisher := base.ZeroAddr
-			if report, err := index.WriteChunk(chain, publisher, chunkPath, appMap, nAppearances, bm.opts.Pin, bm.opts.Remote); err != nil {
+			if report, err := index.WriteChunk(chain, publisher, chunkPath, appMap, nAppearances); err != nil {
 				return err, false
 			} else if report == nil {
 				logger.Fatal("Should not happen, write chunk returned empty report")
@@ -113,9 +112,6 @@ func (bm *BlazeManager) Consolidate(blocks []base.Blknum) (error, bool) {
 				report.FileSize = file.FileSize(chunkPath)
 				report.Report()
 			}
-			if bm.opts.Remote {
-				time.Sleep(250 * time.Millisecond)
-			}
 
 			// reset for next chunk
 			bm.meta, _ = bm.opts.Conn.GetMetaData(bm.IsTestMode())

src/apps/chifra/internal/scrape/scrape_prepare.go

@@ -56,7 +56,7 @@ func (opts *ScrapeOptions) Prepare() (ok bool, err error) {
 
 	logger.Info("Writing block zero allocations for", len(prefunds), "prefunds, nAddresses:", len(appMap))
 	indexPath := index.ToIndexPath(bloomPath)
-	if report, err := index.WriteChunk(chain, opts.PublisherAddr, indexPath, appMap, len(prefunds), opts.Pin, opts.Remote); err != nil {
+	if report, err := index.WriteChunk(chain, opts.PublisherAddr, indexPath, appMap, len(prefunds)); err != nil {
 		return false, err
 	} else if report == nil {
 		logger.Fatal("Should not happen, write chunk returned empty report")

src/apps/chifra/internal/scrape/validate.go

@@ -66,19 +66,6 @@ func (opts *ScrapeOptions) validateScrape() error {
 		fmt.Println(validate.Usage("The index ({0}) is ahead of the chain ({1}).", fmt.Sprintf("%d", m), fmt.Sprintf("%d", meta.Latest)))
 	}
 
-	if opts.Pin {
-		if opts.Remote {
-			pinataKey, pinataSecret := config.GetKey("pinata").ApiKey, config.GetKey("pinata").Secret
-			if pinataKey == "" || pinataSecret == "" {
-				return validate.Usage("The {0} option requires {1}.", "--pin --remote", "an api key")
-			}
-
-		} else if !config.IpfsRunning() {
-			return validate.Usage("The {0} option requires {1}.", "--pin", "a locally running IPFS daemon or --remote")
-
-		}
-	}
-
 	if len(opts.Publisher) > 0 {
 		err := validate.ValidateExactlyOneAddr([]string{opts.Publisher})
 		if err != nil {

src/apps/chifra/pkg/index/chunk_write.go

@@ -14,8 +14,6 @@ import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/index/bloom"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
-	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/manifest"
-	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/pinning"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/version"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -30,8 +28,6 @@ type WriteChunkReport struct {
 	nAppearances int
 	FileSize     int64
 	Snapped      bool
-	Pinned       bool
-	PinRecord    types.SimpleChunkRecord
 }
 
 func (c *WriteChunkReport) Report() {
@@ -41,13 +37,9 @@ func (c *WriteChunkReport) Report() {
 	}
 	report += " (size: {%d} , span: {%d})"
 	logger.Info(colors.ColoredWith(fmt.Sprintf(report, c.nAddresses, c.nAppearances, c.Range, c.FileSize, c.Range.Span()), colors.BrightBlue))
-	if c.Pinned {
-		str := fmt.Sprintf("%sPinned chunk $INDEX/%s.bin (%s,%s)%s", colors.BrightBlue, c.Range, c.PinRecord.IndexHash, c.PinRecord.BloomHash, colors.Off)
-		logger.Info(str)
-	}
 }
 
-func WriteChunk(chain string, publisher base.Address, fileName string, addrAppearanceMap AddressAppearanceMap, nApps int, pin, remote bool) (*WriteChunkReport, error) {
+func WriteChunk(chain string, publisher base.Address, fileName string, addrAppearanceMap AddressAppearanceMap, nApps int) (*WriteChunkReport, error) {
 	// We're going to build two tables. An addressTable and an appearanceTable. We do this as we spin
 	// through the map
 
@@ -146,24 +138,8 @@ func WriteChunk(chain string, publisher base.Address, fileName string, addrAppea
 				Range:        rng,
 				nAddresses:   len(addressTable),
 				nAppearances: len(appearanceTable),
-				Pinned:       pin,
-			}
-
-			if !pin {
-				return &report, nil
 			}
-
-			result, err := pinning.PinOneChunk(chain, ToBloomPath(indexFn), ToIndexPath(indexFn), config.IpfsRunning(), remote)
-			if err != nil {
-				return &report, err
-			}
-
-			rec := ResultToRecord(&result)
-			report.PinRecord.IndexHash = rec.IndexHash
-			report.PinRecord.BloomHash = rec.BloomHash
-			report.PinRecord.IndexSize = rec.IndexSize
-			report.PinRecord.BloomSize = rec.BloomSize
-			return &report, manifest.UpdateManifest(chain, publisher, rec)
+			return &report, nil
 
 		} else {
 			return nil, err
@@ -173,20 +149,3 @@ func WriteChunk(chain string, publisher base.Address, fileName string, addrAppea
 		return nil, err
 	}
 }
-
-func ResultToRecord(result *pinning.Result) types.SimpleChunkRecord {
-	if len(result.Local.BloomHash) > 0 {
-		return types.SimpleChunkRecord{
-			IndexHash: result.Local.IndexHash,
-			IndexSize: result.Local.IndexSize,
-			BloomHash: result.Local.BloomHash,
-			BloomSize: result.Local.BloomSize,
-		}
-	}
-	return types.SimpleChunkRecord{
-		IndexHash: result.Remote.IndexHash,
-		IndexSize: result.Remote.IndexSize,
-		BloomHash: result.Remote.BloomHash,
-		BloomSize: result.Remote.BloomSize,
-	}
-}