Merge pull request #3878 from TrueBlocks/feature/better-manifests

Feature/better manifests
TrueBlocks · Sep 4, 2024 · 6d178c0 · 6d178c0
2 parents 208ccfc + 3a727bb
commit 6d178c0
Show file tree

Hide file tree

Showing 22 changed files with 155 additions and 123 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,22 +1,5 @@
 ## Description
 <!-- Please include a summary of the change and which issue is fixed. -->
 
-## Type of change
-- [ ] Bug fix (non-breaking change which fixes an issue -- include the issue #)
-- [ ] New feature (non-breaking change which adds functionality)
-- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
-- [ ] Other (please specify):
-
-## Checklist
-- [ ] My code follows the style guidelines of this project
-- [ ] I have performed a self-review of my own code
-- [ ] I have added minimal comments to my code in difficult to understand areas
-- [ ] I have made corresponding changes to the documentation
-- [ ] My changes generate no new warnings
-- [ ] I have added tests that prove my fix is effective or that my feature works
-- [ ] New and existing unit tests pass locally with my changes
-- [ ] Any dependent changes have been merged and published in downstream modules
-
 ## PR Target Branch
 - [ ] I am targeting the `develop` branch for non-release changes.
-
diff --git a/.github/workflows/pr-target-check.yml b/.github/workflows/pr-target-check.yml
@@ -9,7 +9,8 @@ jobs:
       - name: Check if PR is targeting the 'develop' branch
         run: |
           if [ "${{ github.event.pull_request.base.ref }}" != "develop" ]; then
-            echo "::warning file=${{ github.event.pull_request.url }}::PR is not targeting 'develop'. Please change the target branch to 'develop'."
+            echo "::error file=${{ github.event.pull_request.url }}::PR is not targeting 'develop'. Please change the target branch to 'develop'."
+            exit 1
           else
             echo "PR is targeting the 'develop' branch. Everything is okay."
           fi
diff --git a/sdk b/sdk
diff --git a/src/apps/chifra/internal/chunks/handle_address.go b/src/apps/chifra/internal/chunks/handle_address.go
@@ -81,7 +81,8 @@ func (opts *ChunksOptions) HandleAddresses(rCtx *output.RenderCtx, blockNums []b
 						Offset:  uint64(obj.Offset),
 						Count:   uint64(obj.Count),
 					}
-					s.RangeDates = tslib.RangeToBounds(chain, &rng)
+					rd := tslib.RangeToBounds(chain, &rng)
+					s.RangeDates = &rd
 
 					modelChan <- &s
 					cnt++

diff --git a/src/apps/chifra/internal/chunks/handle_blooms.go b/src/apps/chifra/internal/chunks/handle_blooms.go
@@ -54,7 +54,8 @@ func (opts *ChunksOptions) HandleBlooms(rCtx *output.RenderCtx, blockNums []base
 				ByteWidth: index.BLOOM_WIDTH_IN_BYTES,
 				NInserted: uint64(nInserted),
 			}
-			s.RangeDates = tslib.RangeToBounds(chain, &rng)
+			rd := tslib.RangeToBounds(chain, &rng)
+			s.RangeDates = &rd
 
 			modelChan <- &s
 			return true, nil

diff --git a/src/apps/chifra/internal/chunks/handle_index.go b/src/apps/chifra/internal/chunks/handle_index.go
@@ -53,7 +53,8 @@ func (opts *ChunksOptions) HandleIndex(rCtx *output.RenderCtx, blockNums []base.
 				NAppearances: uint64(indexChunk.Header.AppearanceCount),
 				Size:         uint64(file.FileSize(fileName)),
 			}
-			s.RangeDates = tslib.RangeToBounds(chain, &rng)
+			rd := tslib.RangeToBounds(chain, &rng)
+			s.RangeDates = &rd
 
 			modelChan <- &s
 			return true, nil

diff --git a/src/apps/chifra/internal/chunks/handle_index_belongs.go b/src/apps/chifra/internal/chunks/handle_index_belongs.go
@@ -176,7 +176,8 @@ func (opts *ChunksOptions) handleResolvedRecords1(modelChan chan types.Modeler,
 				Offset:  uint64(s.AddressRecord.Offset),
 				Range:   rng.String(),
 			}
-			ss.RangeDates = tslib.RangeToBounds(chain, &rng)
+			rd := tslib.RangeToBounds(chain, &rng)
+			ss.RangeDates = &rd
 			modelChan <- &ss
 			cnt++
 		}

diff --git a/src/apps/chifra/internal/chunks/handle_manifest.go b/src/apps/chifra/internal/chunks/handle_manifest.go
@@ -42,7 +42,8 @@ func (opts *ChunksOptions) HandleManifest(rCtx *output.RenderCtx, blockNums []ba
 					IndexHash: chunk.IndexHash,
 					IndexSize: chunk.IndexSize,
 				}
-				s.RangeDates = tslib.RangeToBounds(chain, &rng)
+				rd := tslib.RangeToBounds(chain, &rng)
+				s.RangeDates = &rd
 				modelChan <- &s
 			}
 		}
@@ -65,7 +66,8 @@ func (opts *ChunksOptions) HandleManifest(rCtx *output.RenderCtx, blockNums []ba
 					IndexHash: chunk.IndexHash,
 					IndexSize: chunk.IndexSize,
 				}
-				ch.RangeDates = tslib.RangeToBounds(chain, &rng)
+				rd := tslib.RangeToBounds(chain, &rng)
+				ch.RangeDates = &rd
 				s.Chunks = append(s.Chunks, ch)
 			}
 			modelChan <- &s

diff --git a/src/apps/chifra/internal/chunks/options.go b/src/apps/chifra/internal/chunks/options.go
@@ -312,7 +312,8 @@ func GetChunkStats(chain, path string) (s types.ChunkStats, err error) {
 		ChunkSz: uint64(file.FileSize(index.ToIndexPath(path))),
 		RecWid:  4 + index.BLOOM_WIDTH_IN_BYTES,
 	}
-	s.RangeDates = tslib.RangeToBounds(chain, &rng)
+	rd := tslib.RangeToBounds(chain, &rng)
+	s.RangeDates = &rd
 
 	if s.NBlocks > 0 {
 		s.AddrsPerBlock = float64(s.NAddrs) / float64(s.NBlocks)

diff --git a/src/apps/chifra/pkg/types/types_chunkaddress.go b/src/apps/chifra/pkg/types/types_chunkaddress.go
@@ -22,7 +22,7 @@ type ChunkAddress struct {
 	Count      uint64       `json:"count"`
 	Offset     uint64       `json:"offset"`
 	Range      string       `json:"range"`
-	RangeDates RangeDates   `json:"rangeDates"`
+	RangeDates *RangeDates  `json:"rangeDates,omitempty"`
 	// EXISTING_CODE
 	// EXISTING_CODE
 }
@@ -50,15 +50,18 @@ func (s *ChunkAddress) Model(chain, format string, verbose bool, extraOpts map[s
 		"count",
 	}
 
-	if verbose {
-		if format == "json" {
+	if verbose && format == "json" {
+		if s.RangeDates != nil {
 			model["rangeDates"] = s.RangeDates.Model(chain, format, verbose, extraOpts).Data
-		} else {
+		}
+	} else if verbose {
+		model["firstTs"], model["firstDate"], model["lastTs"], model["lastDate"] = 0, "", 0, ""
+		order = append(order, []string{"firstTs", "firstDate", "lastTs", "lastDate"}...)
+		if s.RangeDates != nil {
 			model["firstTs"] = s.RangeDates.FirstTs
 			model["firstDate"] = s.RangeDates.FirstDate
 			model["lastTs"] = s.RangeDates.LastTs
 			model["lastDate"] = s.RangeDates.LastDate
-			order = append(order, []string{"firstTs", "firstDate", "lastTs", "lastDate"}...)
 		}
 	}
 	// EXISTING_CODE

diff --git a/src/apps/chifra/pkg/types/types_chunkbloom.go b/src/apps/chifra/pkg/types/types_chunkbloom.go
@@ -20,14 +20,14 @@ import (
 // EXISTING_CODE
 
 type ChunkBloom struct {
-	ByteWidth  uint64     `json:"byteWidth"`
-	Hash       base.Hash  `json:"hash"`
-	Magic      string     `json:"magic"`
-	NBlooms    uint64     `json:"nBlooms"`
-	NInserted  uint64     `json:"nInserted"`
-	Range      string     `json:"range"`
-	RangeDates RangeDates `json:"rangeDates"`
-	Size       uint64     `json:"size"`
+	ByteWidth  uint64      `json:"byteWidth"`
+	Hash       base.Hash   `json:"hash"`
+	Magic      string      `json:"magic"`
+	NBlooms    uint64      `json:"nBlooms"`
+	NInserted  uint64      `json:"nInserted"`
+	Range      string      `json:"range"`
+	RangeDates *RangeDates `json:"rangeDates,omitempty"`
+	Size       uint64      `json:"size"`
 	// EXISTING_CODE
 	// EXISTING_CODE
 }
@@ -65,15 +65,18 @@ func (s *ChunkBloom) Model(chain, format string, verbose bool, extraOpts map[str
 		model["hashValue"] = FormattedTag(verbose, s.Hash)
 	}
 
-	if verbose {
-		if format == "json" {
+	if verbose && format == "json" {
+		if s.RangeDates != nil {
 			model["rangeDates"] = s.RangeDates.Model(chain, format, verbose, extraOpts).Data
-		} else {
+		}
+	} else if verbose {
+		model["firstTs"], model["firstDate"], model["lastTs"], model["lastDate"] = 0, "", 0, ""
+		order = append(order, []string{"firstTs", "firstDate", "lastTs", "lastDate"}...)
+		if s.RangeDates != nil {
 			model["firstTs"] = s.RangeDates.FirstTs
 			model["firstDate"] = s.RangeDates.FirstDate
 			model["lastTs"] = s.RangeDates.LastTs
 			model["lastDate"] = s.RangeDates.LastDate
-			order = append(order, []string{"firstTs", "firstDate", "lastTs", "lastDate"}...)
 		}
 	}
 	// EXISTING_CODE

diff --git a/src/apps/chifra/pkg/types/types_chunkindex.go b/src/apps/chifra/pkg/types/types_chunkindex.go
@@ -18,13 +18,13 @@ import (
 // EXISTING_CODE
 
 type ChunkIndex struct {
-	Hash         base.Hash  `json:"hash"`
-	Magic        string     `json:"magic"`
-	NAddresses   uint64     `json:"nAddresses"`
-	NAppearances uint64     `json:"nAppearances"`
-	Range        string     `json:"range"`
-	RangeDates   RangeDates `json:"rangeDates"`
-	Size         uint64     `json:"size"`
+	Hash         base.Hash   `json:"hash"`
+	Magic        string      `json:"magic"`
+	NAddresses   uint64      `json:"nAddresses"`
+	NAppearances uint64      `json:"nAppearances"`
+	Range        string      `json:"range"`
+	RangeDates   *RangeDates `json:"rangeDates,omitempty"`
+	Size         uint64      `json:"size"`
 	// EXISTING_CODE
 	// EXISTING_CODE
 }
@@ -60,15 +60,18 @@ func (s *ChunkIndex) Model(chain, format string, verbose bool, extraOpts map[str
 		model["hashValue"] = FormattedTag(verbose, s.Hash)
 	}
 
-	if verbose {
-		if format == "json" {
+	if verbose && format == "json" {
+		if s.RangeDates != nil {
 			model["rangeDates"] = s.RangeDates.Model(chain, format, verbose, extraOpts).Data
-		} else {
+		}
+	} else if verbose {
+		model["firstTs"], model["firstDate"], model["lastTs"], model["lastDate"] = 0, "", 0, ""
+		order = append(order, []string{"firstTs", "firstDate", "lastTs", "lastDate"}...)
+		if s.RangeDates != nil {
 			model["firstTs"] = s.RangeDates.FirstTs
 			model["firstDate"] = s.RangeDates.FirstDate
 			model["lastTs"] = s.RangeDates.LastTs
 			model["lastDate"] = s.RangeDates.LastDate
-			order = append(order, []string{"firstTs", "firstDate", "lastTs", "lastDate"}...)
 		}
 	}
 	// EXISTING_CODE

diff --git a/src/apps/chifra/pkg/types/types_chunkrecord.go b/src/apps/chifra/pkg/types/types_chunkrecord.go
@@ -23,7 +23,7 @@ type ChunkRecord struct {
 	IndexHash  base.IpfsHash `json:"indexHash"`
 	IndexSize  int64         `json:"indexSize"`
 	Range      string        `json:"range"`
-	RangeDates RangeDates    `json:"rangeDates"`
+	RangeDates *RangeDates   `json:"rangeDates,omitempty"`
 	// EXISTING_CODE
 	// EXISTING_CODE
 }
@@ -53,15 +53,18 @@ func (s *ChunkRecord) Model(chain, format string, verbose bool, extraOpts map[st
 		"indexSize",
 	}
 
-	if verbose {
-		if format == "json" {
+	if verbose && format == "json" {
+		if s.RangeDates != nil {
 			model["rangeDates"] = s.RangeDates.Model(chain, format, verbose, extraOpts).Data
-		} else {
+		}
+	} else if verbose {
+		model["firstTs"], model["firstDate"], model["lastTs"], model["lastDate"] = 0, "", 0, ""
+		order = append(order, []string{"firstTs", "firstDate", "lastTs", "lastDate"}...)
+		if s.RangeDates != nil {
 			model["firstTs"] = s.RangeDates.FirstTs
 			model["firstDate"] = s.RangeDates.FirstDate
 			model["lastTs"] = s.RangeDates.LastTs
 			model["lastDate"] = s.RangeDates.LastDate
-			order = append(order, []string{"firstTs", "firstDate", "lastTs", "lastDate"}...)
 		}
 	}
 	// EXISTING_CODE

diff --git a/src/apps/chifra/pkg/types/types_chunkrecord_sort.go b/src/apps/chifra/pkg/types/types_chunkrecord_sort.go
@@ -46,7 +46,16 @@ func ChunkRecordBy(field ChunkRecordField, order SortOrder) func(p1, p2 ChunkRec
 		}
 	case ChunkRecordRangeDates: // RangeDates
 		return func(p1, p2 ChunkRecord) bool {
-			cmp := p1.RangeDates.Cmp(p2.RangeDates)
+			if p1.RangeDates == nil && p2.RangeDates == nil {
+				return false
+			}
+			if p1.RangeDates == nil {
+				return order == Ascending
+			}
+			if p2.RangeDates == nil {
+				return order != Ascending
+			}
+			cmp := p1.RangeDates.Cmp(*p2.RangeDates)
 			if order == Ascending {
 				return cmp == -1
 			}

diff --git a/src/apps/chifra/pkg/types/types_chunkstats.go b/src/apps/chifra/pkg/types/types_chunkstats.go
@@ -14,19 +14,19 @@ import "encoding/json"
 // EXISTING_CODE
 
 type ChunkStats struct {
-	AddrsPerBlock float64    `json:"addrsPerBlock"`
-	AppsPerAddr   float64    `json:"appsPerAddr"`
-	AppsPerBlock  float64    `json:"appsPerBlock"`
-	BloomSz       uint64     `json:"bloomSz"`
-	ChunkSz       uint64     `json:"chunkSz"`
-	NAddrs        uint64     `json:"nAddrs"`
-	NApps         uint64     `json:"nApps"`
-	NBlocks       uint64     `json:"nBlocks"`
-	NBlooms       uint64     `json:"nBlooms"`
-	Range         string     `json:"range"`
-	RangeDates    RangeDates `json:"rangeDates"`
-	Ratio         float64    `json:"ratio"`
-	RecWid        uint64     `json:"recWid"`
+	AddrsPerBlock float64     `json:"addrsPerBlock"`
+	AppsPerAddr   float64     `json:"appsPerAddr"`
+	AppsPerBlock  float64     `json:"appsPerBlock"`
+	BloomSz       uint64      `json:"bloomSz"`
+	ChunkSz       uint64      `json:"chunkSz"`
+	NAddrs        uint64      `json:"nAddrs"`
+	NApps         uint64      `json:"nApps"`
+	NBlocks       uint64      `json:"nBlocks"`
+	NBlooms       uint64      `json:"nBlooms"`
+	Range         string      `json:"range"`
+	RangeDates    *RangeDates `json:"rangeDates,omitempty"`
+	Ratio         float64     `json:"ratio"`
+	RecWid        uint64      `json:"recWid"`
 	// EXISTING_CODE
 	// EXISTING_CODE
 }
@@ -70,15 +70,17 @@ func (s *ChunkStats) Model(chain, format string, verbose bool, extraOpts map[str
 		"ratio",
 	}
 
-	if verbose {
-		if format == "json" {
+	if verbose && format == "json" {
+		if s.RangeDates != nil {
 			model["rangeDates"] = s.RangeDates.Model(chain, format, verbose, extraOpts).Data
-		} else {
+		}
+	} else if verbose {
+		order = append(order, []string{"firstTs", "firstDate", "lastTs", "lastDate"}...)
+		if s.RangeDates != nil {
 			model["firstTs"] = s.RangeDates.FirstTs
 			model["firstDate"] = s.RangeDates.FirstDate
 			model["lastTs"] = s.RangeDates.LastTs
 			model["lastDate"] = s.RangeDates.LastDate
-			order = append(order, []string{"firstTs", "firstDate", "lastTs", "lastDate"}...)
 		}
 	}
 	// EXISTING_CODE

diff --git a/src/apps/chifra/pkg/types/types_chunkstats_sort.go b/src/apps/chifra/pkg/types/types_chunkstats_sort.go
@@ -103,7 +103,16 @@ func ChunkStatsBy(field ChunkStatsField, order SortOrder) func(p1, p2 ChunkStats
 		}
 	case ChunkStatsRangeDates: // RangeDates
 		return func(p1, p2 ChunkStats) bool {
-			cmp := p1.RangeDates.Cmp(p2.RangeDates)
+			if p1.RangeDates == nil && p2.RangeDates == nil {
+				return false
+			}
+			if p1.RangeDates == nil {
+				return order == Ascending
+			}
+			if p2.RangeDates == nil {
+				return order != Ascending
+			}
+			cmp := p1.RangeDates.Cmp(*p2.RangeDates)
 			if order == Ascending {
 				return cmp == -1
 			}

diff --git a/src/dev_tools/goMaker/templates/classDefinitions/fields/chunkaddress.csv b/src/dev_tools/goMaker/templates/classDefinitions/fields/chunkaddress.csv
@@ -1,6 +1,6 @@
-name       ,type       ,strDefault ,attributes ,docOrder ,description
-address    ,address    ,           ,           ,       1 ,the address in this record
-range      ,blkrange   ,           ,           ,       2 ,the block range of the chunk from which this address record was taken
-offset     ,uint64     ,           ,           ,       3 ,the offset into the appearance table of the first record for this address
-count      ,uint64     ,           ,           ,       4 ,the number of records in teh appearance table for this address
-rangeDates ,RangeDates ,           ,           ,       5 ,if verbose&#44; the block and timestamp bounds of the chunk
+name       ,type        ,strDefault ,attributes ,docOrder ,description
+address    ,address     ,           ,           ,       1 ,the address in this record
+range      ,blkrange    ,           ,           ,       2 ,the block range of the chunk from which this address record was taken
+offset     ,uint64      ,           ,           ,       3 ,the offset into the appearance table of the first record for this address
+count      ,uint64      ,           ,           ,       4 ,the number of records in teh appearance table for this address
+rangeDates ,*RangeDates ,           ,omitempty  ,       5 ,if verbose&#44; the block and timestamp bounds of the chunk (may be null)
diff --git a/src/dev_tools/goMaker/templates/classDefinitions/fields/chunkbloom.csv b/src/dev_tools/goMaker/templates/classDefinitions/fields/chunkbloom.csv
@@ -1,9 +1,9 @@
-name       ,type       ,strDefault ,attributes ,docOrder ,description
-range      ,blkrange   ,           ,           ,       1 ,the block range (inclusive) covered by this chunk
-magic      ,string     ,           ,           ,       2 ,an internal use only magic number to indicate file format
-hash       ,hash       ,           ,           ,       3 ,the hash of the specification under which this chunk was generated
-nBlooms    ,uint64     ,           ,           ,       4 ,the number of individual bloom filters in this bloom file
-nInserted  ,uint64     ,           ,           ,       5 ,the number of addresses inserted into the bloom file
-size       ,uint64     ,           ,           ,       6 ,the size on disc in bytes of this bloom file
-byteWidth  ,uint64     ,           ,           ,       7 ,the width of the bloom filter
-rangeDates ,RangeDates ,           ,           ,       8 ,if verbose&#44; the block and timestamp bounds of the chunk
+name       ,type        ,strDefault ,attributes ,docOrder ,description
+range      ,blkrange    ,           ,           ,       1 ,the block range (inclusive) covered by this chunk
+magic      ,string      ,           ,           ,       2 ,an internal use only magic number to indicate file format
+hash       ,hash        ,           ,           ,       3 ,the hash of the specification under which this chunk was generated
+nBlooms    ,uint64      ,           ,           ,       4 ,the number of individual bloom filters in this bloom file
+nInserted  ,uint64      ,           ,           ,       5 ,the number of addresses inserted into the bloom file
+size       ,uint64      ,           ,           ,       6 ,the size on disc in bytes of this bloom file
+byteWidth  ,uint64      ,           ,           ,       7 ,the width of the bloom filter
+rangeDates ,*RangeDates ,           ,omitempty  ,       8 ,if verbose&#44; the block and timestamp bounds of the chunk (may be null)
diff --git a/src/dev_tools/goMaker/templates/classDefinitions/fields/chunkindex.csv b/src/dev_tools/goMaker/templates/classDefinitions/fields/chunkindex.csv
@@ -1,8 +1,8 @@
-name         ,type       ,strDefault ,attributes ,docOrder ,description
-range        ,blkrange   ,           ,           ,       1 ,the block range (inclusive) covered by this chunk
-magic        ,string     ,           ,           ,       2 ,an internal use only magic number to indicate file format
-hash         ,hash       ,           ,           ,       3 ,the hash of the specification under which this chunk was generated
-nAddresses   ,uint64     ,           ,           ,       4 ,the number of addresses in this chunk
-nAppearances ,uint64     ,           ,           ,       5 ,the number of appearances in this chunk
-size         ,uint64     ,           ,           ,       6 ,the size of the chunk in bytes
-rangeDates   ,RangeDates ,           ,           ,       7 ,if verbose&#44; the block and timestamp bounds of the chunk
+name         ,type        ,strDefault ,attributes ,docOrder ,description
+range        ,blkrange    ,           ,           ,       1 ,the block range (inclusive) covered by this chunk
+magic        ,string      ,           ,           ,       2 ,an internal use only magic number to indicate file format
+hash         ,hash        ,           ,           ,       3 ,the hash of the specification under which this chunk was generated
+nAddresses   ,uint64      ,           ,           ,       4 ,the number of addresses in this chunk
+nAppearances ,uint64      ,           ,           ,       5 ,the number of appearances in this chunk
+size         ,uint64      ,           ,           ,       6 ,the size of the chunk in bytes
+rangeDates   ,*RangeDates ,           ,omitempty  ,       7 ,if verbose&#44; the block and timestamp bounds of the chunk (may be null)
+1 −1		go.mod
+2 −2		go.sum
+1 −1		typescript/src/types/chunkAddress.ts
+1 −1		typescript/src/types/chunkBloom.ts
+1 −1		typescript/src/types/chunkIndex.ts
+1 −1		typescript/src/types/chunkRecord.ts
+1 −1		typescript/src/types/chunkStats.ts