From c93ac057f44b9ee5d01a93e7a91ab80da9e2a852 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Mon, 5 Aug 2024 20:06:05 +0200 Subject: [PATCH 1/3] fix: augment PieceAggregateCommP to return the resulting aggregate size doing so here *massively* simplifies downstream code in lotus --- commd.go | 20 +++++++++++--------- commd_test.go | 2 +- writer/writer.go | 4 ++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/commd.go b/commd.go index d849f18..903a05a 100644 --- a/commd.go +++ b/commd.go @@ -25,13 +25,13 @@ type stackFrame struct { // This function makes no assumptions, other than maximum size, about the pieces that you include. // To create a correctly formed UnsealedCID (CommD) for a sector using this method, you should first // ensure that the pieces add up to the required size. -func PieceAggregateCommP(proofType abi.RegisteredSealProof, pieceInfos []abi.PieceInfo) (cid.Cid, error) { +func PieceAggregateCommP(proofType abi.RegisteredSealProof, pieceInfos []abi.PieceInfo) (cid.Cid, abi.PaddedPieceSize, error) { spi, found := abi.SealProofInfos[proofType] if !found { - return cid.Undef, fmt.Errorf("unknown seal proof type %d", proofType) + return cid.Undef, 0, fmt.Errorf("unknown seal proof type %d", proofType) } if len(pieceInfos) == 0 { - return cid.Undef, errors.New("no pieces provided") + return cid.Undef, 0, errors.New("no pieces provided") } maxSize := uint64(spi.SectorSize) @@ -41,18 +41,18 @@ func PieceAggregateCommP(proofType abi.RegisteredSealProof, pieceInfos []abi.Pie // sancheck everything for i, p := range pieceInfos { if p.Size < 128 { - return cid.Undef, fmt.Errorf("invalid Size of PieceInfo %d: value %d is too small", i, p.Size) + return cid.Undef, 0, fmt.Errorf("invalid Size of PieceInfo %d: value %d is too small", i, p.Size) } if uint64(p.Size) > maxSize { - return cid.Undef, fmt.Errorf("invalid Size of PieceInfo %d: value %d is larger than sector size of SealProofType %d", i, p.Size, proofType) + return cid.Undef, 0, fmt.Errorf("invalid Size of PieceInfo %d: value %d is larger than sector size of SealProofType %d", i, p.Size, proofType) } if bits.OnesCount64(uint64(p.Size)) != 1 { - return cid.Undef, fmt.Errorf("invalid Size of PieceInfo %d: value %d is not a power of 2", i, p.Size) + return cid.Undef, 0, fmt.Errorf("invalid Size of PieceInfo %d: value %d is not a power of 2", i, p.Size) } cp, err := commcid.CIDToPieceCommitmentV1(p.PieceCID) if err != nil { - return cid.Undef, fmt.Errorf("invalid PieceCid for PieceInfo %d: %w", i, err) + return cid.Undef, 0, fmt.Errorf("invalid PieceCid for PieceInfo %d: %w", i, err) } todo[i] = stackFrame{size: uint64(p.Size), commP: cp} } @@ -106,10 +106,12 @@ func PieceAggregateCommP(proofType abi.RegisteredSealProof, pieceInfos []abi.Pie } if stack[0].size > maxSize { - return cid.Undef, fmt.Errorf("provided pieces sum up to %d bytes, which is larger than sector size of SealProofType %d", stack[0].size, proofType) + return cid.Undef, 0, fmt.Errorf("provided pieces sum up to %d bytes, which is larger than sector size of SealProofType %d", stack[0].size, proofType) } - return commcid.PieceCommitmentV1ToCID(stack[0].commP) + fincid, err := commcid.PieceCommitmentV1ToCID(stack[0].commP) + + return fincid, abi.PaddedPieceSize(stack[0].size), err } var s256pool = sync.Pool{New: func() any { return sha256simd.New() }} diff --git a/commd_test.go b/commd_test.go index 79579da..95eb8e3 100644 --- a/commd_test.go +++ b/commd_test.go @@ -26,7 +26,7 @@ func TestGenerateUnsealedCID(t *testing.T) { expCommD := cidMustParse("baga6ea4seaqiw3gbmstmexb7sqwkc5r23o3i7zcyx5kr76pfobpykes3af62kca") - commD, _ := commp.PieceAggregateCommP( + commD, _, _ := commp.PieceAggregateCommP( abi.RegisteredSealProof_StackedDrg32GiBV1_1, // 32G sector SP []abi.PieceInfo{ {PieceCID: cidMustParse("baga6ea4seaqknzm22isnhsxt2s4dnw45kfywmhenngqq3nc7jvecakoca6ksyhy"), Size: 256 << 20}, // https://filfox.info/en/deal/3755444 diff --git a/writer/writer.go b/writer/writer.go index 6c8f092..d3b6bc4 100644 --- a/writer/writer.go +++ b/writer/writer.go @@ -143,14 +143,14 @@ func (w *Writer) Sum() (DataCIDSize, error) { } } - p, err := commp.PieceAggregateCommP(abi.RegisteredSealProof_StackedDrg64GiBV1, pieces) + p, sz, err := commp.PieceAggregateCommP(abi.RegisteredSealProof_StackedDrg64GiBV1, pieces) if err != nil { return DataCIDSize{}, xerrors.Errorf("generating unsealed CID: %w", err) } return DataCIDSize{ PayloadSize: rawLen, - PieceSize: abi.PaddedPieceSize(len(leaves)) * commPBufPad, + PieceSize: sz, PieceCID: p, }, nil } From 69aeca1892cbac6113780d168bfc1a257def06a8 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Tue, 6 Aug 2024 05:47:19 +0200 Subject: [PATCH 2/3] (squash-this) intentionally break size to demonstrate coverage --- commd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commd.go b/commd.go index 903a05a..b3addc3 100644 --- a/commd.go +++ b/commd.go @@ -111,7 +111,7 @@ func PieceAggregateCommP(proofType abi.RegisteredSealProof, pieceInfos []abi.Pie fincid, err := commcid.PieceCommitmentV1ToCID(stack[0].commP) - return fincid, abi.PaddedPieceSize(stack[0].size), err + return fincid, abi.PaddedPieceSize(stack[0].size / 2), err } var s256pool = sync.Pool{New: func() any { return sha256simd.New() }} From c9a710d70a92a167ce1fc77ec3c408792ffc26fd Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Tue, 6 Aug 2024 05:50:21 +0200 Subject: [PATCH 3/3] (squash this) add an extra test --- commd.go | 2 +- commd_test.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/commd.go b/commd.go index b3addc3..903a05a 100644 --- a/commd.go +++ b/commd.go @@ -111,7 +111,7 @@ func PieceAggregateCommP(proofType abi.RegisteredSealProof, pieceInfos []abi.Pie fincid, err := commcid.PieceCommitmentV1ToCID(stack[0].commP) - return fincid, abi.PaddedPieceSize(stack[0].size / 2), err + return fincid, abi.PaddedPieceSize(stack[0].size), err } var s256pool = sync.Pool{New: func() any { return sha256simd.New() }} diff --git a/commd_test.go b/commd_test.go index 95eb8e3..9136bca 100644 --- a/commd_test.go +++ b/commd_test.go @@ -26,7 +26,7 @@ func TestGenerateUnsealedCID(t *testing.T) { expCommD := cidMustParse("baga6ea4seaqiw3gbmstmexb7sqwkc5r23o3i7zcyx5kr76pfobpykes3af62kca") - commD, _, _ := commp.PieceAggregateCommP( + commD, sizeD, _ := commp.PieceAggregateCommP( abi.RegisteredSealProof_StackedDrg32GiBV1_1, // 32G sector SP []abi.PieceInfo{ {PieceCID: cidMustParse("baga6ea4seaqknzm22isnhsxt2s4dnw45kfywmhenngqq3nc7jvecakoca6ksyhy"), Size: 256 << 20}, // https://filfox.info/en/deal/3755444 @@ -49,6 +49,10 @@ func TestGenerateUnsealedCID(t *testing.T) { if commD != expCommD { t.Fatalf("calculated commd for sector f01392893:139074 as %s, expected %s", commD, expCommD) } + + if sizeD != 32<<30 { + t.Fatalf("calculated size for sector f01392893:139074 as %d, expected 32GiB", sizeD) + } } // Delete below when this ships in a stable release https://github.com/ipfs/go-cid/pull/139