diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod index d6074ea44ab..8e485c73bcf 100644 --- a/cmd/cli/go.mod +++ b/cmd/cli/go.mod @@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli go 1.23.2 require ( - github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1 + github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602 github.com/fatih/color v1.18.0 github.com/json-iterator/go v1.1.12 github.com/onsi/ginkgo/v2 v2.21.0 diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum index b4ddb9a8ffd..2aae7a9ad8a 100644 --- a/cmd/cli/go.sum +++ b/cmd/cli/go.sum @@ -1,7 +1,7 @@ code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1 h1:SO4oqmdxqvhi/unWsN7SJsejbh/M7P3TlXXEo3evrLw= -github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw= +github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602 h1:Ypfy9sLjPti1CcebiPGFadXObWDPNPrE+gkAX902Sm8= +github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= diff --git a/cmd/ishard/go.mod b/cmd/ishard/go.mod index eb796a5c522..ea21aff5e56 100644 --- a/cmd/ishard/go.mod +++ b/cmd/ishard/go.mod @@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/ishard go 1.23.2 require ( - github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1 + github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602 github.com/json-iterator/go v1.1.12 github.com/vbauerster/mpb/v4 v4.12.2 ) diff --git a/cmd/ishard/go.sum b/cmd/ishard/go.sum index a9d5bfdb179..babd49e7383 100644 --- a/cmd/ishard/go.sum +++ b/cmd/ishard/go.sum @@ -1,6 +1,6 @@ code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= -github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1 h1:SO4oqmdxqvhi/unWsN7SJsejbh/M7P3TlXXEo3evrLw= -github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw= +github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602 h1:Ypfy9sLjPti1CcebiPGFadXObWDPNPrE+gkAX902Sm8= +github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= diff --git a/cmn/cos/cksum.go b/cmn/cos/cksum.go index 399bca99fa5..c2db5d91eb7 100644 --- a/cmn/cos/cksum.go +++ b/cmn/cos/cksum.go @@ -20,11 +20,14 @@ import ( jsoniter "github.com/json-iterator/go" ) -// NOTE: not supporting SHA-3 family is its current golang.org/x/crypto/sha3 source -// doesn't implement BinaryMarshaler & BinaryUnmarshaler interfaces -// (see also https://golang.org/pkg/encoding) +// [NOTE] +// - currently, we have only two crypto-secure types: sha256 and sha512 +// - see related object comparison logic in cmn/objattrs -// checksums +// [TODO] +// revisit and maybe add SHA-3 family (see golang.org/x/crypto/sha3 for: `BinaryMarshaler`) + +// supported checksums const ( ChecksumNone = "none" ChecksumXXHash = "xxhash" diff --git a/cmn/objattrs.go b/cmn/objattrs.go index 0a221283a4b..d76ce3cf849 100644 --- a/cmn/objattrs.go +++ b/cmn/objattrs.go @@ -14,6 +14,7 @@ import ( "github.com/NVIDIA/aistore/api/apc" "github.com/NVIDIA/aistore/cmn/cos" "github.com/NVIDIA/aistore/cmn/debug" + "github.com/NVIDIA/aistore/cmn/feat" ) // LOM custom metadata stored under `lomCustomMD`. @@ -280,19 +281,28 @@ func (oa *ObjAttrs) CheckEq(rem cos.OAH) error { } // checksum check - if a, b := rem.Checksum(), oa.Cksum; !a.IsEmpty() && !b.IsEmpty() && a.Ty() == b.Ty() { - if !a.Equal(b) { - return fmt.Errorf("%s checksum %s != %s remote", a.Ty(), b, a) - } - cksumVal = a.Val() - // - // NOTE: including xxhash in trusted checksums - // - switch a.Ty() { - case cos.ChecksumXXHash, cos.ChecksumSHA256, cos.ChecksumSHA512: - sameCksum = true + if a, b := rem.Checksum(), oa.Cksum; a != nil && b != nil { + cksumType := a.Ty() + if !a.IsEmpty() && !b.IsEmpty() && cksumType == b.Ty() { + if !a.Equal(b) { + return fmt.Errorf("%s checksum %s != %s remote", cksumType, b, a) + } + cksumVal = a.Val() + + // [NOTE] + // unless overridden via feature flag + // trust two checksums, namely md5 and xxhash, that are _not_ cryptographically secure + + switch { + case Rom.Features().IsSet(feat.TrustCryptoSafeChecksums): + sameCksum = (cksumType == cos.ChecksumSHA256 || cksumType == cos.ChecksumSHA512) + default: + debug.Assert(cksumType != cos.ChecksumNone) + sameCksum = cksumType != cos.ChecksumCRC32C + } + + count++ } - count++ } // custom MD: ETag check (ignoring enclosing quotes) diff --git a/docs/feature_flags.md b/docs/feature_flags.md index 6ce56087413..c9324a0dfdb 100644 --- a/docs/feature_flags.md +++ b/docs/feature_flags.md @@ -51,11 +51,11 @@ By default, all features are disabled, and the corresponding 64-bit field is set ```console $ ais config cluster features -Enforce-IntraCluster-Access Fsync-PUT Ignore-LimitedCoexistence-Conflicts -Skip-Loading-VersionChecksum-MD LZ4-Block-1MB S3-Presigned-Request -Do-not-Auto-Detect-FileShare LZ4-Frame-Checksum Do-not-Optimize-Listing-Virtual-Dirs -S3-API-via-Root Do-not-Allow-Passing-FQN-to-ETL Disable-Cold-GET -S3-Reverse-Proxy none +Enforce-IntraCluster-Access LZ4-Block-1MB Do-not-Optimize-Listing-Virtual-Dirs Do-not-Delete-When-Rebalancing +Skip-Loading-VersionChecksum-MD LZ4-Frame-Checksum Disable-Cold-GET Do-not-Set-Control-Plane-ToS +Do-not-Auto-Detect-FileShare Do-not-Allow-Passing-FQN-to-ETL Streaming-Cold-GET none +S3-API-via-Root Ignore-LimitedCoexistence-Conflicts S3-Reverse-Proxy +Fsync-PUT S3-Presigned-Request S3-Use-Path-Style ``` For example: