Add crc64nvme support for flexible checksums

Update checksum tests to add user-provided checksum tests from SEP

aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs

@@ -217,6 +217,10 @@ where
                     cfg.interceptor_state()
                         .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc32c);
                 }
+                ChecksumAlgorithm::Crc64Nvme => {
+                    cfg.interceptor_state()
+                        .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc64);
+                }
                 #[allow(deprecated)]
                 ChecksumAlgorithm::Md5 => {
                     tracing::warn!(more_info = "Unsupported ChecksumAlgorithm MD5 set");

aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt

@@ -382,7 +382,7 @@ internal class HttpChecksumTest {
                 """
                 //${testDef.docs}
                 ##[::tokio::test]
-                async fn ${algoLower}_response_checksums_fails_correctly() {
+                async fn ${algoLower}_response_checksums_works() {
                     let (http_client, _rx) = #{capture_request}(Some(
                         http::Response::builder()
                             .header("x-amz-checksum-$algoLower", "${testDef.checksumHeaderValue}")
@@ -428,7 +428,7 @@ internal class HttpChecksumTest {
                 """
                 //${testDef.docs}
                 ##[::tokio::test]
-                async fn ${algoLower}_response_checksums_work() {
+                async fn ${algoLower}_response_checksums_fail_correctly() {
                     let (http_client, _rx) = #{capture_request}(Some(
                         http::Response::builder()
                             .header("x-amz-checksum-$algoLower", "${testDef.checksumHeaderValue}")
@@ -709,16 +709,14 @@ val checksumRequestTests =
             "crUfeA==",
             "V",
         ),
-        /* We do not yet support Crc64Nvme checksums
-         RequestChecksumCalculationTest(
-         "CRC64NVME checksum calculation works.",
-         "Hello world",
-         "Crc64Nvme",
-         "CRC64NVME",
-         "uc8X9yrZrD4=",
-         "W",
-         ),
-         */
+        RequestChecksumCalculationTest(
+            "CRC64NVME checksum calculation works.",
+            "Hello world",
+            "Crc64Nvme",
+            "CRC64NVME",
+            "OOJZ0D8xKts=",
+            "W",
+        ),
         RequestChecksumCalculationTest(
             "SHA1 checksum calculation works.",
             "Hello world",
@@ -758,12 +756,12 @@ val streamingRequestTests =
             "Crc32C",
             "crUfeA==",
         ),
-//    StreamingRequestChecksumCalculationTest(
-//        "CRC64NVME streaming checksum calculation works.",
-//        "Hello world",
-//        "Crc64Nvme",
-//        "uc8X9yrZrD4=",
-//    ),
+        StreamingRequestChecksumCalculationTest(
+            "CRC64NVME streaming checksum calculation works.",
+            "Hello world",
+            "Crc64Nvme",
+            "OOJZ0D8xKts=",
+        ),
         StreamingRequestChecksumCalculationTest(
             "SHA1 streaming checksum calculation works.",
             "Hello world",
@@ -799,13 +797,12 @@ val checksumResponseSuccTests =
             "Crc32C",
             "crUfeA==",
         ),
-        /*
         ResponseChecksumValidationSuccessTest(
             "Successful payload validation with Crc64Nvme checksum.",
             "Hello world",
             "Crc64Nvme",
-            "uc8X9yrZrD4=",
-        ),*/
+            "OOJZ0D8xKts=",
+        ),
         ResponseChecksumValidationSuccessTest(
             "Successful payload validation with Sha1 checksum.",
             "Hello world",
@@ -844,14 +841,13 @@ val checksumResponseFailTests =
             "bm90LWEtY2hlY2tzdW0=",
             "crUfeA==",
         ),
-        /*
         ResponseChecksumValidationFailureTest(
             "Failed payload validation with CRC64NVME checksum.",
             "Hello world",
             "Crc64Nvme",
             "bm90LWEtY2hlY2tzdW0=",
-            "uc8X9yrZrD4=",
-        ),*/
+            "OOJZ0D8xKts=",
+        ),
         ResponseChecksumValidationFailureTest(
             "Failed payload validation with SHA1 checksum.",
             "Hello world",

rust-runtime/aws-smithy-checksums/Cargo.toml

@@ -18,6 +18,7 @@ aws-smithy-types = { path = "../aws-smithy-types" }
 bytes = "1"
 crc32c = "0.6.8"
 crc32fast = "1.3"
+crc64fast-nvme = "1.0.0"
 hex = "0.4.3"
 http = "0.2.8"
 http-body = "0.4.4"

rust-runtime/aws-smithy-checksums/src/http.rs

@@ -9,8 +9,8 @@ use aws_smithy_types::base64;
 use http::header::{HeaderMap, HeaderValue};
 
 use crate::{
-    Checksum, Crc32, Crc32c, Md5, Sha1, Sha256, CRC_32_C_NAME, CRC_32_NAME, SHA_1_NAME,
-    SHA_256_NAME,
+    Checksum, Crc32, Crc32c, Crc64Nvme, Md5, Sha1, Sha256, CRC_32_C_NAME, CRC_32_NAME,
+    CRC_64_NVME_NAME, SHA_1_NAME, SHA_256_NAME,
 };
 
 pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32";
@@ -25,8 +25,13 @@ pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
 /// When a response has to be checksum-verified, we have to check possible headers until we find the
 /// header with the precalculated checksum. Because a service may send back multiple headers, we have
 /// to check them in order based on how fast each checksum is to calculate.
-pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 4] =
-    [CRC_32_C_NAME, CRC_32_NAME, SHA_1_NAME, SHA_256_NAME];
+pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] = [
+    CRC_64_NVME_NAME,
+    CRC_32_C_NAME,
+    CRC_32_NAME,
+    SHA_1_NAME,
+    SHA_256_NAME,
+];
 
 /// Checksum algorithms are use to validate the integrity of data. Structs that implement this trait
 /// can be used as checksum calculators. This trait requires Send + Sync because these checksums are
@@ -82,6 +87,12 @@ impl HttpChecksum for Crc32c {
     }
 }
 
+impl HttpChecksum for Crc64Nvme {
+    fn header_name(&self) -> &'static str {
+        CRC_64_NVME_HEADER_NAME
+    }
+}
+
 impl HttpChecksum for Sha1 {
     fn header_name(&self) -> &'static str {
         SHA_1_HEADER_NAME
@@ -105,7 +116,10 @@ mod tests {
     use aws_smithy_types::base64;
     use bytes::Bytes;
 
-    use crate::{ChecksumAlgorithm, CRC_32_C_NAME, CRC_32_NAME, SHA_1_NAME, SHA_256_NAME};
+    use crate::{
+        http::CRC_64_NVME_HEADER_NAME, ChecksumAlgorithm, CRC_32_C_NAME, CRC_32_NAME,
+        CRC_64_NVME_NAME, SHA_1_NAME, SHA_256_NAME,
+    };
 
     use super::HttpChecksum;
 
@@ -157,6 +171,30 @@ mod tests {
         assert_eq!(expected_value, actual_value)
     }
 
+    #[test]
+    fn test_trailer_length_of_crc64nvme_checksum_body() {
+        let checksum = CRC_64_NVME_NAME
+            .parse::<ChecksumAlgorithm>()
+            .unwrap()
+            .into_impl();
+        let expected_size = 37;
+        let actual_size = HttpChecksum::size(&*checksum);
+        assert_eq!(expected_size, actual_size)
+    }
+
+    #[test]
+    fn test_trailer_value_of_crc64nvme_checksum_body() {
+        let checksum = CRC_64_NVME_NAME
+            .parse::<ChecksumAlgorithm>()
+            .unwrap()
+            .into_impl();
+        // The CRC64NVME of an empty string is all zeroes
+        let expected_value = Bytes::from_static(b"\0\0\0\0\0\0\0\0");
+        let expected_value = base64::encode(&expected_value);
+        let actual_value = checksum.header_value();
+        assert_eq!(expected_value, actual_value)
+    }
+
     #[test]
     fn test_trailer_length_of_sha1_checksum_body() {
         let checksum = SHA_1_NAME.parse::<ChecksumAlgorithm>().unwrap().into_impl();

rust-runtime/aws-smithy-checksums/src/lib.rs

@@ -19,7 +19,7 @@
 use crate::error::UnknownChecksumAlgorithmError;
 
 use bytes::Bytes;
-use std::str::FromStr;
+use std::{fmt::Debug, str::FromStr};
 
 pub mod body;
 pub mod error;
@@ -28,6 +28,7 @@ pub mod http;
 // Valid checksum algorithm names
 pub const CRC_32_NAME: &str = "crc32";
 pub const CRC_32_C_NAME: &str = "crc32c";
+pub const CRC_64_NVME_NAME: &str = "crc64nvme";
 pub const SHA_1_NAME: &str = "sha1";
 pub const SHA_256_NAME: &str = "sha256";
 pub const MD5_NAME: &str = "md5";
@@ -42,6 +43,7 @@ pub enum ChecksumAlgorithm {
     Md5,
     Sha1,
     Sha256,
+    Crc64Nvme,
 }
 
 impl FromStr for ChecksumAlgorithm {
@@ -50,6 +52,7 @@ impl FromStr for ChecksumAlgorithm {
     /// Create a new `ChecksumAlgorithm` from an algorithm name. Valid algorithm names are:
     /// - "crc32"
     /// - "crc32c"
+    /// - "crc64nvme"
     /// - "sha1"
     /// - "sha256"
     ///
@@ -66,6 +69,8 @@ impl FromStr for ChecksumAlgorithm {
         } else if checksum_algorithm.eq_ignore_ascii_case(MD5_NAME) {
             // MD5 is now an alias for the default Crc32 since it is deprecated
             Ok(Self::Crc32)
+        } else if checksum_algorithm.eq_ignore_ascii_case(CRC_64_NVME_NAME) {
+            Ok(Self::Crc64Nvme)
         } else {
             Err(UnknownChecksumAlgorithmError::new(checksum_algorithm))
         }
@@ -78,6 +83,7 @@ impl ChecksumAlgorithm {
         match self {
             Self::Crc32 => Box::<Crc32>::default(),
             Self::Crc32c => Box::<Crc32c>::default(),
+            Self::Crc64Nvme => Box::<Crc64Nvme>::default(),
             #[allow(deprecated)]
             Self::Md5 => Box::<Crc32>::default(),
             Self::Sha1 => Box::<Sha1>::default(),
@@ -90,6 +96,7 @@ impl ChecksumAlgorithm {
         match self {
             Self::Crc32 => CRC_32_NAME,
             Self::Crc32c => CRC_32_C_NAME,
+            Self::Crc64Nvme => CRC_64_NVME_NAME,
             #[allow(deprecated)]
             Self::Md5 => MD5_NAME,
             Self::Sha1 => SHA_1_NAME,
@@ -188,6 +195,45 @@ impl Checksum for Crc32c {
     }
 }
 
+#[derive(Default)]
+struct Crc64Nvme {
+    hasher: crc64fast_nvme::Digest,
+}
+
+// crc64fast_nvme::Digest doesn't impl Debug so we can't derive the impl
+impl Debug for Crc64Nvme {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Crc64Nvme").finish()
+    }
+}
+
+impl Crc64Nvme {
+    fn update(&mut self, bytes: &[u8]) {
+        self.hasher.write(bytes);
+    }
+
+    fn finalize(self) -> Bytes {
+        Bytes::copy_from_slice(self.hasher.sum64().to_be_bytes().as_slice())
+    }
+
+    // Size of the checksum in bytes
+    fn size() -> u64 {
+        8
+    }
+}
+
+impl Checksum for Crc64Nvme {
+    fn update(&mut self, bytes: &[u8]) {
+        Self::update(self, bytes)
+    }
+    fn finalize(self: Box<Self>) -> Bytes {
+        Self::finalize(*self)
+    }
+    fn size(&self) -> u64 {
+        Self::size()
+    }
+}
+
 #[derive(Debug, Default)]
 struct Sha1 {
     hasher: sha1::Sha1,
@@ -304,8 +350,11 @@ mod tests {
         Crc32, Crc32c, Md5, Sha1, Sha256,
     };
 
-    use crate::http::HttpChecksum;
     use crate::ChecksumAlgorithm;
+    use crate::{
+        http::{HttpChecksum, CRC_64_NVME_HEADER_NAME},
+        Crc64Nvme,
+    };
     use aws_smithy_types::base64;
     use http::HeaderValue;
     use pretty_assertions::assert_eq;
@@ -354,6 +403,19 @@ mod tests {
         assert_eq!(decoded_checksum, expected_checksum);
     }
 
+    #[test]
+    fn test_crc64nvme_checksum() {
+        let mut checksum = Crc64Nvme::default();
+        checksum.update(TEST_DATA.as_bytes());
+        let checksum_result = Box::new(checksum).headers();
+        let encoded_checksum = checksum_result.get(CRC_64_NVME_HEADER_NAME).unwrap();
+        let decoded_checksum = base64_encoded_checksum_to_hex_string(encoded_checksum);
+
+        let expected_checksum = "0xAECAF3AF9C98A855";
+
+        assert_eq!(decoded_checksum, expected_checksum);
+    }
+
     #[test]
     fn test_sha1_checksum() {
         let mut checksum = Sha1::default();