feat(ic-http-gateway): enable validation of long assets' chunks

Cargo.toml

@@ -33,9 +33,11 @@ http-body-util = "0.1"
 bytes = "1"
 base64 = "0.22"
 lazy_static = "1"
+rand_chacha = "0.3"
 regex = "1"
 serde = "1"
 serde_cbor = "0.11"
+sha2 = "0.10"
 tokio = { version = "1", features = ["full"] }
 hyper = { version = "1", features = ["full"] }
 hyper-util = "0.1"
@@ -47,8 +49,11 @@ ic-utils = "0.38.2"
 candid = "0.10"
 pocket-ic = "=3.0.0"
 assert_matches = "1.5"
+rstest = "0.18"
 
-ic-http-certification = "2"
-ic-response-verification = "2"
+ic-response-verification = { git = "https://github.com/dfinity/response-verification", rev="dcbd7d471e1a5f95d8b1e8a66c4a35fd2a68268f" }
+ic-certification = { git = "https://github.com/dfinity/response-verification", rev="dcbd7d471e1a5f95d8b1e8a66c4a35fd2a68268f" }
+ic-http-certification = { git = "https://github.com/dfinity/response-verification", rev="dcbd7d471e1a5f95d8b1e8a66c4a35fd2a68268f" }
+ic-asset-certification = { git = "https://github.com/dfinity/response-verification", rev="dcbd7d471e1a5f95d8b1e8a66c4a35fd2a68268f" }
 
 ic-http-gateway = { path = "./packages/ic-http-gateway", version = "0.0.0" }

examples/http-gateway/canister/src/custom_assets/Cargo.toml

@@ -8,14 +8,18 @@ crate-type = ["cdylib"]
 
 [dependencies]
 candid.workspace = true
+http.workspace = true
 ic-cdk.workspace = true
 ic-cdk-macros.workspace = true
+regex.workspace = true
 serde.workspace = true
 serde_cbor.workspace = true
+sha2.workspace = true
 lazy_static.workspace = true
 base64.workspace = true
 include_dir = { version = "0.7", features = ["glob"] }
+rand_chacha.workspace = true
 
-ic-certification = { git = "https://github.com/dfinity/response-verification" }
-ic-http-certification = { git = "https://github.com/dfinity/response-verification", features = ["serde"] }
-ic-asset-certification = { git = "https://github.com/dfinity/response-verification" }
+ic-certification.workspace = true
+ic-http-certification.workspace = true
+ic-asset-certification.workspace = true

examples/http-gateway/canister/src/custom_assets/src/lib.rs

@@ -1,13 +1,11 @@
-use base64::engine::general_purpose::STANDARD as BASE64;
-use base64::Engine;
 use ic_asset_certification::{Asset, AssetConfig, AssetFallbackConfig, AssetRouter};
 use ic_cdk::{
     api::{data_certificate, set_certified_data},
     *,
 };
-use ic_certification::HashTree;
-use ic_http_certification::{HeaderField, HttpRequest, HttpResponse};
-use serde::Serialize;
+use ic_http_certification::{HeaderField, HttpRequest, HttpResponse, HttpResponseBuilder};
+use rand_chacha::rand_core::{RngCore, SeedableRng};
+use rand_chacha::ChaCha20Rng;
 use std::cell::RefCell;
 
 #[init]
@@ -22,48 +20,184 @@ fn post_upgrade() {
 
 #[query]
 fn http_request(req: HttpRequest) -> HttpResponse {
-    serve_asset(&req)
+    let mut response = serve_asset(&req);
+    if let Some(index) = chunk_corruption_requested(&req) {
+        let current_chunk = current_chunk_index(&response);
+        if current_chunk == index {
+            // Create a response with a corrupted body.
+            response = HttpResponseBuilder::new()
+                .with_status_code(response.status_code())
+                .with_headers(response.headers().to_vec())
+                .with_upgrade(response.upgrade().unwrap_or(false))
+                .with_body({
+                    let mut body = response.body().to_vec();
+                    body[0] += 1;
+                    body
+                })
+                .build();
+        }
+    }
+    if let Some(index) = cert_corruption_requested(&req) {
+        let current_chunk = current_chunk_index(&response);
+        if current_chunk == index {
+            let body = response.body().to_owned();
+            // Create a response with a corrupted certificate.
+            response = HttpResponseBuilder::new()
+                .with_status_code(response.status_code())
+                .with_upgrade(response.upgrade().unwrap_or(false))
+                .with_body(body)
+                .with_headers({
+                    let mut headers = response.headers().to_owned();
+                    for (key, value) in headers.iter_mut() {
+                        if key == "IC-Certificate" {
+                            value.insert(15, char::from(42));
+                        }
+                    }
+                    headers.to_vec()
+                })
+                .build();
+        }
+    }
+    response
+}
+
+fn current_chunk_index(resp: &HttpResponse) -> usize {
+    if let Some(content_range_header_value) = get_header_value(resp.headers(), "Content-Range") {
+        get_content_range_begin(&content_range_header_value) / ASSET_CHUNK_SIZE
+    } else {
+        // Not a range-response, the asset is a single chunk
+        0
+    }
+}
+
+fn chunk_corruption_requested(req: &HttpRequest) -> Option<usize> {
+    if let Some(corrupted_chunk_index) = get_header_value(req.headers(), "Test-CorruptedChunkIndex")
+    {
+        Some(
+            corrupted_chunk_index
+                .parse()
+                .expect("invalid index of chunk to corrupt"),
+        )
+    } else {
+        None
+    }
+}
+
+fn cert_corruption_requested(req: &HttpRequest) -> Option<usize> {
+    if let Some(corrupted_cert_chunk_index) =
+        get_header_value(req.headers(), "Test-CorruptedCertificate")
+    {
+        Some(
+            corrupted_cert_chunk_index
+                .parse()
+                .expect("invalid index of chunk to corrupt the certificate"),
+        )
+    } else {
+        None
+    }
+}
+
+fn get_header_value(headers: &[HeaderField], header_name: &str) -> Option<String> {
+    for (name, value) in headers.iter() {
+        if name.to_lowercase().eq(&header_name.to_lowercase()) {
+            return Some(value.to_string());
+        }
+    }
+    None
+}
+
+fn get_content_range_begin(content_range_header_value: &str) -> usize {
+    // expected format: `bytes 21010-47021/47022`
+    let re = regex::Regex::new(r"bytes\s+(\d+)-(\d+)/(\d+)").expect("invalid RE");
+    let caps = re
+        .captures(content_range_header_value)
+        .expect("malformed Content-Range header");
+    caps.get(1)
+        .expect("missing range-begin")
+        .as_str()
+        .parse()
+        .expect("malformed range-begin")
 }
 
 thread_local! {
     static ASSET_ROUTER: RefCell<AssetRouter<'static>> = Default::default();
 }
 
-static ASSET_206_BODY: &[u8; 64] =
-    b"<html><body>Some asset that returns a 206-response</body></html>";
+const ASSET_CHUNK_SIZE: usize = 2_000_000;
+
+const ONE_CHUNK_ASSET_LEN: usize = ASSET_CHUNK_SIZE;
+const TWO_CHUNKS_ASSET_LEN: usize = ASSET_CHUNK_SIZE + 1;
+const SIX_CHUNKS_ASSET_LEN: usize = 5 * ASSET_CHUNK_SIZE + 12;
+const TEN_CHUNKS_ASSET_LEN: usize = 10 * ASSET_CHUNK_SIZE;
+
+const ONE_CHUNK_ASSET_NAME: &str = "long_asset_one_chunk";
+const TWO_CHUNKS_ASSET_NAME: &str = "long_asset_two_chunks";
+const SIX_CHUNKS_ASSET_NAME: &str = "long_asset_six_chunks";
+const TEN_CHUNKS_ASSET_NAME: &str = "long_asset_ten_chunks";
+
+use ic_certification::Hash;
+use sha2::{Digest, Sha256};
+
+pub fn hash<T>(data: T) -> Hash
+where
+    T: AsRef<[u8]>,
+{
+    let mut hasher = Sha256::new();
+    hasher.update(data);
+    hasher.finalize().into()
+}
+
+fn long_asset_body(asset_name: &str) -> Vec<u8> {
+    let asset_length = match asset_name {
+        ONE_CHUNK_ASSET_NAME => ONE_CHUNK_ASSET_LEN,
+        TWO_CHUNKS_ASSET_NAME => TWO_CHUNKS_ASSET_LEN,
+        SIX_CHUNKS_ASSET_NAME => SIX_CHUNKS_ASSET_LEN,
+        TEN_CHUNKS_ASSET_NAME => TEN_CHUNKS_ASSET_LEN,
+        _ => ASSET_CHUNK_SIZE * 3 + 1,
+    };
+    let mut rng = ChaCha20Rng::from_seed(hash(asset_name));
+    let mut body = vec![0u8; asset_length];
+    rng.fill_bytes(&mut body);
+    body
+}
 
 fn certify_all_assets() {
-    let asset_configs = vec![
-        AssetConfig::File {
-            path: "index.html".to_string(),
-            content_type: Some("text/html".to_string()),
-            headers: get_asset_headers(vec![(
-                "cache-control".to_string(),
-                "public, no-cache, no-store".to_string(),
-            )]),
-            fallback_for: vec![AssetFallbackConfig {
-                scope: "/".to_string(),
-            }],
-            aliased_by: vec!["/".to_string()],
-            encodings: vec![],
-        },
-        AssetConfig::File {
-            path: "asset_206".to_string(),
-            content_type: Some("text/html".to_string()),
+    let mut assets = vec![Asset::new(
+        "index.html",
+        b"<html><body>Hello, world!</body></html>",
+    )];
+    let mut asset_configs = vec![AssetConfig::File {
+        path: "index.html".to_string(),
+        content_type: Some("text/html".to_string()),
+        headers: get_asset_headers(vec![(
+            "cache-control".to_string(),
+            "public, no-cache, no-store".to_string(),
+        )]),
+        fallback_for: vec![AssetFallbackConfig {
+            scope: "/".to_string(),
+        }],
+        aliased_by: vec!["/".to_string()],
+        encodings: vec![],
+    }];
+    for asset_name in [
+        ONE_CHUNK_ASSET_NAME,
+        TWO_CHUNKS_ASSET_NAME,
+        SIX_CHUNKS_ASSET_NAME,
+        TEN_CHUNKS_ASSET_NAME,
+    ] {
+        asset_configs.push(AssetConfig::File {
+            path: asset_name.to_string(),
+            content_type: Some("application/octet-stream".to_string()),
             headers: get_asset_headers(vec![(
                 "cache-control".to_string(),
                 "public, no-cache, no-store".to_string(),
             )]),
             fallback_for: vec![],
             aliased_by: vec![],
             encodings: vec![],
-        },
-    ];
-
-    let assets = vec![
-        Asset::new("index.html", b"<html><body>Hello, world!</body></html>"),
-        Asset::new("asset_206", ASSET_206_BODY),
-    ];
+        });
+        assets.push(Asset::new(asset_name, long_asset_body(asset_name)));
+    }
 
     ASSET_ROUTER.with_borrow_mut(|asset_router| {
         if let Err(err) = asset_router.certify_assets(assets, asset_configs) {
@@ -76,51 +210,13 @@ fn certify_all_assets() {
 
 fn serve_asset(req: &HttpRequest) -> HttpResponse<'static> {
     ASSET_ROUTER.with_borrow(|asset_router| {
-        if let Ok((mut response, witness, expr_path)) = asset_router.serve_asset(req) {
-            add_certificate_header(&mut response, &witness, &expr_path);
-            // 'asset_206' is split into two chunks, to test "chunk-wise" serving of assets.
-            if req.url().contains("asset_206") {
-                const FIRST_CHUNK_LEN: usize = 42;
-                let mut builder = HttpResponse::builder()
-                    .with_status_code(206)
-                    .with_headers(
-                        response
-                            .headers()
-                            .iter()
-                            .filter(|(name, _)| name.to_ascii_lowercase() != "content-length")
-                            .map(|(name, value)| (name.into(), value.into()))
-                            .collect::<Vec<HeaderField>>(),
-                    )
-                    .with_upgrade(response.upgrade().unwrap_or(false));
-                let range_header = ("Range".to_string(), format!("bytes={}-", FIRST_CHUNK_LEN));
-                let range_header_lowercase =
-                    ("range".to_string(), format!("bytes={}-", FIRST_CHUNK_LEN));
-                let content_range = if req.headers().contains(&range_header)
-                    || req.headers().contains(&range_header_lowercase)
-                {
-                    builder = builder.with_body(ASSET_206_BODY[FIRST_CHUNK_LEN..].to_vec());
-                    format!(
-                        "bytes {}-{}/{}",
-                        FIRST_CHUNK_LEN,
-                        ASSET_206_BODY.len() - 1,
-                        ASSET_206_BODY.len()
-                    )
-                } else {
-                    builder = builder.with_body(ASSET_206_BODY[..FIRST_CHUNK_LEN].to_vec());
-                    format!("bytes 0-{}/{}", FIRST_CHUNK_LEN - 1, ASSET_206_BODY.len())
-                };
-                let mut response_206 = builder.build();
-                response_206.add_header((
-                    "Content-Length".to_string(),
-                    response_206.body().len().to_string(),
-                ));
-                response_206.add_header(("Content-Range".to_string(), content_range));
-                response_206
-            } else {
-                response
-            }
+        if let Ok(response) = asset_router.serve_asset(
+            &data_certificate().expect("No data certificate available"),
+            req,
+        ) {
+            response
         } else {
-            ic_cdk::trap("Failed to serve asset");
+            ic_cdk::trap(&format!("Failed to serve asset for request {:?}", req));
         }
     })
 }
@@ -140,31 +236,3 @@ fn get_asset_headers(additional_headers: Vec<HeaderField>) -> Vec<HeaderField> {
 
     headers
 }
-
-const IC_CERTIFICATE_HEADER: &str = "IC-Certificate";
-fn add_certificate_header(response: &mut HttpResponse, witness: &HashTree, expr_path: &[String]) {
-    let certified_data = data_certificate().expect("No data certificate available");
-    let witness = cbor_encode(witness);
-    let expr_path = cbor_encode(&expr_path);
-
-    response.add_header((
-        IC_CERTIFICATE_HEADER.to_string(),
-        format!(
-            "certificate=:{}:, tree=:{}:, expr_path=:{}:, version=2",
-            BASE64.encode(certified_data),
-            BASE64.encode(witness),
-            BASE64.encode(expr_path)
-        ),
-    ));
-}
-
-fn cbor_encode(value: &impl Serialize) -> Vec<u8> {
-    let mut serializer = serde_cbor::Serializer::new(Vec::new());
-    serializer
-        .self_describe()
-        .expect("Failed to self describe CBOR");
-    value
-        .serialize(&mut serializer)
-        .expect("Failed to serialize value");
-    serializer.into_inner()
-}

packages/ic-http-gateway/Cargo.toml

@@ -41,3 +41,6 @@ pocket-ic.workspace = true
 reqwest = "0.12.8"
 testcontainers = "0.23.1"
 tokio.workspace = true
+rand_chacha.workspace = true
+rstest.workspace = true
+sha2.workspace = true