From a0af510988723cd317f05ccf7ae79c04aa835885 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 12 Dec 2024 12:44:20 +0000 Subject: [PATCH 01/17] Initial /jwks endpoint --- app/src/service_endpoints.h | 69 +++++++++++++++++++++++++++++++++++-- pyscitt/pyscitt/client.py | 3 ++ test/requirements.txt | 2 ++ test/test_ccf.py | 28 +++++++++++++++ 4 files changed, 100 insertions(+), 2 deletions(-) diff --git a/app/src/service_endpoints.h b/app/src/service_endpoints.h index dd99c603..daa18d4e 100644 --- a/app/src/service_endpoints.h +++ b/app/src/service_endpoints.h @@ -49,9 +49,51 @@ namespace scitt } /** - * An indexing strategy collecting all past and present service certificates - * and makes them immediately available. + * An indexing strategy collecting service keys used to sign receipts. */ + class ServiceKeyIndexingStrategy + : public VisitEachEntryInValueTyped + { + public: + ServiceKeyIndexingStrategy() : + VisitEachEntryInValueTyped(ccf::Tables::SERVICE) + {} + + nlohmann::json get_jwks() const + { + std::lock_guard guard(lock); + + std::vector jwks; + for (const auto& service_certificate : service_certificates) + { + auto verifier = ccf::crypto::make_unique_verifier(service_certificate); + jwks.push_back(verifier->public_key_jwk()); + } + nlohmann::json jwks_json; + jwks_json["keys"] = jwks; + return jwks_json; + } + + protected: + void visit_entry( + const ccf::TxID& tx_id, const ccf::ServiceInfo& service_info) override + { + std::lock_guard guard(lock); + + // It is possible for multiple entries in the ServiceInfo table to contain + // the same certificate, eg. if the service status changes. Using an + // std::set removes duplicates. + service_certificates.insert(service_info.cert); + } + + private: + mutable std::mutex lock; + + std::set service_certificates; + }; /** + * An indexing strategy collecting all past and present service + * certificates and makes them immediately available. + */ class ServiceCertificateIndexingStrategy : public VisitEachEntryInValueTyped { @@ -160,6 +202,16 @@ namespace scitt return index->get_did_document(*cfg.service_identifier); } + + static nlohmann::json get_jwks( + const std::shared_ptr& index, + ccf::endpoints::EndpointContext& ctx, + nlohmann::json&& params) + { + // Like get_did_document(), this is not right when the indexer is not up + // to date, which needs fixing + return index->get_jwks(); + } } static void register_service_endpoints( @@ -172,9 +224,13 @@ namespace scitt auto service_certificate_index = std::make_shared(); + auto service_key_index = std::make_shared(); + context.get_indexing_strategies().install_strategy( service_certificate_index); + context.get_indexing_strategies().install_strategy(service_key_index); + registry .make_endpoint( "/parameters", @@ -237,5 +293,14 @@ namespace scitt endpoints::get_did_document, service_certificate_index, _1, _2)), {ccf::empty_auth_policy}) .install(); + + registry + .make_endpoint( + "/jwks", + HTTP_GET, + ccf::json_adapter( + std::bind(endpoints::get_jwks, service_key_index, _1, _2)), + {ccf::empty_auth_policy}) + .install(); } } diff --git a/pyscitt/pyscitt/client.py b/pyscitt/pyscitt/client.py index 49d59b2d..1178470c 100644 --- a/pyscitt/pyscitt/client.py +++ b/pyscitt/pyscitt/client.py @@ -502,6 +502,9 @@ def get_did_document(self, did: str) -> dict: # Note: This endpoint only returns data for did:web DIDs. return self.get(f"/did/{did}").json()["did_document"] + def get_jwks(self) -> dict: + return self.get(f"/jwks").json() + def submit_signed_statement( self, signed_statement: bytes, diff --git a/test/requirements.txt b/test/requirements.txt index ba2b45c9..3b3be1b5 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -6,3 +6,5 @@ loguru aiotools ccf==6.0.0-dev8 cryptography==44.* +jwcrypto==1.5.* +types-jwcrypto \ No newline at end of file diff --git a/test/test_ccf.py b/test/test_ccf.py index a9a33f81..d9e2e0ad 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -1,12 +1,40 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import json + import pytest +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from jwcrypto import jwk from pyscitt import crypto from pyscitt.client import Client from pyscitt.verify import verify_transparent_statement +def test_jwks(client: Client): + """ + Test that the JWKS endpoint returns the expected keys. + """ + + jwks = client.get_jwks() + assert len(jwks["keys"]) == 1 + + cert_pem = client.get("/node/network").json()["service_certificate"] + cert = x509.load_pem_x509_certificate(cert_pem.encode(), default_backend()) + + pkey_pem = cert.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + key = jwk.JWK.from_pem(pkey_pem) + jwk_json = key.export(private_key=False) + jwk_from_cert = json.loads(jwk_json) + del jwk_from_cert["kid"] + assert jwk_from_cert == jwks["keys"][0] + + @pytest.mark.parametrize( "params", [ From e7e797b089216b3c9983121aa090d1292a891761 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 12 Dec 2024 17:30:03 +0000 Subject: [PATCH 02/17] recovery --- test/test_ccf.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/test/test_ccf.py b/test/test_ccf.py index d9e2e0ad..38c88e90 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -13,6 +13,18 @@ from pyscitt.verify import verify_transparent_statement +def pem_cert_to_ccf_jwk(cert_pem: str) -> dict: + cert = x509.load_pem_x509_certificate(cert_pem.encode(), default_backend()) + pkey_pem = cert.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + key = jwk.JWK.from_pem(pkey_pem) + jwk_from_cert = json.loads(key.export(private_key=False)) + del jwk_from_cert["kid"] + return jwk_from_cert + + def test_jwks(client: Client): """ Test that the JWKS endpoint returns the expected keys. @@ -22,17 +34,8 @@ def test_jwks(client: Client): assert len(jwks["keys"]) == 1 cert_pem = client.get("/node/network").json()["service_certificate"] - cert = x509.load_pem_x509_certificate(cert_pem.encode(), default_backend()) - - pkey_pem = cert.public_key().public_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PublicFormat.SubjectPublicKeyInfo, - ) - key = jwk.JWK.from_pem(pkey_pem) - jwk_json = key.export(private_key=False) - jwk_from_cert = json.loads(jwk_json) - del jwk_from_cert["kid"] - assert jwk_from_cert == jwks["keys"][0] + svc_jwk = pem_cert_to_ccf_jwk(cert_pem) + assert svc_jwk == jwks["keys"][0] @pytest.mark.parametrize( @@ -71,14 +74,21 @@ def test_recovery(client, trusted_ca, restart_service): old_network = client.get("/node/network").json() assert old_network["recovery_count"] == 0 + old_jwk = pem_cert_to_ccf_jwk(old_network["service_certificate"]) restart_service() new_network = client.get("/node/network").json() assert new_network["recovery_count"] == 1 assert new_network["service_certificate"] != old_network["service_certificate"] + new_jwk = pem_cert_to_ccf_jwk(new_network["service_certificate"]) # Check that the service is still operating correctly client.register_signed_statement( crypto.sign_json_statement(identity, {"foo": "hello"}) ) + + jwks = client.get_jwks() + assert len(jwks["keys"]) == 2 + assert old_jwk in jwks["keys"] + assert new_jwk in jwks["keys"] From 3da5d2792c0ecbca21f75c82278dacb4f42f0582 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 12 Dec 2024 17:54:12 +0000 Subject: [PATCH 03/17] kids --- app/src/service_endpoints.h | 6 +++++- test/test_ccf.py | 18 +++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app/src/service_endpoints.h b/app/src/service_endpoints.h index daa18d4e..e5be7f32 100644 --- a/app/src/service_endpoints.h +++ b/app/src/service_endpoints.h @@ -67,7 +67,11 @@ namespace scitt for (const auto& service_certificate : service_certificates) { auto verifier = ccf::crypto::make_unique_verifier(service_certificate); - jwks.push_back(verifier->public_key_jwk()); + auto kid = + ccf::crypto::Sha256Hash(verifier->public_key_der()).hex_str(); + nlohmann::json json_jwk = verifier->public_key_jwk(); + json_jwk["kid"] = kid; + jwks.emplace_back(std::move(json_jwk)); } nlohmann::json jwks_json; jwks_json["keys"] = jwks; diff --git a/test/test_ccf.py b/test/test_ccf.py index 38c88e90..6b4cc281 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. import json +from hashlib import sha256 import pytest from cryptography import x509 @@ -21,7 +22,22 @@ def pem_cert_to_ccf_jwk(cert_pem: str) -> dict: ) key = jwk.JWK.from_pem(pkey_pem) jwk_from_cert = json.loads(key.export(private_key=False)) - del jwk_from_cert["kid"] + # jwcrypto sets the kid to the JWK thumbprint, which is not what CCF does + # because it would not work in CBOR/COSE contexts. Instead, CCF uses the + # SHA-256 hash of the DER-encoded public key, encoded as hex. + # The kid is to be used as an opaque handler by clients, but we want this + # test to be precise. + ccf_kid = ( + sha256( + cert.public_key().public_bytes( + serialization.Encoding.DER, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + ) + .digest() + .hex() + ) + jwk_from_cert["kid"] = ccf_kid return jwk_from_cert From 22f503f8c1f4ee1b7bb7f4cb37d059d80b0af226 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 13 Dec 2024 14:26:16 +0000 Subject: [PATCH 04/17] Config skeleton --- app/src/service_endpoints.h | 70 +++++++++++++++++++++++++++++++++++++ test/requirements.txt | 3 +- test/test_ccf.py | 31 ++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/app/src/service_endpoints.h b/app/src/service_endpoints.h index e5be7f32..431e60ae 100644 --- a/app/src/service_endpoints.h +++ b/app/src/service_endpoints.h @@ -11,6 +11,8 @@ #include #include #include +// TODO: needs to be made public in CCF +#include namespace scitt { @@ -216,6 +218,66 @@ namespace scitt // to date, which needs fixing return index->get_jwks(); } + + static void get_transparency_config( + ccf::endpoints::ReadOnlyEndpointContext& ctx) + { + // TODO: a COSESignaturesSubsystem so the endpoint can access the issuer + // cleanly + nlohmann::json config; + config["issuer"] = "TBD"; + + const auto accept = + ctx.rpc_ctx->get_request_header(ccf::http::headers::ACCEPT); + if (accept.has_value()) + { + const auto accept_options = ::http::parse_accept_header(accept.value()); + if (accept_options.empty()) + { + throw ccf::RpcException( + HTTP_STATUS_NOT_ACCEPTABLE, + ccf::errors::UnsupportedContentType, + fmt::format( + "No supported content type in accept header: {}\nOnly {} is " + "currently supported", + accept.value(), + ccf::http::headervalues::contenttype::JSON)); + } + + for (const auto& option : accept_options) + { + // return CBOR eagerly if it is compatible with Accept + if (option.matches(ccf::http::headervalues::contenttype::CBOR)) + { + ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); + ctx.rpc_ctx->set_response_header( + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::CBOR); + ctx.rpc_ctx->set_response_body(nlohmann::json::to_cbor(config)); + return; + } + + // JSON if compatible with Accept + if (option.matches(ccf::http::headervalues::contenttype::JSON)) + { + ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); + ctx.rpc_ctx->set_response_header( + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); + ctx.rpc_ctx->set_response_body(config.dump()); + return; + } + } + } + + // If not Accept, default to CBOR + ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); + ctx.rpc_ctx->set_response_header( + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::CBOR); + ctx.rpc_ctx->set_response_body(nlohmann::json::to_cbor(config)); + return; + } } static void register_service_endpoints( @@ -306,5 +368,13 @@ namespace scitt std::bind(endpoints::get_jwks, service_key_index, _1, _2)), {ccf::empty_auth_policy}) .install(); + + registry + .make_read_only_endpoint( + "/.well-known/transparency-configuration", + HTTP_GET, + endpoints::get_transparency_config, + {ccf::empty_auth_policy}) + .install(); } } diff --git a/test/requirements.txt b/test/requirements.txt index 3b3be1b5..72249140 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -7,4 +7,5 @@ aiotools ccf==6.0.0-dev8 cryptography==44.* jwcrypto==1.5.* -types-jwcrypto \ No newline at end of file +types-jwcrypto +cbor2 \ No newline at end of file diff --git a/test/test_ccf.py b/test/test_ccf.py index 6b4cc281..2da61568 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -3,6 +3,7 @@ import json from hashlib import sha256 +import cbor2 import pytest from cryptography import x509 from cryptography.hazmat.backends import default_backend @@ -108,3 +109,33 @@ def test_recovery(client, trusted_ca, restart_service): assert len(jwks["keys"]) == 2 assert old_jwk in jwks["keys"] assert new_jwk in jwks["keys"] + + +def test_transparency_configuration(client): + config = client.get( + "/.well-known/transparency-configuration", + headers={"Accept": "application/json"}, + ) + assert config.status_code == 200 + assert config.headers["Content-Type"] == "application/json" + assert config.json() == {"issuer": "TBD"} + + config = client.get( + "/.well-known/transparency-configuration", + headers={"Accept": "application/cbor"}, + ) + assert config.status_code == 200 + assert config.headers["Content-Type"] == "application/cbor" + assert cbor2.loads(config.content) == {"issuer": "TBD"} + + config = client.get( + "/.well-known/transparency-configuration", headers={"Accept": "*/*"} + ) + assert config.status_code == 200 + assert config.headers["Content-Type"] == "application/cbor" + assert cbor2.loads(config.content) == {"issuer": "TBD"} + + config = client.get("/.well-known/transparency-configuration") + assert config.status_code == 200 + assert config.headers["Content-Type"] == "application/cbor" + assert cbor2.loads(config.content) == {"issuer": "TBD"} From 12f33da70bef906c33e7fea4addd5e80fa67842b Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 17 Dec 2024 17:37:36 +0000 Subject: [PATCH 05/17] Add issuer to config --- .devcontainer/Dockerfile | 2 +- .github/workflows/build-test.yml | 4 +- .github/workflows/codeql.yml | 2 +- .github/workflows/long-test.yml | 2 +- .pipelines/pullrequest.yml | 2 +- DEVELOPMENT.md | 8 +- app/src/service_endpoints.h | 137 +++++++++++++++++-------------- build.sh | 2 +- docker/snp.Dockerfile | 2 +- docker/virtual.Dockerfile | 2 +- pyscitt/setup.py | 2 +- test/requirements.txt | 2 +- 12 files changed, 89 insertions(+), 78 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f55a2e41..983c0129 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1 +1 @@ -FROM ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev8 +FROM ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev10 diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 83b10a47..b113ef56 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -13,7 +13,7 @@ jobs: checks: name: Format and License Checks runs-on: ubuntu-20.04 - container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev8 + container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev10 steps: - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Checkout repository @@ -37,7 +37,7 @@ jobs: unit_tests_enabled: OFF runs-on: ${{ matrix.platform.nodes }} container: - image: ghcr.io/microsoft/ccf/app/dev/${{ matrix.platform.image }}:ccf-6.0.0-dev8 + image: ghcr.io/microsoft/ccf/app/dev/${{ matrix.platform.image }}:ccf-6.0.0-dev10 options: ${{ matrix.platform.options }} env: # Helps to distinguish between CI and local builds. diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a3fffb38..6f55e434 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -17,7 +17,7 @@ jobs: name: Analyze runs-on: ubuntu-latest - container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev8 + container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev10 permissions: actions: read diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 35563f27..08fefe2a 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -56,7 +56,7 @@ jobs: name: fuzz if: ${{ contains(github.event.pull_request.labels.*.name, 'run-fuzz-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} runs-on: ubuntu-20.04 - container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev8 + container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev10 env: PLATFORM: virtual steps: diff --git a/.pipelines/pullrequest.yml b/.pipelines/pullrequest.yml index 83d7dded..5c01a8a4 100644 --- a/.pipelines/pullrequest.yml +++ b/.pipelines/pullrequest.yml @@ -8,7 +8,7 @@ parameters: # parameters are shown up in ADO UI in a build queue time - name: CCF_VERSION displayName: Target CCF version to build for type: string - default: 6.0.0-dev8 + default: 6.0.0-dev10 variables: SCITT_CI: 1 # used in scitt builds and tests diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 118a76b3..9cb69a23 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -39,10 +39,10 @@ It is expected that you have Ubuntu 20.04. Follow the steps below to setup your 2. Install dependencies: ```sh - wget https://github.com/microsoft/CCF/archive/refs/tags/ccf-6.0.0-dev8.tar.gz - tar xvzf ccf-6.0.0-dev8.tar.gz - cd CCF-ccf-6.0.0-dev8/getting_started/setup_vm/ - ./run.sh app-dev.yml -e ccf_ver=6.0.0-dev8 -e platform= -e clang_version=15 + wget https://github.com/microsoft/CCF/archive/refs/tags/ccf-6.0.0-dev10.tar.gz + tar xvzf ccf-6.0.0-dev10.tar.gz + cd CCF-ccf-6.0.0-dev10/getting_started/setup_vm/ + ./run.sh app-dev.yml -e ccf_ver=6.0.0-dev10 -e platform= -e clang_version=15 ``` ## Compiling diff --git a/app/src/service_endpoints.h b/app/src/service_endpoints.h index 431e60ae..2da10328 100644 --- a/app/src/service_endpoints.h +++ b/app/src/service_endpoints.h @@ -7,12 +7,12 @@ #include "visit_each_entry_in_value.h" #include +#include #include #include +#include #include #include -// TODO: needs to be made public in CCF -#include namespace scitt { @@ -218,66 +218,6 @@ namespace scitt // to date, which needs fixing return index->get_jwks(); } - - static void get_transparency_config( - ccf::endpoints::ReadOnlyEndpointContext& ctx) - { - // TODO: a COSESignaturesSubsystem so the endpoint can access the issuer - // cleanly - nlohmann::json config; - config["issuer"] = "TBD"; - - const auto accept = - ctx.rpc_ctx->get_request_header(ccf::http::headers::ACCEPT); - if (accept.has_value()) - { - const auto accept_options = ::http::parse_accept_header(accept.value()); - if (accept_options.empty()) - { - throw ccf::RpcException( - HTTP_STATUS_NOT_ACCEPTABLE, - ccf::errors::UnsupportedContentType, - fmt::format( - "No supported content type in accept header: {}\nOnly {} is " - "currently supported", - accept.value(), - ccf::http::headervalues::contenttype::JSON)); - } - - for (const auto& option : accept_options) - { - // return CBOR eagerly if it is compatible with Accept - if (option.matches(ccf::http::headervalues::contenttype::CBOR)) - { - ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); - ctx.rpc_ctx->set_response_header( - ccf::http::headers::CONTENT_TYPE, - ccf::http::headervalues::contenttype::CBOR); - ctx.rpc_ctx->set_response_body(nlohmann::json::to_cbor(config)); - return; - } - - // JSON if compatible with Accept - if (option.matches(ccf::http::headervalues::contenttype::JSON)) - { - ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); - ctx.rpc_ctx->set_response_header( - ccf::http::headers::CONTENT_TYPE, - ccf::http::headervalues::contenttype::JSON); - ctx.rpc_ctx->set_response_body(config.dump()); - return; - } - } - } - - // If not Accept, default to CBOR - ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); - ctx.rpc_ctx->set_response_header( - ccf::http::headers::CONTENT_TYPE, - ccf::http::headervalues::contenttype::CBOR); - ctx.rpc_ctx->set_response_body(nlohmann::json::to_cbor(config)); - return; - } } static void register_service_endpoints( @@ -297,6 +237,77 @@ namespace scitt context.get_indexing_strategies().install_strategy(service_key_index); + auto get_transparency_config = + [&](ccf::endpoints::ReadOnlyEndpointContext& ctx) { + auto subsystem = + context.get_subsystem(); + if (!subsystem) + { + ctx.rpc_ctx->set_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + ccf::errors::InternalError, + "COSE signatures subsystem not available"); + return; + } + auto cfg = subsystem->get_cose_signatures_config(); + + nlohmann::json config; + config["issuer"] = cfg.issuer; + config["jwks_uri"] = fmt::format("https://{}/jwks", cfg.issuer); + + const auto accept = + ctx.rpc_ctx->get_request_header(ccf::http::headers::ACCEPT); + if (accept.has_value()) + { + const auto accept_options = + ccf::http::parse_accept_header(accept.value()); + if (accept_options.empty()) + { + throw ccf::RpcException( + HTTP_STATUS_NOT_ACCEPTABLE, + ccf::errors::UnsupportedContentType, + fmt::format( + "No supported content type in accept header: {}\nOnly {} is " + "currently supported", + accept.value(), + ccf::http::headervalues::contenttype::JSON)); + } + + for (const auto& option : accept_options) + { + // return CBOR eagerly if it is compatible with Accept + if (option.matches(ccf::http::headervalues::contenttype::CBOR)) + { + ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); + ctx.rpc_ctx->set_response_header( + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::CBOR); + ctx.rpc_ctx->set_response_body(nlohmann::json::to_cbor(config)); + return; + } + + // JSON if compatible with Accept + if (option.matches(ccf::http::headervalues::contenttype::JSON)) + { + ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); + ctx.rpc_ctx->set_response_header( + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::JSON); + ctx.rpc_ctx->set_response_body(config.dump()); + return; + } + } + } + + // If not Accept, default to CBOR + ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK); + ctx.rpc_ctx->set_response_header( + ccf::http::headers::CONTENT_TYPE, + ccf::http::headervalues::contenttype::CBOR); + ctx.rpc_ctx->set_response_body(nlohmann::json::to_cbor(config)); + return; + }; + registry .make_endpoint( "/parameters", @@ -373,7 +384,7 @@ namespace scitt .make_read_only_endpoint( "/.well-known/transparency-configuration", HTTP_GET, - endpoints::get_transparency_config, + get_transparency_config, {ccf::empty_auth_policy}) .install(); } diff --git a/build.sh b/build.sh index 2d9ec8e8..986efac9 100755 --- a/build.sh +++ b/build.sh @@ -19,7 +19,7 @@ if [ "$PLATFORM" != "virtual" ] && [ "$PLATFORM" != "snp" ]; then fi if [ "$BUILD_CCF_FROM_SOURCE" = "ON" ]; then - CCF_SOURCE_VERSION="6.0.0-dev8" + CCF_SOURCE_VERSION="6.0.0-dev10" echo "Cloning CCF sources" rm -rf ccf-source git clone --single-branch -b "ccf-${CCF_SOURCE_VERSION}" https://github.com/microsoft/CCF ccf-source diff --git a/docker/snp.Dockerfile b/docker/snp.Dockerfile index adf9b5a1..267df3b9 100644 --- a/docker/snp.Dockerfile +++ b/docker/snp.Dockerfile @@ -1,4 +1,4 @@ -ARG CCF_VERSION=6.0.0-dev8 +ARG CCF_VERSION=6.0.0-dev10 FROM ghcr.io/microsoft/ccf/app/dev/snp:ccf-${CCF_VERSION} as builder ARG CCF_VERSION ARG SCITT_VERSION_OVERRIDE diff --git a/docker/virtual.Dockerfile b/docker/virtual.Dockerfile index a7771ab8..a4505f8a 100644 --- a/docker/virtual.Dockerfile +++ b/docker/virtual.Dockerfile @@ -1,4 +1,4 @@ -ARG CCF_VERSION=6.0.0-dev8 +ARG CCF_VERSION=6.0.0-dev10 FROM ghcr.io/microsoft/ccf/app/dev/virtual:ccf-${CCF_VERSION} as builder ARG CCF_VERSION ARG SCITT_VERSION_OVERRIDE diff --git a/pyscitt/setup.py b/pyscitt/setup.py index 1f53836c..522faedd 100644 --- a/pyscitt/setup.py +++ b/pyscitt/setup.py @@ -25,7 +25,7 @@ }, python_requires=">=3.8", install_requires=[ - "ccf==6.0.0-dev8", + "ccf==6.0.0-dev10", "cryptography==44.*", # needs to match ccf "httpx", "cbor2==5.4.*", diff --git a/test/requirements.txt b/test/requirements.txt index 72249140..b69a84df 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -4,7 +4,7 @@ httpx pytest loguru aiotools -ccf==6.0.0-dev8 +ccf==6.0.0-dev10 cryptography==44.* jwcrypto==1.5.* types-jwcrypto From 52e3e1c762362e7c498968456a22320b24ff8c37 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Tue, 17 Dec 2024 18:28:21 +0000 Subject: [PATCH 06/17] test --- test/test_ccf.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_ccf.py b/test/test_ccf.py index 2da61568..50fec36e 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -112,13 +112,15 @@ def test_recovery(client, trusted_ca, restart_service): def test_transparency_configuration(client): + reference = {"issuer": "127.0.0.1:0", "jwks_uri": "https://127.0.0.1:0/jwks"} + config = client.get( "/.well-known/transparency-configuration", headers={"Accept": "application/json"}, ) assert config.status_code == 200 assert config.headers["Content-Type"] == "application/json" - assert config.json() == {"issuer": "TBD"} + assert config.json() == reference config = client.get( "/.well-known/transparency-configuration", @@ -126,16 +128,16 @@ def test_transparency_configuration(client): ) assert config.status_code == 200 assert config.headers["Content-Type"] == "application/cbor" - assert cbor2.loads(config.content) == {"issuer": "TBD"} + assert cbor2.loads(config.content) == reference config = client.get( "/.well-known/transparency-configuration", headers={"Accept": "*/*"} ) assert config.status_code == 200 assert config.headers["Content-Type"] == "application/cbor" - assert cbor2.loads(config.content) == {"issuer": "TBD"} + assert cbor2.loads(config.content) == reference config = client.get("/.well-known/transparency-configuration") assert config.status_code == 200 assert config.headers["Content-Type"] == "application/cbor" - assert cbor2.loads(config.content) == {"issuer": "TBD"} + assert cbor2.loads(config.content) == reference From 78e6a25f78d060fd74feb842b6be7edcb8100058 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 18 Dec 2024 11:04:53 +0000 Subject: [PATCH 07/17] fix port --- test/infra/cchost.py | 19 +++++++++++++++++++ test/test_ccf.py | 6 ++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/test/infra/cchost.py b/test/infra/cchost.py index e74f1da6..eedee6d0 100644 --- a/test/infra/cchost.py +++ b/test/infra/cchost.py @@ -5,8 +5,10 @@ import asyncio import json import os +import random import shutil import signal +import socket import ssl import subprocess from pathlib import Path @@ -27,6 +29,20 @@ CCHOST_PID_FILE_NAME = "cchost.pid" +def unused_tcp_port(host) -> int: + MAX_TRIES = 10 + for _ in range(MAX_TRIES): + port = random.randint(1024, 65535) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind((host, port)) + return port + except socket.error: + pass + else: + raise TimeoutError(f"Could not find unused port after {MAX_TRIES} tries") + + class CCHost(EventLoopThread): binary: str enclave_file: Path @@ -73,6 +89,9 @@ def __init__( self.binary = binary self.listen_rpc_port = rpc_port + # A predictable port is needed for the SCITT issuer name to be resolvable + if self.listen_rpc_port == 0: + self.listen_rpc_port = unused_tcp_port("localhost") self.listen_node_port = node_port self.platform = platform diff --git a/test/test_ccf.py b/test/test_ccf.py index 50fec36e..b6339ff2 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -111,8 +111,10 @@ def test_recovery(client, trusted_ca, restart_service): assert new_jwk in jwks["keys"] -def test_transparency_configuration(client): - reference = {"issuer": "127.0.0.1:0", "jwks_uri": "https://127.0.0.1:0/jwks"} +@pytest.mark.isolated_test +def test_transparency_configuration(client, cchost): + issuer = f"127.0.0.1:{cchost.rpc_port}" + reference = {"issuer": issuer, "jwks_uri": f"https://{issuer}/jwks"} config = client.get( "/.well-known/transparency-configuration", From 1eeda5cc654e47a93d7123cca1cd7fa9a2a99287 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 18 Dec 2024 11:12:53 +0000 Subject: [PATCH 08/17] Remove unused resolvers --- pyscitt/pyscitt/verify.py | 63 --------------------------------------- 1 file changed, 63 deletions(-) diff --git a/pyscitt/pyscitt/verify.py b/pyscitt/pyscitt/verify.py index f75c1593..6f267b21 100644 --- a/pyscitt/pyscitt/verify.py +++ b/pyscitt/pyscitt/verify.py @@ -175,66 +175,3 @@ def lookup(self, phdr) -> ServiceParameters: return self.services[service_id] else: raise ValueError(f"Unknown service identity {service_id!r}") - - -class DIDDocumentTrustStore(TrustStore): - """ - A trust store backed by a single DID-document. - - The trust store will use the KID found in the protected headers to select - the appropriate assertion method from the document. - """ - - document: dict - services: dict - - def __init__(self, document: dict): - self.document = document - - def lookup(self, phdr) -> ServiceParameters: - issuer = phdr.get(SCITTIssuer) - if issuer != self.document.get("id"): - raise ValueError( - f"Incorrect issuer {issuer!r}, expected {self.document['id']}" - ) - - if pycose.headers.KID in phdr: - kid = phdr[pycose.headers.KID].decode("ascii") - else: - kid = None - - method = did.find_assertion_method(self.document, kid) - jwk = method["publicKeyJwk"] - # TODO parse jwk without using x5c as well - if len(jwk.get("x5c", [])) < 1: - raise ValueError("Missing x5c parameter in service JWK") - certificate = base64.b64decode(jwk["x5c"][0]) - - return ServiceParameters("CCF", "ES256", certificate) - - -class DIDResolverTrustStore(TrustStore): - """ - A trust store which uses the issuer found in receipts to dynamically - resolve the service parameters. - - The trust store does not restrict which issuers are allowed, only that the - receipt signature matches the identifier. - """ - - services: dict - - def __init__(self, resolver: Optional[did.Resolver] = None): - if resolver is not None: - self.resolver = resolver - else: - self.resolver = did.Resolver() - - def lookup(self, phdr) -> ServiceParameters: - if SCITTIssuer not in phdr: - raise ValueError("Receipt does not have an issuer") - - issuer = phdr[SCITTIssuer] - document = self.resolver.resolve(issuer) - - return DIDDocumentTrustStore(document).lookup(phdr) From 122fea0c4ce4bc9c870565a1529e227c498679d6 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 18 Dec 2024 11:25:21 +0000 Subject: [PATCH 09/17] kid lookup --- pyscitt/pyscitt/verify.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pyscitt/pyscitt/verify.py b/pyscitt/pyscitt/verify.py index 6f267b21..a3f6aa4b 100644 --- a/pyscitt/pyscitt/verify.py +++ b/pyscitt/pyscitt/verify.py @@ -15,6 +15,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from cryptography.x509 import load_der_x509_certificate +from pycose.headers import KID from pycose.keys.cosekey import CoseKey from pycose.messages import Sign1Message @@ -99,6 +100,11 @@ def verify_receipt( decoded_receipt.verify(msg, service_params) +def get_kid(cose_sign1: bytes) -> bytes: + parsed_receipt = Sign1Message.decode(cose_sign1) + return parsed_receipt.phdr[KID] + + def verify_transparent_statement( transparent_statement: bytes, service_trust_store: TrustStore, @@ -108,15 +114,18 @@ def verify_transparent_statement( for _, service_params in service_trust_store.services.items(): cert = load_der_x509_certificate(service_params.certificate, default_backend()) key = cert.public_key() - kid = sha256( - key.public_bytes(Encoding.DER, PublicFormat.SubjectPublicKeyInfo) - ).digest() + kid = ( + sha256(key.public_bytes(Encoding.DER, PublicFormat.SubjectPublicKeyInfo)) + .digest() + .hex() + .encode() + ) trust_store_keys[kid] = key - # Assume a single service key - service_key = list(trust_store_keys.values())[0] st = Sign1Message.decode(transparent_statement) for receipt in st.uhdr[crypto.SCITTReceipts]: + kid = get_kid(receipt) + service_key = trust_store_keys[kid] ccf.cose.verify_receipt( receipt, service_key, sha256(input_signed_statement).digest() ) From be01a998b36dea75830897aff689649b0b4c266d Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 18 Dec 2024 11:39:06 +0000 Subject: [PATCH 10/17] jwks_uri -> jwksUri to align with typespec --- app/src/service_endpoints.h | 2 +- test/test_ccf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/service_endpoints.h b/app/src/service_endpoints.h index 2da10328..2893c993 100644 --- a/app/src/service_endpoints.h +++ b/app/src/service_endpoints.h @@ -253,7 +253,7 @@ namespace scitt nlohmann::json config; config["issuer"] = cfg.issuer; - config["jwks_uri"] = fmt::format("https://{}/jwks", cfg.issuer); + config["jwksUri"] = fmt::format("https://{}/jwks", cfg.issuer); const auto accept = ctx.rpc_ctx->get_request_header(ccf::http::headers::ACCEPT); diff --git a/test/test_ccf.py b/test/test_ccf.py index b6339ff2..0df3e63c 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -114,7 +114,7 @@ def test_recovery(client, trusted_ca, restart_service): @pytest.mark.isolated_test def test_transparency_configuration(client, cchost): issuer = f"127.0.0.1:{cchost.rpc_port}" - reference = {"issuer": issuer, "jwks_uri": f"https://{issuer}/jwks"} + reference = {"issuer": issuer, "jwksUri": f"https://{issuer}/jwks"} config = client.get( "/.well-known/transparency-configuration", From 6f3ede469d02be768a8820add78bce85e7e6536b Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 18 Dec 2024 13:07:57 +0000 Subject: [PATCH 11/17] update lookup in trust store --- pyscitt/pyscitt/cli/retrieve_signed_claims.py | 4 +- pyscitt/pyscitt/verify.py | 77 ++++++------------- 2 files changed, 25 insertions(+), 56 deletions(-) diff --git a/pyscitt/pyscitt/cli/retrieve_signed_claims.py b/pyscitt/pyscitt/cli/retrieve_signed_claims.py index fe73fffa..8b22ee03 100644 --- a/pyscitt/pyscitt/cli/retrieve_signed_claims.py +++ b/pyscitt/pyscitt/cli/retrieve_signed_claims.py @@ -6,7 +6,7 @@ from typing import Optional from ..client import Client -from ..verify import StaticTrustStore, verify_receipt +from ..verify import StaticTrustStore, verify_transparent_statement from .client_arguments import add_client_arguments, create_client @@ -29,7 +29,7 @@ def retrieve_signed_claimsets( path = base_path / f"{tx}.cose" if service_trust_store: - verify_receipt(claim, service_trust_store=service_trust_store) + verify_transparent_statement(claim, service_trust_store, claim) with open(path, "wb") as f: f.write(claim) diff --git a/pyscitt/pyscitt/verify.py b/pyscitt/pyscitt/verify.py index a3f6aa4b..02243651 100644 --- a/pyscitt/pyscitt/verify.py +++ b/pyscitt/pyscitt/verify.py @@ -13,6 +13,7 @@ import ccf.cose import pycose from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.types import CertificatePublicKeyTypes from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from cryptography.x509 import load_der_x509_certificate from pycose.headers import KID @@ -61,9 +62,9 @@ def services(self): pass @abstractmethod - def lookup(self, phdr) -> ServiceParameters: + def get_key(self, receipt: bytes) -> CertificatePublicKeyTypes: """ - Look up a service's parameters based on the protected headers from a + Look up a service's public key based on the protected headers from a receipt. Raises an exception if not matching service is found. """ raise NotImplementedError() @@ -76,56 +77,14 @@ def verify_cose_sign1(buf: bytes, cert_pem: str): raise ValueError("signature is invalid") -def verify_receipt( - buf: bytes, - service_trust_store: TrustStore, - receipt: Union[Receipt, bytes, None] = None, -) -> None: - msg = Sign1Message.decode(buf) - - if isinstance(receipt, Receipt): - decoded_receipt = receipt - else: - if receipt is None: - embedded_receipt = crypto.get_last_embedded_receipt_from_cose(buf) - if not embedded_receipt: - raise ValueError("No embedded receipt found in COSE message") - parsed_receipt = cbor2.loads(embedded_receipt) - else: - parsed_receipt = cbor2.loads(receipt) - - decoded_receipt = Receipt.from_cose_obj(parsed_receipt) - - service_params = service_trust_store.lookup(decoded_receipt.phdr) - decoded_receipt.verify(msg, service_params) - - -def get_kid(cose_sign1: bytes) -> bytes: - parsed_receipt = Sign1Message.decode(cose_sign1) - return parsed_receipt.phdr[KID] - - def verify_transparent_statement( transparent_statement: bytes, service_trust_store: TrustStore, input_signed_statement: bytes, ): - trust_store_keys = {} - for _, service_params in service_trust_store.services.items(): - cert = load_der_x509_certificate(service_params.certificate, default_backend()) - key = cert.public_key() - kid = ( - sha256(key.public_bytes(Encoding.DER, PublicFormat.SubjectPublicKeyInfo)) - .digest() - .hex() - .encode() - ) - trust_store_keys[kid] = key - st = Sign1Message.decode(transparent_statement) for receipt in st.uhdr[crypto.SCITTReceipts]: - kid = get_kid(receipt) - service_key = trust_store_keys[kid] + service_key = service_trust_store.get_key(receipt) ccf.cose.verify_receipt( receipt, service_key, sha256(input_signed_statement).digest() ) @@ -137,9 +96,24 @@ class StaticTrustStore(TrustStore): """ services: Dict[str, ServiceParameters] = {} + trust_store_keys: dict = {} def __init__(self, services: Dict[str, ServiceParameters]): self.services = services + for _, service_params in self.services.items(): + cert = load_der_x509_certificate( + service_params.certificate, default_backend() + ) + key = cert.public_key() + kid = ( + sha256( + key.public_bytes(Encoding.DER, PublicFormat.SubjectPublicKeyInfo) + ) + .digest() + .hex() + .encode() + ) + self.trust_store_keys[kid] = key @staticmethod def load(path: Path) -> "StaticTrustStore": @@ -175,12 +149,7 @@ def _parse_service_param(param: dict): return StaticTrustStore(store) - def lookup(self, phdr) -> ServiceParameters: - if "service_id" not in phdr: - raise ValueError("Receipt does not have a service identity.") - - service_id = phdr["service_id"] - if service_id in self.services: - return self.services[service_id] - else: - raise ValueError(f"Unknown service identity {service_id!r}") + def get_key(self, receipt: bytes) -> CertificatePublicKeyTypes: + parsed = Sign1Message.decode(receipt) + kid = parsed.phdr[KID] + return self.trust_store_keys[kid] From 62581f6d5c6a39b403d97ae17fb2341c3300ba64 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Wed, 18 Dec 2024 14:38:03 +0000 Subject: [PATCH 12/17] DynamicTrustStore --- pyscitt/pyscitt/verify.py | 43 +++++++++++++++++++++++++++++++++++++-- test/infra/cchost.py | 4 ++++ test/test_ccf.py | 25 ++++++++++++++++------- 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/pyscitt/pyscitt/verify.py b/pyscitt/pyscitt/verify.py index 02243651..87d63df6 100644 --- a/pyscitt/pyscitt/verify.py +++ b/pyscitt/pyscitt/verify.py @@ -11,17 +11,23 @@ import cbor2 import ccf.cose +import httpx import pycose from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric.types import CertificatePublicKeyTypes -from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat +from cryptography.hazmat.primitives.serialization import ( + Encoding, + PublicFormat, + load_pem_public_key, +) from cryptography.x509 import load_der_x509_certificate +from jwcrypto import jwk from pycose.headers import KID from pycose.keys.cosekey import CoseKey from pycose.messages import Sign1Message from . import crypto, did -from .crypto import SCITTIssuer +from .crypto import CWT_ISS, CWTClaims from .receipt import Receipt @@ -153,3 +159,36 @@ def get_key(self, receipt: bytes) -> CertificatePublicKeyTypes: parsed = Sign1Message.decode(receipt) kid = parsed.phdr[KID] return self.trust_store_keys[kid] + + +class DynamicTrustStore(TrustStore): + """ + A dynamic trust store, based on a single service identity used to retrieve + all keys from the service's transparency configuration endpoint. + """ + + services: Dict[str, ServiceParameters] = {} + + def __init__(self): + self.services = {} + + def get_key(self, receipt: bytes) -> CertificatePublicKeyTypes: + parsed = Sign1Message.decode(receipt) + cwt = parsed.phdr[CWTClaims] + issuer = cwt[CWT_ISS] + + session = httpx.Client(verify=False) + tc = session.get( + f"https://{issuer}/.well-known/transparency-configuration", + headers={"Accept": "application/json"}, + ) + tc.raise_for_status() + jwks_uri = tc.json()["jwksUri"] + ju = session.get(jwks_uri) + ju.raise_for_status() + jwks = ju.json()["keys"] + keys = {key["kid"].encode(): key for key in jwks} + key = keys[parsed.phdr[KID]] + pem_key = jwk.JWK.from_json(json.dumps(key)).export_to_pem() + key = load_pem_public_key(pem_key, default_backend()) + return key diff --git a/test/infra/cchost.py b/test/infra/cchost.py index eedee6d0..38af5b16 100644 --- a/test/infra/cchost.py +++ b/test/infra/cchost.py @@ -395,6 +395,10 @@ def _populate_workspace(self, start=True): "recover": { "initial_service_certificate_validity_days": 1, "previous_service_identity_file": str(previous_service_cert), + # "cose_signatures": { + # "issuer": f"127.0.0.1:{self.listen_rpc_port + 1}", + # "subject": "scitt.ccf.signature.v1", + # }, }, } diff --git a/test/test_ccf.py b/test/test_ccf.py index 0df3e63c..e1c965e4 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -12,7 +12,7 @@ from pyscitt import crypto from pyscitt.client import Client -from pyscitt.verify import verify_transparent_statement +from pyscitt.verify import DynamicTrustStore, verify_transparent_statement def pem_cert_to_ccf_jwk(cert_pem: str) -> dict: @@ -80,14 +80,20 @@ def test_make_signed_statement_transparent( ).response_bytes verify_transparent_statement(transparent_statement, trust_store, signed_statement) + dynamic_trust_store = DynamicTrustStore() + verify_transparent_statement( + transparent_statement, dynamic_trust_store, signed_statement + ) + @pytest.mark.isolated_test def test_recovery(client, trusted_ca, restart_service): identity = trusted_ca.create_identity(alg="PS384", kty="rsa") - client.register_signed_statement( - crypto.sign_json_statement(identity, {"foo": "bar"}) - ) + first_signed_statement = crypto.sign_json_statement(identity, {"foo": "bar"}) + first_transparent_statement = client.register_signed_statement( + first_signed_statement + ).response_bytes old_network = client.get("/node/network").json() assert old_network["recovery_count"] == 0 @@ -101,15 +107,20 @@ def test_recovery(client, trusted_ca, restart_service): new_jwk = pem_cert_to_ccf_jwk(new_network["service_certificate"]) # Check that the service is still operating correctly - client.register_signed_statement( - crypto.sign_json_statement(identity, {"foo": "hello"}) - ) + second_signed_statement = crypto.sign_json_statement(identity, {"foo": "hello"}) + second_signed_statement = client.register_signed_statement( + second_signed_statement + ).response_bytes jwks = client.get_jwks() assert len(jwks["keys"]) == 2 assert old_jwk in jwks["keys"] assert new_jwk in jwks["keys"] + dynamic_trust_store = DynamicTrustStore() + # verify_transparent_statement(first_transparent_statement, dynamic_trust_store, first_signed_statement) + # verify_transparent_statement(second_signed_statement, dynamic_trust_store, second_signed_statement) + @pytest.mark.isolated_test def test_transparency_configuration(client, cchost): From 689b0773b5a7b3d51a8cd6d1a512d2cc3c2338d6 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 19 Dec 2024 10:38:53 +0000 Subject: [PATCH 13/17] external get --- pyscitt/pyscitt/verify.py | 16 ++++++++-------- test/test_ccf.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pyscitt/pyscitt/verify.py b/pyscitt/pyscitt/verify.py index 87d63df6..5c2300dd 100644 --- a/pyscitt/pyscitt/verify.py +++ b/pyscitt/pyscitt/verify.py @@ -169,24 +169,24 @@ class DynamicTrustStore(TrustStore): services: Dict[str, ServiceParameters] = {} - def __init__(self): + def __init__(self, getter): self.services = {} + self.getter = getter def get_key(self, receipt: bytes) -> CertificatePublicKeyTypes: parsed = Sign1Message.decode(receipt) cwt = parsed.phdr[CWTClaims] issuer = cwt[CWT_ISS] - session = httpx.Client(verify=False) - tc = session.get( + transparency_configuration = self.getter( f"https://{issuer}/.well-known/transparency-configuration", headers={"Accept": "application/json"}, ) - tc.raise_for_status() - jwks_uri = tc.json()["jwksUri"] - ju = session.get(jwks_uri) - ju.raise_for_status() - jwks = ju.json()["keys"] + transparency_configuration.raise_for_status() + jwks_uri = transparency_configuration.json()["jwksUri"] + jwk_set = self.getter(jwks_uri) + jwk_set.raise_for_status() + jwks = jwk_set.json()["keys"] keys = {key["kid"].encode(): key for key in jwks} key = keys[parsed.phdr[KID]] pem_key = jwk.JWK.from_json(json.dumps(key)).export_to_pem() diff --git a/test/test_ccf.py b/test/test_ccf.py index e1c965e4..cdd9b05e 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -80,7 +80,7 @@ def test_make_signed_statement_transparent( ).response_bytes verify_transparent_statement(transparent_statement, trust_store, signed_statement) - dynamic_trust_store = DynamicTrustStore() + dynamic_trust_store = DynamicTrustStore(client.get) verify_transparent_statement( transparent_statement, dynamic_trust_store, signed_statement ) @@ -117,7 +117,7 @@ def test_recovery(client, trusted_ca, restart_service): assert old_jwk in jwks["keys"] assert new_jwk in jwks["keys"] - dynamic_trust_store = DynamicTrustStore() + dynamic_trust_store = DynamicTrustStore(client.get) # verify_transparent_statement(first_transparent_statement, dynamic_trust_store, first_signed_statement) # verify_transparent_statement(second_signed_statement, dynamic_trust_store, second_signed_statement) From abd6ae3096f8eaa071a18e156bc29379505eebe7 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 9 Jan 2025 13:50:32 +0000 Subject: [PATCH 14/17] test receipts across recoveries --- test/test_ccf.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/test_ccf.py b/test/test_ccf.py index cdd9b05e..29c0bca9 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -108,7 +108,7 @@ def test_recovery(client, trusted_ca, restart_service): # Check that the service is still operating correctly second_signed_statement = crypto.sign_json_statement(identity, {"foo": "hello"}) - second_signed_statement = client.register_signed_statement( + second_transparent_statement = client.register_signed_statement( second_signed_statement ).response_bytes @@ -118,8 +118,12 @@ def test_recovery(client, trusted_ca, restart_service): assert new_jwk in jwks["keys"] dynamic_trust_store = DynamicTrustStore(client.get) - # verify_transparent_statement(first_transparent_statement, dynamic_trust_store, first_signed_statement) - # verify_transparent_statement(second_signed_statement, dynamic_trust_store, second_signed_statement) + verify_transparent_statement( + first_transparent_statement, dynamic_trust_store, first_signed_statement + ) + verify_transparent_statement( + second_transparent_statement, dynamic_trust_store, second_signed_statement + ) @pytest.mark.isolated_test From 92078562a713744a0eef8e3213dd2be4390e99c5 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 9 Jan 2025 14:03:56 +0000 Subject: [PATCH 15/17] 406 on unsupported content types --- app/src/service_endpoints.h | 23 +++++++++++------------ test/test_ccf.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/app/src/service_endpoints.h b/app/src/service_endpoints.h index dbad8e76..a09b78de 100644 --- a/app/src/service_endpoints.h +++ b/app/src/service_endpoints.h @@ -261,18 +261,6 @@ namespace scitt { const auto accept_options = ccf::http::parse_accept_header(accept.value()); - if (accept_options.empty()) - { - throw ccf::RpcException( - HTTP_STATUS_NOT_ACCEPTABLE, - ccf::errors::UnsupportedContentType, - fmt::format( - "No supported content type in accept header: {}\nOnly {} is " - "currently supported", - accept.value(), - ccf::http::headervalues::contenttype::JSON)); - } - for (const auto& option : accept_options) { // return CBOR eagerly if it is compatible with Accept @@ -297,6 +285,17 @@ namespace scitt return; } } + + // If no compatible content type, return 406 + throw ccf::RpcException( + HTTP_STATUS_NOT_ACCEPTABLE, + ccf::errors::UnsupportedContentType, + fmt::format( + "No supported content type in accept header: {}\nOnly {} and {} " + "are currently supported", + accept.value(), + ccf::http::headervalues::contenttype::JSON, + ccf::http::headervalues::contenttype::CBOR)); } // If not Accept, default to CBOR diff --git a/test/test_ccf.py b/test/test_ccf.py index 29c0bca9..2ccc3d38 100644 --- a/test/test_ccf.py +++ b/test/test_ccf.py @@ -14,6 +14,8 @@ from pyscitt.client import Client from pyscitt.verify import DynamicTrustStore, verify_transparent_statement +from .infra.assertions import service_error + def pem_cert_to_ccf_jwk(cert_pem: str) -> dict: cert = x509.load_pem_x509_certificate(cert_pem.encode(), default_backend()) @@ -131,6 +133,20 @@ def test_transparency_configuration(client, cchost): issuer = f"127.0.0.1:{cchost.rpc_port}" reference = {"issuer": issuer, "jwksUri": f"https://{issuer}/jwks"} + # Unsupported Accept header + with service_error("UnsupportedContentType"): + client.get( + "/.well-known/transparency-configuration", + headers={"Accept": "application/text"}, + ) + + # Empty Accept header + with service_error("UnsupportedContentType"): + client.get( + "/.well-known/transparency-configuration", + headers={"Accept": ""}, + ) + config = client.get( "/.well-known/transparency-configuration", headers={"Accept": "application/json"}, From dc70917e6930744d815e609dc52ac812dd83d85a Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 9 Jan 2025 15:38:32 +0000 Subject: [PATCH 16/17] dev++ --- .devcontainer/Dockerfile | 2 +- .github/workflows/build-test.yml | 4 ++-- .github/workflows/codeql.yml | 2 +- .github/workflows/long-test.yml | 2 +- .pipelines/pullrequest.yml | 2 +- DEVELOPMENT.md | 8 ++++---- build.sh | 2 +- docker/snp.Dockerfile | 2 +- docker/virtual.Dockerfile | 2 +- pyscitt/setup.py | 2 +- test/requirements.txt | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 983c0129..116c645e 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1 +1 @@ -FROM ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev10 +FROM ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev11 diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b113ef56..da053ac4 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -13,7 +13,7 @@ jobs: checks: name: Format and License Checks runs-on: ubuntu-20.04 - container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev10 + container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev11 steps: - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Checkout repository @@ -37,7 +37,7 @@ jobs: unit_tests_enabled: OFF runs-on: ${{ matrix.platform.nodes }} container: - image: ghcr.io/microsoft/ccf/app/dev/${{ matrix.platform.image }}:ccf-6.0.0-dev10 + image: ghcr.io/microsoft/ccf/app/dev/${{ matrix.platform.image }}:ccf-6.0.0-dev11 options: ${{ matrix.platform.options }} env: # Helps to distinguish between CI and local builds. diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6f55e434..db470fc9 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -17,7 +17,7 @@ jobs: name: Analyze runs-on: ubuntu-latest - container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev10 + container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev11 permissions: actions: read diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 08fefe2a..b0a1fbbe 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -56,7 +56,7 @@ jobs: name: fuzz if: ${{ contains(github.event.pull_request.labels.*.name, 'run-fuzz-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} runs-on: ubuntu-20.04 - container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev10 + container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev11 env: PLATFORM: virtual steps: diff --git a/.pipelines/pullrequest.yml b/.pipelines/pullrequest.yml index 5c01a8a4..09f432d8 100644 --- a/.pipelines/pullrequest.yml +++ b/.pipelines/pullrequest.yml @@ -8,7 +8,7 @@ parameters: # parameters are shown up in ADO UI in a build queue time - name: CCF_VERSION displayName: Target CCF version to build for type: string - default: 6.0.0-dev10 + default: 6.0.0-dev11 variables: SCITT_CI: 1 # used in scitt builds and tests diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 9cb69a23..6300490a 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -39,10 +39,10 @@ It is expected that you have Ubuntu 20.04. Follow the steps below to setup your 2. Install dependencies: ```sh - wget https://github.com/microsoft/CCF/archive/refs/tags/ccf-6.0.0-dev10.tar.gz - tar xvzf ccf-6.0.0-dev10.tar.gz - cd CCF-ccf-6.0.0-dev10/getting_started/setup_vm/ - ./run.sh app-dev.yml -e ccf_ver=6.0.0-dev10 -e platform= -e clang_version=15 + wget https://github.com/microsoft/CCF/archive/refs/tags/ccf-6.0.0-dev11.tar.gz + tar xvzf ccf-6.0.0-dev11.tar.gz + cd CCF-ccf-6.0.0-dev11/getting_started/setup_vm/ + ./run.sh app-dev.yml -e ccf_ver=6.0.0-dev11 -e platform= -e clang_version=15 ``` ## Compiling diff --git a/build.sh b/build.sh index 986efac9..e5dc11a8 100755 --- a/build.sh +++ b/build.sh @@ -19,7 +19,7 @@ if [ "$PLATFORM" != "virtual" ] && [ "$PLATFORM" != "snp" ]; then fi if [ "$BUILD_CCF_FROM_SOURCE" = "ON" ]; then - CCF_SOURCE_VERSION="6.0.0-dev10" + CCF_SOURCE_VERSION="6.0.0-dev11" echo "Cloning CCF sources" rm -rf ccf-source git clone --single-branch -b "ccf-${CCF_SOURCE_VERSION}" https://github.com/microsoft/CCF ccf-source diff --git a/docker/snp.Dockerfile b/docker/snp.Dockerfile index 267df3b9..70a908f1 100644 --- a/docker/snp.Dockerfile +++ b/docker/snp.Dockerfile @@ -1,4 +1,4 @@ -ARG CCF_VERSION=6.0.0-dev10 +ARG CCF_VERSION=6.0.0-dev11 FROM ghcr.io/microsoft/ccf/app/dev/snp:ccf-${CCF_VERSION} as builder ARG CCF_VERSION ARG SCITT_VERSION_OVERRIDE diff --git a/docker/virtual.Dockerfile b/docker/virtual.Dockerfile index a4505f8a..8a56b726 100644 --- a/docker/virtual.Dockerfile +++ b/docker/virtual.Dockerfile @@ -1,4 +1,4 @@ -ARG CCF_VERSION=6.0.0-dev10 +ARG CCF_VERSION=6.0.0-dev11 FROM ghcr.io/microsoft/ccf/app/dev/virtual:ccf-${CCF_VERSION} as builder ARG CCF_VERSION ARG SCITT_VERSION_OVERRIDE diff --git a/pyscitt/setup.py b/pyscitt/setup.py index 522faedd..e2bcbef0 100644 --- a/pyscitt/setup.py +++ b/pyscitt/setup.py @@ -25,7 +25,7 @@ }, python_requires=">=3.8", install_requires=[ - "ccf==6.0.0-dev10", + "ccf==6.0.0-dev11", "cryptography==44.*", # needs to match ccf "httpx", "cbor2==5.4.*", diff --git a/test/requirements.txt b/test/requirements.txt index b69a84df..d367158d 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -4,7 +4,7 @@ httpx pytest loguru aiotools -ccf==6.0.0-dev10 +ccf==6.0.0-dev11 cryptography==44.* jwcrypto==1.5.* types-jwcrypto From 472dd89c5dc427a52168e770046c9aed2dae0145 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Thu, 9 Jan 2025 15:41:05 +0000 Subject: [PATCH 17/17] did --- pyscitt/pyscitt/verify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyscitt/pyscitt/verify.py b/pyscitt/pyscitt/verify.py index 5c2300dd..182c960f 100644 --- a/pyscitt/pyscitt/verify.py +++ b/pyscitt/pyscitt/verify.py @@ -26,7 +26,7 @@ from pycose.keys.cosekey import CoseKey from pycose.messages import Sign1Message -from . import crypto, did +from . import crypto from .crypto import CWT_ISS, CWTClaims from .receipt import Receipt