From 1310a3cec1c9f5403252e376f0826851d1d8afb4 Mon Sep 17 00:00:00 2001 From: Aaron Todd Date: Thu, 6 Jun 2024 08:55:33 -0400 Subject: [PATCH] additional service specific protocol tests (#3682) ## Motivation and Context There are several service specific http request/response tests from Smithy. These are similar to the protocol tests but apply only to a specific service. We were tasked with ensuring some of the S3 URI related tests were captured. ## Description We currently were running the API gateway related tests as part of the `aws/sdk-adhoc-test` package. There are also tests for [machinelearning](https://github.com/smithy-lang/smithy/blob/main/smithy-aws-protocol-tests/model/awsJson1_1/services/machinelearning.smithy), [glacier](https://github.com/smithy-lang/smithy/blob/main/smithy-aws-protocol-tests/model/restJson1/services/glacier.smithy), and [s3](https://github.com/smithy-lang/smithy/blob/main/smithy-aws-protocol-tests/model/restXml/services/s3.smithy). I attempted to add all of these but hit issues. 1. `machinelearning` can be enabled but we have yet to actual implement the [customization](https://github.com/smithy-lang/smithy-rs/issues/139) that would allow this test to pass. 2. `s3` pulls in `aws-config` as a dev dependency for _reasons_ and this causes conflicts between relocated and non-relocated runtime crates. The `sdk-adhoc-test` package does not implement any of the runtime relocation and Cargo.toml processing that the `sdk` build does (nor do I want to pursue that). For now I've enabled the `glacier` tests and copied the S3 specific tests into `s3-tests.smithy` so that they are built and tested with the actual S3 model. In the future we can either (1) fix the build issues allowing us to remove these tests in favor of the ones defined in Smithy or (2) ideally these tests get implemented in the real upstream service models and we don't have to do anything special to get them. ## Testing Tested locally and CI ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- aws/sdk-adhoc-test/build.gradle.kts | 21 ++- .../rustsdk/IntegrationTestDependencies.kt | 2 +- aws/sdk/aws-models/s3-tests.smithy | 122 ++++++++++++++++++ 3 files changed, 139 insertions(+), 6 deletions(-) diff --git a/aws/sdk-adhoc-test/build.gradle.kts b/aws/sdk-adhoc-test/build.gradle.kts index 1f2b7fde5a..2fbfe88ec8 100644 --- a/aws/sdk-adhoc-test/build.gradle.kts +++ b/aws/sdk-adhoc-test/build.gradle.kts @@ -53,11 +53,6 @@ fun baseTest(service: String, module: String, imports: List = listOf()): } val allCodegenTests = listOf( - baseTest( - "com.amazonaws.apigateway#BackplaneControlService", - "apigateway", - imports = listOf("models/apigateway-rules.smithy"), - ), baseTest( "com.amazonaws.testservice#TestService", "endpoint-test-service", @@ -68,6 +63,22 @@ val allCodegenTests = listOf( "required-values", imports = listOf("models/required-value-test.smithy"), ), + // service specific protocol tests + baseTest( + "com.amazonaws.apigateway#BackplaneControlService", + "apigateway", + imports = listOf("models/apigateway-rules.smithy"), + ), + baseTest( + "com.amazonaws.glacier#Glacier", + "glacier", + ), + // TODO(https://github.com/smithy-lang/smithy-rs/issues/139) - we assume this will be handled by EP2.0 rules but + // the machinelearning service model has yet to be updated to include rules that handle the expected customization + // baseTest( + // "com.amazonaws.machinelearning#AmazonML_20141212", + // "machinelearning", + // ), ) project.registerGenerateSmithyBuildTask(rootProject, pluginName, allCodegenTests) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/IntegrationTestDependencies.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/IntegrationTestDependencies.kt index c333d4d532..4516a0ac28 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/IntegrationTestDependencies.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/IntegrationTestDependencies.kt @@ -57,7 +57,7 @@ class IntegrationTestDecorator : ClientCodegenDecorator { "correctly in the smithy-build.json." } - val moduleName = codegenContext.moduleName.substring("aws-sdk-".length) + val moduleName = codegenContext.moduleName.removePrefix("aws-sdk-") val testPackagePath = integrationTestPath.resolve(moduleName) return if (Files.exists(testPackagePath) && Files.exists(testPackagePath.resolve("Cargo.toml"))) { val hasTests = Files.exists(testPackagePath.resolve("tests")) diff --git a/aws/sdk/aws-models/s3-tests.smithy b/aws/sdk/aws-models/s3-tests.smithy index f4fa10dccd..59ced8bc10 100644 --- a/aws/sdk/aws-models/s3-tests.smithy +++ b/aws/sdk/aws-models/s3-tests.smithy @@ -383,3 +383,125 @@ apply ListObjectVersions @httpResponseTests([ } }] ) + + +// TODO(https://github.com/smithy-lang/smithy-rs/issues/157) - Remove duplicated tests if these make it into the actual model or otherwise become easier +// to integrate. +// Protocol tests below are duplicated from +// https://github.com/smithy-lang/smithy/blob/main/smithy-aws-protocol-tests/model/restXml/services/s3.smithy +// NOTE: These are duplicated because of currently difficult to replicate structural differences in the build. +// S3 pulls in `aws-config` which requires all runtime crates to point to `build` dir. This makes adding the protocol tests +// to `sdk-adhoc-test` difficult as it does not replicate relocating runtimes and re-processing Cargo.toml files. + +apply DeleteObjectTagging @httpRequestTests([ + { + id: "S3EscapeObjectKeyInUriLabel", + documentation: """ + S3 clients should escape special characters in Object Keys + when the Object Key is used as a URI label binding. + """, + protocol: "aws.protocols#restXml", + method: "DELETE", + uri: "/my%20key.txt", + host: "s3.us-west-2.amazonaws.com", + resolvedHost: "mybucket.s3.us-west-2.amazonaws.com", + body: "", + queryParams: [ + "tagging" + ], + params: { + Bucket: "mybucket", + Key: "my key.txt" + }, + vendorParams: { + scopedConfig: { + client: { + region: "us-west-2", + }, + }, + }, + }, + { + id: "S3EscapePathObjectKeyInUriLabel", + documentation: """ + S3 clients should preserve an Object Key representing a path + when the Object Key is used as a URI label binding, but still + escape special characters. + """, + protocol: "aws.protocols#restXml", + method: "DELETE", + uri: "/foo/bar/my%20key.txt", + host: "s3.us-west-2.amazonaws.com", + resolvedHost: "mybucket.s3.us-west-2.amazonaws.com", + body: "", + queryParams: [ + "tagging" + ], + params: { + Bucket: "mybucket", + Key: "foo/bar/my key.txt" + }, + vendorParams: { + scopedConfig: { + client: { + region: "us-west-2", + }, + }, + }, + } +]) + +apply GetObject @httpRequestTests([ + { + id: "S3PreservesLeadingDotSegmentInUriLabel", + documentation: """ + S3 clients should not remove dot segments from request paths. + """, + protocol: "aws.protocols#restXml", + method: "GET", + uri: "/../key.txt", + host: "s3.us-west-2.amazonaws.com", + resolvedHost: "mybucket.s3.us-west-2.amazonaws.com", + body: "", + params: { + Bucket: "mybucket", + Key: "../key.txt" + }, + vendorParams: { + scopedConfig: { + client: { + region: "us-west-2", + s3: { + addressing_style: "virtual", + }, + }, + }, + }, + }, + { + id: "S3PreservesEmbeddedDotSegmentInUriLabel", + documentation: """ + S3 clients should not remove dot segments from request paths. + """, + protocol: "aws.protocols#restXml", + method: "GET", + uri: "/foo/../key.txt", + host: "s3.us-west-2.amazonaws.com", + resolvedHost: "mybucket.s3.us-west-2.amazonaws.com", + body: "", + params: { + Bucket: "mybucket", + Key: "foo/../key.txt" + }, + vendorParams: { + scopedConfig: { + client: { + region: "us-west-2", + s3: { + addressing_style: "virtual", + }, + }, + }, + }, + } +])