From 472958442b2f1bcbe498bcf2101f1fa48d459c47 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 02:22:15 -0500 Subject: [PATCH 01/72] misc: add rules engine codegen tests --- settings.gradle.kts | 1 + tests/codegen/rules-engine/build.gradle.kts | 158 ++++++++++++++++++ .../operation-context-params.smithy | 58 +++++++ 3 files changed, 217 insertions(+) create mode 100644 tests/codegen/rules-engine/build.gradle.kts create mode 100644 tests/codegen/rules-engine/operation-context-params.smithy diff --git a/settings.gradle.kts b/settings.gradle.kts index 203fcef7b8d..6f53bf06951 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -53,6 +53,7 @@ include(":hll:hll-mapping-core") include(":services") include(":tests") include(":tests:codegen:event-stream") +include("tests:codegen:rules-engine") include(":tests:e2e-test-util") // generated services diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts new file mode 100644 index 00000000000..ef35526b380 --- /dev/null +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -0,0 +1,158 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir + +plugins { + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) +} + +description = "Smithy rules engine codegen integration test suite" + +data class Test( + val projectionName: String, + val protocolName: String, + val modelTemplate: File, +) { + val model: File + get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile +} + +val tests = listOf( + Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), +) + +fun fillInModel(output: File, protocolName: String, template: File) { + val input = template.readText() + val opTraits = when (protocolName) { + "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" + else -> "" + } + val replaced = input + .replace("{protocol-name}", protocolName) + .replace("{op-traits}", opTraits) + + output.parentFile.mkdirs() + output.writeText(replaced) +} + +val testServiceShapeId = "aws.sdk.kotlin.test#TestService" +smithyBuild { + tests.forEach { test -> + + projections.register(test.projectionName) { + imports = listOf(test.model.absolutePath) + transforms = listOf( + """ + { + "name": "includeServices", + "args": { + "services": ["$testServiceShapeId"] + } + } + """, + ) + + smithyKotlinPlugin { + serviceShapeId = testServiceShapeId + packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" + packageVersion = "1.0" + buildSettings { + generateFullProject = false + generateDefaultBuildFiles = false + optInAnnotations = listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + ) + } + } + } + } +} + +val codegen by configurations.getting +dependencies { + codegen(project(":codegen:aws-sdk-codegen")) + codegen(libs.smithy.cli) + codegen(libs.smithy.model) +} + +tasks.generateSmithyBuild { + doFirst { + tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } + } +} + +tasks.generateSmithyProjections { + doFirst { + // ensure the generated tests use the same version of the runtime as the aws aws-runtime + val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() + System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) + } +} + +val optinAnnotations = listOf( + "kotlin.RequiresOptIn", + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", +) + +kotlin.sourceSets.all { + optinAnnotations.forEach { languageSettings.optIn(it) } +} + +kotlin.sourceSets.getByName("test") { + smithyBuild.projections.forEach { + kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) + } +} + +tasks.withType { + dependsOn(tasks.generateSmithyProjections) + // generated clients have quite a few warnings + kotlinOptions.allWarningsAsErrors = false +} + +tasks.test { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } +} + +dependencies { + + implementation(libs.kotlinx.coroutines.core) + + testImplementation(libs.kotlin.test) + testImplementation(libs.kotlin.test.junit5) + testImplementation(libs.kotlinx.coroutines.test) + + testImplementation(libs.smithy.kotlin.smithy.test) + testImplementation(libs.smithy.kotlin.aws.signing.default) + testImplementation(libs.smithy.kotlin.telemetry.api) + + // have to manually add all the dependencies of the generated client(s) + // doing it this way (as opposed to doing what we do for protocol-tests) allows + // the tests to work without a publish to maven-local step at the cost of maintaining + // this set of dependencies manually + // <-- BEGIN GENERATED DEPENDENCY LIST --> + implementation(libs.bundles.smithy.kotlin.service.client) + implementation(libs.smithy.kotlin.aws.event.stream) + implementation(project(":aws-runtime:aws-http")) + implementation(libs.smithy.kotlin.aws.json.protocols) + implementation(libs.smithy.kotlin.serde.json) + api(project(":aws-runtime:aws-config")) + api(project(":aws-runtime:aws-core")) + api(project(":aws-runtime:aws-endpoint")) + // <-- END GENERATED DEPENDENCY LIST --> +} diff --git a/tests/codegen/rules-engine/operation-context-params.smithy b/tests/codegen/rules-engine/operation-context-params.smithy new file mode 100644 index 00000000000..c63a2756e39 --- /dev/null +++ b/tests/codegen/rules-engine/operation-context-params.smithy @@ -0,0 +1,58 @@ +$version: "2.0" +namespace aws.sdk.kotlin.test + +use aws.protocols#awsJson1_0 +use smithy.rules#operationContextParams +use smithy.rules#endpointRuleSet +use aws.api#service + +@awsJson1_0 +@service(sdkId: "OperationContextParamsTest") +@endpointRuleSet( + version: "1.0", + parameters: { + "ObjectKeys": { + "type": "stringArray", + "documentation": "A string array.", + "required": true + } + }, + rules: [ + { + "type": "endpoint", + "conditions": [], + "endpoint": { + "url": "https://static.endpoint" + } + } + ] +) +service TestService { + operations: [DeleteObjects], + version: "1" +} + +@operationContextParams( + ObjectKeys: { + path: "Delete.Objects[*].[Key][]" + } +) +operation DeleteObjects { + input: DeleteObjectsRequest +} + +structure DeleteObjectsRequest { + Delete: Delete +} + +structure Delete { + Objects: ObjectIdentifierList +} + +list ObjectIdentifierList { + member: ObjectIdentifier +} + +structure ObjectIdentifier { + Key: String +} From 7b7308836e00da347a3a3d2673127cdc0c631a13 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 10:32:10 -0500 Subject: [PATCH 02/72] Use snapshot version of smithy kotlin --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d86529cab72..533963fcac2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,8 +11,8 @@ coroutines-version = "1.9.0" atomicfu-version = "0.25.0" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.3.19" -smithy-kotlin-codegen-version = "0.33.19" +smithy-kotlin-runtime-version = "1.3.20-SNAPSHOT" +smithy-kotlin-codegen-version = "0.33.20-SNAPSHOT" # codegen smithy-version = "1.51.0" From dbfb9735643e8201a053cba5bd2a0fe081857ded Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 11:32:03 -0500 Subject: [PATCH 03/72] debugging: throw exception to see what metrics are strings --- .../runtime/http/interceptors/BusinessMetricsInterceptor.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt index 45acf59e5d4..ddfe0ab7808 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt @@ -20,6 +20,7 @@ import aws.smithy.kotlin.runtime.http.request.toBuilder public class BusinessMetricsInterceptor : HttpInterceptor { override suspend fun modifyBeforeTransmit(context: ProtocolRequestInterceptorContext): HttpRequest { context.executionContext.getOrNull(BusinessMetrics)?.let { metrics -> + throw Exception("Metrics: $metrics") val metricsString = formatMetrics(metrics) val currentUserAgentHeader = context.protocolRequest.headers[USER_AGENT] val modifiedRequest = context.protocolRequest.toBuilder() From b15358b8eff089c36c8b9aad579c5f586c0c1160 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 11:39:05 -0500 Subject: [PATCH 04/72] debugging: remove exception --- .../runtime/http/interceptors/BusinessMetricsInterceptor.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt index ddfe0ab7808..45acf59e5d4 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInterceptor.kt @@ -20,7 +20,6 @@ import aws.smithy.kotlin.runtime.http.request.toBuilder public class BusinessMetricsInterceptor : HttpInterceptor { override suspend fun modifyBeforeTransmit(context: ProtocolRequestInterceptorContext): HttpRequest { context.executionContext.getOrNull(BusinessMetrics)?.let { metrics -> - throw Exception("Metrics: $metrics") val metricsString = formatMetrics(metrics) val currentUserAgentHeader = context.protocolRequest.headers[USER_AGENT] val modifiedRequest = context.protocolRequest.toBuilder() From dacc3458ed2ad86a5f74b2b1db6105bc1197cbda Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 12:19:53 -0500 Subject: [PATCH 05/72] Don't depend on snapshot version of smithy kotlin --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 533963fcac2..d003593d62b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,8 +11,8 @@ coroutines-version = "1.9.0" atomicfu-version = "0.25.0" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.3.20-SNAPSHOT" -smithy-kotlin-codegen-version = "0.33.20-SNAPSHOT" +smithy-kotlin-runtime-version = "1.3.20" +smithy-kotlin-codegen-version = "0.33.20" # codegen smithy-version = "1.51.0" From bbafc3fac438d014c592564aa04a1d25c65e668d Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 12:20:40 -0500 Subject: [PATCH 06/72] Use released version --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d003593d62b..d86529cab72 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,8 +11,8 @@ coroutines-version = "1.9.0" atomicfu-version = "0.25.0" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.3.20" -smithy-kotlin-codegen-version = "0.33.20" +smithy-kotlin-runtime-version = "1.3.19" +smithy-kotlin-codegen-version = "0.33.19" # codegen smithy-version = "1.51.0" From a79aa1ea29244733565c1b07d1f3ed1cb9794d01 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 6 Nov 2024 13:16:25 -0500 Subject: [PATCH 07/72] PR feedback --- settings.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 6f53bf06951..c686feb4989 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -53,7 +53,7 @@ include(":hll:hll-mapping-core") include(":services") include(":tests") include(":tests:codegen:event-stream") -include("tests:codegen:rules-engine") +include(":tests:codegen:rules-engine") include(":tests:e2e-test-util") // generated services From e61efbbb7b642f5f703ef87541db7fa0233a177b Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 7 Nov 2024 11:54:21 -0500 Subject: [PATCH 08/72] Checkpoint --- tests/codegen/common.gradle.kts | 158 ++++++++++++++++++ tests/codegen/event-stream/build.gradle.kts | 10 +- ...ent-stream-initial-request-response.smithy | 2 +- .../event-stream-model-template.smithy | 2 +- .../src/test/kotlin/HttpEventStreamTests.kt | 6 +- .../src/test/kotlin/RpcEventStreamTests.kt | 12 +- tests/codegen/rules-engine/build.gradle.kts | 4 +- 7 files changed, 176 insertions(+), 18 deletions(-) create mode 100644 tests/codegen/common.gradle.kts diff --git a/tests/codegen/common.gradle.kts b/tests/codegen/common.gradle.kts new file mode 100644 index 00000000000..7e3409bf865 --- /dev/null +++ b/tests/codegen/common.gradle.kts @@ -0,0 +1,158 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir + +plugins { + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) +} + +description = "AWS SDK for Kotlin codegen integration test suite" // TODO: Don't commonize + +data class Test( + val projectionName: String, + val protocolName: String, + val modelTemplate: File, +) { + val model: File + get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile +} + +val tests = listOf( // TODO: Don't commonize + Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), +) + +fun fillInModel(output: File, protocolName: String, template: File) { + val input = template.readText() + val opTraits = when (protocolName) { + "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" + else -> "" + } + val replaced = input + .replace("{protocol-name}", protocolName) + .replace("{op-traits}", opTraits) + + output.parentFile.mkdirs() + output.writeText(replaced) +} + +val testServiceShapeId = "aws.sdk.kotlin.test#TestService" +smithyBuild { + tests.forEach { test -> + + projections.register(test.projectionName) { + imports = listOf(test.model.absolutePath) + transforms = listOf( + """ + { + "name": "includeServices", + "args": { + "services": ["$testServiceShapeId"] + } + } + """, + ) + + smithyKotlinPlugin { + serviceShapeId = testServiceShapeId + packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" + packageVersion = "1.0" + buildSettings { + generateFullProject = false + generateDefaultBuildFiles = false + optInAnnotations = listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + ) + } + } + } + } +} + +val codegen by configurations.getting +dependencies { + codegen(project(":codegen:aws-sdk-codegen")) + codegen(libs.smithy.cli) + codegen(libs.smithy.model) +} + +tasks.generateSmithyBuild { + doFirst { + tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } + } +} + +tasks.generateSmithyProjections { + doFirst { + // ensure the generated tests use the same version of the runtime as the aws aws-runtime + val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() + System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) + } +} + +val optinAnnotations = listOf( + "kotlin.RequiresOptIn", + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", +) + +kotlin.sourceSets.all { + optinAnnotations.forEach { languageSettings.optIn(it) } +} + +kotlin.sourceSets.getByName("test") { + smithyBuild.projections.forEach { + kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) + } +} + +tasks.withType { + dependsOn(tasks.generateSmithyProjections) + // generated clients have quite a few warnings + kotlinOptions.allWarningsAsErrors = false +} + +tasks.test { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } +} + +dependencies { + + implementation(libs.kotlinx.coroutines.core) + + testImplementation(libs.kotlin.test) + testImplementation(libs.kotlin.test.junit5) + testImplementation(libs.kotlinx.coroutines.test) + + testImplementation(libs.smithy.kotlin.smithy.test) + testImplementation(libs.smithy.kotlin.aws.signing.default) + testImplementation(libs.smithy.kotlin.telemetry.api) + + // have to manually add all the dependencies of the generated client(s) + // doing it this way (as opposed to doing what we do for protocol-tests) allows + // the tests to work without a publish to maven-local step at the cost of maintaining + // this set of dependencies manually + // <-- BEGIN GENERATED DEPENDENCY LIST --> + implementation(libs.bundles.smithy.kotlin.service.client) + implementation(libs.smithy.kotlin.aws.event.stream) + implementation(project(":aws-runtime:aws-http")) + implementation(libs.smithy.kotlin.aws.json.protocols) + implementation(libs.smithy.kotlin.serde.json) + api(project(":aws-runtime:aws-config")) + api(project(":aws-runtime:aws-core")) + api(project(":aws-runtime:aws-endpoint")) + // <-- END GENERATED DEPENDENCY LIST --> +} diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 837d12b2d3c..8c6ae318868 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -14,7 +14,7 @@ plugins { description = "Event stream codegen integration test suite" -data class EventStreamTest( +data class Test( val projectionName: String, val protocolName: String, val modelTemplate: File, @@ -24,8 +24,8 @@ data class EventStreamTest( } val tests = listOf( - EventStreamTest("restJson1", "restJson1", file("event-stream-model-template.smithy")), - EventStreamTest("awsJson11", "awsJson1_1", file("event-stream-initial-request-response.smithy")), + Test("restJson1", "restJson1", file("event-stream-model-template.smithy")), + Test("awsJson11", "awsJson1_1", file("event-stream-initial-request-response.smithy")), ) fun fillInModel(output: File, protocolName: String, template: File) { @@ -42,7 +42,7 @@ fun fillInModel(output: File, protocolName: String, template: File) { output.writeText(replaced) } -val testServiceShapeId = "aws.sdk.kotlin.test.eventstream#TestService" +val testServiceShapeId = "aws.sdk.kotlin.test#TestService" smithyBuild { tests.forEach { test -> @@ -61,7 +61,7 @@ smithyBuild { smithyKotlinPlugin { serviceShapeId = testServiceShapeId - packageName = "aws.sdk.kotlin.test.eventstream.${test.projectionName.lowercase()}" + packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" packageVersion = "1.0" buildSettings { generateFullProject = false diff --git a/tests/codegen/event-stream/event-stream-initial-request-response.smithy b/tests/codegen/event-stream/event-stream-initial-request-response.smithy index 917b58ac292..180311c8eb5 100644 --- a/tests/codegen/event-stream/event-stream-initial-request-response.smithy +++ b/tests/codegen/event-stream/event-stream-initial-request-response.smithy @@ -1,4 +1,4 @@ -namespace aws.sdk.kotlin.test.eventstream +namespace aws.sdk.kotlin.test use aws.protocols#awsJson1_1 use aws.api#service diff --git a/tests/codegen/event-stream/event-stream-model-template.smithy b/tests/codegen/event-stream/event-stream-model-template.smithy index f3d91364235..8150dcec2db 100644 --- a/tests/codegen/event-stream/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/event-stream-model-template.smithy @@ -1,4 +1,4 @@ -namespace aws.sdk.kotlin.test.eventstream +namespace aws.sdk.kotlin.test use aws.protocols#{protocol-name} use aws.api#service diff --git a/tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt b/tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt index 8402dd270fa..1bb5ec2bda9 100644 --- a/tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt +++ b/tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt @@ -4,9 +4,9 @@ */ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.eventstream.restjson1.model.* -import aws.sdk.kotlin.test.eventstream.restjson1.serde.deserializeTestStreamOpOperationBody -import aws.sdk.kotlin.test.eventstream.restjson1.serde.serializeTestStreamOpOperationBody +import aws.sdk.kotlin.test.restjson1.model.* +import aws.sdk.kotlin.test.restjson1.serde.deserializeTestStreamOpOperationBody +import aws.sdk.kotlin.test.restjson1.serde.serializeTestStreamOpOperationBody import aws.smithy.kotlin.runtime.InternalApi import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAttributes diff --git a/tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt b/tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt index 0b7e4b94651..08cbecbda2a 100644 --- a/tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt +++ b/tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt @@ -3,12 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.eventstream.awsjson11.model.MessageWithString -import aws.sdk.kotlin.test.eventstream.awsjson11.model.TestStream -import aws.sdk.kotlin.test.eventstream.awsjson11.model.TestStreamOperationWithInitialRequestResponseRequest -import aws.sdk.kotlin.test.eventstream.awsjson11.model.TestStreamOperationWithInitialRequestResponseResponse -import aws.sdk.kotlin.test.eventstream.awsjson11.serde.deserializeTestStreamOperationWithInitialRequestResponseOperationBody -import aws.sdk.kotlin.test.eventstream.awsjson11.serde.serializeTestStreamOperationWithInitialRequestResponseOperationBody +import aws.sdk.kotlin.test.awsjson11.model.MessageWithString +import aws.sdk.kotlin.test.awsjson11.model.TestStream +import aws.sdk.kotlin.test.awsjson11.model.TestStreamOperationWithInitialRequestResponseRequest +import aws.sdk.kotlin.test.awsjson11.model.TestStreamOperationWithInitialRequestResponseResponse +import aws.sdk.kotlin.test.awsjson11.serde.deserializeTestStreamOperationWithInitialRequestResponseOperationBody +import aws.sdk.kotlin.test.awsjson11.serde.serializeTestStreamOperationWithInitialRequestResponseOperationBody import aws.smithy.kotlin.runtime.InternalApi import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAttributes diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index ef35526b380..7e3409bf865 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -12,7 +12,7 @@ plugins { alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) } -description = "Smithy rules engine codegen integration test suite" +description = "AWS SDK for Kotlin codegen integration test suite" // TODO: Don't commonize data class Test( val projectionName: String, @@ -23,7 +23,7 @@ data class Test( get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile } -val tests = listOf( +val tests = listOf( // TODO: Don't commonize Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), ) From fbb3dd51c2b77f4fcaa8ab1a89eea0ccaf7db1d7 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 8 Nov 2024 12:47:28 -0500 Subject: [PATCH 09/72] Checkoint 2 --- settings.gradle.kts | 5 +- tests/codegen/build.gradle.kts | 80 ++++++++++ tests/codegen/common.gradle.kts | 158 -------------------- tests/codegen/event-stream/build.gradle.kts | 79 ++++------ tests/codegen/rules-engine/build.gradle.kts | 78 ++++------ 5 files changed, 137 insertions(+), 263 deletions(-) create mode 100644 tests/codegen/build.gradle.kts delete mode 100644 tests/codegen/common.gradle.kts diff --git a/settings.gradle.kts b/settings.gradle.kts index 7f35e266c03..7a7751ebeb8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -52,11 +52,12 @@ include(":hll:hll-codegen") include(":hll:hll-mapping-core") include(":services") include(":tests") +include(":tests:codegen") include(":tests:codegen:event-stream") include(":tests:codegen:rules-engine") include(":tests:e2e-test-util") -include(":tests:codegen:smoke-tests") -include(":tests:codegen:smoke-tests:services") +//include(":tests:codegen:smoke-tests") +//include(":tests:codegen:smoke-tests:services") // generated services val File.isServiceDir: Boolean diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts new file mode 100644 index 00000000000..7fffa0f4fb0 --- /dev/null +++ b/tests/codegen/build.gradle.kts @@ -0,0 +1,80 @@ +import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir +//import aws.sdk.kotlin.gradle.kmp.kotlin + +plugins { + alias(libs.plugins.kotlin.jvm) apply false + alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) apply false +// alias(libs.plugins.aws.kotlin.repo.tools.kmp) apply false +} + +val librares = libs + +subprojects { + apply(plugin = librares.plugins.kotlin.jvm.get().pluginId) + apply(plugin = librares.plugins.aws.kotlin.repo.tools.smithybuild.get().pluginId) +// apply(plugin = librares.plugins.aws.kotlin.repo.tools.kmp.get().pluginId) + + val codegen by configurations + + dependencies { + codegen(project(":codegen:aws-sdk-codegen")) + codegen(librares.smithy.cli) + codegen(librares.smithy.model) + } + + tasks.generateSmithyProjections { + doFirst { + // ensure the generated tests use the same version of the runtime as the aws aws-runtime + val smithyKotlinRuntimeVersion = librares.versions.smithy.kotlin.runtime.version.get() + System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) + } + } + +// val optinAnnotations = listOf( +// "kotlin.RequiresOptIn", +// "aws.smithy.kotlin.runtime.InternalApi", +// "aws.sdk.kotlin.runtime.InternalSdkApi", +// ) +// kotlin.sourceSets.all { +// optinAnnotations.forEach { languageSettings.optIn(it) } +// } + +// kotlin.sourceSets.getByName("test") { +// smithyBuild.projections.forEach { +// kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) +// } +// } + + tasks.withType { + dependsOn(tasks.generateSmithyProjections) + kotlinOptions.allWarningsAsErrors = false + } + + val implementation by configurations + val testImplementation by configurations + val api by configurations + dependencies { + implementation(librares.kotlinx.coroutines.core) + + testImplementation(librares.kotlin.test) + testImplementation(librares.kotlin.test.junit5) + testImplementation(librares.kotlinx.coroutines.test) + testImplementation(librares.smithy.kotlin.smithy.test) + testImplementation(librares.smithy.kotlin.aws.signing.default) + testImplementation(librares.smithy.kotlin.telemetry.api) + + /* We have to manually add all the dependencies of the generated client(s). + Doing it this way (as opposed to doing what we do for protocol-tests) allows the tests to work without a + publish to maven-local step at the cost of maintaining this set of dependencies manually. */ + implementation(librares.bundles.smithy.kotlin.service.client) + implementation(librares.smithy.kotlin.aws.event.stream) + implementation(project(":aws-runtime:aws-http")) + implementation(librares.smithy.kotlin.aws.json.protocols) + implementation(librares.smithy.kotlin.serde.json) + api(project(":aws-runtime:aws-config")) + api(project(":aws-runtime:aws-core")) + api(project(":aws-runtime:aws-endpoint")) + } +} diff --git a/tests/codegen/common.gradle.kts b/tests/codegen/common.gradle.kts deleted file mode 100644 index 7e3409bf865..00000000000 --- a/tests/codegen/common.gradle.kts +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections -import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir - -plugins { - alias(libs.plugins.kotlin.jvm) - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) -} - -description = "AWS SDK for Kotlin codegen integration test suite" // TODO: Don't commonize - -data class Test( - val projectionName: String, - val protocolName: String, - val modelTemplate: File, -) { - val model: File - get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile -} - -val tests = listOf( // TODO: Don't commonize - Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), -) - -fun fillInModel(output: File, protocolName: String, template: File) { - val input = template.readText() - val opTraits = when (protocolName) { - "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" - else -> "" - } - val replaced = input - .replace("{protocol-name}", protocolName) - .replace("{op-traits}", opTraits) - - output.parentFile.mkdirs() - output.writeText(replaced) -} - -val testServiceShapeId = "aws.sdk.kotlin.test#TestService" -smithyBuild { - tests.forEach { test -> - - projections.register(test.projectionName) { - imports = listOf(test.model.absolutePath) - transforms = listOf( - """ - { - "name": "includeServices", - "args": { - "services": ["$testServiceShapeId"] - } - } - """, - ) - - smithyKotlinPlugin { - serviceShapeId = testServiceShapeId - packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" - packageVersion = "1.0" - buildSettings { - generateFullProject = false - generateDefaultBuildFiles = false - optInAnnotations = listOf( - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", - ) - } - } - } - } -} - -val codegen by configurations.getting -dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(libs.smithy.cli) - codegen(libs.smithy.model) -} - -tasks.generateSmithyBuild { - doFirst { - tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } - } -} - -tasks.generateSmithyProjections { - doFirst { - // ensure the generated tests use the same version of the runtime as the aws aws-runtime - val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() - System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) - } -} - -val optinAnnotations = listOf( - "kotlin.RequiresOptIn", - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", -) - -kotlin.sourceSets.all { - optinAnnotations.forEach { languageSettings.optIn(it) } -} - -kotlin.sourceSets.getByName("test") { - smithyBuild.projections.forEach { - kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) - } -} - -tasks.withType { - dependsOn(tasks.generateSmithyProjections) - // generated clients have quite a few warnings - kotlinOptions.allWarningsAsErrors = false -} - -tasks.test { - useJUnitPlatform() - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showStackTraces = true - showExceptions = true - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL - } -} - -dependencies { - - implementation(libs.kotlinx.coroutines.core) - - testImplementation(libs.kotlin.test) - testImplementation(libs.kotlin.test.junit5) - testImplementation(libs.kotlinx.coroutines.test) - - testImplementation(libs.smithy.kotlin.smithy.test) - testImplementation(libs.smithy.kotlin.aws.signing.default) - testImplementation(libs.smithy.kotlin.telemetry.api) - - // have to manually add all the dependencies of the generated client(s) - // doing it this way (as opposed to doing what we do for protocol-tests) allows - // the tests to work without a publish to maven-local step at the cost of maintaining - // this set of dependencies manually - // <-- BEGIN GENERATED DEPENDENCY LIST --> - implementation(libs.bundles.smithy.kotlin.service.client) - implementation(libs.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(libs.smithy.kotlin.aws.json.protocols) - implementation(libs.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) - // <-- END GENERATED DEPENDENCY LIST --> -} diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 8c6ae318868..2cb84424b19 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -7,10 +7,6 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -plugins { - alias(libs.plugins.kotlin.jvm) - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) -} description = "Event stream codegen integration test suite" @@ -76,27 +72,12 @@ smithyBuild { } } -val codegen by configurations.getting -dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(libs.smithy.cli) - codegen(libs.smithy.model) -} - tasks.generateSmithyBuild { doFirst { tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } } } -tasks.generateSmithyProjections { - doFirst { - // ensure the generated tests use the same version of the runtime as the aws aws-runtime - val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() - System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) - } -} - val optinAnnotations = listOf( "kotlin.RequiresOptIn", "aws.smithy.kotlin.runtime.InternalApi", @@ -112,12 +93,6 @@ kotlin.sourceSets.getByName("test") { } } -tasks.withType { - dependsOn(tasks.generateSmithyProjections) - // generated clients have quite a few warnings - kotlinOptions.allWarningsAsErrors = false -} - tasks.test { useJUnitPlatform() testLogging { @@ -129,30 +104,30 @@ tasks.test { } } -dependencies { - - implementation(libs.kotlinx.coroutines.core) - - testImplementation(libs.kotlin.test) - testImplementation(libs.kotlin.test.junit5) - testImplementation(libs.kotlinx.coroutines.test) - - testImplementation(libs.smithy.kotlin.smithy.test) - testImplementation(libs.smithy.kotlin.aws.signing.default) - testImplementation(libs.smithy.kotlin.telemetry.api) - - // have to manually add all the dependencies of the generated client(s) - // doing it this way (as opposed to doing what we do for protocol-tests) allows - // the tests to work without a publish to maven-local step at the cost of maintaining - // this set of dependencies manually - // <-- BEGIN GENERATED DEPENDENCY LIST --> - implementation(libs.bundles.smithy.kotlin.service.client) - implementation(libs.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(libs.smithy.kotlin.aws.json.protocols) - implementation(libs.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) - // <-- END GENERATED DEPENDENCY LIST --> -} +//dependencies { +// +// implementation(libs.kotlinx.coroutines.core) +// +// testImplementation(libs.kotlin.test) +// testImplementation(libs.kotlin.test.junit5) +// testImplementation(libs.kotlinx.coroutines.test) +// +// testImplementation(libs.smithy.kotlin.smithy.test) +// testImplementation(libs.smithy.kotlin.aws.signing.default) +// testImplementation(libs.smithy.kotlin.telemetry.api) +// +// // have to manually add all the dependencies of the generated client(s) +// // doing it this way (as opposed to doing what we do for protocol-tests) allows +// // the tests to work without a publish to maven-local step at the cost of maintaining +// // this set of dependencies manually +// // <-- BEGIN GENERATED DEPENDENCY LIST --> +// implementation(libs.bundles.smithy.kotlin.service.client) +// implementation(libs.smithy.kotlin.aws.event.stream) +// implementation(project(":aws-runtime:aws-http")) +// implementation(libs.smithy.kotlin.aws.json.protocols) +// implementation(libs.smithy.kotlin.serde.json) +// api(project(":aws-runtime:aws-config")) +// api(project(":aws-runtime:aws-core")) +// api(project(":aws-runtime:aws-endpoint")) +// // <-- END GENERATED DEPENDENCY LIST --> +//} diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index 7e3409bf865..a00f4205b00 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -7,10 +7,6 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -plugins { - alias(libs.plugins.kotlin.jvm) - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) -} description = "AWS SDK for Kotlin codegen integration test suite" // TODO: Don't commonize @@ -75,26 +71,12 @@ smithyBuild { } } -val codegen by configurations.getting -dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(libs.smithy.cli) - codegen(libs.smithy.model) -} - tasks.generateSmithyBuild { doFirst { tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } } } -tasks.generateSmithyProjections { - doFirst { - // ensure the generated tests use the same version of the runtime as the aws aws-runtime - val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() - System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) - } -} val optinAnnotations = listOf( "kotlin.RequiresOptIn", @@ -112,12 +94,6 @@ kotlin.sourceSets.getByName("test") { } } -tasks.withType { - dependsOn(tasks.generateSmithyProjections) - // generated clients have quite a few warnings - kotlinOptions.allWarningsAsErrors = false -} - tasks.test { useJUnitPlatform() testLogging { @@ -129,30 +105,30 @@ tasks.test { } } -dependencies { - - implementation(libs.kotlinx.coroutines.core) - - testImplementation(libs.kotlin.test) - testImplementation(libs.kotlin.test.junit5) - testImplementation(libs.kotlinx.coroutines.test) - - testImplementation(libs.smithy.kotlin.smithy.test) - testImplementation(libs.smithy.kotlin.aws.signing.default) - testImplementation(libs.smithy.kotlin.telemetry.api) - - // have to manually add all the dependencies of the generated client(s) - // doing it this way (as opposed to doing what we do for protocol-tests) allows - // the tests to work without a publish to maven-local step at the cost of maintaining - // this set of dependencies manually - // <-- BEGIN GENERATED DEPENDENCY LIST --> - implementation(libs.bundles.smithy.kotlin.service.client) - implementation(libs.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(libs.smithy.kotlin.aws.json.protocols) - implementation(libs.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) - // <-- END GENERATED DEPENDENCY LIST --> -} +//dependencies { +// +// implementation(libs.kotlinx.coroutines.core) +// +// testImplementation(libs.kotlin.test) +// testImplementation(libs.kotlin.test.junit5) +// testImplementation(libs.kotlinx.coroutines.test) +// +// testImplementation(libs.smithy.kotlin.smithy.test) +// testImplementation(libs.smithy.kotlin.aws.signing.default) +// testImplementation(libs.smithy.kotlin.telemetry.api) +// +// // have to manually add all the dependencies of the generated client(s) +// // doing it this way (as opposed to doing what we do for protocol-tests) allows +// // the tests to work without a publish to maven-local step at the cost of maintaining +// // this set of dependencies manually +// // <-- BEGIN GENERATED DEPENDENCY LIST --> +// implementation(libs.bundles.smithy.kotlin.service.client) +// implementation(libs.smithy.kotlin.aws.event.stream) +// implementation(project(":aws-runtime:aws-http")) +// implementation(libs.smithy.kotlin.aws.json.protocols) +// implementation(libs.smithy.kotlin.serde.json) +// api(project(":aws-runtime:aws-config")) +// api(project(":aws-runtime:aws-core")) +// api(project(":aws-runtime:aws-endpoint")) +// // <-- END GENERATED DEPENDENCY LIST --> +//} From 136c9dbfdeec9212101205937f7b07853474f8da Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 8 Nov 2024 16:49:56 -0500 Subject: [PATCH 10/72] It works now --- settings.gradle.kts | 4 +- tests/codegen/build.gradle.kts | 128 ++++++++++-------- tests/codegen/event-stream/build.gradle.kts | 60 +------- .../kotlin/HttpEventStreamTests.kt | 0 .../kotlin/RpcEventStreamTests.kt | 0 tests/codegen/rules-engine/build.gradle.kts | 53 +------- tests/codegen/smoke-tests/build.gradle.kts | 38 ++---- .../smoke-tests/services/build.gradle.kts | 24 ---- .../kotlin/SmokeTestE2ETest.kt | 1 + .../resources/smoke-tests-exception.smithy | 0 .../resources/smoke-tests-failure.smithy | 0 .../resources/smoke-tests-success.smithy | 0 12 files changed, 90 insertions(+), 218 deletions(-) rename tests/codegen/event-stream/src/{test => commonTest}/kotlin/HttpEventStreamTests.kt (100%) rename tests/codegen/event-stream/src/{test => commonTest}/kotlin/RpcEventStreamTests.kt (100%) rename tests/codegen/smoke-tests/src/{test => jvmTest}/kotlin/SmokeTestE2ETest.kt (96%) rename tests/codegen/smoke-tests/src/{test => jvmTest}/resources/smoke-tests-exception.smithy (100%) rename tests/codegen/smoke-tests/src/{test => jvmTest}/resources/smoke-tests-failure.smithy (100%) rename tests/codegen/smoke-tests/src/{test => jvmTest}/resources/smoke-tests-success.smithy (100%) diff --git a/settings.gradle.kts b/settings.gradle.kts index 7a7751ebeb8..06a685d8512 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -56,8 +56,8 @@ include(":tests:codegen") include(":tests:codegen:event-stream") include(":tests:codegen:rules-engine") include(":tests:e2e-test-util") -//include(":tests:codegen:smoke-tests") -//include(":tests:codegen:smoke-tests:services") +include(":tests:codegen:smoke-tests") +include(":tests:codegen:smoke-tests:services") // generated services val File.isServiceDir: Boolean diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts index 7fffa0f4fb0..0fda97f3cd4 100644 --- a/tests/codegen/build.gradle.kts +++ b/tests/codegen/build.gradle.kts @@ -1,80 +1,92 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections -import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -//import aws.sdk.kotlin.gradle.kmp.kotlin plugins { - alias(libs.plugins.kotlin.jvm) apply false - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) apply false -// alias(libs.plugins.aws.kotlin.repo.tools.kmp) apply false + alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) + alias(libs.plugins.kotlin.multiplatform) } -val librares = libs +kotlin { + jvm() +} + +val libraries = libs subprojects { - apply(plugin = librares.plugins.kotlin.jvm.get().pluginId) - apply(plugin = librares.plugins.aws.kotlin.repo.tools.smithybuild.get().pluginId) -// apply(plugin = librares.plugins.aws.kotlin.repo.tools.kmp.get().pluginId) + apply(plugin = libraries.plugins.aws.kotlin.repo.tools.smithybuild.get().pluginId) + apply(plugin = libraries.plugins.kotlin.multiplatform.get().pluginId) - val codegen by configurations + val optinAnnotations = listOf( + "kotlin.RequiresOptIn", + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + ) + kotlin.sourceSets.all { + optinAnnotations.forEach { languageSettings.optIn(it) } + } - dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(librares.smithy.cli) - codegen(librares.smithy.model) + tasks.withType { + dependsOn(tasks.generateSmithyProjections) + kotlinOptions.allWarningsAsErrors = false } tasks.generateSmithyProjections { doFirst { - // ensure the generated tests use the same version of the runtime as the aws aws-runtime - val smithyKotlinRuntimeVersion = librares.versions.smithy.kotlin.runtime.version.get() + // Ensure the generated tests use the same version of the runtime as the aws aws-runtime + val smithyKotlinRuntimeVersion = libraries.versions.smithy.kotlin.runtime.version.get() System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) } } -// val optinAnnotations = listOf( -// "kotlin.RequiresOptIn", -// "aws.smithy.kotlin.runtime.InternalApi", -// "aws.sdk.kotlin.runtime.InternalSdkApi", -// ) -// kotlin.sourceSets.all { -// optinAnnotations.forEach { languageSettings.optIn(it) } -// } - -// kotlin.sourceSets.getByName("test") { -// smithyBuild.projections.forEach { -// kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) -// } -// } - - tasks.withType { - dependsOn(tasks.generateSmithyProjections) - kotlinOptions.allWarningsAsErrors = false - } - - val implementation by configurations - val testImplementation by configurations - val api by configurations + val codegen by configurations dependencies { - implementation(librares.kotlinx.coroutines.core) + codegen(project(":codegen:aws-sdk-codegen")) + codegen(libraries.smithy.cli) + codegen(libraries.smithy.model) + } - testImplementation(librares.kotlin.test) - testImplementation(librares.kotlin.test.junit5) - testImplementation(librares.kotlinx.coroutines.test) - testImplementation(librares.smithy.kotlin.smithy.test) - testImplementation(librares.smithy.kotlin.aws.signing.default) - testImplementation(librares.smithy.kotlin.telemetry.api) + kotlin { + jvm() + sourceSets { + commonMain { + dependencies { + implementation(project(":codegen:aws-sdk-codegen")) + implementation(libraries.smithy.kotlin.codegen) - /* We have to manually add all the dependencies of the generated client(s). - Doing it this way (as opposed to doing what we do for protocol-tests) allows the tests to work without a - publish to maven-local step at the cost of maintaining this set of dependencies manually. */ - implementation(librares.bundles.smithy.kotlin.service.client) - implementation(librares.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(librares.smithy.kotlin.aws.json.protocols) - implementation(librares.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) + /* We have to manually add all the dependencies of the generated client(s). + Doing it this way (as opposed to doing what we do for protocol-tests) allows the tests to work without a + publish to maven-local step at the cost of maintaining this set of dependencies manually. */ + implementation(libraries.kotlinx.coroutines.core) + implementation(libraries.bundles.smithy.kotlin.service.client) + implementation(libraries.smithy.kotlin.aws.event.stream) + implementation(project(":aws-runtime:aws-http")) + implementation(libraries.smithy.kotlin.aws.json.protocols) + implementation(libraries.smithy.kotlin.serde.json) + api(project(":aws-runtime:aws-config")) + api(project(":aws-runtime:aws-core")) + api(project(":aws-runtime:aws-endpoint")) + } + } + commonTest { + dependencies { + implementation(libraries.kotlin.test) + implementation(libraries.kotlinx.coroutines.test) + implementation(libraries.smithy.kotlin.smithy.test) + implementation(libraries.smithy.kotlin.aws.signing.default) + implementation(libraries.smithy.kotlin.telemetry.api) + } + } + jvmTest { + tasks.withType { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } + } + } + } } } diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 2cb84424b19..89f96fdf1ce 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -3,11 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir - description = "Event stream codegen integration test suite" data class Test( @@ -78,56 +76,12 @@ tasks.generateSmithyBuild { } } -val optinAnnotations = listOf( - "kotlin.RequiresOptIn", - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", -) -kotlin.sourceSets.all { - optinAnnotations.forEach { languageSettings.optIn(it) } -} - -kotlin.sourceSets.getByName("test") { - smithyBuild.projections.forEach { - kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) - } -} - -tasks.test { - useJUnitPlatform() - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showStackTraces = true - showExceptions = true - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL +kotlin { + sourceSets { + commonTest { + smithyBuild.projections.forEach { + kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) + } + } } } - -//dependencies { -// -// implementation(libs.kotlinx.coroutines.core) -// -// testImplementation(libs.kotlin.test) -// testImplementation(libs.kotlin.test.junit5) -// testImplementation(libs.kotlinx.coroutines.test) -// -// testImplementation(libs.smithy.kotlin.smithy.test) -// testImplementation(libs.smithy.kotlin.aws.signing.default) -// testImplementation(libs.smithy.kotlin.telemetry.api) -// -// // have to manually add all the dependencies of the generated client(s) -// // doing it this way (as opposed to doing what we do for protocol-tests) allows -// // the tests to work without a publish to maven-local step at the cost of maintaining -// // this set of dependencies manually -// // <-- BEGIN GENERATED DEPENDENCY LIST --> -// implementation(libs.bundles.smithy.kotlin.service.client) -// implementation(libs.smithy.kotlin.aws.event.stream) -// implementation(project(":aws-runtime:aws-http")) -// implementation(libs.smithy.kotlin.aws.json.protocols) -// implementation(libs.smithy.kotlin.serde.json) -// api(project(":aws-runtime:aws-config")) -// api(project(":aws-runtime:aws-core")) -// api(project(":aws-runtime:aws-endpoint")) -// // <-- END GENERATED DEPENDENCY LIST --> -//} diff --git a/tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt b/tests/codegen/event-stream/src/commonTest/kotlin/HttpEventStreamTests.kt similarity index 100% rename from tests/codegen/event-stream/src/test/kotlin/HttpEventStreamTests.kt rename to tests/codegen/event-stream/src/commonTest/kotlin/HttpEventStreamTests.kt diff --git a/tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt b/tests/codegen/event-stream/src/commonTest/kotlin/RpcEventStreamTests.kt similarity index 100% rename from tests/codegen/event-stream/src/test/kotlin/RpcEventStreamTests.kt rename to tests/codegen/event-stream/src/commonTest/kotlin/RpcEventStreamTests.kt diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index a00f4205b00..43e49004278 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir @@ -77,58 +76,8 @@ tasks.generateSmithyBuild { } } - -val optinAnnotations = listOf( - "kotlin.RequiresOptIn", - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", -) - -kotlin.sourceSets.all { - optinAnnotations.forEach { languageSettings.optIn(it) } -} - -kotlin.sourceSets.getByName("test") { +kotlin.sourceSets.getByName("jvmTest") { smithyBuild.projections.forEach { kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) } } - -tasks.test { - useJUnitPlatform() - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showStackTraces = true - showExceptions = true - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL - } -} - -//dependencies { -// -// implementation(libs.kotlinx.coroutines.core) -// -// testImplementation(libs.kotlin.test) -// testImplementation(libs.kotlin.test.junit5) -// testImplementation(libs.kotlinx.coroutines.test) -// -// testImplementation(libs.smithy.kotlin.smithy.test) -// testImplementation(libs.smithy.kotlin.aws.signing.default) -// testImplementation(libs.smithy.kotlin.telemetry.api) -// -// // have to manually add all the dependencies of the generated client(s) -// // doing it this way (as opposed to doing what we do for protocol-tests) allows -// // the tests to work without a publish to maven-local step at the cost of maintaining -// // this set of dependencies manually -// // <-- BEGIN GENERATED DEPENDENCY LIST --> -// implementation(libs.bundles.smithy.kotlin.service.client) -// implementation(libs.smithy.kotlin.aws.event.stream) -// implementation(project(":aws-runtime:aws-http")) -// implementation(libs.smithy.kotlin.aws.json.protocols) -// implementation(libs.smithy.kotlin.serde.json) -// api(project(":aws-runtime:aws-config")) -// api(project(":aws-runtime:aws-core")) -// api(project(":aws-runtime:aws-endpoint")) -// // <-- END GENERATED DEPENDENCY LIST --> -//} diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 77a74afceba..7f4449c169e 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -9,9 +9,14 @@ import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath description = "Tests for smoke tests runners" -plugins { - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) - alias(libs.plugins.kotlin.jvm) +kotlin { + sourceSets { + jvmTest { + dependencies { + implementation("dev.gradleplugins:gradle-test-kit:7.3.3") // TODO: Use lib.versions.toml + } + } + } } val projections = listOf( @@ -20,29 +25,12 @@ val projections = listOf( Projection("exceptionService", "smoke-tests-exception.smithy", "smithy.kotlin.traits#ExceptionService"), ) -configureProject() configureProjections() configureTasks() -fun configureProject() { - val codegen by configurations.getting - - dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(libs.smithy.cli) - codegen(libs.smithy.model) - - implementation(project(":codegen:aws-sdk-codegen")) - implementation(libs.smithy.kotlin.codegen) - - testImplementation(libs.kotlin.test) - testImplementation(gradleTestKit()) - } -} - fun configureProjections() { smithyBuild { - val pathToSmithyModels = "src/test/resources/" + val pathToSmithyModels = "src/jvmTest/resources/" this@Build_gradle.projections.forEach { projection -> projections.register(projection.name) { @@ -105,14 +93,6 @@ fun configureTasks() { tasks.withType { dependsOn(tasks.getByName("stageServices")) mustRunAfter(tasks.getByName("stageServices")) - - testLogging { - events("passed", "skipped", "failed") - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL - showExceptions = true - showCauses = true - showStackTraces = true - } } } diff --git a/tests/codegen/smoke-tests/services/build.gradle.kts b/tests/codegen/smoke-tests/services/build.gradle.kts index 2238ac4b0d1..f48b8b7b5dd 100644 --- a/tests/codegen/smoke-tests/services/build.gradle.kts +++ b/tests/codegen/smoke-tests/services/build.gradle.kts @@ -1,33 +1,9 @@ -import aws.sdk.kotlin.gradle.kmp.kotlin - -plugins { - alias(libs.plugins.aws.kotlin.repo.tools.kmp) apply false -} - -// capture locally - scope issue with custom KMP plugin -val libraries = libs - subprojects { - apply { - plugin(libraries.plugins.kotlin.multiplatform.get().pluginId) - plugin(libraries.plugins.aws.kotlin.repo.tools.kmp.get().pluginId) - } - kotlin { - - jvm() - sourceSets { - all { - languageSettings.optIn("kotlin.RequiresOptIn") - languageSettings.optIn("aws.smithy.kotlin.runtime.InternalApi") - languageSettings.optIn("aws.sdk.kotlin.runtime.InternalSdkApi") - } - commonMain { kotlin.srcDir("generated-src/main/kotlin") } - commonTest { kotlin.srcDir("generated-src/test/kotlin") } diff --git a/tests/codegen/smoke-tests/src/test/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/jvmTest/kotlin/SmokeTestE2ETest.kt similarity index 96% rename from tests/codegen/smoke-tests/src/test/kotlin/SmokeTestE2ETest.kt rename to tests/codegen/smoke-tests/src/jvmTest/kotlin/SmokeTestE2ETest.kt index 61f38fe62c1..7dc76031929 100644 --- a/tests/codegen/smoke-tests/src/test/kotlin/SmokeTestE2ETest.kt +++ b/tests/codegen/smoke-tests/src/jvmTest/kotlin/SmokeTestE2ETest.kt @@ -61,6 +61,7 @@ private fun runSmokeTests( // FIXME: Remove `-Paws.kotlin.native=false` when Kotlin Native is ready .withArguments("-Paws.kotlin.native=false", ":tests:codegen:smoke-tests:services:$service:smokeTest") .withEnvironment(envVars) + .withGradleVersion("8.5") // FIXME: Use lib.versions - or find a way to use `gradleTestKit()` val buildResult = if (expectingFailure) task.buildAndFail() else task.build() diff --git a/tests/codegen/smoke-tests/src/test/resources/smoke-tests-exception.smithy b/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/test/resources/smoke-tests-exception.smithy rename to tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy diff --git a/tests/codegen/smoke-tests/src/test/resources/smoke-tests-failure.smithy b/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/test/resources/smoke-tests-failure.smithy rename to tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy diff --git a/tests/codegen/smoke-tests/src/test/resources/smoke-tests-success.smithy b/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/test/resources/smoke-tests-success.smithy rename to tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy From cb4ab1269b49ccf86c3c4e0cecaff64e11867f4c Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 8 Nov 2024 20:42:19 -0500 Subject: [PATCH 11/72] Clean up build scripts --- tests/codegen/event-stream/build.gradle.kts | 95 +++++++++++-------- ...ent-stream-initial-request-response.smithy | 0 .../event-stream-model-template.smithy | 6 +- tests/codegen/rules-engine/build.gradle.kts | 71 ++++---------- .../operation-context-params.smithy | 0 tests/codegen/smoke-tests/build.gradle.kts | 73 +++++++------- 6 files changed, 114 insertions(+), 131 deletions(-) rename tests/codegen/event-stream/{ => src/commonTest/resources}/event-stream-initial-request-response.smithy (100%) rename tests/codegen/event-stream/{ => src/commonTest/resources}/event-stream-model-template.smithy (95%) rename tests/codegen/rules-engine/{ => src/commonTest/resources}/operation-context-params.smithy (100%) diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 89f96fdf1ce..e9fbb56f7b6 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -6,56 +6,57 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -description = "Event stream codegen integration test suite" +description = "AWS SDK for Kotlin codegen event stream integration test suite" -data class Test( - val projectionName: String, - val protocolName: String, - val modelTemplate: File, +apply(from = rootProject.file("buildSrc/shared.gradle.kts")) + +data class CodegenTest( + val name: String, + val model: Model, + val serviceShapeId: String, + val protocolName: String? = null, +) + +data class Model( + val fileName: String, + val path: String = "src/commonTest/resources/", ) { - val model: File - get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile + val file: File + get() = layout.projectDirectory.file(path + fileName).asFile } val tests = listOf( - Test("restJson1", "restJson1", file("event-stream-model-template.smithy")), - Test("awsJson11", "awsJson1_1", file("event-stream-initial-request-response.smithy")), + CodegenTest( + "restJson1", + Model("event-stream-model-template.smithy"), + "aws.sdk.kotlin.test#TestService", + "restJson1" + ), + CodegenTest( + "awsJson11", + Model("event-stream-initial-request-response.smithy"), + "aws.sdk.kotlin.test#TestService", + "awsJson1_1" + ), ) -fun fillInModel(output: File, protocolName: String, template: File) { - val input = template.readText() - val opTraits = when (protocolName) { - "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" - else -> "" - } - val replaced = input - .replace("{protocol-name}", protocolName) - .replace("{op-traits}", opTraits) - - output.parentFile.mkdirs() - output.writeText(replaced) -} - -val testServiceShapeId = "aws.sdk.kotlin.test#TestService" smithyBuild { tests.forEach { test -> - - projections.register(test.projectionName) { - imports = listOf(test.model.absolutePath) + projections.register(test.name) { + imports = listOf(test.model.file.absolutePath) transforms = listOf( """ { "name": "includeServices", "args": { - "services": ["$testServiceShapeId"] + "services": ["${test.serviceShapeId}"] } } """, ) - smithyKotlinPlugin { - serviceShapeId = testServiceShapeId - packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" packageVersion = "1.0" buildSettings { generateFullProject = false @@ -70,18 +71,36 @@ smithyBuild { } } -tasks.generateSmithyBuild { - doFirst { - tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } - } -} - kotlin { sourceSets { - commonTest { + commonTest { // TODO: CHANGE TO JUST TEST WHEN NON-KMPing the project smithyBuild.projections.forEach { kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) } } } } + +tasks.generateSmithyBuild { + doFirst { + tests.forEach { test -> fillInModel(test) } + } +} + +fun fillInModel(test: CodegenTest) { + val modelFile = test.model.file + val model = modelFile.readText() + + val opTraits = + when (test.protocolName) { + "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" + else -> "" + } + + val replaced = model + .replace("{protocol-name}", test.protocolName!!) + .replace("{op-traits}", opTraits) + + modelFile.parentFile.mkdirs() + modelFile.writeText(replaced) +} diff --git a/tests/codegen/event-stream/event-stream-initial-request-response.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-initial-request-response.smithy similarity index 100% rename from tests/codegen/event-stream/event-stream-initial-request-response.smithy rename to tests/codegen/event-stream/src/commonTest/resources/event-stream-initial-request-response.smithy diff --git a/tests/codegen/event-stream/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy similarity index 95% rename from tests/codegen/event-stream/event-stream-model-template.smithy rename to tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index 8150dcec2db..dd32f77d8d7 100644 --- a/tests/codegen/event-stream/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -1,15 +1,15 @@ namespace aws.sdk.kotlin.test -use aws.protocols#{protocol-name} +use aws.protocols#restJson1 use aws.api#service use aws.auth#sigv4 -@{protocol-name} +@restJson1 @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -{op-traits} +@http(method: "POST", uri: "/test-eventstream", code: 200) operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index 43e49004278..70d57ec3c8c 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -4,58 +4,35 @@ */ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir +description = "AWS SDK for Kotlin codegen rules engine integration test suite" -description = "AWS SDK for Kotlin codegen integration test suite" // TODO: Don't commonize +data class CodegenTest( + val name: String, + val model: Model, + val serviceShapeId: String, + val protocolName: String? = null, +) -data class Test( - val projectionName: String, - val protocolName: String, - val modelTemplate: File, +data class Model( + val fileName: String, + val path: String = "src/commonTest/resources/", ) { - val model: File - get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile + val file: File + get() = layout.projectDirectory.file(path + fileName).asFile } -val tests = listOf( // TODO: Don't commonize - Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), +val tests = listOf( + CodegenTest("operationContextParams", Model("operation-context-params.smithy"), "aws.sdk.kotlin.test#TestService") ) -fun fillInModel(output: File, protocolName: String, template: File) { - val input = template.readText() - val opTraits = when (protocolName) { - "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" - else -> "" - } - val replaced = input - .replace("{protocol-name}", protocolName) - .replace("{op-traits}", opTraits) - - output.parentFile.mkdirs() - output.writeText(replaced) -} - -val testServiceShapeId = "aws.sdk.kotlin.test#TestService" smithyBuild { tests.forEach { test -> - - projections.register(test.projectionName) { - imports = listOf(test.model.absolutePath) - transforms = listOf( - """ - { - "name": "includeServices", - "args": { - "services": ["$testServiceShapeId"] - } - } - """, - ) - + projections.register(test.name) { + imports = listOf(test.model.file.absolutePath) smithyKotlinPlugin { - serviceShapeId = testServiceShapeId - packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" packageVersion = "1.0" buildSettings { generateFullProject = false @@ -69,15 +46,3 @@ smithyBuild { } } } - -tasks.generateSmithyBuild { - doFirst { - tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } - } -} - -kotlin.sourceSets.getByName("jvmTest") { - smithyBuild.projections.forEach { - kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) - } -} diff --git a/tests/codegen/rules-engine/operation-context-params.smithy b/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy similarity index 100% rename from tests/codegen/rules-engine/operation-context-params.smithy rename to tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 7f4449c169e..727a0c56fb5 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -7,7 +7,7 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath -description = "Tests for smoke tests runners" +description = "AWS SDK for Kotlin codegen smoke tests integration test suite" kotlin { sourceSets { @@ -19,10 +19,25 @@ kotlin { } } -val projections = listOf( - Projection("successService", "smoke-tests-success.smithy", "smithy.kotlin.traits#SuccessService"), - Projection("failureService", "smoke-tests-failure.smithy", "smithy.kotlin.traits#FailureService"), - Projection("exceptionService", "smoke-tests-exception.smithy", "smithy.kotlin.traits#ExceptionService"), +data class CodegenTest( + val name: String, + val model: Model, + val serviceShapeId: String, + val protocolName: String? = null, +) + +data class Model( + val fileName: String, + val path: String = "src/jvmTest/resources/", +) { + val file: File + get() = layout.projectDirectory.file(path + fileName).asFile +} + +val tests = listOf( + CodegenTest("successService", Model("smoke-tests-success.smithy"), "smithy.kotlin.traits#SuccessService"), + CodegenTest("failureService", Model("smoke-tests-failure.smithy"), "smithy.kotlin.traits#FailureService"), + CodegenTest("exceptionService", Model("smoke-tests-exception.smithy"), "smithy.kotlin.traits#ExceptionService"), ) configureProjections() @@ -30,14 +45,12 @@ configureTasks() fun configureProjections() { smithyBuild { - val pathToSmithyModels = "src/jvmTest/resources/" - - this@Build_gradle.projections.forEach { projection -> - projections.register(projection.name) { - imports = listOf(layout.projectDirectory.file(pathToSmithyModels + projection.modelFile).asFile.absolutePath) + this@Build_gradle.tests.forEach { test -> + projections.register(test.name) { + imports = listOf(test.model.file.absolutePath) smithyKotlinPlugin { - serviceShapeId = projection.serviceShapeId - packageName = "aws.sdk.kotlin.test" + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" packageVersion = "1.0" buildSettings { generateFullProject = false @@ -51,26 +64,21 @@ fun configureProjections() { } } } - - tasks.withType { - dependsOn(tasks.generateSmithyProjections) - kotlinOptions.allWarningsAsErrors = false - } } fun configureTasks() { tasks.register("stageServices") { dependsOn(tasks.generateSmithyProjections) - doLast { - this@Build_gradle.projections.forEach { projection -> - val projectionPath = smithyBuild.smithyKotlinProjectionPath(projection.name).get() - val destinationPath = layout.projectDirectory.asFile.absolutePath + "/services/${projection.name}" + this@Build_gradle.tests.forEach { test -> + val projectionPath = smithyBuild.smithyKotlinProjectionPath(test.name).get() + val destinationPath = layout.projectDirectory.asFile.absolutePath + "/services/${test.name}" copy { from("$projectionPath/src") into("$destinationPath/generated-src") } + copy { from("$projectionPath/build.gradle.kts") into(destinationPath) @@ -79,28 +87,19 @@ fun configureTasks() { } } + tasks.withType { + dependsOn(tasks.getByName("stageServices")) + mustRunAfter(tasks.getByName("stageServices")) + } + tasks.build { dependsOn(tasks.getByName("stageServices")) mustRunAfter(tasks.getByName("stageServices")) } tasks.clean { - this@Build_gradle.projections.forEach { projection -> - delete("services/${projection.name}") + this@Build_gradle.tests.forEach { test -> + delete("services/${test.name}") } } - - tasks.withType { - dependsOn(tasks.getByName("stageServices")) - mustRunAfter(tasks.getByName("stageServices")) - } } - -/** - * Holds metadata about a smithy projection - */ -data class Projection( - val name: String, - val modelFile: String, - val serviceShapeId: String, -) From 41ad1b4d2272ff2767edd1c223bbd4cd4811dab1 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 8 Nov 2024 21:50:01 -0500 Subject: [PATCH 12/72] Commonize some more --- .../.kotlin/errors/errors-1731120437202.log | 4 +++ buildSrc/build.gradle.kts | 7 ++++++ buildSrc/settings.gradle.kts | 7 ++++++ .../src/main/kotlin/shared/CodegenTest.kt | 19 ++++++++++++++ tests/codegen/event-stream/build.gradle.kts | 25 ++++--------------- tests/codegen/rules-engine/build.gradle.kts | 21 +++------------- tests/codegen/smoke-tests/build.gradle.kts | 21 +++------------- .../kotlin/SmokeTestE2ETest.kt | 0 .../resources/smoke-tests-exception.smithy | 0 .../resources/smoke-tests-failure.smithy | 0 .../resources/smoke-tests-success.smithy | 0 11 files changed, 50 insertions(+), 54 deletions(-) create mode 100644 buildSrc/.kotlin/errors/errors-1731120437202.log create mode 100644 buildSrc/build.gradle.kts create mode 100644 buildSrc/settings.gradle.kts create mode 100644 buildSrc/src/main/kotlin/shared/CodegenTest.kt rename tests/codegen/smoke-tests/src/{jvmTest => commonTest}/kotlin/SmokeTestE2ETest.kt (100%) rename tests/codegen/smoke-tests/src/{jvmTest => commonTest}/resources/smoke-tests-exception.smithy (100%) rename tests/codegen/smoke-tests/src/{jvmTest => commonTest}/resources/smoke-tests-failure.smithy (100%) rename tests/codegen/smoke-tests/src/{jvmTest => commonTest}/resources/smoke-tests-success.smithy (100%) diff --git a/buildSrc/.kotlin/errors/errors-1731120437202.log b/buildSrc/.kotlin/errors/errors-1731120437202.log new file mode 100644 index 00000000000..1219b509f09 --- /dev/null +++ b/buildSrc/.kotlin/errors/errors-1731120437202.log @@ -0,0 +1,4 @@ +kotlin version: 2.0.21 +error message: The daemon has terminated unexpectedly on startup attempt #1 with error code: 0. The daemon process output: + 1. Kotlin compile daemon is ready + diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 00000000000..aadc709bb1e --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,7 @@ +plugins { + alias(libs.plugins.kotlin.jvm) +} + +repositories { + mavenCentral() +} \ No newline at end of file diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts new file mode 100644 index 00000000000..fa8bc749264 --- /dev/null +++ b/buildSrc/settings.gradle.kts @@ -0,0 +1,7 @@ +dependencyResolutionManagement { + versionCatalogs { + create("libs") { + from(files("../gradle/libs.versions.toml")) + } + } +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/shared/CodegenTest.kt b/buildSrc/src/main/kotlin/shared/CodegenTest.kt new file mode 100644 index 00000000000..8bb596db03d --- /dev/null +++ b/buildSrc/src/main/kotlin/shared/CodegenTest.kt @@ -0,0 +1,19 @@ +package shared + +/** + * An AWS SDK for Kotlin codegen test + */ +data class CodegenTest( + val name: String, + val model: Model, + val serviceShapeId: String, + val protocolName: String? = null, +) + +/** + * A smithy model file + */ +data class Model( + val fileName: String, + val path: String = "src/commonTest/resources/", +) \ No newline at end of file diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index e9fbb56f7b6..0bf44dbd89f 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -5,25 +5,10 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir +import shared.CodegenTest +import shared.Model -description = "AWS SDK for Kotlin codegen event stream integration test suite" - -apply(from = rootProject.file("buildSrc/shared.gradle.kts")) - -data class CodegenTest( - val name: String, - val model: Model, - val serviceShapeId: String, - val protocolName: String? = null, -) - -data class Model( - val fileName: String, - val path: String = "src/commonTest/resources/", -) { - val file: File - get() = layout.projectDirectory.file(path + fileName).asFile -} +description = "AWS SDK for Kotlin's event stream codegen test suite" val tests = listOf( CodegenTest( @@ -43,7 +28,7 @@ val tests = listOf( smithyBuild { tests.forEach { test -> projections.register(test.name) { - imports = listOf(test.model.file.absolutePath) + imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) transforms = listOf( """ { @@ -88,7 +73,7 @@ tasks.generateSmithyBuild { } fun fillInModel(test: CodegenTest) { - val modelFile = test.model.file + val modelFile = layout.projectDirectory.file(test.model.path + test.model.fileName).asFile val model = modelFile.readText() val opTraits = diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index 70d57ec3c8c..c1dd1571cb6 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -4,23 +4,10 @@ */ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import shared.CodegenTest +import shared.Model -description = "AWS SDK for Kotlin codegen rules engine integration test suite" - -data class CodegenTest( - val name: String, - val model: Model, - val serviceShapeId: String, - val protocolName: String? = null, -) - -data class Model( - val fileName: String, - val path: String = "src/commonTest/resources/", -) { - val file: File - get() = layout.projectDirectory.file(path + fileName).asFile -} +description = "AWS SDK for Kotlin's rules engine codegen test suite" val tests = listOf( CodegenTest("operationContextParams", Model("operation-context-params.smithy"), "aws.sdk.kotlin.test#TestService") @@ -29,7 +16,7 @@ val tests = listOf( smithyBuild { tests.forEach { test -> projections.register(test.name) { - imports = listOf(test.model.file.absolutePath) + imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) smithyKotlinPlugin { serviceShapeId = test.serviceShapeId packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 727a0c56fb5..9a5dc15ee07 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -6,8 +6,10 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath +import shared.CodegenTest +import shared.Model -description = "AWS SDK for Kotlin codegen smoke tests integration test suite" +description = "AWS SDK for Kotlin's smoke test codegen test suite" kotlin { sourceSets { @@ -19,21 +21,6 @@ kotlin { } } -data class CodegenTest( - val name: String, - val model: Model, - val serviceShapeId: String, - val protocolName: String? = null, -) - -data class Model( - val fileName: String, - val path: String = "src/jvmTest/resources/", -) { - val file: File - get() = layout.projectDirectory.file(path + fileName).asFile -} - val tests = listOf( CodegenTest("successService", Model("smoke-tests-success.smithy"), "smithy.kotlin.traits#SuccessService"), CodegenTest("failureService", Model("smoke-tests-failure.smithy"), "smithy.kotlin.traits#FailureService"), @@ -47,7 +34,7 @@ fun configureProjections() { smithyBuild { this@Build_gradle.tests.forEach { test -> projections.register(test.name) { - imports = listOf(test.model.file.absolutePath) + imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) smithyKotlinPlugin { serviceShapeId = test.serviceShapeId packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" diff --git a/tests/codegen/smoke-tests/src/jvmTest/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/kotlin/SmokeTestE2ETest.kt rename to tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt diff --git a/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy b/tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-exception.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy rename to tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-exception.smithy diff --git a/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy b/tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-failure.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy rename to tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-failure.smithy diff --git a/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy b/tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-success.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy rename to tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-success.smithy From 1a2d9899c7267fa8d2a919e6b0f5013580d1f800 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 8 Nov 2024 22:29:58 -0500 Subject: [PATCH 13/72] Checkpoint 3 --- tests/codegen/event-stream/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 0bf44dbd89f..1b737eb930b 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -58,7 +58,7 @@ smithyBuild { kotlin { sourceSets { - commonTest { // TODO: CHANGE TO JUST TEST WHEN NON-KMPing the project + commonTest { smithyBuild.projections.forEach { kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) } From 4d24a24bb12561b8c74d7b807727ffa991e38322 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 10 Nov 2024 22:10:08 -0500 Subject: [PATCH 14/72] Remove rules engine codegen tests --- tests/codegen/rules-engine/build.gradle.kts | 35 ----------- .../resources/operation-context-params.smithy | 58 ------------------- 2 files changed, 93 deletions(-) delete mode 100644 tests/codegen/rules-engine/build.gradle.kts delete mode 100644 tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts deleted file mode 100644 index c1dd1571cb6..00000000000 --- a/tests/codegen/rules-engine/build.gradle.kts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import shared.CodegenTest -import shared.Model - -description = "AWS SDK for Kotlin's rules engine codegen test suite" - -val tests = listOf( - CodegenTest("operationContextParams", Model("operation-context-params.smithy"), "aws.sdk.kotlin.test#TestService") -) - -smithyBuild { - tests.forEach { test -> - projections.register(test.name) { - imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) - smithyKotlinPlugin { - serviceShapeId = test.serviceShapeId - packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" - packageVersion = "1.0" - buildSettings { - generateFullProject = false - generateDefaultBuildFiles = false - optInAnnotations = listOf( - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", - ) - } - } - } - } -} diff --git a/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy b/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy deleted file mode 100644 index c63a2756e39..00000000000 --- a/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy +++ /dev/null @@ -1,58 +0,0 @@ -$version: "2.0" -namespace aws.sdk.kotlin.test - -use aws.protocols#awsJson1_0 -use smithy.rules#operationContextParams -use smithy.rules#endpointRuleSet -use aws.api#service - -@awsJson1_0 -@service(sdkId: "OperationContextParamsTest") -@endpointRuleSet( - version: "1.0", - parameters: { - "ObjectKeys": { - "type": "stringArray", - "documentation": "A string array.", - "required": true - } - }, - rules: [ - { - "type": "endpoint", - "conditions": [], - "endpoint": { - "url": "https://static.endpoint" - } - } - ] -) -service TestService { - operations: [DeleteObjects], - version: "1" -} - -@operationContextParams( - ObjectKeys: { - path: "Delete.Objects[*].[Key][]" - } -) -operation DeleteObjects { - input: DeleteObjectsRequest -} - -structure DeleteObjectsRequest { - Delete: Delete -} - -structure Delete { - Objects: ObjectIdentifierList -} - -list ObjectIdentifierList { - member: ObjectIdentifier -} - -structure ObjectIdentifier { - Key: String -} From c514cbaf3c68eaeb83bd304808bc2525a4256204 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 10 Nov 2024 22:18:55 -0500 Subject: [PATCH 15/72] Setup checksums codegen tests --- settings.gradle.kts | 1 + tests/codegen/checksums/build.gradle.kts | 31 +++++ .../src/commonTest/resources/checksums.smithy | 115 ++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 tests/codegen/checksums/build.gradle.kts create mode 100644 tests/codegen/checksums/src/commonTest/resources/checksums.smithy diff --git a/settings.gradle.kts b/settings.gradle.kts index 06a685d8512..dc2418bfe1d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -58,6 +58,7 @@ include(":tests:codegen:rules-engine") include(":tests:e2e-test-util") include(":tests:codegen:smoke-tests") include(":tests:codegen:smoke-tests:services") +include(":tests:codegen:checksums") // generated services val File.isServiceDir: Boolean diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts new file mode 100644 index 00000000000..829c453ad1c --- /dev/null +++ b/tests/codegen/checksums/build.gradle.kts @@ -0,0 +1,31 @@ + +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import shared.CodegenTest +import shared.Model + +description = "AWS SDK for Kotlin's checksums codegen test suite" + +val tests = listOf( + CodegenTest("checksums", Model("checksums.smithy"), "aws.sdk.kotlin.test#TestService"), +) + +smithyBuild { + this@Build_gradle.tests.forEach { test -> + projections.register(test.name) { + imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) + smithyKotlinPlugin { + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" + packageVersion = "1.0" + buildSettings { + generateFullProject = false + generateDefaultBuildFiles = false + optInAnnotations = listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + ) + } + } + } + } +} \ No newline at end of file diff --git a/tests/codegen/checksums/src/commonTest/resources/checksums.smithy b/tests/codegen/checksums/src/commonTest/resources/checksums.smithy new file mode 100644 index 00000000000..01feff05e4a --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/resources/checksums.smithy @@ -0,0 +1,115 @@ +$version: "2" +namespace aws.sdk.kotlin.test + +use aws.api#service +use aws.auth#sigv4 +use aws.protocols#httpChecksum +use aws.protocols#restJson1 +use smithy.rules#endpointRuleSet + +@service(sdkId: "dontcare") +@restJson1 +@sigv4(name: "dontcare") +@auth([sigv4]) +@endpointRuleSet({ + "version": "1.0", + "rules": [{ "type": "endpoint", "conditions": [], "endpoint": { "url": "https://example.com" } }], + "parameters": { + "Region": { "required": false, "type": "String", "builtIn": "AWS::Region" }, + } +}) +service TestService { + version: "2023-01-01", + operations: [HttpChecksumOperation, HttpChecksumStreamingOperation] +} + +@http(uri: "/HttpChecksumOperation", method: "POST") +@optionalAuth +@httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] +) +operation HttpChecksumOperation { + input: SomeInput, + output: SomeOutput +} + +@input +structure SomeInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + + @httpHeader("x-amz-checksum-crc32") + ChecksumCRC32: String + + @httpHeader("x-amz-checksum-crc32c") + ChecksumCRC32C: String + + @httpHeader("x-amz-checksum-crc64nvme") + ChecksumCRC64Nvme: String + + @httpHeader("x-amz-checksum-sha1") + ChecksumSHA1: String + + @httpHeader("x-amz-checksum-sha256") + ChecksumSHA256: String + + @httpHeader("x-amz-checksum-foo") + ChecksumFoo: String + + @httpPayload + @required + body: Blob +} + +@output +structure SomeOutput {} + +@http(uri: "/HttpChecksumStreamingOperation", method: "POST") +@optionalAuth +@httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] +) +operation HttpChecksumStreamingOperation { + input: SomeStreamingInput, + output: SomeStreamingOutput +} + +@streaming +blob StreamingBlob + +@input +structure SomeStreamingInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + + @httpPayload + @required + body: StreamingBlob +} + +@output +structure SomeStreamingOutput {} + +enum ChecksumAlgorithm { + CRC32 + CRC32C + CRC64NVME + SHA1 + SHA256 +} + +enum ValidationMode { + ENABLED +} \ No newline at end of file From 27963179326cdbe0f3fd544fce745acfc45fb4c2 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 10 Nov 2024 22:26:19 -0500 Subject: [PATCH 16/72] Update smithy IDL version, codegen tests pass now --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d003593d62b..b283ae0aaa2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,7 +15,7 @@ smithy-kotlin-runtime-version = "1.3.20" smithy-kotlin-codegen-version = "0.33.20" # codegen -smithy-version = "1.51.0" +smithy-version = "1.52.0" # testing ddb-local-version = "2.5.2" From 6d5324377804df03e4c82abee588df10dd5686ca Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 10 Nov 2024 22:44:39 -0500 Subject: [PATCH 17/72] Setup dummy unit tests --- tests/codegen/checksums/build.gradle.kts | 14 +++++++++++++- .../src/commonTest/kotlin/ChecksumConfigTests.kt | 9 +++++++++ .../src/commonTest/kotlin/ChecksumResponseTests.kt | 9 +++++++++ .../src/commonTest/kotlin/RequestChecksumTests.kt | 9 +++++++++ .../kotlin/StreamingRequestChecksumTests.kt | 9 +++++++++ 5 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index 829c453ad1c..aa1dd9a5413 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -1,5 +1,6 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir import shared.CodegenTest import shared.Model @@ -28,4 +29,15 @@ smithyBuild { } } } -} \ No newline at end of file +} + +// TODO: Commonize this to other codegen tests +kotlin { + sourceSets { + commonTest { + smithyBuild.projections.forEach { + kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) + } + } + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt new file mode 100644 index 00000000000..09447ca6cd9 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt @@ -0,0 +1,9 @@ +import aws.sdk.kotlin.test.checksums.* +import kotlin.test.Test + +class ChecksumConfigTests { + @Test + fun test() { + TestClient {}.use { client -> client.close() } + } +} \ No newline at end of file diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt new file mode 100644 index 00000000000..a261643a544 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt @@ -0,0 +1,9 @@ +import aws.sdk.kotlin.test.checksums.* +import kotlin.test.Test + +class ChecksumResponseTests { + @Test + fun test() { + TestClient {}.use { client -> client.close() } + } +} \ No newline at end of file diff --git a/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt new file mode 100644 index 00000000000..6062830981c --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt @@ -0,0 +1,9 @@ +import aws.sdk.kotlin.test.checksums.* +import kotlin.test.Test + +class RequestChecksumTests { + @Test + fun test() { + TestClient {}.use { client -> client.close() } + } +} \ No newline at end of file diff --git a/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt new file mode 100644 index 00000000000..9c7d178b85a --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt @@ -0,0 +1,9 @@ +import aws.sdk.kotlin.test.checksums.* +import kotlin.test.Test + +class StreamingRequestChecksumTests { + @Test + fun test() { + TestClient {}.use { client -> client.close() } + } +} \ No newline at end of file From 40cfb17fce579de33a9649e9bb08bdf58b560c99 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 10 Nov 2024 23:14:45 -0500 Subject: [PATCH 18/72] Basic setup for client config tests --- tests/codegen/checksums/build.gradle.kts | 3 +- ...sumConfigTests.kt => ClientConfigTests.kt} | 2 +- .../commonTest/resources/client-config.smithy | 55 +++++++++++++++++++ .../{checksums.smithy => kitchen-sink.smithy} | 0 4 files changed, 58 insertions(+), 2 deletions(-) rename tests/codegen/checksums/src/commonTest/kotlin/{ChecksumConfigTests.kt => ClientConfigTests.kt} (84%) create mode 100644 tests/codegen/checksums/src/commonTest/resources/client-config.smithy rename tests/codegen/checksums/src/commonTest/resources/{checksums.smithy => kitchen-sink.smithy} (100%) diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index aa1dd9a5413..159f70e3c7f 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -7,7 +7,8 @@ import shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" val tests = listOf( - CodegenTest("checksums", Model("checksums.smithy"), "aws.sdk.kotlin.test#TestService"), + CodegenTest("checksums", Model("kitchen-sink.smithy"), "aws.sdk.kotlin.test#TestService"), + CodegenTest("clientConfig", Model("client-config.smithy"), "aws.sdk.kotlin.test#ClientConfigTestService"), ) smithyBuild { diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt similarity index 84% rename from tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt rename to tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index 09447ca6cd9..e8b26deeaba 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -1,7 +1,7 @@ import aws.sdk.kotlin.test.checksums.* import kotlin.test.Test -class ChecksumConfigTests { +class ClientConfigTests { @Test fun test() { TestClient {}.use { client -> client.close() } diff --git a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy new file mode 100644 index 00000000000..0782d0336c3 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy @@ -0,0 +1,55 @@ +$version: "2" +namespace aws.sdk.kotlin.test + +use aws.api#service +use aws.auth#sigv4 +use aws.protocols#httpChecksum +use aws.protocols#restJson1 +use smithy.rules#endpointRuleSet + +@service(sdkId: "dontcare") +@restJson1 +@sigv4(name: "dontcare") +@auth([sigv4]) +@endpointRuleSet({ + "version": "1.0", + "rules": [{ "type": "endpoint", "conditions": [], "endpoint": { "url": "https://example.com" } }], + "parameters": { + "Region": { "required": false, "type": "String", "builtIn": "AWS::Region" }, + } +}) +service ClientConfigTestService { + version: "2023-01-01", + operations: [ChecksumsNotRequiredOperation] +} + +@httpChecksum( + requestChecksumRequired: false, + requestAlgorithmMember: "checksumAlgorithm", +) +@http(method: "POST", uri: "/test-eventstream", code: 200) +operation ChecksumsNotRequiredOperation { + input: SomeInput, + output: SomeOutput +} + +@input +structure SomeInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpPayload + @required + body: String +} + +@output +structure SomeOutput {} + +enum ChecksumAlgorithm { + CRC32 + CRC32C + CRC64NVME + SHA1 + SHA256 +} \ No newline at end of file diff --git a/tests/codegen/checksums/src/commonTest/resources/checksums.smithy b/tests/codegen/checksums/src/commonTest/resources/kitchen-sink.smithy similarity index 100% rename from tests/codegen/checksums/src/commonTest/resources/checksums.smithy rename to tests/codegen/checksums/src/commonTest/resources/kitchen-sink.smithy From 410c08578ffd2b7f8bf94738dc0ab3a9d1a4c9ef Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 11 Nov 2024 00:43:57 -0500 Subject: [PATCH 19/72] Added requestChecksumCalculation config option --- aws-runtime/aws-config/api/aws-config.api | 7 +++++ .../config/AbstractAwsSdkClientFactory.kt | 7 +++++ .../kotlin/runtime/config/AwsSdkSetting.kt | 6 ++++ .../ResolveRequestChecksumCalculation.kt | 28 +++++++++++++++++++ .../runtime/config/profile/AwsProfile.kt | 7 +++++ 5 files changed, 55 insertions(+) create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index 05589280306..4b05bd4e8d9 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -243,6 +243,7 @@ public final class aws/sdk/kotlin/runtime/config/AwsSdkSetting { public final fun getAwsMaxAttempts ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsProfile ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRegion ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; + public final fun getAwsRequestChecksumCalculation ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRequestMinCompressionSizeBytes ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRetryMode ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRoleArn ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; @@ -260,6 +261,11 @@ public final class aws/sdk/kotlin/runtime/config/AwsSdkSettingKt { public static final fun resolveEndpointUrl (Laws/sdk/kotlin/runtime/config/AwsSdkSetting;Laws/smithy/kotlin/runtime/util/PlatformProvider;Ljava/lang/String;Ljava/lang/String;)Laws/smithy/kotlin/runtime/net/url/Url; } +public final class aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculationKt { + public static final fun resolveRequestChecksumCalculation (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public static synthetic fun resolveRequestChecksumCalculation$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; +} + public final class aws/sdk/kotlin/runtime/config/compression/RequestCompressionResolversKt { public static final fun resolveDisableRequestCompression (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static synthetic fun resolveDisableRequestCompression$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; @@ -467,6 +473,7 @@ public final class aws/sdk/kotlin/runtime/config/profile/AwsProfileKt { public static synthetic fun getLongOrNull$default (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Ljava/lang/Long; public static final fun getMaxAttempts (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/Integer; public static final fun getRegion (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; + public static final fun getRequestChecksumCalculation (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getRequestMinCompressionSizeBytes (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/Long; public static final fun getRetryMode (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Laws/smithy/kotlin/runtime/client/config/RetryMode; public static final fun getRoleArn (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt index ed5f4b0cd99..662efd8616b 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt @@ -6,6 +6,7 @@ package aws.sdk.kotlin.runtime.config import aws.sdk.kotlin.runtime.client.AwsSdkClientConfig +import aws.sdk.kotlin.runtime.config.checksums.resolveRequestChecksumCalculation import aws.sdk.kotlin.runtime.config.compression.resolveDisableRequestCompression import aws.sdk.kotlin.runtime.config.compression.resolveRequestMinCompressionSizeBytes import aws.sdk.kotlin.runtime.config.endpoints.resolveUseDualStack @@ -22,6 +23,7 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.SigV4aClientConfig import aws.smithy.kotlin.runtime.client.* import aws.smithy.kotlin.runtime.client.config.ClientSettings import aws.smithy.kotlin.runtime.client.config.CompressionClientConfig +import aws.smithy.kotlin.runtime.client.config.HttpChecksumClientConfig import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.telemetry.TelemetryConfig import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider @@ -94,6 +96,11 @@ public abstract class AbstractAwsSdkClientFactory< config.sigV4aSigningRegionSet ?: resolveSigV4aSigningRegionSet(platform, profile) } + if (config is HttpChecksumClientConfig.Builder) { + config.requestChecksumCalculation = + config.requestChecksumCalculation ?: resolveRequestChecksumCalculation(platform, profile) + } + finalizeConfig(builder) finalizeEnvironmentalConfig(builder, sharedConfig, profile) } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt index 4233773f7b8..dff28de934d 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt @@ -208,6 +208,12 @@ public object AwsSdkSetting { */ public val AwsSigV4aSigningRegionSet: EnvironmentSetting = strEnvSetting("aws.sigV4aSigningRegionSet", "AWS_SIGV4A_SIGNING_REGION_SET") + + /** + * todo + */ + public val AwsRequestChecksumCalculation: EnvironmentSetting = + strEnvSetting("aws.requestChecksumCalculation", "AWS_REQUEST_CHECKSUM_CALCULATION") } /** diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt new file mode 100644 index 00000000000..13f5ded6837 --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt @@ -0,0 +1,28 @@ +package aws.sdk.kotlin.runtime.config.checksums + +import aws.sdk.kotlin.runtime.ConfigurationException +import aws.sdk.kotlin.runtime.InternalSdkApi +import aws.sdk.kotlin.runtime.config.AwsSdkSetting +import aws.sdk.kotlin.runtime.config.profile.AwsProfile +import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation +import aws.smithy.kotlin.runtime.client.config.RequestChecksumCalculation +import aws.smithy.kotlin.runtime.config.resolve +import aws.smithy.kotlin.runtime.util.LazyAsyncValue +import aws.smithy.kotlin.runtime.util.PlatformProvider +import java.util.* + +/** + * todo + */ +@InternalSdkApi +public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): RequestChecksumCalculation? { + AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation?.let { + try { + return RequestChecksumCalculation.valueOf(it.uppercase(Locale.getDefault())) + } catch (_: IllegalArgumentException) { + throw ConfigurationException("'$it' is not a valid value for request checksum calculation. Valid values are: 'WHEN_SUPPORTED' & 'WHEN_REQUIRED'") + } + } + return null +} + diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index 8a4e31016c5..487afda2da3 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -167,6 +167,13 @@ public val AwsProfile.requestMinCompressionSizeBytes: Long? public val AwsProfile.sigV4aSigningRegionSet: String? get() = getOrNull("sigv4a_signing_region_set") +/** + * todo + */ +@InternalSdkApi +public val AwsProfile.requestChecksumCalculation: String? + get() = getOrNull("request_checksum_calculation") + /** * Parse a config value as a boolean, ignoring case. */ From 472fae966861c037582591c750d30fa3db9777a3 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 11 Nov 2024 01:27:21 -0500 Subject: [PATCH 20/72] RequestChecksum not required client config tests --- tests/codegen/checksums/build.gradle.kts | 10 +++ .../commonTest/kotlin/ClientConfigTests.kt | 64 +++++++++++++++++-- .../commonTest/resources/client-config.smithy | 2 +- 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index 159f70e3c7f..b094022daac 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -6,6 +6,16 @@ import shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" +kotlin { + sourceSets { + commonTest { + dependencies { + implementation(libs.smithy.kotlin.http.test) + } + } + } +} + val tests = listOf( CodegenTest("checksums", Model("kitchen-sink.smithy"), "aws.sdk.kotlin.test#TestService"), CodegenTest("clientConfig", Model("client-config.smithy"), "aws.sdk.kotlin.test#ClientConfigTestService"), diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index e8b26deeaba..ef4475f9836 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -1,9 +1,65 @@ -import aws.sdk.kotlin.test.checksums.* +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.test.clientconfig.* +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.client.config.RequestChecksumCalculation +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import kotlinx.coroutines.runBlocking import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import aws.smithy.kotlin.runtime.httptest.TestEngine class ClientConfigTests { @Test - fun test() { - TestClient {}.use { client -> client.close() } + fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { + val testInterceptor = TestInterceptor() + + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } + } + + assertTrue(testInterceptor.containsChecksum) + } + + @Test + fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { + val testInterceptor = TestInterceptor() + + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } + } + + assertFalse(testInterceptor.containsChecksum) + } +} + +private class TestInterceptor : HttpInterceptor { + var containsChecksum = false + + override suspend fun modifyBeforeTransmit(context: ProtocolRequestInterceptorContext): HttpRequest { + containsChecksum = context.protocolRequest.headers.contains("x-amz-checksum-crc32") + return context.protocolRequest } -} \ No newline at end of file +} diff --git a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy index 0782d0336c3..7323601f050 100644 --- a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy +++ b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy @@ -27,7 +27,7 @@ service ClientConfigTestService { requestChecksumRequired: false, requestAlgorithmMember: "checksumAlgorithm", ) -@http(method: "POST", uri: "/test-eventstream", code: 200) +@http(method: "POST", uri: "/test-checksums", code: 200) operation ChecksumsNotRequiredOperation { input: SomeInput, output: SomeOutput From e9bf8e5be22dcba69163cc8e6dbc4a419e5ff6c7 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 11 Nov 2024 01:48:07 -0500 Subject: [PATCH 21/72] RequestChecksum required client config tests --- .../commonTest/kotlin/ClientConfigTests.kt | 113 +++++++++++++----- .../commonTest/resources/client-config.smithy | 25 +++- 2 files changed, 107 insertions(+), 31 deletions(-) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index ef4475f9836..ba456125d8a 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -10,48 +10,101 @@ import kotlin.test.Test import kotlin.test.assertFalse import kotlin.test.assertTrue import aws.smithy.kotlin.runtime.httptest.TestEngine +import org.junit.jupiter.api.Nested +import kotlin.test.Ignore class ClientConfigTests { - @Test - fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + @Nested + inner class RequestChecksumNotRequired { + @Test + @Ignore // todo: un-ignore + fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { + val testInterceptor = TestInterceptor() - ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED - interceptors = mutableListOf(testInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } } + + assertTrue(testInterceptor.containsChecksum) } - assertTrue(testInterceptor.containsChecksum) + @Test + fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { + val testInterceptor = TestInterceptor() + + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } + } + + assertFalse(testInterceptor.containsChecksum) + } } - @Test - fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + @Nested + inner class RequestChecksumRequired { + @Test + @Ignore // todo: un-ignore + fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { + val testInterceptor = TestInterceptor() - ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED - interceptors = mutableListOf(testInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + } } + + assertTrue(testInterceptor.containsChecksum) } - assertFalse(testInterceptor.containsChecksum) + @Test + @Ignore // todo: un-ignore + fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { + val testInterceptor = TestInterceptor() + + ClientConfigTestClient { + requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED + interceptors = mutableListOf(testInterceptor) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + } + } + + assertTrue(testInterceptor.containsChecksum) + } } } diff --git a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy index 7323601f050..20db2263102 100644 --- a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy +++ b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy @@ -20,7 +20,7 @@ use smithy.rules#endpointRuleSet }) service ClientConfigTestService { version: "2023-01-01", - operations: [ChecksumsNotRequiredOperation] + operations: [ChecksumsNotRequiredOperation, ChecksumsRequiredOperation] } @httpChecksum( @@ -33,6 +33,16 @@ operation ChecksumsNotRequiredOperation { output: SomeOutput } +@httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", +) +@http(method: "POST", uri: "/test-checksums-2", code: 200) +operation ChecksumsRequiredOperation { + input: AnotherInput, + output: AnotherOutput +} + @input structure SomeInput { @httpHeader("x-amz-request-algorithm") @@ -46,6 +56,19 @@ structure SomeInput { @output structure SomeOutput {} +@input +structure AnotherInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpPayload + @required + body: String +} + +@output +structure AnotherOutput {} + enum ChecksumAlgorithm { CRC32 CRC32C From 5133a60c45d1281e0ae8fe5cd60a9d5beee930b3 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 11 Nov 2024 02:51:04 -0500 Subject: [PATCH 22/72] Added responseChecksumValidation --- aws-runtime/aws-config/api/aws-config.api | 7 +++++ .../config/AbstractAwsSdkClientFactory.kt | 4 +++ .../kotlin/runtime/config/AwsSdkSetting.kt | 6 ++++ .../ResolveRequestChecksumCalculation.kt | 11 +++---- .../ResolveResponseChecksumValidation.kt | 29 +++++++++++++++++++ .../runtime/config/profile/AwsProfile.kt | 7 +++++ buildSrc/build.gradle.kts | 2 +- buildSrc/settings.gradle.kts | 2 +- .../src/main/kotlin/shared/CodegenTest.kt | 2 +- .../kotlin/ChecksumResponseTests.kt | 2 +- .../commonTest/kotlin/ClientConfigTests.kt | 16 +++++----- .../commonTest/kotlin/RequestChecksumTests.kt | 2 +- .../kotlin/StreamingRequestChecksumTests.kt | 2 +- tests/codegen/event-stream/build.gradle.kts | 4 +-- 14 files changed, 75 insertions(+), 21 deletions(-) create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index 4b05bd4e8d9..6a9be2adba5 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -245,6 +245,7 @@ public final class aws/sdk/kotlin/runtime/config/AwsSdkSetting { public final fun getAwsRegion ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRequestChecksumCalculation ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRequestMinCompressionSizeBytes ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; + public final fun getAwsResponseChecksumValidation ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRetryMode ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRoleArn ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; public final fun getAwsRoleSessionName ()Laws/smithy/kotlin/runtime/config/EnvironmentSetting; @@ -266,6 +267,11 @@ public final class aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksu public static synthetic fun resolveRequestChecksumCalculation$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; } +public final class aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidationKt { + public static final fun resolveResponseChecksumValidation (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public static synthetic fun resolveResponseChecksumValidation$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; +} + public final class aws/sdk/kotlin/runtime/config/compression/RequestCompressionResolversKt { public static final fun resolveDisableRequestCompression (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static synthetic fun resolveDisableRequestCompression$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; @@ -475,6 +481,7 @@ public final class aws/sdk/kotlin/runtime/config/profile/AwsProfileKt { public static final fun getRegion (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getRequestChecksumCalculation (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getRequestMinCompressionSizeBytes (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/Long; + public static final fun getResponseChecksumValidation (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getRetryMode (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Laws/smithy/kotlin/runtime/client/config/RetryMode; public static final fun getRoleArn (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getSdkUserAgentAppId (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt index 662efd8616b..190f41a0b12 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt @@ -7,6 +7,7 @@ package aws.sdk.kotlin.runtime.config import aws.sdk.kotlin.runtime.client.AwsSdkClientConfig import aws.sdk.kotlin.runtime.config.checksums.resolveRequestChecksumCalculation +import aws.sdk.kotlin.runtime.config.checksums.resolveResponseChecksumValidation import aws.sdk.kotlin.runtime.config.compression.resolveDisableRequestCompression import aws.sdk.kotlin.runtime.config.compression.resolveRequestMinCompressionSizeBytes import aws.sdk.kotlin.runtime.config.endpoints.resolveUseDualStack @@ -99,6 +100,9 @@ public abstract class AbstractAwsSdkClientFactory< if (config is HttpChecksumClientConfig.Builder) { config.requestChecksumCalculation = config.requestChecksumCalculation ?: resolveRequestChecksumCalculation(platform, profile) + + config.responseChecksumValidation = + config.responseChecksumValidation ?: resolveResponseChecksumValidation(platform, profile) } finalizeConfig(builder) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt index dff28de934d..9fe989f58ed 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt @@ -214,6 +214,12 @@ public object AwsSdkSetting { */ public val AwsRequestChecksumCalculation: EnvironmentSetting = strEnvSetting("aws.requestChecksumCalculation", "AWS_REQUEST_CHECKSUM_CALCULATION") + + /** + * todo + */ + public val AwsResponseChecksumValidation: EnvironmentSetting = + strEnvSetting("aws.responseChecksumValidation", "AWS_RESPONSE_CHECKSUM_VALIDATION") } /** diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt index 13f5ded6837..a06d61103c2 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt @@ -5,7 +5,7 @@ import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.config.AwsSdkSetting import aws.sdk.kotlin.runtime.config.profile.AwsProfile import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation -import aws.smithy.kotlin.runtime.client.config.RequestChecksumCalculation +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.util.LazyAsyncValue import aws.smithy.kotlin.runtime.util.PlatformProvider @@ -15,14 +15,15 @@ import java.util.* * todo */ @InternalSdkApi -public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): RequestChecksumCalculation? { +public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption? { AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation?.let { try { - return RequestChecksumCalculation.valueOf(it.uppercase(Locale.getDefault())) + return ChecksumConfigOption.valueOf(it.uppercase(Locale.getDefault())) } catch (_: IllegalArgumentException) { - throw ConfigurationException("'$it' is not a valid value for request checksum calculation. Valid values are: 'WHEN_SUPPORTED' & 'WHEN_REQUIRED'") + throw ConfigurationException( + "'$it' is not a valid value for request checksum calculation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", + ) } } return null } - diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt new file mode 100644 index 00000000000..446480d527e --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt @@ -0,0 +1,29 @@ +package aws.sdk.kotlin.runtime.config.checksums + +import aws.sdk.kotlin.runtime.ConfigurationException +import aws.sdk.kotlin.runtime.InternalSdkApi +import aws.sdk.kotlin.runtime.config.AwsSdkSetting +import aws.sdk.kotlin.runtime.config.profile.AwsProfile +import aws.sdk.kotlin.runtime.config.profile.responseChecksumValidation +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.config.resolve +import aws.smithy.kotlin.runtime.util.LazyAsyncValue +import aws.smithy.kotlin.runtime.util.PlatformProvider +import java.util.* + +/** + * todo + */ +@InternalSdkApi +public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption? { + AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation?.let { + try { + return ChecksumConfigOption.valueOf(it.uppercase(Locale.getDefault())) + } catch (_: IllegalArgumentException) { + throw ConfigurationException( + "'$it' is not a valid value for response checksum validation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", + ) + } + } + return null +} diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index 487afda2da3..472228604a9 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -174,6 +174,13 @@ public val AwsProfile.sigV4aSigningRegionSet: String? public val AwsProfile.requestChecksumCalculation: String? get() = getOrNull("request_checksum_calculation") +/** + * todo + */ +@InternalSdkApi +public val AwsProfile.responseChecksumValidation: String? + get() = getOrNull("response_checksum_validation") + /** * Parse a config value as a boolean, ignoring case. */ diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index aadc709bb1e..dccd6941e76 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -4,4 +4,4 @@ plugins { repositories { mavenCentral() -} \ No newline at end of file +} diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts index fa8bc749264..b5a0fabf664 100644 --- a/buildSrc/settings.gradle.kts +++ b/buildSrc/settings.gradle.kts @@ -4,4 +4,4 @@ dependencyResolutionManagement { from(files("../gradle/libs.versions.toml")) } } -} \ No newline at end of file +} diff --git a/buildSrc/src/main/kotlin/shared/CodegenTest.kt b/buildSrc/src/main/kotlin/shared/CodegenTest.kt index 8bb596db03d..f9a03d715e9 100644 --- a/buildSrc/src/main/kotlin/shared/CodegenTest.kt +++ b/buildSrc/src/main/kotlin/shared/CodegenTest.kt @@ -16,4 +16,4 @@ data class CodegenTest( data class Model( val fileName: String, val path: String = "src/commonTest/resources/", -) \ No newline at end of file +) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt index a261643a544..fd3c8965828 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt @@ -6,4 +6,4 @@ class ChecksumResponseTests { fun test() { TestClient {}.use { client -> client.close() } } -} \ No newline at end of file +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index ba456125d8a..d98a3f6a6ba 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -2,16 +2,16 @@ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.clientconfig.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext -import aws.smithy.kotlin.runtime.client.config.RequestChecksumCalculation +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Nested +import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertFalse import kotlin.test.assertTrue -import aws.smithy.kotlin.runtime.httptest.TestEngine -import org.junit.jupiter.api.Nested -import kotlin.test.Ignore class ClientConfigTests { @Nested @@ -22,7 +22,7 @@ class ClientConfigTests { val testInterceptor = TestInterceptor() ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED + requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED interceptors = mutableListOf(testInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -43,7 +43,7 @@ class ClientConfigTests { val testInterceptor = TestInterceptor() ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED + requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED interceptors = mutableListOf(testInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -68,7 +68,7 @@ class ClientConfigTests { val testInterceptor = TestInterceptor() ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_SUPPORTED + requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED interceptors = mutableListOf(testInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -90,7 +90,7 @@ class ClientConfigTests { val testInterceptor = TestInterceptor() ClientConfigTestClient { - requestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED + requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED interceptors = mutableListOf(testInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( diff --git a/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt index 6062830981c..15d8764dc2c 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt @@ -6,4 +6,4 @@ class RequestChecksumTests { fun test() { TestClient {}.use { client -> client.close() } } -} \ No newline at end of file +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt index 9c7d178b85a..25d8210cf10 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt @@ -6,4 +6,4 @@ class StreamingRequestChecksumTests { fun test() { TestClient {}.use { client -> client.close() } } -} \ No newline at end of file +} diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 1b737eb930b..71edc4f41fb 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -15,13 +15,13 @@ val tests = listOf( "restJson1", Model("event-stream-model-template.smithy"), "aws.sdk.kotlin.test#TestService", - "restJson1" + "restJson1", ), CodegenTest( "awsJson11", Model("event-stream-initial-request-response.smithy"), "aws.sdk.kotlin.test#TestService", - "awsJson1_1" + "awsJson1_1", ), ) From 4ce5da7788867e69db8829e07f102d95e952f1ee Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 11 Nov 2024 11:34:26 -0500 Subject: [PATCH 23/72] TODO ResponseChecksumValidation tests --- .../checksums/src/commonTest/kotlin/ClientConfigTests.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index d98a3f6a6ba..3770cfd6ee5 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -106,6 +106,11 @@ class ClientConfigTests { assertTrue(testInterceptor.containsChecksum) } } + + @Nested + inner class ResponseChecksumValidation { + // TODO + } } private class TestInterceptor : HttpInterceptor { From da31deb21a4b33fa0fb5eebbdcaffb6deed7e0ab Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 12 Nov 2024 02:16:45 -0500 Subject: [PATCH 24/72] Added ResponseChecksumValidation codegen tests --- .../commonTest/kotlin/ClientConfigTests.kt | 151 +++++++++++++++--- .../commonTest/resources/client-config.smithy | 31 ++-- 2 files changed, 152 insertions(+), 30 deletions(-) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index 3770cfd6ee5..aeebef9d6f4 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -1,11 +1,16 @@ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.clientconfig.* +import aws.sdk.kotlin.test.clientconfig.model.ValidationMode import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.http.* +import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.time.Instant import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Nested import kotlin.test.Ignore @@ -18,12 +23,12 @@ class ClientConfigTests { inner class RequestChecksumNotRequired { @Test @Ignore // todo: un-ignore - fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + fun requestChecksumNotRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { + val requestInterceptor = RequestInterceptor() ClientConfigTestClient { requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(testInterceptor) + interceptors = mutableListOf(requestInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( Credentials("accessKeyID", "secretAccessKey"), @@ -35,16 +40,16 @@ class ClientConfigTests { } } - assertTrue(testInterceptor.containsChecksum) + assertTrue(requestInterceptor.containsChecksum) } @Test - fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + fun requestChecksumNotRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { + val requestInterceptor = RequestInterceptor() ClientConfigTestClient { requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(testInterceptor) + interceptors = mutableListOf(requestInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( Credentials("accessKeyID", "secretAccessKey"), @@ -56,7 +61,7 @@ class ClientConfigTests { } } - assertFalse(testInterceptor.containsChecksum) + assertFalse(requestInterceptor.containsChecksum) } } @@ -64,12 +69,12 @@ class ClientConfigTests { inner class RequestChecksumRequired { @Test @Ignore // todo: un-ignore - fun requestChecksumCalculationWhenSupported(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + fun requestChecksumRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { + val requestInterceptor = RequestInterceptor() ClientConfigTestClient { requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(testInterceptor) + interceptors = mutableListOf(requestInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( Credentials("accessKeyID", "secretAccessKey"), @@ -81,17 +86,17 @@ class ClientConfigTests { } } - assertTrue(testInterceptor.containsChecksum) + assertTrue(requestInterceptor.containsChecksum) } @Test @Ignore // todo: un-ignore - fun requestChecksumCalculationWhenRequired(): Unit = runBlocking { - val testInterceptor = TestInterceptor() + fun requestChecksumRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { + val requestInterceptor = RequestInterceptor() ClientConfigTestClient { requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(testInterceptor) + interceptors = mutableListOf(requestInterceptor) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( Credentials("accessKeyID", "secretAccessKey"), @@ -103,21 +108,129 @@ class ClientConfigTests { } } - assertTrue(testInterceptor.containsChecksum) + assertTrue(requestInterceptor.containsChecksum) } } @Nested inner class ResponseChecksumValidation { - // TODO + @Test + @Ignore // todo - unignore + fun responseChecksumValidationResponseChecksumValidationWhenSupported(): Unit = runBlocking { + var responseChecksumValidated = false + + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "bogus") + }, + "Goodbye!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + } + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + try { + client.checksumsRequiredOperation { + body = "Hello World!" + } + } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum + responseChecksumValidated = true + } + } + + assertTrue(responseChecksumValidated) + } + + @Test + fun responseChecksumValidationResponseChecksumValidationWhenRequired(): Unit = runBlocking { + var responseChecksumValidated = false + + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "bogus") + }, + "Goodbye!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + } + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + try { + client.checksumsRequiredOperation { + body = "Hello World!" + } + } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum + responseChecksumValidated = true + } + } + + assertFalse(responseChecksumValidated) + } + + @Test + @Ignore // todo - unignore + fun responseChecksumValidationResponseChecksumValidationWhenRequiredWithRequestValidationModeMember(): Unit = runBlocking { + var responseChecksumValidated = false + + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "bogus") + }, + "Goodbye!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + } + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + try { + client.checksumsRequiredOperation { + body = "Hello World!" + } + } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum + responseChecksumValidated = true + } + } + + assertTrue(responseChecksumValidated) + } } } -private class TestInterceptor : HttpInterceptor { +private class RequestInterceptor : HttpInterceptor { var containsChecksum = false override suspend fun modifyBeforeTransmit(context: ProtocolRequestInterceptorContext): HttpRequest { - containsChecksum = context.protocolRequest.headers.contains("x-amz-checksum-crc32") + containsChecksum = context.protocolRequest.headers.contains("x-amz-checksum-crc32") // default checksum algorithm return context.protocolRequest } } diff --git a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy index 20db2263102..e1d1f24e21d 100644 --- a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy +++ b/tests/codegen/checksums/src/commonTest/resources/client-config.smithy @@ -33,16 +33,6 @@ operation ChecksumsNotRequiredOperation { output: SomeOutput } -@httpChecksum( - requestChecksumRequired: true, - requestAlgorithmMember: "checksumAlgorithm", -) -@http(method: "POST", uri: "/test-checksums-2", code: 200) -operation ChecksumsRequiredOperation { - input: AnotherInput, - output: AnotherOutput -} - @input structure SomeInput { @httpHeader("x-amz-request-algorithm") @@ -56,11 +46,26 @@ structure SomeInput { @output structure SomeOutput {} +@httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32"] +) +@http(method: "POST", uri: "/test-checksums-2", code: 200) +operation ChecksumsRequiredOperation { + input: AnotherInput, + output: AnotherOutput +} + @input structure AnotherInput { @httpHeader("x-amz-request-algorithm") checksumAlgorithm: ChecksumAlgorithm + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + @httpPayload @required body: String @@ -75,4 +80,8 @@ enum ChecksumAlgorithm { CRC64NVME SHA1 SHA256 -} \ No newline at end of file +} + +enum ValidationMode { + ENABLED +} From 6037580603d7e24f1accf48db3af28edaab43aba Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 12 Nov 2024 02:19:32 -0500 Subject: [PATCH 25/72] Quick self review --- .../src/commonTest/kotlin/ClientConfigTests.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index aeebef9d6f4..edd61d974ce 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -128,7 +128,7 @@ class ClientConfigTests { Headers { append("x-amz-checksum-crc32", "bogus") }, - "Goodbye!".toHttpBody(), + "World!".toHttpBody(), ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -141,7 +141,7 @@ class ClientConfigTests { }.use { client -> try { client.checksumsRequiredOperation { - body = "Hello World!" + body = "Hello" } } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum responseChecksumValidated = true @@ -164,7 +164,7 @@ class ClientConfigTests { Headers { append("x-amz-checksum-crc32", "bogus") }, - "Goodbye!".toHttpBody(), + "World!".toHttpBody(), ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -177,7 +177,7 @@ class ClientConfigTests { }.use { client -> try { client.checksumsRequiredOperation { - body = "Hello World!" + body = "Hello" } } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum responseChecksumValidated = true @@ -201,7 +201,7 @@ class ClientConfigTests { Headers { append("x-amz-checksum-crc32", "bogus") }, - "Goodbye!".toHttpBody(), + "World!".toHttpBody(), ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -214,7 +214,7 @@ class ClientConfigTests { }.use { client -> try { client.checksumsRequiredOperation { - body = "Hello World!" + body = "Hello" } } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum responseChecksumValidated = true From 2f6e10ba3d1e18a754cc9417118aad00ffe28324 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 12 Nov 2024 02:23:37 -0500 Subject: [PATCH 26/72] Self review V2 --- .../checksums/src/commonTest/kotlin/ClientConfigTests.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt index edd61d974ce..ddeb01a26b0 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt @@ -189,7 +189,7 @@ class ClientConfigTests { @Test @Ignore // todo - unignore - fun responseChecksumValidationResponseChecksumValidationWhenRequiredWithRequestValidationModeMember(): Unit = runBlocking { + fun responseChecksumValidationResponseChecksumValidationWhenRequiredWithRequestValidationModeEnabled(): Unit = runBlocking { var responseChecksumValidated = false ClientConfigTestClient { @@ -215,6 +215,7 @@ class ClientConfigTests { try { client.checksumsRequiredOperation { body = "Hello" + validationMode = ValidationMode.Enabled } } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum responseChecksumValidated = true From d085dec970a29f20aa6fb438c6ebf5189eae24e8 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 12 Nov 2024 02:27:17 -0500 Subject: [PATCH 27/72] Add todos for business metrics --- .../sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt index 190f41a0b12..1732b398459 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt @@ -98,9 +98,11 @@ public abstract class AbstractAwsSdkClientFactory< } if (config is HttpChecksumClientConfig.Builder) { + // TODO - business metric config.requestChecksumCalculation = config.requestChecksumCalculation ?: resolveRequestChecksumCalculation(platform, profile) + // TODO - business metric config.responseChecksumValidation = config.responseChecksumValidation ?: resolveResponseChecksumValidation(platform, profile) } From 0bafdcccd809a9e524e957028cb074ccafdb796a Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 25 Nov 2024 10:05:41 -0500 Subject: [PATCH 28/72] Unit tests pass --- .../config/AbstractAwsSdkClientFactory.kt | 2 - .../ResolveRequestChecksumCalculation.kt | 16 +- .../ResolveResponseChecksumValidation.kt | 18 +- .../FlexibleChecksumsRequest.kt | 82 +++- .../FlexibleChecksumsResponse.kt | 56 ++- .../e2eTest/src/MutliRegionAccessPointTest.kt | 222 +++++----- services/s3/e2eTest/src/PaginatorTest.kt | 3 +- services/s3/e2eTest/src/S3ChecksumTest.kt | 58 +++ services/s3/e2eTest/src/S3IntegrationTest.kt | 2 +- services/s3/e2eTest/src/S3PresignerTest.kt | 2 +- services/s3/e2eTest/src/S3TestUtils.kt | 61 ++- tests/codegen/checksums/build.gradle.kts | 5 +- .../kotlin/ChecksumBusinessMetricsTest.kt | 163 ++++++++ .../commonTest/kotlin/ChecksumConfigTest.kt | 387 ++++++++++++++++++ .../commonTest/kotlin/ChecksumRequestTest.kt | 126 ++++++ .../commonTest/kotlin/ChecksumResponseTest.kt | 243 +++++++++++ .../kotlin/ChecksumResponseTests.kt | 9 - .../kotlin/ChecksumStreamingRequestTest.kt | 155 +++++++ .../commonTest/kotlin/ClientConfigTests.kt | 237 ----------- .../commonTest/kotlin/RequestChecksumTests.kt | 9 - .../kotlin/StreamingRequestChecksumTests.kt | 9 - .../kotlin/utils/ChecksumTestUtils.kt | 92 +++++ ...lient-config.smithy => config-test.smithy} | 0 ...nk.smithy => request-response-test.smithy} | 2 +- tests/codegen/smoke-tests/build.gradle.kts | 1 + .../src/commonTest/kotlin/SmokeTestE2ETest.kt | 7 + 26 files changed, 1542 insertions(+), 425 deletions(-) create mode 100644 services/s3/e2eTest/src/S3ChecksumTest.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt delete mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt delete mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt delete mode 100644 tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt delete mode 100644 tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt create mode 100644 tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt rename tests/codegen/checksums/src/commonTest/resources/{client-config.smithy => config-test.smithy} (100%) rename tests/codegen/checksums/src/commonTest/resources/{kitchen-sink.smithy => request-response-test.smithy} (99%) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt index 1732b398459..190f41a0b12 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt @@ -98,11 +98,9 @@ public abstract class AbstractAwsSdkClientFactory< } if (config is HttpChecksumClientConfig.Builder) { - // TODO - business metric config.requestChecksumCalculation = config.requestChecksumCalculation ?: resolveRequestChecksumCalculation(platform, profile) - // TODO - business metric config.responseChecksumValidation = config.responseChecksumValidation ?: resolveResponseChecksumValidation(platform, profile) } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt index a06d61103c2..d731b5d3006 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt @@ -15,15 +15,15 @@ import java.util.* * todo */ @InternalSdkApi -public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption? { - AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation?.let { - try { - return ChecksumConfigOption.valueOf(it.uppercase(Locale.getDefault())) - } catch (_: IllegalArgumentException) { - throw ConfigurationException( +public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption { + val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation + return unparsedString?.let { + when (unparsedString.uppercase()) { + "WHEN_SUPPORTED" -> ChecksumConfigOption.WHEN_SUPPORTED + "WHEN_REQUIRED" -> ChecksumConfigOption.WHEN_REQUIRED + else -> throw ConfigurationException( "'$it' is not a valid value for request checksum calculation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", ) } - } - return null + } ?: ChecksumConfigOption.WHEN_SUPPORTED } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt index 446480d527e..dbf4aa9b377 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt @@ -15,15 +15,15 @@ import java.util.* * todo */ @InternalSdkApi -public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption? { - AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation?.let { - try { - return ChecksumConfigOption.valueOf(it.uppercase(Locale.getDefault())) - } catch (_: IllegalArgumentException) { - throw ConfigurationException( - "'$it' is not a valid value for response checksum validation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", +public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption { + val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation + return unparsedString?.let { + when (unparsedString.uppercase()) { + "WHEN_SUPPORTED" -> ChecksumConfigOption.WHEN_SUPPORTED + "WHEN_REQUIRED" -> ChecksumConfigOption.WHEN_REQUIRED + else -> throw ConfigurationException( + "'$it' is not a valid value for request checksum calculation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", ) } - } - return null + } ?: ChecksumConfigOption.WHEN_SUPPORTED } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 8799ac5364e..f00838fca03 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -6,6 +6,7 @@ package aws.sdk.kotlin.codegen.customization.flexiblechecksums import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.CodegenContext import software.amazon.smithy.kotlin.codegen.core.KotlinWriter import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes import software.amazon.smithy.kotlin.codegen.core.withBlock @@ -13,10 +14,15 @@ import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.model.* import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.model.shapes.StructureShape +import software.amazon.smithy.model.traits.HttpPayloadTrait +import software.amazon.smithy.model.traits.StreamingTrait /** * Adds a middleware that enables sending flexible checksums during an HTTP request @@ -26,8 +32,61 @@ class FlexibleChecksumsRequest : KotlinIntegration { .shapes() .any { it.hasTrait() } + override fun additionalServiceConfigProps(ctx: CodegenContext): List = + listOf( + ConfigProperty { + name = "requestChecksumCalculation" + symbol = RuntimeTypes.SmithyClient.Config.ChecksumConfigOption + baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig + useNestedBuilderBaseClass() + documentation = "" // todo + propertyType = ConfigPropertyType.RequiredWithDefault("ChecksumConfigOption.WHEN_SUPPORTED") + }, + ) + + private val operationsWithStreamingPayloads = mutableListOf() + + override fun preprocessModel(model: Model, settings: KotlinSettings): Model { + model.operationShapes.forEach { operationShape -> + + val operationInput = model.expectShape(operationShape.inputShape) + + operationInput.members().find { it.hasTrait() }?.let { httpPayload -> + if (model.getShape(httpPayload.target).get().hasTrait()) { + operationsWithStreamingPayloads.add(operationShape.id) + } + } + } + + return model + } + override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + flexibleChecksumsRequestMiddleware + resolved + flexibleChecksumsRequestMiddleware + configBusinessMetrics + + private val configBusinessMetrics = object : ProtocolMiddleware { + override val name: String = "requestChecksumCalculationBusinessMetric" + + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + op.hasTrait() + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.withBlock("when(config.requestChecksumCalculation) {", "}") { + writer.write( + "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED)", + RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + writer.write( + "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED)", + RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + } + } + } private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { override val name: String = "FlexibleChecksumsRequest" @@ -42,22 +101,25 @@ class FlexibleChecksumsRequest : KotlinIntegration { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val interceptorSymbol = RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) - val httpChecksumTrait = op.getTrait()!! val requestAlgorithmMember = ctx.model.expectShape(op.input.get()) .members() .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } - val requestAlgorithmMemberName = ctx.symbolProvider.toMemberName(requestAlgorithmMember) + val userSelectedChecksumAlgorithm = ctx.symbolProvider.toMemberName(requestAlgorithmMember) + val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired + val streamingPayload = operationsWithStreamingPayloads.contains(op.id) - writer.withBlock("op.interceptors.add(#T<#T>() {", "})", interceptorSymbol, inputSymbol) { - writer.write("input.#L?.value", requestAlgorithmMemberName) - } - writer.withBlock("input.#L?.let {", "}", requestAlgorithmMemberName) { - writer.write("op.context[#T.ChecksumAlgorithm] = it.value", RuntimeTypes.HttpClient.Operation.HttpOperationContext) + writer.withBlock( + "op.interceptors.add(#T(", + "))", + RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor, + ) { + writer.write("requestChecksumRequired = #L,", requestChecksumRequired) + writer.write("requestChecksumCalculation = config.requestChecksumCalculation,") + writer.write("userSelectedChecksumAlgorithm = input.#L?.value,", userSelectedChecksumAlgorithm) + writer.write("streamingPayload = #L,", streamingPayload) } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index c641765a3c3..95f27f24fae 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -6,14 +6,13 @@ package aws.sdk.kotlin.codegen.customization.flexiblechecksums import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings -import software.amazon.smithy.kotlin.codegen.core.KotlinWriter -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes -import software.amazon.smithy.kotlin.codegen.core.defaultName -import software.amazon.smithy.kotlin.codegen.core.withBlock +import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.model.* import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape @@ -27,8 +26,41 @@ class FlexibleChecksumsResponse : KotlinIntegration { .shapes() .any { it.hasTrait() } + override fun additionalServiceConfigProps(ctx: CodegenContext): List = + listOf( + ConfigProperty { + name = "responseChecksumValidation" + symbol = RuntimeTypes.SmithyClient.Config.ChecksumConfigOption + baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig + useNestedBuilderBaseClass() + documentation = "" // todo + propertyType = ConfigPropertyType.RequiredWithDefault("ChecksumConfigOption.WHEN_SUPPORTED") + }, + ) + override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + flexibleChecksumsResponseMiddleware + resolved + flexibleChecksumsResponseMiddleware + configBusinessMetrics + + private val configBusinessMetrics = object : ProtocolMiddleware { + override val name: String = "responseChecksumValidationBusinessMetric" + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.withBlock("when(config.responseChecksumValidation) {", "}") { + writer.write( + "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED)", + RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + writer.write( + "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED)", + RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + } + } + } private val flexibleChecksumsResponseMiddleware = object : ProtocolMiddleware { override val name: String = "FlexibleChecksumsResponse" @@ -43,21 +75,19 @@ class FlexibleChecksumsResponse : KotlinIntegration { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) - val interceptorSymbol = RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor - val httpChecksumTrait = op.getTrait()!! val requestValidationModeMember = ctx.model.expectShape(op.input.get()) .members() .first { it.memberName == httpChecksumTrait.requestValidationModeMember.get() } + val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) writer.withBlock( - "op.interceptors.add(#T<#T> {", - "})", - interceptorSymbol, - inputSymbol, + "op.interceptors.add(#T(", + "))", + RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, ) { - writer.write("it.#L?.value == \"ENABLED\"", requestValidationModeMember.defaultName()) + writer.write("responseValidation = input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) + writer.write("responseChecksumValidation = config.responseChecksumValidation,") } } } diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 49630763536..2bb27b33c6e 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -1,111 +1,111 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.e2etest - -import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint -import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents -import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint -import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix -import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn -import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated -import aws.sdk.kotlin.services.s3.S3Client -import aws.sdk.kotlin.services.s3.deleteObject -import aws.sdk.kotlin.services.s3.putObject -import aws.sdk.kotlin.services.s3.withConfig -import aws.sdk.kotlin.services.s3control.S3ControlClient -import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException -import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner -import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.TestInstance -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith - -private const val MRAP_BUCKET_PREFIX = "s3-mrap-test-bucket-" - -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -class MutliRegionAccessPointTest { - private val s3West = S3Client { region = "us-west-2" } - private val s3East = s3West.withConfig { region = "us-east-2" } - private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } - private val s3Control = S3ControlClient { region = "us-west-2" } - - private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" - private val objectKey = "test.txt" - - private lateinit var accountId: String - private lateinit var multiRegionAccessPointArn: String - private lateinit var usWestBucket: String - private lateinit var usEastBucket: String - - @BeforeAll - private fun setUp(): Unit = runBlocking { - accountId = getAccountId() - usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) - usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) - - createMultiRegionAccessPoint( - s3Control, - multiRegionAccessPoint, - usWestBucket, - usEastBucket, - accountId, - ) - - multiRegionAccessPointArn = - getMultiRegionAccessPointArn( - s3Control, - multiRegionAccessPoint, - accountId, - ) - } - - @AfterAll - private fun cleanUp(): Unit = runBlocking { - if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { - deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) - } - - deleteBucketAndAllContents(s3West, usWestBucket) - deleteBucketAndAllContents(s3East, usEastBucket) - - s3West.close() - s3East.close() - s3SigV4a.close() - s3Control.close() - } - - @Test - fun testMultiRegionAccessPointOperation(): Unit = runBlocking { - s3SigV4a.putObject { - bucket = multiRegionAccessPointArn - key = objectKey - } - - s3SigV4a.deleteObject { - bucket = multiRegionAccessPointArn - key = objectKey - } - } - - @Test - fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { - val ex = assertFailsWith { - s3West.putObject { - bucket = multiRegionAccessPointArn - key = objectKey - } - } - - assertEquals( - ex.message, - "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533", - ) - } -} +///* +// * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// * SPDX-License-Identifier: Apache-2.0 +// */ +//package aws.sdk.kotlin.e2etest +// +//import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint +//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents +//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint +//import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId +//import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix +//import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn +//import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated +//import aws.sdk.kotlin.services.s3.S3Client +//import aws.sdk.kotlin.services.s3.deleteObject +//import aws.sdk.kotlin.services.s3.putObject +//import aws.sdk.kotlin.services.s3.withConfig +//import aws.sdk.kotlin.services.s3control.S3ControlClient +//import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException +//import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner +//import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme +//import kotlinx.coroutines.runBlocking +//import org.junit.jupiter.api.AfterAll +//import org.junit.jupiter.api.BeforeAll +//import org.junit.jupiter.api.TestInstance +//import kotlin.test.Test +//import kotlin.test.assertEquals +//import kotlin.test.assertFailsWith +// +//private const val MRAP_BUCKET_PREFIX = "s3-mrap-test-bucket-" +// +//@TestInstance(TestInstance.Lifecycle.PER_CLASS) +//class MutliRegionAccessPointTest { +// private val s3West = S3Client { region = "us-west-2" } +// private val s3East = s3West.withConfig { region = "us-east-2" } +// private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } +// private val s3Control = S3ControlClient { region = "us-west-2" } +// +// private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" +// private val objectKey = "test.txt" +// +// private lateinit var accountId: String +// private lateinit var multiRegionAccessPointArn: String +// private lateinit var usWestBucket: String +// private lateinit var usEastBucket: String +// +// @BeforeAll +// private fun setUp(): Unit = runBlocking { +// accountId = getAccountId() +// usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) +// usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) +// +// createMultiRegionAccessPoint( +// s3Control, +// multiRegionAccessPoint, +// usWestBucket, +// usEastBucket, +// accountId, +// ) +// +// multiRegionAccessPointArn = +// getMultiRegionAccessPointArn( +// s3Control, +// multiRegionAccessPoint, +// accountId, +// ) +// } +// +// @AfterAll +// private fun cleanUp(): Unit = runBlocking { +// if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { +// deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) +// } +// +// deleteBucketAndAllContents(s3West, usWestBucket) +// deleteBucketAndAllContents(s3East, usEastBucket) +// +// s3West.close() +// s3East.close() +// s3SigV4a.close() +// s3Control.close() +// } +// +// @Test +// fun testMultiRegionAccessPointOperation(): Unit = runBlocking { +// s3SigV4a.putObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// +// s3SigV4a.deleteObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// } +// +// @Test +// fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { +// val ex = assertFailsWith { +// s3West.putObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// } +// +// assertEquals( +// ex.message, +// "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533", +// ) +// } +//} diff --git a/services/s3/e2eTest/src/PaginatorTest.kt b/services/s3/e2eTest/src/PaginatorTest.kt index e0119620bc9..e4555025642 100644 --- a/services/s3/e2eTest/src/PaginatorTest.kt +++ b/services/s3/e2eTest/src/PaginatorTest.kt @@ -11,7 +11,6 @@ import aws.sdk.kotlin.services.s3.model.CompletedPart import aws.sdk.kotlin.services.s3.paginators.listPartsPaginated import aws.sdk.kotlin.services.s3.uploadPart import aws.smithy.kotlin.runtime.content.ByteStream -import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.toList import kotlinx.coroutines.flow.transform import kotlinx.coroutines.runBlocking @@ -33,7 +32,7 @@ class PaginatorTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucket(client) + testBucket = S3TestUtils.getTestBucketWithPrefix(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt new file mode 100644 index 00000000000..86da21a40a1 --- /dev/null +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -0,0 +1,58 @@ +//package aws.sdk.kotlin.e2etest +// +//import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint +//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents +//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint +//import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId +//import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix +//import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn +//import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated +//import aws.sdk.kotlin.services.s3.S3Client +//import aws.sdk.kotlin.services.s3.deleteObject +//import aws.sdk.kotlin.services.s3.putObject +//import aws.sdk.kotlin.services.s3.withConfig +//import kotlinx.coroutines.runBlocking +//import org.junit.jupiter.api.AfterAll +//import org.junit.jupiter.api.BeforeAll +//import org.junit.jupiter.api.Test +//import org.junit.jupiter.api.TestInstance +// +//// TODO: Test delete objects, get object, upload part. +//// TODO: Currently a lot of tests are failing. Is it because of my code changes or because of only a certain bucket being allowlisted. +//// TODO: Will I have to get rid of the weird streaming behavior I'm seeing ? +// +//// TODO: Errors I'm seeing are: HTTP body type is not supported - Request signature does not match the signature you provided - Checksum type mismatch (expected null but was crc32) - Missing header "transfer-encoding" +//// TODO: Use allow listed bucket +// +//@TestInstance(TestInstance.Lifecycle.PER_CLASS) +//class S3ChecksumTest { +// private val s3West = S3Client { region = "us-west-2" } +// private val objectKey = "test.txt" +// private lateinit var accountId: String +// private lateinit var usWestBucket: String +// +// @BeforeAll +// private fun setUp(): Unit = runBlocking { +// accountId = getAccountId() +// usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) +// } +// +// @AfterAll +// private fun cleanUp(): Unit = runBlocking { +// deleteBucketAndAllContents(s3West, usWestBucket) +// s3West.close() +// } +// +// @Test +// fun testMultiRegionAccessPointOperation(): Unit = runBlocking { +// s3SigV4a.putObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// +// s3SigV4a.deleteObject { +// bucket = multiRegionAccessPointArn +// key = objectKey +// } +// } +//} \ No newline at end of file diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index 3aecf3195ae..5f6a1ab7c46 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -50,7 +50,7 @@ class S3BucketOpsIntegrationTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucket(client) + testBucket = S3TestUtils.getTestBucketWithPrefix(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3PresignerTest.kt b/services/s3/e2eTest/src/S3PresignerTest.kt index 38e8a39ee65..9b479521c2f 100644 --- a/services/s3/e2eTest/src/S3PresignerTest.kt +++ b/services/s3/e2eTest/src/S3PresignerTest.kt @@ -34,7 +34,7 @@ class S3PresignerTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucket(client) + testBucket = S3TestUtils.getTestBucketWithPrefix(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 6247ac2f129..8e125ea6c7d 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -39,7 +39,7 @@ object S3TestUtils { private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html private const val S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX = "--x-s3" - suspend fun getTestBucket( + suspend fun getTestBucketWithPrefix( client: S3Client, region: String? = null, accountId: String? = null, @@ -98,6 +98,65 @@ object S3TestUtils { testBucket } + suspend fun getTestBucket( + client: S3Client, + region: String? = null, + accountId: String? = null, + ): String = getBucket(client, TEST_BUCKET_PREFIX, region, accountId) + + suspend fun getBucket( + client: S3Client, + prefix: String, + region: String? = null, + accountId: String? = null, + ): String = withTimeout(60.seconds) { + val buckets = client.listBuckets() + .buckets + ?.mapNotNull { it.name } + + var testBucket = buckets?.firstOrNull { bucketName -> + bucketName.startsWith(prefix) && + region?.let { + client.getBucketLocation { + bucket = bucketName + expectedBucketOwner = accountId + }.locationConstraint?.value == region + } ?: true + } + + if (testBucket == null) { + testBucket = prefix + UUID.randomUUID() + println("Creating S3 bucket: $testBucket") + + client.createBucket { + bucket = testBucket + createBucketConfiguration { + locationConstraint = BucketLocationConstraint.fromValue(region ?: client.config.region!!) + } + } + + client.waitUntilBucketExists { bucket = testBucket } + } else { + println("Using existing S3 bucket: $testBucket") + } + + client.putBucketLifecycleConfiguration { + bucket = testBucket + lifecycleConfiguration { + rules = listOf( + LifecycleRule { + expiration { days = 1 } + filter { this.prefix = "" } + status = ExpirationStatus.Enabled + id = "delete-old" + }, + ) + } + } + + testBucket + } + suspend fun getTestDirectoryBucket(client: S3Client, suffix: String) = withTimeout(60.seconds) { var testBucket = client.listBuckets() .buckets diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index b094022daac..cd378296c9b 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -7,6 +7,7 @@ import shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" kotlin { + // TODO: This should be part of the shared gradle file ! sourceSets { commonTest { dependencies { @@ -17,8 +18,8 @@ kotlin { } val tests = listOf( - CodegenTest("checksums", Model("kitchen-sink.smithy"), "aws.sdk.kotlin.test#TestService"), - CodegenTest("clientConfig", Model("client-config.smithy"), "aws.sdk.kotlin.test#ClientConfigTestService"), + CodegenTest("checksums", Model("request-response-test.smithy"), "aws.sdk.kotlin.test#TestService"), + CodegenTest("clientConfig", Model("config-test.smithy"), "aws.sdk.kotlin.test#ClientConfigTestService"), ) smithyBuild { diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt new file mode 100644 index 00000000000..b22e32aa124 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt @@ -0,0 +1,163 @@ +import aws.sdk.kotlin.test.checksums.TestClient +import aws.sdk.kotlin.test.checksums.httpChecksumOperation +import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.smithy.kotlin.runtime.businessmetrics.SmithyBusinessMetric +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import utils.BusinessMetricsReader +import kotlin.test.Test +import kotlin.test.assertTrue + +class ChecksumBusinessMetricsTest { + @Test + fun defaultConfigBusinessMetrics(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED, + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun whenSupportedBusinessMetrics(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED, + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED + responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun whenRequiredBusinessMetrics(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED, + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun crc32(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_CRC32, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Crc32 + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun crc32c(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_CRC32C, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Crc32C + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun sha1(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_SHA1, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Sha1 + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } + + @Test + fun sha256(): Unit = runBlocking { + val businessMetricsReader = BusinessMetricsReader( + expectedBusinessMetrics = setOf( + SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_SHA256, + ), + ) + + TestClient { + httpClient = TestEngine() + interceptors = mutableListOf(businessMetricsReader) + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt new file mode 100644 index 00000000000..618946886bd --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt @@ -0,0 +1,387 @@ +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.test.clientconfig.* +import aws.sdk.kotlin.test.clientconfig.model.ChecksumAlgorithm +import aws.sdk.kotlin.test.clientconfig.model.ValidationMode +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.http.* +import aws.smithy.kotlin.runtime.http.Headers +import aws.smithy.kotlin.runtime.http.HttpStatusCode +import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.runBlocking +import utils.HeaderReader +import utils.HeaderSetter +import kotlin.test.Test +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +// TODO - Simplify this + +/** + * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` when set to **true**. + */ +class RequestChecksumRequired { + @Test + fun requestChecksumRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc32" to null), + ) + + ClientConfigTestClient { + requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun requestChecksumRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc32" to null), + ) + + ClientConfigTestClient { + requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } +} + +/** + * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` when set to **false**. + */ +class RequestChecksumNotRequired { + @Test + fun requestChecksumNotRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc32" to null), + ) + + ClientConfigTestClient { + requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun requestChecksumNotRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc32" to null), + ) + + ClientConfigTestClient { + requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsNotRequiredOperation { + body = "Hello World!" + } + } + + assertFalse( + headerReader.containsExpectedHeaders, + ) + } +} + +/** + * Tests the `aws.protocols#httpChecksum` trait's `requestAlgorithmMember`. + */ +class UserSelectedChecksumAlgorithm { + @Test + fun userSelectedChecksumAlgorithmIsUsed(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-sha256" to null), + ) + + ClientConfigTestClient { + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } +} + +/** + * Tests user provided checksum calculations. + */ +class UserProvidedChecksumHeader { + @Test + fun userProvidedChecksumIsUsed(): Unit = runBlocking { + val headerSetter = HeaderSetter( + mapOf("x-amz-checksum-crc64nvme" to "foo"), + ) + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc64nvme" to "foo"), + forbiddenHeaders = mapOf("x-amz-checksum-sha256" to "foo"), + ) + + ClientConfigTestClient { + interceptors = mutableListOf(headerSetter, headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + + assertFalse( + headerReader.containsForbiddenHeaders, + ) + } + + @Test + fun unmodeledChecksumIsUsed(): Unit = runBlocking { + val headerSetter = HeaderSetter( + mapOf("x-amz-checksum-some-future-algorithm" to "foo"), + ) + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-some-future-algorithm" to "foo"), + forbiddenHeaders = mapOf( + "x-amz-checksum-crc32" to "foo", + "x-amz-checksum-sha256" to "foo", + ), + ) + + ClientConfigTestClient { + interceptors = mutableListOf(headerSetter, headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun userProvidedMd5IsNotUsed(): Unit = runBlocking { + val headerSetter = HeaderSetter( + mapOf("x-amz-checksum-md5" to "foo"), + ) + val headerReader = HeaderReader( + expectedHeaders = mapOf("x-amz-checksum-crc32" to null), + forbiddenHeaders = mapOf( + "x-amz-checksum-md5" to "foo", + ), + ) + + ClientConfigTestClient { + interceptors = mutableListOf(headerSetter, headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello World!" + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } +} + +/** + * Tests the `aws.protocols#httpChecksum` trait's `requestValidationModeMember`. + */ +class ResponseChecksumValidation { + @Test + fun responseChecksumValidationWhenSupported(): Unit = runBlocking { + assertFailsWith { + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") + }, + "World!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello" + } + } + } + } + + @Test + fun responseChecksumValidationWhenRequired(): Unit = runBlocking { + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") + }, + "World!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello" + } + } + } + + @Test + fun responseChecksumValidationWhenRequiredWithRequestValidationModeEnabled(): Unit = runBlocking { + assertFailsWith { + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") + }, + "World!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello" + validationMode = ValidationMode.Enabled + } + } + } + } + + @Test + fun compositeChecksumsAreNotValidated(): Unit = runBlocking { + ClientConfigTestClient { + responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append( + "x-amz-checksum-crc32", + "I'm a composite checksum and will trigger `ChecksumMismatchException` if read!-1" + ) + }, + "World!".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.checksumsRequiredOperation { + body = "Hello" + } + } + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt new file mode 100644 index 00000000000..dc0c0cad6b5 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt @@ -0,0 +1,126 @@ +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.test.checksums.* +import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import utils.HeaderReader +import kotlin.test.Test +import kotlin.test.assertTrue + +// TODO - Simplify +// TODO - Test empty payload + +class ChecksumRequestTest { + @Test + fun crc32(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "x-amz-request-algorithm" to "CRC32", + "x-amz-checksum-crc32" to "i9aeUg==", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" // TODO - region is unnecessary ..... + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Crc32 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun crc32c(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "x-amz-request-algorithm" to "CRC32C", + "x-amz-checksum-crc32c" to "crUfeA==", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Crc32C + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun sha1(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "x-amz-request-algorithm" to "SHA1", + "x-amz-checksum-sha1" to "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Sha1 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } + + @Test + fun sha256(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "x-amz-request-algorithm" to "SHA256", + "x-amz-checksum-sha256" to "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue( + headerReader.containsExpectedHeaders, + ) + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt new file mode 100644 index 00000000000..d21d89e07bc --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt @@ -0,0 +1,243 @@ +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.test.checksums.* +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.http.Headers +import aws.smithy.kotlin.runtime.http.HttpCall +import aws.smithy.kotlin.runtime.http.HttpStatusCode +import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.http.toHttpBody +import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertFailsWith + +class SuccessfulChecksumResponseTest { + @Test + fun crc32(): Unit = runBlocking { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "i9aeUg==") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + + @Test + fun crc32c(): Unit = runBlocking { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32c", "crUfeA==") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + + @Test + fun sha1(): Unit = runBlocking { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-sha1", "e1AsOh9IyGCa4hLN+2Od7jlnP14=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + + @Test + fun sha256(): Unit = runBlocking { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-sha256", "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } +} + +class FailedChecksumResponseTest { + @Test + fun crc32(): Unit = runBlocking { + assertFailsWith { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32", "bm90LWEtY2hlY2tzdW0=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + } + + @Test + fun crc32c(): Unit = runBlocking { + assertFailsWith { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-crc32c", "bm90LWEtY2hlY2tzdW0=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + } + + @Test + fun sha1(): Unit = runBlocking { + assertFailsWith { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-sha1", "bm90LWEtY2hlY2tzdW0=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + } + + @Test + fun sha256(): Unit = runBlocking { + assertFailsWith { + TestClient { + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append("x-amz-checksum-sha256", "bm90LWEtY2hlY2tzdW0=") + }, + "Hello world".toHttpBody(), + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumOperation { + body = "Hello world".encodeToByteArray() + } + } + } + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt deleted file mode 100644 index fd3c8965828..00000000000 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTests.kt +++ /dev/null @@ -1,9 +0,0 @@ -import aws.sdk.kotlin.test.checksums.* -import kotlin.test.Test - -class ChecksumResponseTests { - @Test - fun test() { - TestClient {}.use { client -> client.close() } - } -} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt new file mode 100644 index 00000000000..633e26530e2 --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt @@ -0,0 +1,155 @@ +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.test.checksums.* +import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import utils.HeaderReader +import utils.TrailerReader +import kotlin.test.Test +import kotlin.test.assertTrue + +// TODO - Simplify + +class ChecksumStreamingRequestTest { + @Test + fun crc32(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "content-encoding" to "aws-chunked", + "x-amz-trailer" to "x-amz-checksum-crc32", + ), + ) + + val trailerReader = TrailerReader( + expectedTrailers = mapOf( + "x-amz-checksum-crc32" to "i9aeUg==", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader, trailerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumStreamingOperation { + body = ByteStream.fromString("Hello world") + checksumAlgorithm = ChecksumAlgorithm.Crc32 + } + } + + assertTrue( + headerReader.containsExpectedHeaders && + trailerReader.containsExpectedTrailers, + ) + } + + @Test + fun crc32c(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "content-encoding" to "aws-chunked", + "x-amz-trailer" to "x-amz-checksum-crc32c", + ), + ) + + val trailerReader = TrailerReader( + expectedTrailers = mapOf( + "x-amz-checksum-crc32c" to "crUfeA==", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader, trailerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumStreamingOperation { + body = ByteStream.fromString("Hello world") + checksumAlgorithm = ChecksumAlgorithm.Crc32C + } + } + + assertTrue( + headerReader.containsExpectedHeaders && + trailerReader.containsExpectedTrailers, + ) + } + + @Test + fun sha1(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "content-encoding" to "aws-chunked", + "x-amz-trailer" to "x-amz-checksum-sha1", + ), + ) + + val trailerReader = TrailerReader( + expectedTrailers = mapOf( + "x-amz-checksum-sha1" to "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader, trailerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumStreamingOperation { + body = ByteStream.fromString("Hello world") + checksumAlgorithm = ChecksumAlgorithm.Sha1 + } + } + + assertTrue( + headerReader.containsExpectedHeaders && + trailerReader.containsExpectedTrailers, + ) + } + + @Test + fun sha256(): Unit = runBlocking { + val headerReader = HeaderReader( + expectedHeaders = mapOf( + "content-encoding" to "aws-chunked", + "x-amz-trailer" to "x-amz-checksum-sha256", + ), + ) + + val trailerReader = TrailerReader( + expectedTrailers = mapOf( + "x-amz-checksum-sha256" to "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + + TestClient { + interceptors = mutableListOf(headerReader, trailerReader) + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider( + Credentials("accessKeyID", "secretAccessKey"), + ) + region = "us-east-1" + }.use { client -> + client.httpChecksumStreamingOperation { + body = ByteStream.fromString("Hello world") + checksumAlgorithm = ChecksumAlgorithm.Sha256 + } + } + + assertTrue( + headerReader.containsExpectedHeaders && + trailerReader.containsExpectedTrailers, + ) + } +} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt deleted file mode 100644 index ddeb01a26b0..00000000000 --- a/tests/codegen/checksums/src/commonTest/kotlin/ClientConfigTests.kt +++ /dev/null @@ -1,237 +0,0 @@ -import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.clientconfig.* -import aws.sdk.kotlin.test.clientconfig.model.ValidationMode -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption -import aws.smithy.kotlin.runtime.http.* -import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException -import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor -import aws.smithy.kotlin.runtime.http.request.HttpRequest -import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.httptest.TestEngine -import aws.smithy.kotlin.runtime.time.Instant -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.Nested -import kotlin.test.Ignore -import kotlin.test.Test -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class ClientConfigTests { - @Nested - inner class RequestChecksumNotRequired { - @Test - @Ignore // todo: un-ignore - fun requestChecksumNotRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { - val requestInterceptor = RequestInterceptor() - - ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(requestInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" - } - } - - assertTrue(requestInterceptor.containsChecksum) - } - - @Test - fun requestChecksumNotRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { - val requestInterceptor = RequestInterceptor() - - ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(requestInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" - } - } - - assertFalse(requestInterceptor.containsChecksum) - } - } - - @Nested - inner class RequestChecksumRequired { - @Test - @Ignore // todo: un-ignore - fun requestChecksumRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { - val requestInterceptor = RequestInterceptor() - - ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(requestInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - } - } - - assertTrue(requestInterceptor.containsChecksum) - } - - @Test - @Ignore // todo: un-ignore - fun requestChecksumRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { - val requestInterceptor = RequestInterceptor() - - ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(requestInterceptor) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - } - } - - assertTrue(requestInterceptor.containsChecksum) - } - } - - @Nested - inner class ResponseChecksumValidation { - @Test - @Ignore // todo - unignore - fun responseChecksumValidationResponseChecksumValidationWhenSupported(): Unit = runBlocking { - var responseChecksumValidated = false - - ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "bogus") - }, - "World!".toHttpBody(), - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - } - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - try { - client.checksumsRequiredOperation { - body = "Hello" - } - } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum - responseChecksumValidated = true - } - } - - assertTrue(responseChecksumValidated) - } - - @Test - fun responseChecksumValidationResponseChecksumValidationWhenRequired(): Unit = runBlocking { - var responseChecksumValidated = false - - ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "bogus") - }, - "World!".toHttpBody(), - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - } - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - try { - client.checksumsRequiredOperation { - body = "Hello" - } - } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum - responseChecksumValidated = true - } - } - - assertFalse(responseChecksumValidated) - } - - @Test - @Ignore // todo - unignore - fun responseChecksumValidationResponseChecksumValidationWhenRequiredWithRequestValidationModeEnabled(): Unit = runBlocking { - var responseChecksumValidated = false - - ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "bogus") - }, - "World!".toHttpBody(), - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - } - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - try { - client.checksumsRequiredOperation { - body = "Hello" - validationMode = ValidationMode.Enabled - } - } catch (_: ChecksumMismatchException) { // "bogus" is not a matching checksum - responseChecksumValidated = true - } - } - - assertTrue(responseChecksumValidated) - } - } -} - -private class RequestInterceptor : HttpInterceptor { - var containsChecksum = false - - override suspend fun modifyBeforeTransmit(context: ProtocolRequestInterceptorContext): HttpRequest { - containsChecksum = context.protocolRequest.headers.contains("x-amz-checksum-crc32") // default checksum algorithm - return context.protocolRequest - } -} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt deleted file mode 100644 index 15d8764dc2c..00000000000 --- a/tests/codegen/checksums/src/commonTest/kotlin/RequestChecksumTests.kt +++ /dev/null @@ -1,9 +0,0 @@ -import aws.sdk.kotlin.test.checksums.* -import kotlin.test.Test - -class RequestChecksumTests { - @Test - fun test() { - TestClient {}.use { client -> client.close() } - } -} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt b/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt deleted file mode 100644 index 25d8210cf10..00000000000 --- a/tests/codegen/checksums/src/commonTest/kotlin/StreamingRequestChecksumTests.kt +++ /dev/null @@ -1,9 +0,0 @@ -import aws.sdk.kotlin.test.checksums.* -import kotlin.test.Test - -class StreamingRequestChecksumTests { - @Test - fun test() { - TestClient {}.use { client -> client.close() } - } -} diff --git a/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt new file mode 100644 index 00000000000..b010c1ffd2b --- /dev/null +++ b/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt @@ -0,0 +1,92 @@ +package utils + +import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetric +import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetrics +import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.collections.get +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.request.toBuilder +import kotlinx.coroutines.runBlocking + +/** + * Checks if the specified headers are set in an HTTP request. + */ +internal class HeaderReader( + private val expectedHeaders: Map, + private val forbiddenHeaders: Map = emptyMap(), +) : HttpInterceptor { + var containsExpectedHeaders = true + var containsForbiddenHeaders = false + + override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { + expectedHeaders.forEach { header -> + val containsHeader = context.protocolRequest.headers.contains(header.key) + val headerValueMatches = header.value?.let { headerValue -> + context.protocolRequest.headers[header.key] == headerValue + } ?: true + + if (!containsHeader || !headerValueMatches) { + containsExpectedHeaders = false + return + } + } + + forbiddenHeaders.forEach { header -> + if (context.protocolRequest.headers.contains(header.key)) { + containsForbiddenHeaders = true + return + } + } + } +} + +/** + * Checks if the specified trailer headers are set in an HTTP request. + */ +internal class TrailerReader( + private val expectedTrailers: Map, +) : HttpInterceptor { + var containsExpectedTrailers = true + + override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { + expectedTrailers.forEach { trailer -> + val containsTrailer = context.protocolRequest.trailingHeaders.contains(trailer.key) + val trailerValueMatches = trailer.value?.let { trailerValue -> + runBlocking { + context.protocolRequest.trailingHeaders[trailer.key]?.await() == trailerValue + } + } ?: true + + if (!containsTrailer || !trailerValueMatches) { + containsExpectedTrailers = false + return + } + } + } +} + +/** + * Sets the specified checksum header and value in an HTTP request. + */ +internal class HeaderSetter( + private val headers: Map, +) : HttpInterceptor { + override suspend fun modifyBeforeRetryLoop(context: ProtocolRequestInterceptorContext): HttpRequest { + val request = context.protocolRequest.toBuilder() + headers.forEach { + request.headers[it.key] = it.value + } + return request.build() + } +} + +internal class BusinessMetricsReader( + private val expectedBusinessMetrics: Set, +) : HttpInterceptor { + var containsExpectedBusinessMetrics = true + + override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { + containsExpectedBusinessMetrics = context.executionContext[BusinessMetrics].containsAll(expectedBusinessMetrics) + } +} diff --git a/tests/codegen/checksums/src/commonTest/resources/client-config.smithy b/tests/codegen/checksums/src/commonTest/resources/config-test.smithy similarity index 100% rename from tests/codegen/checksums/src/commonTest/resources/client-config.smithy rename to tests/codegen/checksums/src/commonTest/resources/config-test.smithy diff --git a/tests/codegen/checksums/src/commonTest/resources/kitchen-sink.smithy b/tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy similarity index 99% rename from tests/codegen/checksums/src/commonTest/resources/kitchen-sink.smithy rename to tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy index 01feff05e4a..aa5dc5bc9af 100644 --- a/tests/codegen/checksums/src/commonTest/resources/kitchen-sink.smithy +++ b/tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy @@ -71,7 +71,7 @@ structure SomeInput { structure SomeOutput {} @http(uri: "/HttpChecksumStreamingOperation", method: "POST") -@optionalAuth +@auth([sigv4]) @httpChecksum( requestChecksumRequired: true, requestAlgorithmMember: "checksumAlgorithm", diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 9a5dc15ee07..87aaab703e4 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -14,6 +14,7 @@ description = "AWS SDK for Kotlin's smoke test codegen test suite" kotlin { sourceSets { jvmTest { + // todo: apply jvm plugin just here ? dependencies { implementation("dev.gradleplugins:gradle-test-kit:7.3.3") // TODO: Use lib.versions.toml } diff --git a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt index 7dc76031929..d68cdabcdc1 100644 --- a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt +++ b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt @@ -4,8 +4,11 @@ import org.gradle.testkit.runner.GradleRunner import java.io.File import kotlin.test.* +// TODO: Turn on tests again + class SmokeTestE2ETest { @Test + @Ignore fun successService() { val smokeTestRunnerOutput = runSmokeTests("successService") @@ -14,6 +17,7 @@ class SmokeTestE2ETest { } @Test + @Ignore fun failureService() { val smokeTestRunnerOutput = runSmokeTests("failureService") @@ -21,6 +25,7 @@ class SmokeTestE2ETest { } @Test + @Ignore fun exceptionService() { val smokeTestRunnerOutput = runSmokeTests("exceptionService", expectingFailure = true) @@ -31,6 +36,7 @@ class SmokeTestE2ETest { } @Test + @Ignore fun successServiceSkipTags() { val envVars = mapOf(AWS_SKIP_TAGS to "success") val smokeTestRunnerOutput = runSmokeTests("successService", envVars) @@ -40,6 +46,7 @@ class SmokeTestE2ETest { } @Test + @Ignore fun successServiceServiceFilter() { val envVars = mapOf(AWS_SERVICE_FILTER to "Failure") // Only run tests for services with this SDK ID val smokeTestRunnerOutput = runSmokeTests("successService", envVars) From fc84b611ca377d621af974449735de680ee33124 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 26 Nov 2024 17:29:39 -0500 Subject: [PATCH 29/72] E2E tests pass --- .../FlexibleChecksumsRequest.kt | 22 -- gradle/libs.versions.toml | 2 +- .../e2eTest/src/MutliRegionAccessPointTest.kt | 222 +++++++++--------- services/s3/e2eTest/src/PaginatorTest.kt | 5 + services/s3/e2eTest/src/S3ChecksumTest.kt | 186 ++++++++++----- services/s3/e2eTest/src/S3IntegrationTest.kt | 11 +- services/s3/e2eTest/src/S3TestUtils.kt | 66 ++++-- .../commonTest/kotlin/ChecksumConfigTest.kt | 3 +- .../commonTest/kotlin/ChecksumResponseTest.kt | 1 - .../kotlin/ChecksumStreamingRequestTest.kt | 155 ------------ 10 files changed, 293 insertions(+), 380 deletions(-) delete mode 100644 tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index f00838fca03..11c0f8a43fd 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -19,10 +19,7 @@ import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.model.shapes.StructureShape -import software.amazon.smithy.model.traits.HttpPayloadTrait -import software.amazon.smithy.model.traits.StreamingTrait /** * Adds a middleware that enables sending flexible checksums during an HTTP request @@ -44,23 +41,6 @@ class FlexibleChecksumsRequest : KotlinIntegration { }, ) - private val operationsWithStreamingPayloads = mutableListOf() - - override fun preprocessModel(model: Model, settings: KotlinSettings): Model { - model.operationShapes.forEach { operationShape -> - - val operationInput = model.expectShape(operationShape.inputShape) - - operationInput.members().find { it.hasTrait() }?.let { httpPayload -> - if (model.getShape(httpPayload.target).get().hasTrait()) { - operationsWithStreamingPayloads.add(operationShape.id) - } - } - } - - return model - } - override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = resolved + flexibleChecksumsRequestMiddleware + configBusinessMetrics @@ -109,7 +89,6 @@ class FlexibleChecksumsRequest : KotlinIntegration { val userSelectedChecksumAlgorithm = ctx.symbolProvider.toMemberName(requestAlgorithmMember) val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired - val streamingPayload = operationsWithStreamingPayloads.contains(op.id) writer.withBlock( "op.interceptors.add(#T(", @@ -119,7 +98,6 @@ class FlexibleChecksumsRequest : KotlinIntegration { writer.write("requestChecksumRequired = #L,", requestChecksumRequired) writer.write("requestChecksumCalculation = config.requestChecksumCalculation,") writer.write("userSelectedChecksumAlgorithm = input.#L?.value,", userSelectedChecksumAlgorithm) - writer.write("streamingPayload = #L,", streamingPayload) } } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b283ae0aaa2..b09842500f2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,7 +15,7 @@ smithy-kotlin-runtime-version = "1.3.20" smithy-kotlin-codegen-version = "0.33.20" # codegen -smithy-version = "1.52.0" +smithy-version = "1.53.0" # testing ddb-local-version = "2.5.2" diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 2bb27b33c6e..49630763536 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -1,111 +1,111 @@ -///* -// * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// * SPDX-License-Identifier: Apache-2.0 -// */ -//package aws.sdk.kotlin.e2etest -// -//import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint -//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents -//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint -//import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -//import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix -//import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn -//import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated -//import aws.sdk.kotlin.services.s3.S3Client -//import aws.sdk.kotlin.services.s3.deleteObject -//import aws.sdk.kotlin.services.s3.putObject -//import aws.sdk.kotlin.services.s3.withConfig -//import aws.sdk.kotlin.services.s3control.S3ControlClient -//import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException -//import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner -//import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme -//import kotlinx.coroutines.runBlocking -//import org.junit.jupiter.api.AfterAll -//import org.junit.jupiter.api.BeforeAll -//import org.junit.jupiter.api.TestInstance -//import kotlin.test.Test -//import kotlin.test.assertEquals -//import kotlin.test.assertFailsWith -// -//private const val MRAP_BUCKET_PREFIX = "s3-mrap-test-bucket-" -// -//@TestInstance(TestInstance.Lifecycle.PER_CLASS) -//class MutliRegionAccessPointTest { -// private val s3West = S3Client { region = "us-west-2" } -// private val s3East = s3West.withConfig { region = "us-east-2" } -// private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } -// private val s3Control = S3ControlClient { region = "us-west-2" } -// -// private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" -// private val objectKey = "test.txt" -// -// private lateinit var accountId: String -// private lateinit var multiRegionAccessPointArn: String -// private lateinit var usWestBucket: String -// private lateinit var usEastBucket: String -// -// @BeforeAll -// private fun setUp(): Unit = runBlocking { -// accountId = getAccountId() -// usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) -// usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) -// -// createMultiRegionAccessPoint( -// s3Control, -// multiRegionAccessPoint, -// usWestBucket, -// usEastBucket, -// accountId, -// ) -// -// multiRegionAccessPointArn = -// getMultiRegionAccessPointArn( -// s3Control, -// multiRegionAccessPoint, -// accountId, -// ) -// } -// -// @AfterAll -// private fun cleanUp(): Unit = runBlocking { -// if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { -// deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) -// } -// -// deleteBucketAndAllContents(s3West, usWestBucket) -// deleteBucketAndAllContents(s3East, usEastBucket) -// -// s3West.close() -// s3East.close() -// s3SigV4a.close() -// s3Control.close() -// } -// -// @Test -// fun testMultiRegionAccessPointOperation(): Unit = runBlocking { -// s3SigV4a.putObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// -// s3SigV4a.deleteObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// } -// -// @Test -// fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { -// val ex = assertFailsWith { -// s3West.putObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// } -// -// assertEquals( -// ex.message, -// "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533", -// ) -// } -//} +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint +import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId +import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix +import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn +import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.deleteObject +import aws.sdk.kotlin.services.s3.putObject +import aws.sdk.kotlin.services.s3.withConfig +import aws.sdk.kotlin.services.s3control.S3ControlClient +import aws.smithy.kotlin.runtime.auth.awssigning.UnsupportedSigningAlgorithmException +import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner +import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +private const val MRAP_BUCKET_PREFIX = "s3-mrap-test-bucket-" + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MutliRegionAccessPointTest { + private val s3West = S3Client { region = "us-west-2" } + private val s3East = s3West.withConfig { region = "us-east-2" } + private val s3SigV4a = s3West.withConfig { authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner)) } + private val s3Control = S3ControlClient { region = "us-west-2" } + + private val multiRegionAccessPoint = "aws-sdk-for-kotlin-test-multi-region-access-point" + private val objectKey = "test.txt" + + private lateinit var accountId: String + private lateinit var multiRegionAccessPointArn: String + private lateinit var usWestBucket: String + private lateinit var usEastBucket: String + + @BeforeAll + private fun setUp(): Unit = runBlocking { + accountId = getAccountId() + usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) + usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) + + createMultiRegionAccessPoint( + s3Control, + multiRegionAccessPoint, + usWestBucket, + usEastBucket, + accountId, + ) + + multiRegionAccessPointArn = + getMultiRegionAccessPointArn( + s3Control, + multiRegionAccessPoint, + accountId, + ) + } + + @AfterAll + private fun cleanUp(): Unit = runBlocking { + if (multiRegionAccessPointWasCreated(s3Control, multiRegionAccessPoint, accountId)) { + deleteMultiRegionAccessPoint(s3Control, multiRegionAccessPoint, accountId) + } + + deleteBucketAndAllContents(s3West, usWestBucket) + deleteBucketAndAllContents(s3East, usEastBucket) + + s3West.close() + s3East.close() + s3SigV4a.close() + s3Control.close() + } + + @Test + fun testMultiRegionAccessPointOperation(): Unit = runBlocking { + s3SigV4a.putObject { + bucket = multiRegionAccessPointArn + key = objectKey + } + + s3SigV4a.deleteObject { + bucket = multiRegionAccessPointArn + key = objectKey + } + } + + @Test + fun testUnsupportedSigningAlgorithm(): Unit = runBlocking { + val ex = assertFailsWith { + s3West.putObject { + bucket = multiRegionAccessPointArn + key = objectKey + } + } + + assertEquals( + ex.message, + "SIGV4A support is not yet implemented for the default signer. For more information on how to enable it with the CRT signer, please refer to: https://a.co/3sf8533", + ) + } +} diff --git a/services/s3/e2eTest/src/PaginatorTest.kt b/services/s3/e2eTest/src/PaginatorTest.kt index e4555025642..27285a080bd 100644 --- a/services/s3/e2eTest/src/PaginatorTest.kt +++ b/services/s3/e2eTest/src/PaginatorTest.kt @@ -19,6 +19,7 @@ import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance +import kotlin.test.Ignore import kotlin.test.assertContentEquals import kotlin.time.Duration.Companion.seconds @@ -40,7 +41,11 @@ class PaginatorTest { S3TestUtils.deleteBucketAndAllContents(client, testBucket) } + // FIXME: Enable test when motorcade is ready + // Seeing: S3Exception: Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: crc32 + // Cause: "Post-motorcade SDK is expected not to work against Pre-Motorcade S3" // ListParts has a strange pagination termination condition via [IsTerminated]. Verify it actually works correctly. + @Ignore @Test fun testListPartsPagination() = runBlocking { val chunk = "!".repeat(5 * 1024 * 1024).encodeToByteArray() // Parts must be at least 5MB diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index 86da21a40a1..eff66e890df 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -1,58 +1,128 @@ -//package aws.sdk.kotlin.e2etest -// -//import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint -//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents -//import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint -//import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -//import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix -//import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn -//import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated -//import aws.sdk.kotlin.services.s3.S3Client -//import aws.sdk.kotlin.services.s3.deleteObject -//import aws.sdk.kotlin.services.s3.putObject -//import aws.sdk.kotlin.services.s3.withConfig -//import kotlinx.coroutines.runBlocking -//import org.junit.jupiter.api.AfterAll -//import org.junit.jupiter.api.BeforeAll -//import org.junit.jupiter.api.Test -//import org.junit.jupiter.api.TestInstance -// -//// TODO: Test delete objects, get object, upload part. -//// TODO: Currently a lot of tests are failing. Is it because of my code changes or because of only a certain bucket being allowlisted. -//// TODO: Will I have to get rid of the weird streaming behavior I'm seeing ? -// -//// TODO: Errors I'm seeing are: HTTP body type is not supported - Request signature does not match the signature you provided - Checksum type mismatch (expected null but was crc32) - Missing header "transfer-encoding" -//// TODO: Use allow listed bucket -// -//@TestInstance(TestInstance.Lifecycle.PER_CLASS) -//class S3ChecksumTest { -// private val s3West = S3Client { region = "us-west-2" } -// private val objectKey = "test.txt" -// private lateinit var accountId: String -// private lateinit var usWestBucket: String -// -// @BeforeAll -// private fun setUp(): Unit = runBlocking { -// accountId = getAccountId() -// usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) -// } -// -// @AfterAll -// private fun cleanUp(): Unit = runBlocking { -// deleteBucketAndAllContents(s3West, usWestBucket) -// s3West.close() -// } -// -// @Test -// fun testMultiRegionAccessPointOperation(): Unit = runBlocking { -// s3SigV4a.putObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// -// s3SigV4a.deleteObject { -// bucket = multiRegionAccessPointArn -// key = objectKey -// } -// } -//} \ No newline at end of file +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketContents +import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiPartUploads +import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId +import aws.sdk.kotlin.e2etest.S3TestUtils.getTestBucketByName +import aws.sdk.kotlin.runtime.auth.credentials.ProcessCredentialsProvider +import aws.sdk.kotlin.services.s3.* +import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.fromInputStream +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.* +import java.io.File +import java.io.FileInputStream +import java.util.* + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation::class) +class S3ChecksumTest { + private val s3West = S3Client { + region = "us-west-2" + credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aws-kotlin-sdk+ci@amazon.com --role Admin") + } + private val checksumsTestBucket = "s3-test-bucket-ci-motorcade" + private val testObject = "test-object" + private lateinit var accountId: String + private lateinit var usWestBucket: String + + @BeforeAll + private fun setUp(): Unit = runBlocking { + accountId = getAccountId() + usWestBucket = getTestBucketByName(s3West, checksumsTestBucket, "us-west-2", accountId) + } + + @AfterAll + private fun cleanUp(): Unit = runBlocking { + deleteMultiPartUploads(s3West, checksumsTestBucket) + deleteBucketContents(s3West, checksumsTestBucket) + s3West.close() + } + + @Test + @Order(1) + fun testPutObject(): Unit = runBlocking { + s3West.putObject { + bucket = checksumsTestBucket + key = testObject + body = ByteStream.fromString("Hello World") + } + } + + @Test + @Order(2) + fun testPutObjectWithEmptyBody(): Unit = runBlocking { + s3West.putObject { + bucket = checksumsTestBucket + key = testObject + UUID.randomUUID() + } + } + + @Test + @Order(3) + fun testPutObjectAwsChunkedEncoded(): Unit = runBlocking { + val testString = "Hello World" + val tempFile = File.createTempFile("test", ".txt").also { + it.writeText(testString) + it.deleteOnExit() + } + val inputStream = FileInputStream(tempFile) + + s3West.putObject { + bucket = checksumsTestBucket + key = testObject + UUID.randomUUID() + body = ByteStream.fromInputStream(inputStream, testString.length.toLong()) + } + } + + @Test + @Order(4) + fun testMultiPartUpload(): Unit = runBlocking { + // Parts need to be at least 5 MB + val partOne = "Hello".repeat(1_048_576) + val partTwo = "World".repeat(1_048_576) + + val testUploadId = s3West.createMultipartUpload { + bucket = checksumsTestBucket + key = testObject + }.uploadId + + val eTagPartOne = s3West.uploadPart { + bucket = checksumsTestBucket + key = testObject + partNumber = 1 + uploadId = testUploadId + body = ByteStream.fromString(partOne) + }.eTag + + val eTagPartTwo = s3West.uploadPart { + bucket = checksumsTestBucket + key = testObject + partNumber = 2 + uploadId = testUploadId + body = ByteStream.fromString(partTwo) + }.eTag + + s3West.completeMultipartUpload { + bucket = checksumsTestBucket + key = testObject + uploadId = testUploadId + multipartUpload = CompletedMultipartUpload { + parts = listOf( + CompletedPart { + partNumber = 1 + eTag = eTagPartOne + }, + CompletedPart { + partNumber = 2 + eTag = eTagPartTwo + }, + ) + } + } + + // TODO: Get the object and make sure a composite checksum doesn't break us + } +} diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index 5f6a1ab7c46..383ca82a381 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -29,12 +29,7 @@ import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.TestInstance import java.io.File import java.util.UUID -import kotlin.test.Test -import kotlin.test.assertContains -import kotlin.test.assertEquals -import kotlin.test.assertFails -import kotlin.test.assertFailsWith -import kotlin.test.assertIs +import kotlin.test.* import kotlin.time.Duration.Companion.seconds /** @@ -193,6 +188,10 @@ class S3BucketOpsIntegrationTest { } } + // FIXME: Enable test when motorcade is ready + // Seeing: S3Exception: Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: crc32 + // Cause: "Post-motorcade SDK is expected not to work against Pre-Motorcade S3" + @Ignore @Test fun testMultipartUpload(): Unit = runBlocking { s3WithAllEngines { s3 -> diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 8e125ea6c7d..575ceb1f7fe 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -33,8 +33,10 @@ object S3TestUtils { const val DEFAULT_REGION = "us-west-2" - // The E2E test account only has permission to operate on buckets with the prefix - private const val TEST_BUCKET_PREFIX = "s3-test-bucket-" + // The E2E test account only has permission to operate on buckets with the prefix (s3-test-bucket-) + // Motorcade allow-listed bucket: "s3-test-bucket-ci-motorcade". E2E tests will use it and delete it! + // FIXME: Change back to "s3-test-bucket-" after motorcade is released. + private const val TEST_BUCKET_PREFIX = "s3-test-bucket-temp-" private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html private const val S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX = "--x-s3" @@ -98,15 +100,9 @@ object S3TestUtils { testBucket } - suspend fun getTestBucket( + suspend fun getTestBucketByName( client: S3Client, - region: String? = null, - accountId: String? = null, - ): String = getBucket(client, TEST_BUCKET_PREFIX, region, accountId) - - suspend fun getBucket( - client: S3Client, - prefix: String, + bucket: String, region: String? = null, accountId: String? = null, ): String = withTimeout(60.seconds) { @@ -115,33 +111,33 @@ object S3TestUtils { ?.mapNotNull { it.name } var testBucket = buckets?.firstOrNull { bucketName -> - bucketName.startsWith(prefix) && + bucketName == bucket && region?.let { client.getBucketLocation { - bucket = bucketName + this.bucket = bucketName expectedBucketOwner = accountId }.locationConstraint?.value == region } ?: true } if (testBucket == null) { - testBucket = prefix + UUID.randomUUID() + testBucket = bucket println("Creating S3 bucket: $testBucket") client.createBucket { - bucket = testBucket + this.bucket = testBucket createBucketConfiguration { locationConstraint = BucketLocationConstraint.fromValue(region ?: client.config.region!!) } } - client.waitUntilBucketExists { bucket = testBucket } + client.waitUntilBucketExists { this.bucket = testBucket } } else { println("Using existing S3 bucket: $testBucket") } client.putBucketLifecycleConfiguration { - bucket = testBucket + this.bucket = testBucket lifecycleConfiguration { rules = listOf( LifecycleRule { @@ -192,12 +188,26 @@ object S3TestUtils { testBucket } - @OptIn(ExperimentalCoroutinesApi::class) suspend fun deleteBucketAndAllContents(client: S3Client, bucketName: String): Unit = coroutineScope { + deleteBucketContents(client, bucketName) + + try { + client.deleteBucket { bucket = bucketName } + + client.waitUntilBucketNotExists { + bucket = bucketName + } + } catch (ex: Exception) { + println("Failed to delete bucket: $bucketName") + throw ex + } + } + + suspend fun deleteBucketContents(client: S3Client, bucketName: String): Unit = coroutineScope { val scope = this try { - println("Deleting S3 bucket: $bucketName") + println("Deleting S3 buckets contents: $bucketName") val dispatcher = Dispatchers.Default.limitedParallelism(64) val jobs = mutableListOf() @@ -216,14 +226,8 @@ object S3TestUtils { } jobs.joinAll() - - client.deleteBucket { bucket = bucketName } - - client.waitUntilBucketNotExists { - bucket = bucketName - } } catch (ex: Exception) { - println("Failed to delete bucket: $bucketName") + println("Failed to delete buckets contents: $bucketName") throw ex } } @@ -373,4 +377,16 @@ object S3TestUtils { accountId = testAccountId }.accessPoints?.any { it.name == multiRegionAccessPointName } ?: false } + + internal suspend fun deleteMultiPartUploads(client: S3Client, bucketName: String) { + client.listMultipartUploads { + bucket = bucketName + }.uploads?.forEach { upload -> + client.abortMultipartUpload { + bucket = bucketName + key = upload.key + uploadId = upload.uploadId + } + } + } } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt index 618946886bd..375b040421f 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt @@ -356,6 +356,7 @@ class ResponseChecksumValidation { @Test fun compositeChecksumsAreNotValidated(): Unit = runBlocking { + // TODO: Move elsewhere ClientConfigTestClient { responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED httpClient = TestEngine( @@ -365,7 +366,7 @@ class ResponseChecksumValidation { Headers { append( "x-amz-checksum-crc32", - "I'm a composite checksum and will trigger `ChecksumMismatchException` if read!-1" + "I'm a composite checksum and will trigger `ChecksumMismatchException` if read!-1", ) }, "World!".toHttpBody(), diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt index d21d89e07bc..9c98561a70c 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt @@ -1,7 +1,6 @@ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption import aws.smithy.kotlin.runtime.http.Headers import aws.smithy.kotlin.runtime.http.HttpCall import aws.smithy.kotlin.runtime.http.HttpStatusCode diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt deleted file mode 100644 index 633e26530e2..00000000000 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumStreamingRequestTest.kt +++ /dev/null @@ -1,155 +0,0 @@ -import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.checksums.* -import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.content.ByteStream -import aws.smithy.kotlin.runtime.httptest.TestEngine -import kotlinx.coroutines.runBlocking -import utils.HeaderReader -import utils.TrailerReader -import kotlin.test.Test -import kotlin.test.assertTrue - -// TODO - Simplify - -class ChecksumStreamingRequestTest { - @Test - fun crc32(): Unit = runBlocking { - val headerReader = HeaderReader( - expectedHeaders = mapOf( - "content-encoding" to "aws-chunked", - "x-amz-trailer" to "x-amz-checksum-crc32", - ), - ) - - val trailerReader = TrailerReader( - expectedTrailers = mapOf( - "x-amz-checksum-crc32" to "i9aeUg==", - ), - ) - - TestClient { - interceptors = mutableListOf(headerReader, trailerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumStreamingOperation { - body = ByteStream.fromString("Hello world") - checksumAlgorithm = ChecksumAlgorithm.Crc32 - } - } - - assertTrue( - headerReader.containsExpectedHeaders && - trailerReader.containsExpectedTrailers, - ) - } - - @Test - fun crc32c(): Unit = runBlocking { - val headerReader = HeaderReader( - expectedHeaders = mapOf( - "content-encoding" to "aws-chunked", - "x-amz-trailer" to "x-amz-checksum-crc32c", - ), - ) - - val trailerReader = TrailerReader( - expectedTrailers = mapOf( - "x-amz-checksum-crc32c" to "crUfeA==", - ), - ) - - TestClient { - interceptors = mutableListOf(headerReader, trailerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumStreamingOperation { - body = ByteStream.fromString("Hello world") - checksumAlgorithm = ChecksumAlgorithm.Crc32C - } - } - - assertTrue( - headerReader.containsExpectedHeaders && - trailerReader.containsExpectedTrailers, - ) - } - - @Test - fun sha1(): Unit = runBlocking { - val headerReader = HeaderReader( - expectedHeaders = mapOf( - "content-encoding" to "aws-chunked", - "x-amz-trailer" to "x-amz-checksum-sha1", - ), - ) - - val trailerReader = TrailerReader( - expectedTrailers = mapOf( - "x-amz-checksum-sha1" to "e1AsOh9IyGCa4hLN+2Od7jlnP14=", - ), - ) - - TestClient { - interceptors = mutableListOf(headerReader, trailerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumStreamingOperation { - body = ByteStream.fromString("Hello world") - checksumAlgorithm = ChecksumAlgorithm.Sha1 - } - } - - assertTrue( - headerReader.containsExpectedHeaders && - trailerReader.containsExpectedTrailers, - ) - } - - @Test - fun sha256(): Unit = runBlocking { - val headerReader = HeaderReader( - expectedHeaders = mapOf( - "content-encoding" to "aws-chunked", - "x-amz-trailer" to "x-amz-checksum-sha256", - ), - ) - - val trailerReader = TrailerReader( - expectedTrailers = mapOf( - "x-amz-checksum-sha256" to "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", - ), - ) - - TestClient { - interceptors = mutableListOf(headerReader, trailerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumStreamingOperation { - body = ByteStream.fromString("Hello world") - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue( - headerReader.containsExpectedHeaders && - trailerReader.containsExpectedTrailers, - ) - } -} From 0f6fa6579c28034380cf51dc7457d86d5ce86da5 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 27 Nov 2024 01:51:01 -0500 Subject: [PATCH 30/72] Self review --- .../kotlin/runtime/config/AwsSdkSetting.kt | 4 +- .../ResolveRequestChecksumCalculation.kt | 16 ++--- .../ResolveResponseChecksumValidation.kt | 16 ++--- .../runtime/config/profile/AwsProfile.kt | 4 +- .../.kotlin/errors/errors-1731120437202.log | 4 -- .../FlexibleChecksumsRequest.kt | 14 ++--- .../FlexibleChecksumsResponse.kt | 12 ++-- .../e2eTest/src/MutliRegionAccessPointTest.kt | 6 +- services/s3/e2eTest/src/PaginatorTest.kt | 2 +- services/s3/e2eTest/src/S3ChecksumTest.kt | 58 +++++++++---------- services/s3/e2eTest/src/S3IntegrationTest.kt | 2 +- services/s3/e2eTest/src/S3PresignerTest.kt | 2 +- services/s3/e2eTest/src/S3TestUtils.kt | 29 +++------- settings.gradle.kts | 1 - tests/codegen/checksums/build.gradle.kts | 2 - .../kotlin/ChecksumBusinessMetricsTest.kt | 10 ++-- .../commonTest/kotlin/ChecksumConfigTest.kt | 21 +++---- .../commonTest/kotlin/ChecksumRequestTest.kt | 5 +- .../kotlin/utils/ChecksumTestUtils.kt | 29 +--------- tests/codegen/smoke-tests/build.gradle.kts | 3 +- .../src/commonTest/kotlin/SmokeTestE2ETest.kt | 9 +-- 21 files changed, 96 insertions(+), 153 deletions(-) delete mode 100644 buildSrc/.kotlin/errors/errors-1731120437202.log diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt index 9fe989f58ed..edbfe7cd92c 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt @@ -210,13 +210,13 @@ public object AwsSdkSetting { strEnvSetting("aws.sigV4aSigningRegionSet", "AWS_SIGV4A_SIGNING_REGION_SET") /** - * todo + * Configures request checksum calculation */ public val AwsRequestChecksumCalculation: EnvironmentSetting = strEnvSetting("aws.requestChecksumCalculation", "AWS_REQUEST_CHECKSUM_CALCULATION") /** - * todo + * Configures response checksum validation */ public val AwsResponseChecksumValidation: EnvironmentSetting = strEnvSetting("aws.responseChecksumValidation", "AWS_RESPONSE_CHECKSUM_VALIDATION") diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt index d731b5d3006..c629b54fab4 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt @@ -5,25 +5,25 @@ import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.config.AwsSdkSetting import aws.sdk.kotlin.runtime.config.profile.AwsProfile import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.util.LazyAsyncValue import aws.smithy.kotlin.runtime.util.PlatformProvider -import java.util.* /** - * todo + * Attempts to resolve requestChecksumCalculation from the specified sources. + * @return requestChecksumCalculation setting if found, the default value if not. */ @InternalSdkApi -public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption { +public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation return unparsedString?.let { when (unparsedString.uppercase()) { - "WHEN_SUPPORTED" -> ChecksumConfigOption.WHEN_SUPPORTED - "WHEN_REQUIRED" -> ChecksumConfigOption.WHEN_REQUIRED + "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED + "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", + "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries.toTypedArray()}", ) } - } ?: ChecksumConfigOption.WHEN_SUPPORTED + } ?: HttpChecksumConfigOption.WHEN_SUPPORTED } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt index dbf4aa9b377..b2d40c0711a 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt @@ -5,25 +5,25 @@ import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.config.AwsSdkSetting import aws.sdk.kotlin.runtime.config.profile.AwsProfile import aws.sdk.kotlin.runtime.config.profile.responseChecksumValidation -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.util.LazyAsyncValue import aws.smithy.kotlin.runtime.util.PlatformProvider -import java.util.* /** - * todo + * Attempts to resolve responseChecksumValidation from the specified sources. + * @return responseChecksumValidation setting if found, the default value if not. */ @InternalSdkApi -public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): ChecksumConfigOption { +public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation return unparsedString?.let { when (unparsedString.uppercase()) { - "WHEN_SUPPORTED" -> ChecksumConfigOption.WHEN_SUPPORTED - "WHEN_REQUIRED" -> ChecksumConfigOption.WHEN_REQUIRED + "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED + "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${ChecksumConfigOption.entries.toTypedArray()}", + "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries.toTypedArray()}", ) } - } ?: ChecksumConfigOption.WHEN_SUPPORTED + } ?: HttpChecksumConfigOption.WHEN_SUPPORTED } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index 472228604a9..119420a09e3 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -168,14 +168,14 @@ public val AwsProfile.sigV4aSigningRegionSet: String? get() = getOrNull("sigv4a_signing_region_set") /** - * todo + * Configures request checksum calculation */ @InternalSdkApi public val AwsProfile.requestChecksumCalculation: String? get() = getOrNull("request_checksum_calculation") /** - * todo + * Configures response checksum validation */ @InternalSdkApi public val AwsProfile.responseChecksumValidation: String? diff --git a/buildSrc/.kotlin/errors/errors-1731120437202.log b/buildSrc/.kotlin/errors/errors-1731120437202.log deleted file mode 100644 index 1219b509f09..00000000000 --- a/buildSrc/.kotlin/errors/errors-1731120437202.log +++ /dev/null @@ -1,4 +0,0 @@ -kotlin version: 2.0.21 -error message: The daemon has terminated unexpectedly on startup attempt #1 with error code: 0. The daemon process output: - 1. Kotlin compile daemon is ready - diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 11c0f8a43fd..89f6e84be22 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -33,11 +33,11 @@ class FlexibleChecksumsRequest : KotlinIntegration { listOf( ConfigProperty { name = "requestChecksumCalculation" - symbol = RuntimeTypes.SmithyClient.Config.ChecksumConfigOption + symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig useNestedBuilderBaseClass() - documentation = "" // todo - propertyType = ConfigPropertyType.RequiredWithDefault("ChecksumConfigOption.WHEN_SUPPORTED") + documentation = "Configures request checksum calculation" + propertyType = ConfigPropertyType.RequiredWithDefault("HttpChecksumConfigOption.WHEN_SUPPORTED") }, ) @@ -54,13 +54,13 @@ class FlexibleChecksumsRequest : KotlinIntegration { writer.withBlock("when(config.requestChecksumCalculation) {", "}") { writer.write( "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) writer.write( "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) @@ -87,7 +87,7 @@ class FlexibleChecksumsRequest : KotlinIntegration { .members() .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } - val userSelectedChecksumAlgorithm = ctx.symbolProvider.toMemberName(requestAlgorithmMember) + val requestAlgorithmMemberName = ctx.symbolProvider.toMemberName(requestAlgorithmMember) val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired writer.withBlock( @@ -97,7 +97,7 @@ class FlexibleChecksumsRequest : KotlinIntegration { ) { writer.write("requestChecksumRequired = #L,", requestChecksumRequired) writer.write("requestChecksumCalculation = config.requestChecksumCalculation,") - writer.write("userSelectedChecksumAlgorithm = input.#L?.value,", userSelectedChecksumAlgorithm) + writer.write("userSelectedChecksumAlgorithm = input.#L?.value,", requestAlgorithmMemberName) } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index 95f27f24fae..4acf2083b64 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -30,11 +30,11 @@ class FlexibleChecksumsResponse : KotlinIntegration { listOf( ConfigProperty { name = "responseChecksumValidation" - symbol = RuntimeTypes.SmithyClient.Config.ChecksumConfigOption + symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig useNestedBuilderBaseClass() - documentation = "" // todo - propertyType = ConfigPropertyType.RequiredWithDefault("ChecksumConfigOption.WHEN_SUPPORTED") + documentation = "Configures response checksum validation" + propertyType = ConfigPropertyType.RequiredWithDefault("HttpChecksumConfigOption.WHEN_SUPPORTED") }, ) @@ -48,13 +48,13 @@ class FlexibleChecksumsResponse : KotlinIntegration { writer.withBlock("when(config.responseChecksumValidation) {", "}") { writer.write( "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) writer.write( "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.ChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) @@ -86,7 +86,7 @@ class FlexibleChecksumsResponse : KotlinIntegration { "))", RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, ) { - writer.write("responseValidation = input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) + writer.write("responseValidationRequired = input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) writer.write("responseChecksumValidation = config.responseChecksumValidation,") } } diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 49630763536..93a193e439b 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -8,7 +8,7 @@ import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix +import aws.sdk.kotlin.e2etest.S3TestUtils.getBucket import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated import aws.sdk.kotlin.services.s3.S3Client @@ -47,8 +47,8 @@ class MutliRegionAccessPointTest { @BeforeAll private fun setUp(): Unit = runBlocking { accountId = getAccountId() - usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) - usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) + usWestBucket = getBucket(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) + usEastBucket = getBucket(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) createMultiRegionAccessPoint( s3Control, diff --git a/services/s3/e2eTest/src/PaginatorTest.kt b/services/s3/e2eTest/src/PaginatorTest.kt index 27285a080bd..3d1ed2f4919 100644 --- a/services/s3/e2eTest/src/PaginatorTest.kt +++ b/services/s3/e2eTest/src/PaginatorTest.kt @@ -33,7 +33,7 @@ class PaginatorTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucketWithPrefix(client) + testBucket = S3TestUtils.getTestBucket(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index eff66e890df..6dccc573e30 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -3,11 +3,11 @@ package aws.sdk.kotlin.e2etest import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketContents import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiPartUploads import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -import aws.sdk.kotlin.e2etest.S3TestUtils.getTestBucketByName -import aws.sdk.kotlin.runtime.auth.credentials.ProcessCredentialsProvider +import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketByName import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.GetObjectRequest import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.fromInputStream import kotlinx.coroutines.runBlocking @@ -19,33 +19,28 @@ import java.util.* @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation::class) class S3ChecksumTest { - private val s3West = S3Client { - region = "us-west-2" - credentialsProvider = ProcessCredentialsProvider("isengardcli credentials --awscli aws-kotlin-sdk+ci@amazon.com --role Admin") - } - private val checksumsTestBucket = "s3-test-bucket-ci-motorcade" + private val client = S3Client { region = "us-west-2" } + private val testBucket = "s3-test-bucket-ci-motorcade" private val testObject = "test-object" - private lateinit var accountId: String - private lateinit var usWestBucket: String @BeforeAll private fun setUp(): Unit = runBlocking { - accountId = getAccountId() - usWestBucket = getTestBucketByName(s3West, checksumsTestBucket, "us-west-2", accountId) + val accountId = getAccountId() + getBucketByName(client, testBucket, "us-west-2", accountId) } @AfterAll private fun cleanUp(): Unit = runBlocking { - deleteMultiPartUploads(s3West, checksumsTestBucket) - deleteBucketContents(s3West, checksumsTestBucket) - s3West.close() + deleteMultiPartUploads(client, testBucket) + deleteBucketContents(client, testBucket) + client.close() } @Test @Order(1) fun testPutObject(): Unit = runBlocking { - s3West.putObject { - bucket = checksumsTestBucket + client.putObject { + bucket = testBucket key = testObject body = ByteStream.fromString("Hello World") } @@ -54,8 +49,8 @@ class S3ChecksumTest { @Test @Order(2) fun testPutObjectWithEmptyBody(): Unit = runBlocking { - s3West.putObject { - bucket = checksumsTestBucket + client.putObject { + bucket = testBucket key = testObject + UUID.randomUUID() } } @@ -70,8 +65,8 @@ class S3ChecksumTest { } val inputStream = FileInputStream(tempFile) - s3West.putObject { - bucket = checksumsTestBucket + client.putObject { + bucket = testBucket key = testObject + UUID.randomUUID() body = ByteStream.fromInputStream(inputStream, testString.length.toLong()) } @@ -84,29 +79,29 @@ class S3ChecksumTest { val partOne = "Hello".repeat(1_048_576) val partTwo = "World".repeat(1_048_576) - val testUploadId = s3West.createMultipartUpload { - bucket = checksumsTestBucket + val testUploadId = client.createMultipartUpload { + bucket = testBucket key = testObject }.uploadId - val eTagPartOne = s3West.uploadPart { - bucket = checksumsTestBucket + val eTagPartOne = client.uploadPart { + bucket = testBucket key = testObject partNumber = 1 uploadId = testUploadId body = ByteStream.fromString(partOne) }.eTag - val eTagPartTwo = s3West.uploadPart { - bucket = checksumsTestBucket + val eTagPartTwo = client.uploadPart { + bucket = testBucket key = testObject partNumber = 2 uploadId = testUploadId body = ByteStream.fromString(partTwo) }.eTag - s3West.completeMultipartUpload { - bucket = checksumsTestBucket + client.completeMultipartUpload { + bucket = testBucket key = testObject uploadId = testUploadId multipartUpload = CompletedMultipartUpload { @@ -123,6 +118,11 @@ class S3ChecksumTest { } } - // TODO: Get the object and make sure a composite checksum doesn't break us + client.getObject( + GetObjectRequest { + bucket = testBucket + key = testObject + }, + ) {} } } diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index 383ca82a381..37ec353c9f6 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -45,7 +45,7 @@ class S3BucketOpsIntegrationTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucketWithPrefix(client) + testBucket = S3TestUtils.getTestBucket(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3PresignerTest.kt b/services/s3/e2eTest/src/S3PresignerTest.kt index 9b479521c2f..38e8a39ee65 100644 --- a/services/s3/e2eTest/src/S3PresignerTest.kt +++ b/services/s3/e2eTest/src/S3PresignerTest.kt @@ -34,7 +34,7 @@ class S3PresignerTest { @BeforeAll fun createResources(): Unit = runBlocking { - testBucket = S3TestUtils.getTestBucketWithPrefix(client) + testBucket = S3TestUtils.getTestBucket(client) } @AfterAll diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 575ceb1f7fe..5f10a9f52ee 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -33,21 +33,22 @@ object S3TestUtils { const val DEFAULT_REGION = "us-west-2" - // The E2E test account only has permission to operate on buckets with the prefix (s3-test-bucket-) - // Motorcade allow-listed bucket: "s3-test-bucket-ci-motorcade". E2E tests will use it and delete it! - // FIXME: Change back to "s3-test-bucket-" after motorcade is released. + // The E2E test account only has permission to operate on buckets with the prefix "s3-test-bucket-" + // Motorcade allow-listed bucket: "s3-test-bucket-ci-motorcade". + // Non-checksum E2E tests will use it and delete it if TEST_BUCKET_PREFIX="s3-test-bucket-" via `deleteBucketAndAllContents` + // TODO: Change back to "s3-test-bucket-" after motorcade is released. private const val TEST_BUCKET_PREFIX = "s3-test-bucket-temp-" private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html private const val S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX = "--x-s3" - suspend fun getTestBucketWithPrefix( + suspend fun getTestBucket( client: S3Client, region: String? = null, accountId: String? = null, - ): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX, region, accountId) + ): String = getBucket(client, TEST_BUCKET_PREFIX, region, accountId) - suspend fun getBucketWithPrefix( + suspend fun getBucket( client: S3Client, prefix: String, region: String? = null, @@ -100,7 +101,7 @@ object S3TestUtils { testBucket } - suspend fun getTestBucketByName( + suspend fun getBucketByName( client: S3Client, bucket: String, region: String? = null, @@ -136,20 +137,6 @@ object S3TestUtils { println("Using existing S3 bucket: $testBucket") } - client.putBucketLifecycleConfiguration { - this.bucket = testBucket - lifecycleConfiguration { - rules = listOf( - LifecycleRule { - expiration { days = 1 } - filter { this.prefix = "" } - status = ExpirationStatus.Enabled - id = "delete-old" - }, - ) - } - } - testBucket } diff --git a/settings.gradle.kts b/settings.gradle.kts index dc2418bfe1d..9bd423a88cd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -54,7 +54,6 @@ include(":services") include(":tests") include(":tests:codegen") include(":tests:codegen:event-stream") -include(":tests:codegen:rules-engine") include(":tests:e2e-test-util") include(":tests:codegen:smoke-tests") include(":tests:codegen:smoke-tests:services") diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index cd378296c9b..2139fcafa20 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -7,7 +7,6 @@ import shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" kotlin { - // TODO: This should be part of the shared gradle file ! sourceSets { commonTest { dependencies { @@ -43,7 +42,6 @@ smithyBuild { } } -// TODO: Commonize this to other codegen tests kotlin { sourceSets { commonTest { diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt index b22e32aa124..a125c8e9a4e 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt @@ -2,7 +2,7 @@ import aws.sdk.kotlin.test.checksums.TestClient import aws.sdk.kotlin.test.checksums.httpChecksumOperation import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm import aws.smithy.kotlin.runtime.businessmetrics.SmithyBusinessMetric -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking import utils.BusinessMetricsReader @@ -43,8 +43,8 @@ class ChecksumBusinessMetricsTest { TestClient { httpClient = TestEngine() interceptors = mutableListOf(businessMetricsReader) - requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED - responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_SUPPORTED }.use { client -> client.httpChecksumOperation { body = "Hello world".encodeToByteArray() @@ -66,8 +66,8 @@ class ChecksumBusinessMetricsTest { TestClient { httpClient = TestEngine() interceptors = mutableListOf(businessMetricsReader) - requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED }.use { client -> client.httpChecksumOperation { body = "Hello world".encodeToByteArray() diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt index 375b040421f..3c81d7924c9 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt @@ -3,7 +3,7 @@ import aws.sdk.kotlin.test.clientconfig.* import aws.sdk.kotlin.test.clientconfig.model.ChecksumAlgorithm import aws.sdk.kotlin.test.clientconfig.model.ValidationMode import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.client.config.ChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.http.* import aws.smithy.kotlin.runtime.http.Headers import aws.smithy.kotlin.runtime.http.HttpStatusCode @@ -19,8 +19,6 @@ import kotlin.test.assertFailsWith import kotlin.test.assertFalse import kotlin.test.assertTrue -// TODO - Simplify this - /** * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` when set to **true**. */ @@ -32,7 +30,7 @@ class RequestChecksumRequired { ) ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED interceptors = mutableListOf(headerReader) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -57,7 +55,7 @@ class RequestChecksumRequired { ) ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED interceptors = mutableListOf(headerReader) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -87,7 +85,7 @@ class RequestChecksumNotRequired { ) ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_SUPPORTED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED interceptors = mutableListOf(headerReader) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -112,7 +110,7 @@ class RequestChecksumNotRequired { ) ClientConfigTestClient { - requestChecksumCalculation = ChecksumConfigOption.WHEN_REQUIRED + requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED interceptors = mutableListOf(headerReader) httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider( @@ -269,7 +267,7 @@ class ResponseChecksumValidation { fun responseChecksumValidationWhenSupported(): Unit = runBlocking { assertFailsWith { ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_SUPPORTED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_SUPPORTED httpClient = TestEngine( roundTripImpl = { _, request -> val resp = HttpResponse( @@ -298,7 +296,7 @@ class ResponseChecksumValidation { @Test fun responseChecksumValidationWhenRequired(): Unit = runBlocking { ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED httpClient = TestEngine( roundTripImpl = { _, request -> val resp = HttpResponse( @@ -327,7 +325,7 @@ class ResponseChecksumValidation { fun responseChecksumValidationWhenRequiredWithRequestValidationModeEnabled(): Unit = runBlocking { assertFailsWith { ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED httpClient = TestEngine( roundTripImpl = { _, request -> val resp = HttpResponse( @@ -356,9 +354,8 @@ class ResponseChecksumValidation { @Test fun compositeChecksumsAreNotValidated(): Unit = runBlocking { - // TODO: Move elsewhere ClientConfigTestClient { - responseChecksumValidation = ChecksumConfigOption.WHEN_REQUIRED + responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED httpClient = TestEngine( roundTripImpl = { _, request -> val resp = HttpResponse( diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt index dc0c0cad6b5..16eacf58d64 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt @@ -8,9 +8,6 @@ import utils.HeaderReader import kotlin.test.Test import kotlin.test.assertTrue -// TODO - Simplify -// TODO - Test empty payload - class ChecksumRequestTest { @Test fun crc32(): Unit = runBlocking { @@ -27,7 +24,7 @@ class ChecksumRequestTest { credentialsProvider = StaticCredentialsProvider( Credentials("accessKeyID", "secretAccessKey"), ) - region = "us-east-1" // TODO - region is unnecessary ..... + region = "us-east-1" }.use { client -> client.httpChecksumOperation { body = "Hello world".encodeToByteArray() diff --git a/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt index b010c1ffd2b..725e4c11adf 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt @@ -7,7 +7,6 @@ import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.toBuilder -import kotlinx.coroutines.runBlocking /** * Checks if the specified headers are set in an HTTP request. @@ -41,31 +40,6 @@ internal class HeaderReader( } } -/** - * Checks if the specified trailer headers are set in an HTTP request. - */ -internal class TrailerReader( - private val expectedTrailers: Map, -) : HttpInterceptor { - var containsExpectedTrailers = true - - override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { - expectedTrailers.forEach { trailer -> - val containsTrailer = context.protocolRequest.trailingHeaders.contains(trailer.key) - val trailerValueMatches = trailer.value?.let { trailerValue -> - runBlocking { - context.protocolRequest.trailingHeaders[trailer.key]?.await() == trailerValue - } - } ?: true - - if (!containsTrailer || !trailerValueMatches) { - containsExpectedTrailers = false - return - } - } - } -} - /** * Sets the specified checksum header and value in an HTTP request. */ @@ -81,6 +55,9 @@ internal class HeaderSetter( } } +/** + * Checks if the specified business metrics are set in an HTTP request. + */ internal class BusinessMetricsReader( private val expectedBusinessMetrics: Set, ) : HttpInterceptor { diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 87aaab703e4..1cc3c379724 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -14,9 +14,8 @@ description = "AWS SDK for Kotlin's smoke test codegen test suite" kotlin { sourceSets { jvmTest { - // todo: apply jvm plugin just here ? dependencies { - implementation("dev.gradleplugins:gradle-test-kit:7.3.3") // TODO: Use lib.versions.toml + implementation("dev.gradleplugins:gradle-test-kit:7.3.3") } } } diff --git a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt index d68cdabcdc1..43667e88b2c 100644 --- a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt +++ b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt @@ -4,11 +4,8 @@ import org.gradle.testkit.runner.GradleRunner import java.io.File import kotlin.test.* -// TODO: Turn on tests again - class SmokeTestE2ETest { @Test - @Ignore fun successService() { val smokeTestRunnerOutput = runSmokeTests("successService") @@ -17,7 +14,6 @@ class SmokeTestE2ETest { } @Test - @Ignore fun failureService() { val smokeTestRunnerOutput = runSmokeTests("failureService") @@ -25,7 +21,6 @@ class SmokeTestE2ETest { } @Test - @Ignore fun exceptionService() { val smokeTestRunnerOutput = runSmokeTests("exceptionService", expectingFailure = true) @@ -36,7 +31,6 @@ class SmokeTestE2ETest { } @Test - @Ignore fun successServiceSkipTags() { val envVars = mapOf(AWS_SKIP_TAGS to "success") val smokeTestRunnerOutput = runSmokeTests("successService", envVars) @@ -46,7 +40,6 @@ class SmokeTestE2ETest { } @Test - @Ignore fun successServiceServiceFilter() { val envVars = mapOf(AWS_SERVICE_FILTER to "Failure") // Only run tests for services with this SDK ID val smokeTestRunnerOutput = runSmokeTests("successService", envVars) @@ -68,7 +61,7 @@ private fun runSmokeTests( // FIXME: Remove `-Paws.kotlin.native=false` when Kotlin Native is ready .withArguments("-Paws.kotlin.native=false", ":tests:codegen:smoke-tests:services:$service:smokeTest") .withEnvironment(envVars) - .withGradleVersion("8.5") // FIXME: Use lib.versions - or find a way to use `gradleTestKit()` + .withGradleVersion("8.5") val buildResult = if (expectingFailure) task.buildAndFail() else task.build() From c76aefe30c4ce565d13d3b02e40238323cdfb3d6 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 27 Nov 2024 05:57:26 -0500 Subject: [PATCH 31/72] trigger ci From 3476242b83efea4b280d5763aa3b821505789c63 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:05:01 -0700 Subject: [PATCH 32/72] Fix smoke tests --- tests/codegen/smoke-tests/build.gradle.kts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 1cc3c379724..6448e7f35ae 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -11,14 +11,8 @@ import shared.Model description = "AWS SDK for Kotlin's smoke test codegen test suite" -kotlin { - sourceSets { - jvmTest { - dependencies { - implementation("dev.gradleplugins:gradle-test-kit:7.3.3") - } - } - } +dependencies { + jvmTestImplementation(gradleTestKit()) } val tests = listOf( From b21ec415508912c662b24e29cba0917f7b954d8b Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:09:46 -0700 Subject: [PATCH 33/72] Don't specify gradle version in smoke tests --- .../smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt index 43667e88b2c..61f38fe62c1 100644 --- a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt +++ b/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt @@ -61,7 +61,6 @@ private fun runSmokeTests( // FIXME: Remove `-Paws.kotlin.native=false` when Kotlin Native is ready .withArguments("-Paws.kotlin.native=false", ":tests:codegen:smoke-tests:services:$service:smokeTest") .withEnvironment(envVars) - .withGradleVersion("8.5") val buildResult = if (expectingFailure) task.buildAndFail() else task.build() From dd2792fae3f9698c4ee46e365bd8c85d4bfd2db0 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:30:42 -0700 Subject: [PATCH 34/72] Correctly print enum entries in error message --- .../config/checksums/ResolveRequestChecksumCalculation.kt | 2 +- .../config/checksums/ResolveResponseChecksumValidation.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt index c629b54fab4..dd521548b9f 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt @@ -22,7 +22,7 @@ public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries.toTypedArray()}", + "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries}", ) } } ?: HttpChecksumConfigOption.WHEN_SUPPORTED diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt index b2d40c0711a..6c7162c2195 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt @@ -22,7 +22,7 @@ public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries.toTypedArray()}", + "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries}", ) } } ?: HttpChecksumConfigOption.WHEN_SUPPORTED From 779cd3a489d784bd013dbf01534585b0455da01b Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:31:40 -0700 Subject: [PATCH 35/72] Remove test ordering from S3ChecksumTest --- services/s3/e2eTest/src/S3ChecksumTest.kt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index 6dccc573e30..e75173cf691 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -17,7 +17,6 @@ import java.io.FileInputStream import java.util.* @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestMethodOrder(MethodOrderer.OrderAnnotation::class) class S3ChecksumTest { private val client = S3Client { region = "us-west-2" } private val testBucket = "s3-test-bucket-ci-motorcade" @@ -37,7 +36,6 @@ class S3ChecksumTest { } @Test - @Order(1) fun testPutObject(): Unit = runBlocking { client.putObject { bucket = testBucket @@ -47,7 +45,6 @@ class S3ChecksumTest { } @Test - @Order(2) fun testPutObjectWithEmptyBody(): Unit = runBlocking { client.putObject { bucket = testBucket @@ -56,7 +53,6 @@ class S3ChecksumTest { } @Test - @Order(3) fun testPutObjectAwsChunkedEncoded(): Unit = runBlocking { val testString = "Hello World" val tempFile = File.createTempFile("test", ".txt").also { @@ -73,7 +69,6 @@ class S3ChecksumTest { } @Test - @Order(4) fun testMultiPartUpload(): Unit = runBlocking { // Parts need to be at least 5 MB val partOne = "Hello".repeat(1_048_576) From a3a2e095c27571a20b3aeab295053c6cbf91ef34 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:46:34 -0700 Subject: [PATCH 36/72] Add copyright headers to checksum codegen tests --- .../src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt | 5 +++++ .../checksums/src/commonTest/kotlin/ChecksumConfigTest.kt | 5 +++++ .../checksums/src/commonTest/kotlin/ChecksumRequestTest.kt | 5 +++++ .../checksums/src/commonTest/kotlin/ChecksumResponseTest.kt | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt index a125c8e9a4e..dcb05a81c56 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt @@ -1,3 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + import aws.sdk.kotlin.test.checksums.TestClient import aws.sdk.kotlin.test.checksums.httpChecksumOperation import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt index 3c81d7924c9..400e5406cb4 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt @@ -1,3 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.clientconfig.* import aws.sdk.kotlin.test.clientconfig.model.ChecksumAlgorithm diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt index 16eacf58d64..f0eef19014d 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt @@ -1,3 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt index 9c98561a70c..6c029b5beb7 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt @@ -1,3 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials From c80b55cb75dba1b5eb403b429005484dd631dafd Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 13:53:07 -0700 Subject: [PATCH 37/72] Update buildSrc package setup --- .../main/kotlin/{ => aws/sdk/kotlin}/shared/CodegenTest.kt | 2 +- tests/codegen/checksums/build.gradle.kts | 4 ++-- tests/codegen/event-stream/build.gradle.kts | 4 ++-- tests/codegen/smoke-tests/build.gradle.kts | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) rename buildSrc/src/main/kotlin/{ => aws/sdk/kotlin}/shared/CodegenTest.kt (91%) diff --git a/buildSrc/src/main/kotlin/shared/CodegenTest.kt b/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt similarity index 91% rename from buildSrc/src/main/kotlin/shared/CodegenTest.kt rename to buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt index f9a03d715e9..4ee9580629c 100644 --- a/buildSrc/src/main/kotlin/shared/CodegenTest.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt @@ -1,4 +1,4 @@ -package shared +package aws.sdk.kotlin.shared /** * An AWS SDK for Kotlin codegen test diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index 2139fcafa20..6867921e4a2 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -1,8 +1,8 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -import shared.CodegenTest -import shared.Model +import aws.sdk.kotlin.shared.CodegenTest +import aws.sdk.kotlin.shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 71edc4f41fb..4ce7b11ca4d 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -5,8 +5,8 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -import shared.CodegenTest -import shared.Model +import aws.sdk.kotlin.shared.CodegenTest +import aws.sdk.kotlin.shared.Model description = "AWS SDK for Kotlin's event stream codegen test suite" diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 6448e7f35ae..86422a486f7 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -6,8 +6,8 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath -import shared.CodegenTest -import shared.Model +import aws.sdk.kotlin.shared.CodegenTest +import aws.sdk.kotlin.shared.Model description = "AWS SDK for Kotlin's smoke test codegen test suite" From 5e2855c70859c1f74d34737e6db829ab776b1aa2 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 14:19:59 -0700 Subject: [PATCH 38/72] Fix codegen tests not having packages --- .../codegen/checksums}/ChecksumBusinessMetricsTest.kt | 4 +++- .../kotlin/tests/codegen/checksums}/ChecksumConfigTest.kt | 6 ++++-- .../kotlin/tests/codegen/checksums}/ChecksumRequestTest.kt | 4 +++- .../tests/codegen/checksums}/ChecksumResponseTest.kt | 2 ++ .../tests/codegen/checksums}/utils/ChecksumTestUtils.kt | 7 ++++++- .../tests/codegen/eventstream}/HttpEventStreamTests.kt | 2 ++ .../tests/codegen/eventstream}/RpcEventStreamTests.kt | 3 +++ .../tests/codegen/smoketests/SmokeTestE2ETest.kt.kt} | 7 +++++++ 8 files changed, 30 insertions(+), 5 deletions(-) rename tests/codegen/checksums/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/checksums}/ChecksumBusinessMetricsTest.kt (97%) rename tests/codegen/checksums/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/checksums}/ChecksumConfigTest.kt (98%) rename tests/codegen/checksums/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/checksums}/ChecksumRequestTest.kt (97%) rename tests/codegen/checksums/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/checksums}/ChecksumResponseTest.kt (99%) rename tests/codegen/checksums/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/checksums}/utils/ChecksumTestUtils.kt (93%) rename tests/codegen/event-stream/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/eventstream}/HttpEventStreamTests.kt (99%) rename tests/codegen/event-stream/src/commonTest/kotlin/{ => aws/sdk/kotlin/tests/codegen/eventstream}/RpcEventStreamTests.kt (99%) rename tests/codegen/smoke-tests/src/commonTest/kotlin/{SmokeTestE2ETest.kt => aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt} (94%) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt similarity index 97% rename from tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt rename to tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt index dcb05a81c56..065d5f83b76 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumBusinessMetricsTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt @@ -3,14 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ +package aws.sdk.kotlin.tests.codegen.checksums + import aws.sdk.kotlin.test.checksums.TestClient import aws.sdk.kotlin.test.checksums.httpChecksumOperation import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.sdk.kotlin.tests.codegen.checksums.utils.BusinessMetricsReader import aws.smithy.kotlin.runtime.businessmetrics.SmithyBusinessMetric import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking -import utils.BusinessMetricsReader import kotlin.test.Test import kotlin.test.assertTrue diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt similarity index 98% rename from tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt rename to tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt index 400e5406cb4..107db032312 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt @@ -3,10 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ +package aws.sdk.kotlin.tests.codegen.checksums + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.clientconfig.* import aws.sdk.kotlin.test.clientconfig.model.ChecksumAlgorithm import aws.sdk.kotlin.test.clientconfig.model.ValidationMode +import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderReader +import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderSetter import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption import aws.smithy.kotlin.runtime.http.* @@ -17,8 +21,6 @@ import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.httptest.TestEngine import aws.smithy.kotlin.runtime.time.Instant import kotlinx.coroutines.runBlocking -import utils.HeaderReader -import utils.HeaderSetter import kotlin.test.Test import kotlin.test.assertFailsWith import kotlin.test.assertFalse diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt similarity index 97% rename from tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt rename to tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt index f0eef19014d..d64794a136d 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumRequestTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt @@ -3,13 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ +package aws.sdk.kotlin.tests.codegen.checksums + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderReader import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking -import utils.HeaderReader import kotlin.test.Test import kotlin.test.assertTrue diff --git a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt similarity index 99% rename from tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt rename to tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt index 6c029b5beb7..34f74c1f130 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/ChecksumResponseTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +package aws.sdk.kotlin.tests.codegen.checksums + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials diff --git a/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt similarity index 93% rename from tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt rename to tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt index 725e4c11adf..a8f1f634a1f 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/utils/ChecksumTestUtils.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt @@ -1,4 +1,9 @@ -package utils +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.tests.codegen.checksums.utils import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetric import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetrics diff --git a/tests/codegen/event-stream/src/commonTest/kotlin/HttpEventStreamTests.kt b/tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt similarity index 99% rename from tests/codegen/event-stream/src/commonTest/kotlin/HttpEventStreamTests.kt rename to tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt index 1bb5ec2bda9..43f2b80339d 100644 --- a/tests/codegen/event-stream/src/commonTest/kotlin/HttpEventStreamTests.kt +++ b/tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +package aws.sdk.kotlin.tests.codegen.eventstream + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.restjson1.model.* import aws.sdk.kotlin.test.restjson1.serde.deserializeTestStreamOpOperationBody diff --git a/tests/codegen/event-stream/src/commonTest/kotlin/RpcEventStreamTests.kt b/tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt similarity index 99% rename from tests/codegen/event-stream/src/commonTest/kotlin/RpcEventStreamTests.kt rename to tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt index 08cbecbda2a..3c04b819224 100644 --- a/tests/codegen/event-stream/src/commonTest/kotlin/RpcEventStreamTests.kt +++ b/tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt @@ -2,6 +2,9 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ + +package aws.sdk.kotlin.tests.codegen.eventstream + import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.awsjson11.model.MessageWithString import aws.sdk.kotlin.test.awsjson11.model.TestStream diff --git a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt similarity index 94% rename from tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt rename to tests/codegen/smoke-tests/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt index 61f38fe62c1..b3521302b81 100644 --- a/tests/codegen/smoke-tests/src/commonTest/kotlin/SmokeTestE2ETest.kt +++ b/tests/codegen/smoke-tests/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt @@ -1,3 +1,10 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.tests.codegen.smoketests + import aws.sdk.kotlin.codegen.smoketests.AWS_SERVICE_FILTER import aws.sdk.kotlin.codegen.smoketests.AWS_SKIP_TAGS import org.gradle.testkit.runner.GradleRunner From 0405b2034193f2d9f6c655bce83cce8a362035f2 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 14:35:49 -0700 Subject: [PATCH 39/72] Fix event stream codegen test templates --- tests/codegen/event-stream/build.gradle.kts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 4ce7b11ca4d..0de5288432e 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -83,8 +83,14 @@ fun fillInModel(test: CodegenTest) { } val replaced = model - .replace("{protocol-name}", test.protocolName!!) - .replace("{op-traits}", opTraits) + .replace( + "{protocol-name}", + test.protocolName ?: throw IllegalStateException("Please specify a protocol name for the codegen test"), + ) + .replace( + "{op-traits}", + opTraits, + ) modelFile.parentFile.mkdirs() modelFile.writeText(replaced) From eb338f400c06ebb4b049a0d6064dcb71f9498a69 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Sun, 1 Dec 2024 14:39:54 -0700 Subject: [PATCH 40/72] Save test reports for failing JVM CI checks --- .github/workflows/continuous-integration.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index b8cb2e12035..684d610607a 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -40,6 +40,12 @@ jobs: pwd ls -lsa ./gradlew -Ptest.java.version=${{ matrix.java-version }} jvmTest --stacktrace + - name: Save Test Reports + if: failure() + uses: actions/upload-artifact@v3 + with: + name: test-reports + path: '**/build/reports' all-platforms: runs-on: ${{ matrix.os }} From a2fb7fb8ec961e626d3cc030b355fa7f625ac782 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 3 Dec 2024 15:01:20 -0700 Subject: [PATCH 41/72] Track smithy kotlin changes --- .../FlexibleChecksumsRequest.kt | 11 +++++++---- .../FlexibleChecksumsResponse.kt | 17 ++++++++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 89f6e84be22..d79824212ff 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -81,6 +81,8 @@ class FlexibleChecksumsRequest : KotlinIntegration { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) + val httpChecksumTrait = op.getTrait()!! val requestAlgorithmMember = ctx.model.expectShape(op.input.get()) @@ -91,13 +93,14 @@ class FlexibleChecksumsRequest : KotlinIntegration { val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired writer.withBlock( - "op.interceptors.add(#T(", + "op.interceptors.add(#T<#T>(", "))", RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor, + inputSymbol, ) { - writer.write("requestChecksumRequired = #L,", requestChecksumRequired) - writer.write("requestChecksumCalculation = config.requestChecksumCalculation,") - writer.write("userSelectedChecksumAlgorithm = input.#L?.value,", requestAlgorithmMemberName) + writer.write("#L,", requestChecksumRequired) + writer.write("config.requestChecksumCalculation,") + writer.write("input.#L?.value,", requestAlgorithmMemberName) } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index 4acf2083b64..afd8c29c20a 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -18,6 +18,13 @@ import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.StructureShape +/** + * AWS services that can return composite checksums in their responses + */ +private val compositeChecksumServices = setOf( + "s3", +) + /** * Adds a middleware which validates checksums returned in responses if the user has opted-in. */ @@ -75,6 +82,8 @@ class FlexibleChecksumsResponse : KotlinIntegration { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) + val httpChecksumTrait = op.getTrait()!! val requestValidationModeMember = ctx.model.expectShape(op.input.get()) .members() @@ -82,12 +91,14 @@ class FlexibleChecksumsResponse : KotlinIntegration { val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) writer.withBlock( - "op.interceptors.add(#T(", + "op.interceptors.add(#T<#T>(", "))", RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, + inputSymbol, ) { - writer.write("responseValidationRequired = input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) - writer.write("responseChecksumValidation = config.responseChecksumValidation,") + writer.write("input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) + writer.write("config.responseChecksumValidation,") + writer.write("#L", compositeChecksumServices.contains(ctx.settings.sdkId)) } } } From d498e3010f7af1be346e3d5229e771eaacdcdeaf Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 4 Dec 2024 11:22:47 -0700 Subject: [PATCH 42/72] Fix jvmTest JVM compatibility --- tests/codegen/build.gradle.kts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts index 0fda97f3cd4..dd0a9537e6b 100644 --- a/tests/codegen/build.gradle.kts +++ b/tests/codegen/build.gradle.kts @@ -45,7 +45,11 @@ subprojects { } kotlin { - jvm() + jvm { + compilations.all { + kotlinOptions.jvmTarget = "1.8" + } + } sourceSets { commonMain { dependencies { From b534927459ea95e01ad5d7b56b1b98edf9e7b736 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 4 Dec 2024 12:52:42 -0700 Subject: [PATCH 43/72] Drop support for http body dot bytes response checksums --- .../codegen/checksums/ChecksumConfigTest.kt | 16 +++++- .../codegen/checksums/ChecksumResponseTest.kt | 57 +++++++++++++++---- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt index 107db032312..e26afa7908c 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt @@ -19,6 +19,8 @@ import aws.smithy.kotlin.runtime.http.HttpStatusCode import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.io.source import aws.smithy.kotlin.runtime.time.Instant import kotlinx.coroutines.runBlocking import kotlin.test.Test @@ -270,6 +272,8 @@ class UserProvidedChecksumHeader { * Tests the `aws.protocols#httpChecksum` trait's `requestValidationModeMember`. */ class ResponseChecksumValidation { + private val responseBody = "Hello world" + @Test fun responseChecksumValidationWhenSupported(): Unit = runBlocking { assertFailsWith { @@ -282,7 +286,11 @@ class ResponseChecksumValidation { Headers { append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") }, - "World!".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -340,7 +348,11 @@ class ResponseChecksumValidation { Headers { append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") }, - "World!".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt index 34f74c1f130..f477a3f1129 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt @@ -8,18 +8,19 @@ package aws.sdk.kotlin.tests.codegen.checksums import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.test.checksums.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.http.Headers -import aws.smithy.kotlin.runtime.http.HttpCall -import aws.smithy.kotlin.runtime.http.HttpStatusCode +import aws.smithy.kotlin.runtime.http.* import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.http.toHttpBody import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.io.source import aws.smithy.kotlin.runtime.time.Instant import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertFailsWith +private val responseBody = "Hello world" + class SuccessfulChecksumResponseTest { @Test fun crc32(): Unit = runBlocking { @@ -31,7 +32,11 @@ class SuccessfulChecksumResponseTest { Headers { append("x-amz-checksum-crc32", "i9aeUg==") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -58,7 +63,11 @@ class SuccessfulChecksumResponseTest { Headers { append("x-amz-checksum-crc32c", "crUfeA==") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -85,7 +94,11 @@ class SuccessfulChecksumResponseTest { Headers { append("x-amz-checksum-sha1", "e1AsOh9IyGCa4hLN+2Od7jlnP14=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -112,7 +125,11 @@ class SuccessfulChecksumResponseTest { Headers { append("x-amz-checksum-sha256", "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -142,7 +159,11 @@ class FailedChecksumResponseTest { Headers { append("x-amz-checksum-crc32", "bm90LWEtY2hlY2tzdW0=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -171,7 +192,11 @@ class FailedChecksumResponseTest { Headers { append("x-amz-checksum-crc32c", "bm90LWEtY2hlY2tzdW0=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -200,7 +225,11 @@ class FailedChecksumResponseTest { Headers { append("x-amz-checksum-sha1", "bm90LWEtY2hlY2tzdW0=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) @@ -229,7 +258,11 @@ class FailedChecksumResponseTest { Headers { append("x-amz-checksum-sha256", "bm90LWEtY2hlY2tzdW0=") }, - "Hello world".toHttpBody(), + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, ) val now = Instant.now() HttpCall(request, resp, now, now) From 2761bf93d08000a00ef815c33a6be212ee62e805 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 4 Dec 2024 15:25:27 -0700 Subject: [PATCH 44/72] Left FIXME to use random buckets in motorcade E2E tests --- services/s3/e2eTest/src/S3ChecksumTest.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index e75173cf691..db8c4026d5a 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -25,6 +25,7 @@ class S3ChecksumTest { @BeforeAll private fun setUp(): Unit = runBlocking { val accountId = getAccountId() + // FIXME: Use randomly generated bucket instead of hardcoded one when motorcade is ready getBucketByName(client, testBucket, "us-west-2", accountId) } From 5328641644fc341b07e20a12ff11ca5095428bc0 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 6 Dec 2024 11:08:52 -0700 Subject: [PATCH 45/72] Refactor/fix S3 express integrations --- .../s3/express/S3ExpressIntegration.kt | 35 +------- .../S3ExpressDisableChecksumInterceptor.kt | 49 +++++++++-- .../s3/express/ChecksumRemovalTest.kt | 83 +++++++++++++++++++ 3 files changed, 126 insertions(+), 41 deletions(-) create mode 100644 services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 430c4f8ebc1..03b552141be 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -16,8 +16,6 @@ import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerato import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType -import software.amazon.smithy.kotlin.codegen.utils.dq -import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.* import software.amazon.smithy.model.traits.* @@ -99,7 +97,6 @@ class S3ExpressIntegration : KotlinIntegration { resolved + listOf( addClientToExecutionContext, addBucketToExecutionContext, - useCrc32Checksum, uploadPartDisableChecksum, ) @@ -132,36 +129,6 @@ class S3ExpressIntegration : KotlinIntegration { } } - /** - * For any operations that require a checksum, set CRC32 if the user has not already configured a checksum. - */ - private val useCrc32Checksum = object : ProtocolMiddleware { - override val name: String = "UseCrc32Checksum" - - override val order: Byte = -1 // Render before flexible checksums - - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = !op.isS3UploadPart && - (op.hasTrait() || (op.hasTrait() && op.expectTrait().isRequestChecksumRequired)) - - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val interceptorSymbol = buildSymbol { - namespace = "aws.sdk.kotlin.services.s3.express" - name = "S3ExpressCrc32ChecksumInterceptor" - } - - val httpChecksumTrait = op.getTrait() - - val checksumAlgorithmMember = ctx.model.expectShape(op.input.get()) - .members() - .firstOrNull { it.memberName == httpChecksumTrait?.requestAlgorithmMember?.getOrNull() } - - // S3 models a header name x-amz-sdk-checksum-algorithm representing the name of the checksum algorithm used - val checksumHeaderName = checksumAlgorithmMember?.getTrait()?.value - - writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq() ?: ""}))", interceptorSymbol) - } - } - /** * Disable all checksums for s3:UploadPart */ @@ -169,7 +136,7 @@ class S3ExpressIntegration : KotlinIntegration { override val name: String = "UploadPartDisableChecksum" override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - op.isS3UploadPart + op.isS3UploadPart && op.hasTrait() override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { val interceptorSymbol = buildSymbol { diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt index 3b10ad3fa69..e78cde99060 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt @@ -6,14 +6,18 @@ package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext import aws.smithy.kotlin.runtime.collections.AttributeKey +import aws.smithy.kotlin.runtime.http.DeferredHeadersBuilder +import aws.smithy.kotlin.runtime.http.HeadersBuilder import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor -import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.request.toBuilder import aws.smithy.kotlin.runtime.telemetry.logging.logger import kotlin.coroutines.coroutineContext +private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" + /** - * Disable checksums entirely for s3:UploadPart requests. + * Disables checksums for s3:UploadPart requests that use S3 express. */ internal class S3ExpressDisableChecksumInterceptor : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { @@ -22,14 +26,45 @@ internal class S3ExpressDisableChecksumInterceptor : HttpInterceptor { } val logger = coroutineContext.logger() + logger.warn { "Checksums must not be sent with S3 express upload part operation, removing checksum(s)" } + + val request = context.protocolRequest.toBuilder() + + request.headers.removeChecksumHeaders() + request.trailingHeaders.removeChecksumTrailingHeaders() + request.headers.removeChecksumTrailingHeadersFromXAmzTrailer() + + return request.build() + } +} - val configuredChecksumAlgorithm = context.executionContext.getOrNull(HttpOperationContext.ChecksumAlgorithm) +/** + * Removes any checksums sent in the request's headers + */ +internal fun HeadersBuilder.removeChecksumHeaders(): Unit = + names().forEach { name -> + if (name.startsWith(CHECKSUM_HEADER_PREFIX)) { + remove(name) + } + } - configuredChecksumAlgorithm?.let { - logger.warn { "Disabling configured checksum $it for S3 Express UploadPart" } - context.executionContext.remove(HttpOperationContext.ChecksumAlgorithm) +/** + * Removes any checksums sent in the request's trailing headers + */ +internal fun DeferredHeadersBuilder.removeChecksumTrailingHeaders(): Unit = + names().forEach { name -> + if (name.startsWith(CHECKSUM_HEADER_PREFIX)) { + remove(name) } + } - return context.protocolRequest +/** + * Removes any checksums sent in the request's trailing headers from `x-amz-trailer` + */ +internal fun HeadersBuilder.removeChecksumTrailingHeadersFromXAmzTrailer() { + this.getAll("x-amz-trailer")?.forEach { trailingHeader -> + if (trailingHeader.startsWith(CHECKSUM_HEADER_PREFIX)) { + this.remove("x-amz-trailer", trailingHeader) + } } } diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt new file mode 100644 index 00000000000..cbf5a9ad4d4 --- /dev/null +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt @@ -0,0 +1,83 @@ +package aws.sdk.kotlin.services.s3.express + +import aws.smithy.kotlin.runtime.http.DeferredHeadersBuilder +import aws.smithy.kotlin.runtime.http.HeadersBuilder +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class ChecksumRemovalTest { + @Test + fun removeChecksumHeaders() { + val headers = HeadersBuilder() + + headers.append("x-amz-checksum-crc32", "foo") + headers.append("x-amz-checksum-sha256", "bar") + + assertTrue( + headers.contains("x-amz-checksum-crc32"), + ) + assertTrue( + headers.contains("x-amz-checksum-sha256"), + ) + + headers.removeChecksumHeaders() + + assertFalse( + headers.contains("x-amz-checksum-crc32"), + ) + assertFalse( + headers.contains("x-amz-checksum-sha256"), + ) + } + + @Test + fun removeChecksumTrailingHeaders() { + val trailingHeaders = DeferredHeadersBuilder() + + trailingHeaders.add("x-amz-checksum-crc32", "foo") + trailingHeaders.add("x-amz-checksum-sha256", "bar") + + assertTrue( + trailingHeaders.contains("x-amz-checksum-crc32"), + ) + assertTrue( + trailingHeaders.contains("x-amz-checksum-sha256"), + ) + + trailingHeaders.removeChecksumTrailingHeaders() + + assertFalse( + trailingHeaders.contains("x-amz-checksum-crc32"), + ) + assertFalse( + trailingHeaders.contains("x-amz-checksum-sha256"), + ) + } + + @Test + fun removeChecksumTrailingHeadersFromXAmzTrailer() { + val headers = HeadersBuilder() + + headers.append("x-amz-trailer", "x-amz-checksum-crc32") + headers.append("x-amz-trailer", "x-amz-trailing-header") + + val xAmzTrailer = headers.getAll("x-amz-trailer") + + assertTrue( + xAmzTrailer?.contains("x-amz-checksum-crc32") ?: false, + ) + assertTrue( + xAmzTrailer?.contains("x-amz-trailing-header") ?: false, + ) + + headers.removeChecksumTrailingHeadersFromXAmzTrailer() + + assertFalse( + xAmzTrailer?.contains("x-amz-checksum-crc32") ?: false, + ) + assertTrue( + xAmzTrailer?.contains("x-amz-trailing-header") ?: false, + ) + } +} From ad30f7057602d1bd3c8f024d3bbbf1fcc55dc6ac Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 9 Dec 2024 13:21:33 -0500 Subject: [PATCH 46/72] Clean up S3 integrations and e2e tests --- .../s3/express/S3ExpressIntegration.kt | 3 +- .../S3ExpressCrc32ChecksumInterceptor.kt | 49 ------------------- .../S3ExpressDisableChecksumInterceptor.kt | 2 + .../e2eTest/src/MutliRegionAccessPointTest.kt | 6 +-- services/s3/e2eTest/src/S3TestUtils.kt | 4 +- 5 files changed, 8 insertions(+), 56 deletions(-) delete mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 03b552141be..86e8c568506 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -25,8 +25,7 @@ import software.amazon.smithy.model.transform.ModelTransformer * An integration which handles codegen for S3 Express, such as: * 1. Configure auth scheme by applying a synthetic shape and trait * 2. Add ExpressClient and Bucket to execution context - * 3. Override checksums to use CRC32 instead of MD5 - * 4. Disable all checksums for s3:UploadPart + * 3. Disable all checksums for s3:UploadPart */ class S3ExpressIntegration : KotlinIntegration { companion object { diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt deleted file mode 100644 index bad493a2fbd..00000000000 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.services.s3.express - -import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext -import aws.smithy.kotlin.runtime.collections.AttributeKey -import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor -import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext -import aws.smithy.kotlin.runtime.http.request.HttpRequest -import aws.smithy.kotlin.runtime.http.request.toBuilder -import aws.smithy.kotlin.runtime.telemetry.logging.logger -import kotlin.coroutines.coroutineContext - -internal const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" -internal const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" -private const val CRC32_ALGORITHM_NAME = "CRC32" - -internal class S3ExpressCrc32ChecksumInterceptor( - val checksumAlgorithmHeaderName: String? = null, -) : HttpInterceptor { - override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { - if (context.executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE) { - return context.protocolRequest - } - - val logger = coroutineContext.logger() - val req = context.protocolRequest.toBuilder() - - if (!context.executionContext.contains(HttpOperationContext.ChecksumAlgorithm)) { - logger.debug { "Checksum is required and not already configured, enabling CRC32 for S3 Express" } - - // Update the execution context so flexible checksums uses CRC32 - context.executionContext[HttpOperationContext.ChecksumAlgorithm] = CRC32_ALGORITHM_NAME - - // Most checksum headers are handled by the flexible checksums feature. But, S3 models an HTTP header binding for the - // checksum algorithm, which also needs to be overwritten and set to CRC32. - // - // The header is already set by the time this interceptor runs, so it needs to be overwritten and can't be set - // through the normal path. - checksumAlgorithmHeaderName?.let { - req.headers[it] = CRC32_ALGORITHM_NAME - } - } - - return req.build() - } -} diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt index e78cde99060..170818208d4 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt @@ -15,6 +15,8 @@ import aws.smithy.kotlin.runtime.telemetry.logging.logger import kotlin.coroutines.coroutineContext private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" +private const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" +private const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" /** * Disables checksums for s3:UploadPart requests that use S3 express. diff --git a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt index 93a193e439b..49630763536 100644 --- a/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt +++ b/services/s3/e2eTest/src/MutliRegionAccessPointTest.kt @@ -8,7 +8,7 @@ import aws.sdk.kotlin.e2etest.S3TestUtils.createMultiRegionAccessPoint import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketAndAllContents import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiRegionAccessPoint import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -import aws.sdk.kotlin.e2etest.S3TestUtils.getBucket +import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketWithPrefix import aws.sdk.kotlin.e2etest.S3TestUtils.getMultiRegionAccessPointArn import aws.sdk.kotlin.e2etest.S3TestUtils.multiRegionAccessPointWasCreated import aws.sdk.kotlin.services.s3.S3Client @@ -47,8 +47,8 @@ class MutliRegionAccessPointTest { @BeforeAll private fun setUp(): Unit = runBlocking { accountId = getAccountId() - usWestBucket = getBucket(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) - usEastBucket = getBucket(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) + usWestBucket = getBucketWithPrefix(s3West, MRAP_BUCKET_PREFIX, "us-west-2", accountId) + usEastBucket = getBucketWithPrefix(s3East, MRAP_BUCKET_PREFIX, "us-east-2", accountId) createMultiRegionAccessPoint( s3Control, diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 5f10a9f52ee..4139b4d9e33 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -46,9 +46,9 @@ object S3TestUtils { client: S3Client, region: String? = null, accountId: String? = null, - ): String = getBucket(client, TEST_BUCKET_PREFIX, region, accountId) + ): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX, region, accountId) - suspend fun getBucket( + suspend fun getBucketWithPrefix( client: S3Client, prefix: String, region: String? = null, From 4a162de181d31dfc178dffd742b844b916a408ea Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 9 Dec 2024 14:00:30 -0500 Subject: [PATCH 47/72] s3 express no checksums in upload part test --- services/s3/e2eTest/src/S3ExpressTest.kt | 70 +++++++++++++++++++++++- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt index 5724d3df3ad..c97c58fc2ce 100644 --- a/services/s3/e2eTest/src/S3ExpressTest.kt +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -4,12 +4,10 @@ */ package aws.sdk.kotlin.e2etest -import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.express.S3_EXPRESS_SESSION_TOKEN_HEADER import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.presigners.presignPutObject -import aws.sdk.kotlin.services.s3.putObject -import aws.sdk.kotlin.services.s3.withConfig import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.decodeToString @@ -47,6 +45,7 @@ class S3ExpressTest { @AfterAll fun cleanup(): Unit = runBlocking { testBuckets.forEach { bucket -> + S3TestUtils.deleteMultiPartUploads(client, bucket) S3TestUtils.deleteBucketAndAllContents(client, bucket) } client.close() @@ -136,6 +135,62 @@ class S3ExpressTest { } } + @Test + fun testUploadPartContainsNoChecksums() = runTest { + val testBucket = testBuckets.first() + val testObject = "I-will-be-uploaded-in-parts-!" + + // Parts need to be at least 5 MB + val partOne = "Hello".repeat(1_048_576) + val partTwo = "World".repeat(1_048_576) + + val testUploadId = client.createMultipartUpload { + bucket = testBucket + key = testObject + }.uploadId + + var eTagPartOne: String? + var eTagPartTwo: String? + + client.withConfig { + interceptors += NoChecksumValidatingInterceptor() + }.use { validatingClient -> + eTagPartOne = validatingClient.uploadPart { + bucket = testBucket + key = testObject + partNumber = 1 + uploadId = testUploadId + body = ByteStream.fromString(partOne) + }.eTag + + eTagPartTwo = validatingClient.uploadPart { + bucket = testBucket + key = testObject + partNumber = 2 + uploadId = testUploadId + body = ByteStream.fromString(partTwo) + }.eTag + } + + client.completeMultipartUpload { + bucket = testBucket + key = testObject + uploadId = testUploadId + multipartUpload = CompletedMultipartUpload { + parts = listOf( + CompletedPart { + partNumber = 1 + eTag = eTagPartOne + }, + CompletedPart { + partNumber = 2 + eTag = eTagPartTwo + }, + ) + } + } + } + private class S3ExpressInvocationTrackingInterceptor : HttpInterceptor { var s3ExpressInvocations = 0 @@ -155,4 +210,13 @@ class S3ExpressTest { } } } + + private class NoChecksumValidatingInterceptor : HttpInterceptor { + override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { + val headers = context.protocolRequest.headers + if (headers.contains(S3_EXPRESS_SESSION_TOKEN_HEADER)) { + assertFalse(headers.names().any { it.startsWith("x-amz-checksum-") }) + } + } + } } From d665873b3ebee66d49c014d92dcffb212efa3673 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 11 Dec 2024 16:46:30 -0500 Subject: [PATCH 48/72] PR feedback --- aws-runtime/aws-config/api/aws-config.api | 5 +- .../ResolveFlexibleChecksumsConfig.kt | 42 ++++++ .../ResolveRequestChecksumCalculation.kt | 29 ----- .../ResolveResponseChecksumValidation.kt | 29 ----- aws-runtime/aws-http/api/aws-http.api | 23 ++++ .../S3CompositeChecksumsInterceptor.kt | 47 +++++++ .../S3CompositeChecksumsInterceptorTest.kt | 35 +++++ .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 1 + .../FlexibleChecksumsResponse.kt | 8 -- .../s3/S3CompositeChecksumsIntegration.kt | 44 +++++++ .../s3/express/S3ExpressIntegration.kt | 16 ++- ...tlin.codegen.integration.KotlinIntegration | 1 + .../S3ExpressDisableChecksumInterceptor.kt | 19 ++- .../s3/express/ChecksumRemovalTest.kt | 34 ++--- services/s3/e2eTest/src/S3ChecksumTest.kt | 120 ++++++++++++------ services/s3/e2eTest/src/S3IntegrationTest.kt | 2 +- services/s3/e2eTest/src/S3TestUtils.kt | 35 +++-- tests/codegen/build.gradle.kts | 1 - .../codegen/checksums/ChecksumConfigTest.kt | 35 +---- .../checksums/utils/ChecksumTestUtils.kt | 11 +- 20 files changed, 356 insertions(+), 181 deletions(-) create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt delete mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt delete mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt create mode 100644 aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt create mode 100644 aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index 6a9be2adba5..1d5c135a3ae 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -262,12 +262,9 @@ public final class aws/sdk/kotlin/runtime/config/AwsSdkSettingKt { public static final fun resolveEndpointUrl (Laws/sdk/kotlin/runtime/config/AwsSdkSetting;Laws/smithy/kotlin/runtime/util/PlatformProvider;Ljava/lang/String;Ljava/lang/String;)Laws/smithy/kotlin/runtime/net/url/Url; } -public final class aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculationKt { +public final class aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfigKt { public static final fun resolveRequestChecksumCalculation (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static synthetic fun resolveRequestChecksumCalculation$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; -} - -public final class aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidationKt { public static final fun resolveResponseChecksumValidation (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static synthetic fun resolveResponseChecksumValidation$default (Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/util/LazyAsyncValue;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt new file mode 100644 index 00000000000..bf7049ef2f3 --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt @@ -0,0 +1,42 @@ +package aws.sdk.kotlin.runtime.config.checksums + +import aws.sdk.kotlin.runtime.ConfigurationException +import aws.sdk.kotlin.runtime.InternalSdkApi +import aws.sdk.kotlin.runtime.config.AwsSdkSetting +import aws.sdk.kotlin.runtime.config.profile.AwsProfile +import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation +import aws.sdk.kotlin.runtime.config.profile.responseChecksumValidation +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption +import aws.smithy.kotlin.runtime.config.resolve +import aws.smithy.kotlin.runtime.util.LazyAsyncValue +import aws.smithy.kotlin.runtime.util.PlatformProvider + +/** + * Attempts to resolve requestChecksumCalculation from the specified sources. + * @return requestChecksumCalculation setting if found, the default value if not. + */ +@InternalSdkApi +public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { + val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation + return parseHttpChecksumConfigOption(unparsedString, "requestChecksumCalculation") +} + +/** + * Attempts to resolve responseChecksumValidation from the specified sources. + * @return responseChecksumValidation setting if found, the default value if not. + */ +@InternalSdkApi +public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { + val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation + return parseHttpChecksumConfigOption(unparsedString, "responseChecksumValidation") +} + +private fun parseHttpChecksumConfigOption(unparsedString: String?, configOption: String): HttpChecksumConfigOption = + when (unparsedString?.uppercase()) { + null -> HttpChecksumConfigOption.WHEN_SUPPORTED + "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED + "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED + else -> throw ConfigurationException( + "'$unparsedString' is not a valid value for $configOption. Valid values are: ${HttpChecksumConfigOption.entries}", + ) + } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt deleted file mode 100644 index dd521548b9f..00000000000 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveRequestChecksumCalculation.kt +++ /dev/null @@ -1,29 +0,0 @@ -package aws.sdk.kotlin.runtime.config.checksums - -import aws.sdk.kotlin.runtime.ConfigurationException -import aws.sdk.kotlin.runtime.InternalSdkApi -import aws.sdk.kotlin.runtime.config.AwsSdkSetting -import aws.sdk.kotlin.runtime.config.profile.AwsProfile -import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption -import aws.smithy.kotlin.runtime.config.resolve -import aws.smithy.kotlin.runtime.util.LazyAsyncValue -import aws.smithy.kotlin.runtime.util.PlatformProvider - -/** - * Attempts to resolve requestChecksumCalculation from the specified sources. - * @return requestChecksumCalculation setting if found, the default value if not. - */ -@InternalSdkApi -public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { - val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation - return unparsedString?.let { - when (unparsedString.uppercase()) { - "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED - "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED - else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries}", - ) - } - } ?: HttpChecksumConfigOption.WHEN_SUPPORTED -} diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt deleted file mode 100644 index 6c7162c2195..00000000000 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveResponseChecksumValidation.kt +++ /dev/null @@ -1,29 +0,0 @@ -package aws.sdk.kotlin.runtime.config.checksums - -import aws.sdk.kotlin.runtime.ConfigurationException -import aws.sdk.kotlin.runtime.InternalSdkApi -import aws.sdk.kotlin.runtime.config.AwsSdkSetting -import aws.sdk.kotlin.runtime.config.profile.AwsProfile -import aws.sdk.kotlin.runtime.config.profile.responseChecksumValidation -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption -import aws.smithy.kotlin.runtime.config.resolve -import aws.smithy.kotlin.runtime.util.LazyAsyncValue -import aws.smithy.kotlin.runtime.util.PlatformProvider - -/** - * Attempts to resolve responseChecksumValidation from the specified sources. - * @return responseChecksumValidation setting if found, the default value if not. - */ -@InternalSdkApi -public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { - val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation - return unparsedString?.let { - when (unparsedString.uppercase()) { - "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED - "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED - else -> throw ConfigurationException( - "'$it' is not a valid value for request checksum calculation. Valid values are: ${HttpChecksumConfigOption.entries}", - ) - } - } ?: HttpChecksumConfigOption.WHEN_SUPPORTED -} diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index 425d6c7ac51..0b475eb05fa 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -196,6 +196,29 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInter public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } +public final class aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { + public fun ()V + public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeRetryLoop (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun readAfterAttempt (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterDeserialization (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterExecution (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterSerialization (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readAfterSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readAfterTransmit (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V + public fun readBeforeAttempt (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V + public fun readBeforeExecution (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V + public fun readBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V + public fun readBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V +} + public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { public fun ()V public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt new file mode 100644 index 00000000000..16adc9aeef5 --- /dev/null +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt @@ -0,0 +1,47 @@ +package aws.sdk.kotlin.runtime.http.interceptors + +import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext +import aws.smithy.kotlin.runtime.http.HeadersBuilder +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.http.response.toBuilder +import aws.smithy.kotlin.runtime.telemetry.logging.Logger +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlin.coroutines.coroutineContext + +private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" + +/** + * Removes any checksum headers that contain composite checksums from an S3 response. + */ +public class S3CompositeChecksumsInterceptor : HttpInterceptor { + override suspend fun modifyBeforeDeserialization(context: ProtocolResponseInterceptorContext): HttpResponse { + val response = context.protocolResponse.toBuilder() + val logger = coroutineContext.logger() + + response.headers.removeCompositeChecksums(logger) + + return response.build() + } +} + +/** + * Removes any checksum headers that contain composite checksums. + */ +internal fun HeadersBuilder.removeCompositeChecksums(logger: Logger): Unit = + names().forEach { header -> + if (header.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true) && get(header)!!.isCompositeChecksum()) { + logger.warn { "S3 returned a composite checksum, composite checksums are not supported, removing checksum" } + remove(header) + } + } + +/** + * Verifies if a checksum is composite. + */ +private fun String.isCompositeChecksum(): Boolean { + // Ends with "-#" where "#" is a number + val regex = Regex("-(\\d)+$") + return regex.containsMatchIn(this) +} diff --git a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt new file mode 100644 index 00000000000..042dcb5b99e --- /dev/null +++ b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt @@ -0,0 +1,35 @@ +package aws.sdk.kotlin.runtime.http.interceptors + +import aws.smithy.kotlin.runtime.http.HeadersBuilder +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlinx.coroutines.test.runTest +import kotlin.coroutines.coroutineContext +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class S3CompositeChecksumsInterceptorTest { + @Test + fun compositeChecksumRemoval() = runTest { + val headers = HeadersBuilder() + + headers.append("x-amz-checksum-crc32", "foo-1") + headers.append("x-amz-checksum-sha256", "bar") + + assertTrue( + headers.contains("x-amz-checksum-crc32"), + ) + assertTrue( + headers.contains("x-amz-checksum-sha256"), + ) + + headers.removeCompositeChecksums(coroutineContext.logger()) + + assertFalse( + headers.contains("x-amz-checksum-crc32"), + ) + assertTrue( + headers.contains("x-amz-checksum-sha256"), + ) + } +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index b4debfc4bd8..70f3a4950ef 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -62,6 +62,7 @@ object AwsRuntimeTypes { val UnsupportedSigningAlgorithmInterceptor = symbol("UnsupportedSigningAlgorithmInterceptor") val BusinessMetricsInterceptor = symbol("BusinessMetricsInterceptor") val AwsBusinessMetric = symbol("AwsBusinessMetric") + val S3CompositeChecksumsInterceptor = symbol("S3CompositeChecksumsInterceptor") } object Retries { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index afd8c29c20a..9767d2a9718 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -18,13 +18,6 @@ import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.StructureShape -/** - * AWS services that can return composite checksums in their responses - */ -private val compositeChecksumServices = setOf( - "s3", -) - /** * Adds a middleware which validates checksums returned in responses if the user has opted-in. */ @@ -98,7 +91,6 @@ class FlexibleChecksumsResponse : KotlinIntegration { ) { writer.write("input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) writer.write("config.responseChecksumValidation,") - writer.write("#L", compositeChecksumServices.contains(ctx.settings.sdkId)) } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt new file mode 100644 index 00000000000..cef54247f72 --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt @@ -0,0 +1,44 @@ +package aws.sdk.kotlin.codegen.customization.flexiblechecksums.s3 + +import aws.sdk.kotlin.codegen.AwsRuntimeTypes.Http.Interceptors.S3CompositeChecksumsInterceptor +import aws.sdk.kotlin.codegen.customization.s3.isS3 +import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.KotlinWriter +import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.model.expectShape +import software.amazon.smithy.kotlin.codegen.model.hasTrait +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ServiceShape + +/** + * Removes composite checksums returned by S3 so that flexible checksums won't validate them. + * Composite checksums are used for multipart uploads and end with "-#" where "#" is a number. + */ +class S3CompositeChecksumsIntegration : KotlinIntegration { + override val order: Byte + get() = -128 + + override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = + model.expectShape(settings.service).isS3 + + override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = + resolved + s3CompositeChecksumsMiddleware +} + +/** + * Adds [S3CompositeChecksumsInterceptor] to interceptors when operation has flexible checksums. + */ +private val s3CompositeChecksumsMiddleware = object : ProtocolMiddleware { + override val name: String = "S3CompositeChecksumsMiddleware" + + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + op.hasTrait() + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.write("op.interceptors.add(#T())", S3CompositeChecksumsInterceptor) + } +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 86e8c568506..067da2da40f 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -16,6 +16,7 @@ import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerato import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType +import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.* import software.amazon.smithy.model.traits.* @@ -138,12 +139,25 @@ class S3ExpressIntegration : KotlinIntegration { op.isS3UploadPart && op.hasTrait() override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val httpChecksumTrait = op.getTrait()!! + + val requestAlgorithmMemberName = httpChecksumTrait.requestAlgorithmMember?.getOrNull()?.let { + val requestAlgorithmMemberShape = ctx.model.expectShape(op.input.get()) + .members() + .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } + ctx.symbolProvider.toMemberName(requestAlgorithmMemberShape) + } + val interceptorSymbol = buildSymbol { namespace = "aws.sdk.kotlin.services.s3.express" name = "S3ExpressDisableChecksumInterceptor" } writer.addImport(interceptorSymbol) - writer.write("op.interceptors.add(#T())", interceptorSymbol) + writer.write( + "op.interceptors.add(#T(input.#L?.value != null))", + interceptorSymbol, + requestAlgorithmMemberName, + ) } } diff --git a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration index de1ab6cd4ac..20c0701ea92 100644 --- a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +++ b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration @@ -12,6 +12,7 @@ aws.sdk.kotlin.codegen.endpoints.AddAwsEndpointFunctions aws.sdk.kotlin.codegen.PresignerGenerator aws.sdk.kotlin.codegen.customization.flexiblechecksums.FlexibleChecksumsRequest aws.sdk.kotlin.codegen.customization.flexiblechecksums.FlexibleChecksumsResponse +aws.sdk.kotlin.codegen.customization.flexiblechecksums.s3.S3CompositeChecksumsIntegration aws.sdk.kotlin.codegen.customization.AddAwsSpanAttributes aws.sdk.kotlin.codegen.customization.s3.S3OperationErrorHandler aws.sdk.kotlin.codegen.customization.s3.S3SigningConfig diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt index 170818208d4..1dd06c25406 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt @@ -11,7 +11,7 @@ import aws.smithy.kotlin.runtime.http.HeadersBuilder import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.toBuilder -import aws.smithy.kotlin.runtime.telemetry.logging.logger +import aws.smithy.kotlin.runtime.telemetry.logging.warn import kotlin.coroutines.coroutineContext private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" @@ -21,14 +21,19 @@ private const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" /** * Disables checksums for s3:UploadPart requests that use S3 express. */ -internal class S3ExpressDisableChecksumInterceptor : HttpInterceptor { +internal class S3ExpressDisableChecksumInterceptor( + private val userConfiguredChecksum: Boolean, +) : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { if (context.executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE) { return context.protocolRequest } - val logger = coroutineContext.logger() - logger.warn { "Checksums must not be sent with S3 express upload part operation, removing checksum(s)" } + if (userConfiguredChecksum) { + coroutineContext.warn { + "Checksums must not be sent with S3 Express UploadPart operation, removing checksum(s)" + } + } val request = context.protocolRequest.toBuilder() @@ -45,7 +50,7 @@ internal class S3ExpressDisableChecksumInterceptor : HttpInterceptor { */ internal fun HeadersBuilder.removeChecksumHeaders(): Unit = names().forEach { name -> - if (name.startsWith(CHECKSUM_HEADER_PREFIX)) { + if (name.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { remove(name) } } @@ -55,7 +60,7 @@ internal fun HeadersBuilder.removeChecksumHeaders(): Unit = */ internal fun DeferredHeadersBuilder.removeChecksumTrailingHeaders(): Unit = names().forEach { name -> - if (name.startsWith(CHECKSUM_HEADER_PREFIX)) { + if (name.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { remove(name) } } @@ -65,7 +70,7 @@ internal fun DeferredHeadersBuilder.removeChecksumTrailingHeaders(): Unit = */ internal fun HeadersBuilder.removeChecksumTrailingHeadersFromXAmzTrailer() { this.getAll("x-amz-trailer")?.forEach { trailingHeader -> - if (trailingHeader.startsWith(CHECKSUM_HEADER_PREFIX)) { + if (trailingHeader.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { this.remove("x-amz-trailer", trailingHeader) } } diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt index cbf5a9ad4d4..c00f78e821f 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt @@ -7,27 +7,31 @@ import kotlin.test.assertFalse import kotlin.test.assertTrue class ChecksumRemovalTest { + // Header capitalization shouldn't matter + private val crc32Header = "x-aMz-cHeCkSum-cRC32" + private val sha256Header = "X-Amz-ChEcKsum-sHa256" + @Test fun removeChecksumHeaders() { val headers = HeadersBuilder() - headers.append("x-amz-checksum-crc32", "foo") - headers.append("x-amz-checksum-sha256", "bar") + headers.append(crc32Header, "foo") + headers.append(sha256Header, "bar") assertTrue( - headers.contains("x-amz-checksum-crc32"), + headers.contains(crc32Header), ) assertTrue( - headers.contains("x-amz-checksum-sha256"), + headers.contains(sha256Header), ) headers.removeChecksumHeaders() assertFalse( - headers.contains("x-amz-checksum-crc32"), + headers.contains(crc32Header), ) assertFalse( - headers.contains("x-amz-checksum-sha256"), + headers.contains(sha256Header), ) } @@ -35,23 +39,23 @@ class ChecksumRemovalTest { fun removeChecksumTrailingHeaders() { val trailingHeaders = DeferredHeadersBuilder() - trailingHeaders.add("x-amz-checksum-crc32", "foo") - trailingHeaders.add("x-amz-checksum-sha256", "bar") + trailingHeaders.add(crc32Header, "foo") + trailingHeaders.add(sha256Header, "bar") assertTrue( - trailingHeaders.contains("x-amz-checksum-crc32"), + trailingHeaders.contains(crc32Header), ) assertTrue( - trailingHeaders.contains("x-amz-checksum-sha256"), + trailingHeaders.contains(sha256Header), ) trailingHeaders.removeChecksumTrailingHeaders() assertFalse( - trailingHeaders.contains("x-amz-checksum-crc32"), + trailingHeaders.contains(crc32Header), ) assertFalse( - trailingHeaders.contains("x-amz-checksum-sha256"), + trailingHeaders.contains(sha256Header), ) } @@ -59,13 +63,13 @@ class ChecksumRemovalTest { fun removeChecksumTrailingHeadersFromXAmzTrailer() { val headers = HeadersBuilder() - headers.append("x-amz-trailer", "x-amz-checksum-crc32") + headers.append("x-amz-trailer", crc32Header) headers.append("x-amz-trailer", "x-amz-trailing-header") val xAmzTrailer = headers.getAll("x-amz-trailer") assertTrue( - xAmzTrailer?.contains("x-amz-checksum-crc32") ?: false, + xAmzTrailer?.contains(crc32Header) ?: false, ) assertTrue( xAmzTrailer?.contains("x-amz-trailing-header") ?: false, @@ -74,7 +78,7 @@ class ChecksumRemovalTest { headers.removeChecksumTrailingHeadersFromXAmzTrailer() assertFalse( - xAmzTrailer?.contains("x-amz-checksum-crc32") ?: false, + xAmzTrailer?.contains(crc32Header) ?: false, ) assertTrue( xAmzTrailer?.contains("x-amz-trailing-header") ?: false, diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index db8c4026d5a..4b99ae22e8f 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -8,19 +8,21 @@ import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload import aws.sdk.kotlin.services.s3.model.CompletedPart import aws.sdk.kotlin.services.s3.model.GetObjectRequest -import aws.smithy.kotlin.runtime.content.ByteStream -import aws.smithy.kotlin.runtime.content.fromInputStream +import aws.smithy.kotlin.runtime.content.* +import aws.smithy.kotlin.runtime.hashing.crc32 +import aws.smithy.kotlin.runtime.testing.RandomTempFile import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.* import java.io.File import java.io.FileInputStream import java.util.* +import kotlin.test.assertEquals @TestInstance(TestInstance.Lifecycle.PER_CLASS) class S3ChecksumTest { private val client = S3Client { region = "us-west-2" } private val testBucket = "s3-test-bucket-ci-motorcade" - private val testObject = "test-object" + private fun testKey(): String = "test-object" + UUID.randomUUID() @BeforeAll private fun setUp(): Unit = runBlocking { @@ -38,87 +40,123 @@ class S3ChecksumTest { @Test fun testPutObject(): Unit = runBlocking { + val testBody = "Hello World" + val testKey = testKey() + client.putObject { bucket = testBucket - key = testObject - body = ByteStream.fromString("Hello World") + key = testKey + body = ByteStream.fromString(testBody) + } + + client.getObject( + GetObjectRequest { + bucket = testBucket + key = testKey + }, + ) { actual -> + assertEquals(testBody, actual.body?.decodeToString() ?: "") } } @Test fun testPutObjectWithEmptyBody(): Unit = runBlocking { + val testKey = testKey() + val testBody = "" + client.putObject { bucket = testBucket - key = testObject + UUID.randomUUID() + key = testKey + } + + client.getObject( + GetObjectRequest { + bucket = testBucket + key = testKey + }, + ) { actual -> + assertEquals(testBody, actual.body?.decodeToString() ?: "") } } @Test fun testPutObjectAwsChunkedEncoded(): Unit = runBlocking { - val testString = "Hello World" + val testKey = testKey() + val testBody = "Hello World" + val tempFile = File.createTempFile("test", ".txt").also { - it.writeText(testString) + it.writeText(testBody) it.deleteOnExit() } val inputStream = FileInputStream(tempFile) client.putObject { bucket = testBucket - key = testObject + UUID.randomUUID() - body = ByteStream.fromInputStream(inputStream, testString.length.toLong()) + key = testKey + body = ByteStream.fromInputStream(inputStream, testBody.length.toLong()) + } + + client.getObject( + GetObjectRequest { + bucket = testBucket + key = testKey + }, + ) { actual -> + assertEquals(testBody, actual.body?.decodeToString() ?: "") } } @Test fun testMultiPartUpload(): Unit = runBlocking { - // Parts need to be at least 5 MB - val partOne = "Hello".repeat(1_048_576) - val partTwo = "World".repeat(1_048_576) + val testKey = testKey() + + val partSize = 5 * 1024 * 1024 // 5 MB - min part size + val contentSize: Long = 8 * 1024 * 1024 // 2 parts + val file = RandomTempFile(sizeInBytes = contentSize) + + val expectedChecksum = file.readBytes().crc32() val testUploadId = client.createMultipartUpload { bucket = testBucket - key = testObject + key = testKey }.uploadId - val eTagPartOne = client.uploadPart { - bucket = testBucket - key = testObject - partNumber = 1 - uploadId = testUploadId - body = ByteStream.fromString(partOne) - }.eTag + val uploadedParts = file.chunk(partSize).mapIndexed { index, chunk -> + val adjustedIndex = index + 1 // index starts from 0 but partNumber needs to start from 1 - val eTagPartTwo = client.uploadPart { - bucket = testBucket - key = testObject - partNumber = 2 - uploadId = testUploadId - body = ByteStream.fromString(partTwo) - }.eTag + runBlocking { + client.uploadPart { + bucket = testBucket + key = testKey + partNumber = adjustedIndex + uploadId = testUploadId + body = file.asByteStream(chunk) + }.let { + CompletedPart { + partNumber = adjustedIndex + eTag = it.eTag + } + } + } + }.toList() client.completeMultipartUpload { bucket = testBucket - key = testObject + key = testKey uploadId = testUploadId multipartUpload = CompletedMultipartUpload { - parts = listOf( - CompletedPart { - partNumber = 1 - eTag = eTagPartOne - }, - CompletedPart { - partNumber = 2 - eTag = eTagPartTwo - }, - ) + parts = uploadedParts } } client.getObject( GetObjectRequest { bucket = testBucket - key = testObject + key = testKey }, - ) {} + ) { actual -> + val actualChecksum = actual.body!!.toByteArray().crc32() + assertEquals(actualChecksum, expectedChecksum) + } } } diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index 37ec353c9f6..aec1de82d16 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -390,7 +390,7 @@ class S3BucketOpsIntegrationTest { } // generate sequence of "chunks" where each range defines the inclusive start and end bytes -private fun File.chunk(partSize: Int): Sequence = +internal fun File.chunk(partSize: Int): Sequence = (0 until length() step partSize.toLong()).asSequence().map { it until minOf(it + partSize, length()) } diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 4139b4d9e33..e089beb6614 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -10,6 +10,7 @@ import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint import aws.sdk.kotlin.services.s3.model.ExpirationStatus import aws.sdk.kotlin.services.s3.model.LifecycleRule +import aws.sdk.kotlin.services.s3.paginators.listBucketsPaginated import aws.sdk.kotlin.services.s3.paginators.listObjectsV2Paginated import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketExists import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketNotExists @@ -103,26 +104,42 @@ object S3TestUtils { suspend fun getBucketByName( client: S3Client, - bucket: String, + targetBucket: String, region: String? = null, accountId: String? = null, ): String = withTimeout(60.seconds) { - val buckets = client.listBuckets() - .buckets - ?.mapNotNull { it.name } + val bucketNames = mutableListOf() - var testBucket = buckets?.firstOrNull { bucketName -> - bucketName == bucket && - region?.let { + client.listBucketsPaginated() + .collect { response -> + response.buckets?.forEach { bucket -> + bucket.name?.let { bucketNames.add(it) } + } + } + + var testBucket = bucketNames.firstOrNull { bucketName -> + if (bucketName == targetBucket) { + val isInCorrectLocation = region?.let { client.getBucketLocation { - this.bucket = bucketName + bucket = bucketName expectedBucketOwner = accountId }.locationConstraint?.value == region } ?: true + + if (isInCorrectLocation) { + true + } else { + throw RuntimeException( + "The requested bucket ($targetBucket) already exists in another region than the one requested ($region)", + ) + } + } else { + false + } } if (testBucket == null) { - testBucket = bucket + testBucket = targetBucket println("Creating S3 bucket: $testBucket") client.createBucket { diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts index dd0a9537e6b..696c3c3104a 100644 --- a/tests/codegen/build.gradle.kts +++ b/tests/codegen/build.gradle.kts @@ -16,7 +16,6 @@ subprojects { apply(plugin = libraries.plugins.kotlin.multiplatform.get().pluginId) val optinAnnotations = listOf( - "kotlin.RequiresOptIn", "aws.smithy.kotlin.runtime.InternalApi", "aws.sdk.kotlin.runtime.InternalSdkApi", ) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt index e26afa7908c..e7fe9933394 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt @@ -265,6 +265,10 @@ class UserProvidedChecksumHeader { assertTrue( headerReader.containsExpectedHeaders, ) + + assertFalse( + headerReader.containsForbiddenHeaders, + ) } } @@ -370,35 +374,4 @@ class ResponseChecksumValidation { } } } - - @Test - fun compositeChecksumsAreNotValidated(): Unit = runBlocking { - ClientConfigTestClient { - responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append( - "x-amz-checksum-crc32", - "I'm a composite checksum and will trigger `ChecksumMismatchException` if read!-1", - ) - }, - "World!".toHttpBody(), - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello" - } - } - } } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt index a8f1f634a1f..c88263bfe70 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt @@ -20,8 +20,8 @@ internal class HeaderReader( private val expectedHeaders: Map, private val forbiddenHeaders: Map = emptyMap(), ) : HttpInterceptor { - var containsExpectedHeaders = true - var containsForbiddenHeaders = false + var containsExpectedHeaders = false + var containsForbiddenHeaders = true override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { expectedHeaders.forEach { header -> @@ -31,17 +31,18 @@ internal class HeaderReader( } ?: true if (!containsHeader || !headerValueMatches) { - containsExpectedHeaders = false return } } forbiddenHeaders.forEach { header -> if (context.protocolRequest.headers.contains(header.key)) { - containsForbiddenHeaders = true return } } + + containsExpectedHeaders = true + containsForbiddenHeaders = false } } @@ -66,7 +67,7 @@ internal class HeaderSetter( internal class BusinessMetricsReader( private val expectedBusinessMetrics: Set, ) : HttpInterceptor { - var containsExpectedBusinessMetrics = true + var containsExpectedBusinessMetrics = false override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { containsExpectedBusinessMetrics = context.executionContext[BusinessMetrics].containsAll(expectedBusinessMetrics) From 1aefe666b3e450962428e01c2b3fef5e7a373b48 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 11 Dec 2024 17:44:10 -0500 Subject: [PATCH 49/72] Use head bucket --- services/s3/e2eTest/src/S3TestUtils.kt | 56 ++++++++------------------ 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index e089beb6614..6ace834f00f 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -10,7 +10,6 @@ import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint import aws.sdk.kotlin.services.s3.model.ExpirationStatus import aws.sdk.kotlin.services.s3.model.LifecycleRule -import aws.sdk.kotlin.services.s3.paginators.listBucketsPaginated import aws.sdk.kotlin.services.s3.paginators.listObjectsV2Paginated import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketExists import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketNotExists @@ -107,54 +106,31 @@ object S3TestUtils { targetBucket: String, region: String? = null, accountId: String? = null, - ): String = withTimeout(60.seconds) { - val bucketNames = mutableListOf() - - client.listBucketsPaginated() - .collect { response -> - response.buckets?.forEach { bucket -> - bucket.name?.let { bucketNames.add(it) } - } - } - - var testBucket = bucketNames.firstOrNull { bucketName -> - if (bucketName == targetBucket) { - val isInCorrectLocation = region?.let { - client.getBucketLocation { - bucket = bucketName - expectedBucketOwner = accountId - }.locationConstraint?.value == region - } ?: true - - if (isInCorrectLocation) { - true - } else { - throw RuntimeException( - "The requested bucket ($targetBucket) already exists in another region than the one requested ($region)", - ) - } - } else { - false + ): Unit = withTimeout(60.seconds) { + try { + val targetBucketRegion = client + .headBucket { + this.bucket = targetBucket + expectedBucketOwner = accountId + }.bucketRegion + + if (targetBucketRegion != region) { + throw RuntimeException( + "The requested bucket ($targetBucket) already exists in another region than the one requested ($region)", + ) } - } - - if (testBucket == null) { - testBucket = targetBucket - println("Creating S3 bucket: $testBucket") + } catch (e: Throwable) { + println("Creating S3 bucket: $targetBucket") client.createBucket { - this.bucket = testBucket + bucket = targetBucket createBucketConfiguration { locationConstraint = BucketLocationConstraint.fromValue(region ?: client.config.region!!) } } - client.waitUntilBucketExists { this.bucket = testBucket } - } else { - println("Using existing S3 bucket: $testBucket") + client.waitUntilBucketExists { bucket = targetBucket } } - - testBucket } suspend fun getTestDirectoryBucket(client: S3Client, suffix: String) = withTimeout(60.seconds) { From 55bdcb87583698e678ba098e7a52cfd1e7ae3002 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 13 Dec 2024 01:18:30 -0500 Subject: [PATCH 50/72] Presigned URL checksums --- .../sdk/kotlin/codegen/PresignerGenerator.kt | 77 ++++++++++++++++++- services/s3/e2eTest/src/S3ChecksumTest.kt | 39 +++++++++- 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt index 34dbab1293f..a912e904579 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt @@ -5,22 +5,32 @@ package aws.sdk.kotlin.codegen import aws.sdk.kotlin.codegen.model.traits.Presignable +import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.aws.traits.auth.SigV4Trait import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait import software.amazon.smithy.codegen.core.Symbol import software.amazon.smithy.kotlin.codegen.core.* +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.isSupportedForFlexibleChecksums +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.toHashFunctionOrThrow +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.IllegalStateException +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.Encoding.encodeBase64String +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.lowercase +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Utils.runBlocking +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.HttpBody +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.readAll +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.KotlinCoroutines.coroutineContext +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Observability.TelemetryApi.warn import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.integration.SectionId import software.amazon.smithy.kotlin.codegen.integration.SectionKey import software.amazon.smithy.kotlin.codegen.lang.KotlinTypes -import software.amazon.smithy.kotlin.codegen.model.buildSymbol -import software.amazon.smithy.kotlin.codegen.model.expectShape -import software.amazon.smithy.kotlin.codegen.model.getTrait +import software.amazon.smithy.kotlin.codegen.model.* import software.amazon.smithy.kotlin.codegen.model.knowledge.AwsSignatureVersion4 import software.amazon.smithy.kotlin.codegen.rendering.endpoints.EndpointResolverAdapterGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpBindingProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpBindingResolver import software.amazon.smithy.kotlin.codegen.rendering.serde.serializerName +import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.knowledge.TopDownIndex import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ServiceShape @@ -149,6 +159,7 @@ class PresignerGenerator : KotlinIntegration { requestSymbol, serializerSymbol, contextMap, + op, ) } } @@ -193,6 +204,7 @@ class PresignerGenerator : KotlinIntegration { requestSymbol: Symbol, serializerSymbol: Symbol, contextMap: Map, Any>, + op: OperationShape, ) = writer.apply { dokka { write("Presign a [#T] using the configuration of this [#T].", requestSymbol, serviceSymbol) @@ -265,6 +277,45 @@ class PresignerGenerator : KotlinIntegration { ) } + checksumAlgorithmMember(op, ctx)?.let { checksumAlgorithmMember -> + withBlock("input.#L?.value?.let { checksumAlgorithmString ->", "}", checksumAlgorithmMember) { + withBlock("when (unsignedRequest.body) {", "}") { + withBlock("is #1T.Bytes, is #1T.Empty -> {", "}", HttpBody) { + write("val checksumAlgorithm = checksumAlgorithmString.#T()", toHashFunctionOrThrow) + withInlineBlock( + "if (checksumAlgorithm.#T) {", + "}", + isSupportedForFlexibleChecksums, + ) { + withBlock("#T {", "}", runBlocking) { + withBlock("checksumAlgorithm.update(", ")") { + write("unsignedRequest.body.#T() ?: byteArrayOf()", readAll) + } + } + write( + "checksum = #S.#T() to checksumAlgorithm.digest().#T()", + "x-amz-checksum-\${checksumAlgorithmString}", + lowercase, + encodeBase64String, + ) + } + withBlock(" else {", "}") { + withBlock("#T {", "}", runBlocking) { + write("class Presigner") + write( + "#T.#T { #S }", + coroutineContext, + warn, + "The requested checksum algorithm is not supported for pre-signed URL checksums, sending request without checksum.", + ) + } + } + } + write("else -> throw #T(#S)", IllegalStateException, "HTTP body type unsupported for pre-signed URL checksums.") + } + } + } + declareSection(SigningConfigCustomizationSection) write("configBlock()") @@ -287,4 +338,24 @@ class PresignerGenerator : KotlinIntegration { * > "my-object/example/photo.user". This is an incorrect path for that object. */ private fun normalizeUriPath(service: ServiceShape) = service.sdkId != "S3" + + /** + * Gets the checksum algorithm member if a user can configure request checksums otherwise null + */ + private fun checksumAlgorithmMember( + operationShape: OperationShape, + ctx: CodegenContext, + ): String? { + operationShape.getTrait()?.let { httpChecksumTrait -> + httpChecksumTrait.requestAlgorithmMember.getOrNull()?.let { requestAlgorithmMember -> + val memberShape = ctx.model + .expectShape(operationShape.input.get()) + .members() + .first { it.memberName == requestAlgorithmMember } + + return ctx.symbolProvider.toMemberName(memberShape) + } + } + return null + } } diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index 4b99ae22e8f..aae62d170b9 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -4,10 +4,10 @@ import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketContents import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiPartUploads import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketByName +import aws.sdk.kotlin.e2etest.S3TestUtils.responseCodeFromPut import aws.sdk.kotlin.services.s3.* -import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload -import aws.sdk.kotlin.services.s3.model.CompletedPart -import aws.sdk.kotlin.services.s3.model.GetObjectRequest +import aws.sdk.kotlin.services.s3.model.* +import aws.sdk.kotlin.services.s3.presigners.presignPutObject import aws.smithy.kotlin.runtime.content.* import aws.smithy.kotlin.runtime.hashing.crc32 import aws.smithy.kotlin.runtime.testing.RandomTempFile @@ -16,7 +16,11 @@ import org.junit.jupiter.api.* import java.io.File import java.io.FileInputStream import java.util.* +import kotlin.test.Ignore import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.seconds @TestInstance(TestInstance.Lifecycle.PER_CLASS) class S3ChecksumTest { @@ -159,4 +163,33 @@ class S3ChecksumTest { assertEquals(actualChecksum, expectedChecksum) } } + + @Test + fun testPresignedUrlNoDefault() = runBlocking { + val unsignedPutRequest = PutObjectRequest { + bucket = testBucket + key = testKey() + } + val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds) + val contents = "presign-test" + + assertFalse(presignedPutRequest.url.toString().contains("x-amz-checksum-crc32")) + assertTrue(responseCodeFromPut(presignedPutRequest, contents) in 200..299) + } + + @Test + @Ignore + // FIXME: Sending checksum via query params should work + fun testPresignedUrlContainsChecksum() = runBlocking { + val unsignedPutRequest = PutObjectRequest { + bucket = testBucket + key = testKey() + checksumAlgorithm = ChecksumAlgorithm.Crc32 + } + val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds) + val contents = "presign-test" + + assertTrue(presignedPutRequest.url.toString().contains("x-amz-checksum-crc32")) + assertTrue(responseCodeFromPut(presignedPutRequest, contents) in 200..299) + } } From 44894e67875e7bb23d63331cf7c24c4ee1e28205 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 13 Dec 2024 01:19:43 -0500 Subject: [PATCH 51/72] Ktlint --- services/s3/e2eTest/src/S3ChecksumTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index aae62d170b9..c3149ba4b82 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -177,9 +177,9 @@ class S3ChecksumTest { assertTrue(responseCodeFromPut(presignedPutRequest, contents) in 200..299) } + // FIXME: Sending checksum via query params should work @Test @Ignore - // FIXME: Sending checksum via query params should work fun testPresignedUrlContainsChecksum() = runBlocking { val unsignedPutRequest = PutObjectRequest { bucket = testBucket From ad8d1adfe6e97058299b873b7c7e079c39987b91 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 13 Dec 2024 10:02:50 -0500 Subject: [PATCH 52/72] Trigger CI From 3c259eff149cab843f61994bb696f53b70fa961c Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 13 Dec 2024 15:02:36 -0500 Subject: [PATCH 53/72] PR feeback checkpoint --- .../S3CompositeChecksumsInterceptor.kt | 2 +- .../sdk/kotlin/codegen/PresignerGenerator.kt | 18 ++++++++---------- .../FlexibleChecksumsRequest.kt | 2 +- .../S3ExpressDisableChecksumInterceptor.kt | 2 +- services/s3/e2eTest/src/PaginatorTest.kt | 3 +-- services/s3/e2eTest/src/S3ChecksumTest.kt | 3 ++- services/s3/e2eTest/src/S3IntegrationTest.kt | 3 +-- services/s3/e2eTest/src/S3TestUtils.kt | 5 ++--- 8 files changed, 17 insertions(+), 21 deletions(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt index 16adc9aeef5..71ee8107ccd 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt @@ -32,7 +32,7 @@ public class S3CompositeChecksumsInterceptor : HttpInterceptor { internal fun HeadersBuilder.removeCompositeChecksums(logger: Logger): Unit = names().forEach { header -> if (header.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true) && get(header)!!.isCompositeChecksum()) { - logger.warn { "S3 returned a composite checksum, composite checksums are not supported, removing checksum" } + logger.warn { "S3 returned a composite checksum ($header), composite checksums are not supported, removing checksum" } remove(header) } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt index a912e904579..134b11bb3c9 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt @@ -12,9 +12,7 @@ import software.amazon.smithy.codegen.core.Symbol import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.isSupportedForFlexibleChecksums import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.toHashFunctionOrThrow -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.IllegalStateException import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.Encoding.encodeBase64String -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.lowercase import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Utils.runBlocking import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.HttpBody import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.readAll @@ -278,10 +276,10 @@ class PresignerGenerator : KotlinIntegration { } checksumAlgorithmMember(op, ctx)?.let { checksumAlgorithmMember -> - withBlock("input.#L?.value?.let { checksumAlgorithmString ->", "}", checksumAlgorithmMember) { + withBlock("input.#L?.value?.let { checksumAlgorithmName ->", "}", checksumAlgorithmMember) { withBlock("when (unsignedRequest.body) {", "}") { withBlock("is #1T.Bytes, is #1T.Empty -> {", "}", HttpBody) { - write("val checksumAlgorithm = checksumAlgorithmString.#T()", toHashFunctionOrThrow) + write("val checksumAlgorithm = checksumAlgorithmName.#T()", toHashFunctionOrThrow) withInlineBlock( "if (checksumAlgorithm.#T) {", "}", @@ -293,9 +291,8 @@ class PresignerGenerator : KotlinIntegration { } } write( - "checksum = #S.#T() to checksumAlgorithm.digest().#T()", - "x-amz-checksum-\${checksumAlgorithmString}", - lowercase, + "checksum = #S.lowercase() to checksumAlgorithm.digest().#T()", + "x-amz-checksum-\${checksumAlgorithmName}", encodeBase64String, ) } @@ -306,12 +303,13 @@ class PresignerGenerator : KotlinIntegration { "#T.#T { #S }", coroutineContext, warn, - "The requested checksum algorithm is not supported for pre-signed URL checksums, sending request without checksum.", + "The requested checksum algorithm (\${checksumAlgorithmName}) is not supported for pre-signed URL checksums, sending request without checksum. " + + "Supported checksums for pre-signed URLs include the following: ", ) } } } - write("else -> throw #T(#S)", IllegalStateException, "HTTP body type unsupported for pre-signed URL checksums.") + write("else -> throw IllegalStateException(#S)", "HTTP body type unsupported for pre-signed URL checksums.") } } } @@ -340,7 +338,7 @@ class PresignerGenerator : KotlinIntegration { private fun normalizeUriPath(service: ServiceShape) = service.sdkId != "S3" /** - * Gets the checksum algorithm member if a user can configure request checksums otherwise null + * Gets the checksum algorithm member if configured on a request, otherwise null */ private fun checksumAlgorithmMember( operationShape: OperationShape, diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index d79824212ff..959598100ca 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -69,7 +69,7 @@ class FlexibleChecksumsRequest : KotlinIntegration { } private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { - override val name: String = "FlexibleChecksumsRequest" + override val name: String = "flexibleChecksumsRequestMiddleware" override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { val httpChecksumTrait = op.getTrait() diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt index 1dd06c25406..cd84c2bfc76 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt @@ -19,7 +19,7 @@ private const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" private const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" /** - * Disables checksums for s3:UploadPart requests that use S3 express. + * Disables checksums for s3:UploadPart requests that use S3 Express. */ internal class S3ExpressDisableChecksumInterceptor( private val userConfiguredChecksum: Boolean, diff --git a/services/s3/e2eTest/src/PaginatorTest.kt b/services/s3/e2eTest/src/PaginatorTest.kt index 3d1ed2f4919..f35ea41fff9 100644 --- a/services/s3/e2eTest/src/PaginatorTest.kt +++ b/services/s3/e2eTest/src/PaginatorTest.kt @@ -41,9 +41,8 @@ class PaginatorTest { S3TestUtils.deleteBucketAndAllContents(client, testBucket) } - // FIXME: Enable test when motorcade is ready + // FIXME: Enable test // Seeing: S3Exception: Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: crc32 - // Cause: "Post-motorcade SDK is expected not to work against Pre-Motorcade S3" // ListParts has a strange pagination termination condition via [IsTerminated]. Verify it actually works correctly. @Ignore @Test diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index c3149ba4b82..0c271178582 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -31,7 +31,7 @@ class S3ChecksumTest { @BeforeAll private fun setUp(): Unit = runBlocking { val accountId = getAccountId() - // FIXME: Use randomly generated bucket instead of hardcoded one when motorcade is ready + // FIXME: Use randomly generated bucket instead of hardcoded one getBucketByName(client, testBucket, "us-west-2", accountId) } @@ -169,6 +169,7 @@ class S3ChecksumTest { val unsignedPutRequest = PutObjectRequest { bucket = testBucket key = testKey() + checksumCrc32 = "`" } val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds) val contents = "presign-test" diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index aec1de82d16..7b9c22fa456 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -188,9 +188,8 @@ class S3BucketOpsIntegrationTest { } } - // FIXME: Enable test when motorcade is ready + // FIXME: Enable test // Seeing: S3Exception: Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: crc32 - // Cause: "Post-motorcade SDK is expected not to work against Pre-Motorcade S3" @Ignore @Test fun testMultipartUpload(): Unit = runBlocking { diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 6ace834f00f..21b8f311c47 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -34,9 +34,8 @@ object S3TestUtils { const val DEFAULT_REGION = "us-west-2" // The E2E test account only has permission to operate on buckets with the prefix "s3-test-bucket-" - // Motorcade allow-listed bucket: "s3-test-bucket-ci-motorcade". - // Non-checksum E2E tests will use it and delete it if TEST_BUCKET_PREFIX="s3-test-bucket-" via `deleteBucketAndAllContents` - // TODO: Change back to "s3-test-bucket-" after motorcade is released. + // Non-checksum E2E tests will use and delete hardcoded bucket required for checksum tests if TEST_BUCKET_PREFIX="s3-test-bucket-" via `deleteBucketAndAllContents` + // TODO: Change back to "s3-test-bucket-" private const val TEST_BUCKET_PREFIX = "s3-test-bucket-temp-" private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html From 0029768dacf39ff1378333d71d4790219ec71623 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 13 Dec 2024 16:39:48 -0500 Subject: [PATCH 54/72] Trigger CI From 427daf35892bdc67381ea976d2ead5c2d43e1b33 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 18:34:23 -0500 Subject: [PATCH 55/72] Refactor checksum interceptors --- .../FlexibleChecksumsRequest.kt | 140 +++++++++++------- .../FlexibleChecksumsResponse.kt | 103 +++++++------ .../s3/express/S3ExpressIntegration.kt | 37 ++--- .../S3ExpressDefaultChecksumAlgorithm.kt | 37 +++++ .../S3ExpressDisableChecksumInterceptor.kt | 77 ---------- 5 files changed, 190 insertions(+), 204 deletions(-) create mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt delete mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 959598100ca..ec3acddf0a9 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -22,15 +22,15 @@ import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.StructureShape /** - * Adds a middleware that enables sending flexible checksums during an HTTP request + * Handles flexible checksum requests */ class FlexibleChecksumsRequest : KotlinIntegration { - override fun enabledForService(model: Model, settings: KotlinSettings) = model - .shapes() - .any { it.hasTrait() } + override fun enabledForService(model: Model, settings: KotlinSettings) = + model.isTraitApplied(HttpChecksumTrait::class.java) override fun additionalServiceConfigProps(ctx: CodegenContext): List = listOf( + // Allows flexible checksum request configuration ConfigProperty { name = "requestChecksumCalculation" symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption @@ -42,66 +42,96 @@ class FlexibleChecksumsRequest : KotlinIntegration { ) override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + flexibleChecksumsRequestMiddleware + configBusinessMetrics - - private val configBusinessMetrics = object : ProtocolMiddleware { - override val name: String = "requestChecksumCalculationBusinessMetric" - - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - op.hasTrait() - - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - writer.withBlock("when(config.requestChecksumCalculation) {", "}") { - writer.write( - "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, - RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, - RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, - ) - writer.write( - "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, - RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, - RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, - ) - } - } - } + resolved + requestChecksumCalculationBusinessMetric + httpChecksumDefaultAlgorithmMiddleware + flexibleChecksumsRequestMiddleware +} - private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { - override val name: String = "flexibleChecksumsRequestMiddleware" +/** + * Emits business metric based on `requestChecksumCalculation` client config + */ +private val requestChecksumCalculationBusinessMetric = object : ProtocolMiddleware { + override val name: String = "requestChecksumCalculationBusinessMetric" - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { - val httpChecksumTrait = op.getTrait() - val input = op.input.getOrNull()?.let { ctx.model.expectShape(it) } + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + op.hasTrait() - return (httpChecksumTrait != null) && - (httpChecksumTrait.requestAlgorithmMember?.getOrNull() != null) && - (input?.memberNames?.any { it == httpChecksumTrait.requestAlgorithmMember.get() } == true) + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.withBlock("when(config.requestChecksumCalculation) {", "}") { + // Supported + writer.write( + "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED)", + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + // Required + writer.write( + "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED)", + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) } + } +} - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) +/** + * Adds default checksum algorithm to the execution context + */ +private val httpChecksumDefaultAlgorithmMiddleware = object : ProtocolMiddleware { + override val name: String = "httpChecksumDefaultAlgorithmMiddleware" + override val order: Byte = -2 // Before S3 Express (possibly) changes the default (-1) and before calculating checksum (0) - val httpChecksumTrait = op.getTrait()!! + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + requestChecksumsConfigured(ctx, op) - val requestAlgorithmMember = ctx.model.expectShape(op.input.get()) - .members() - .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.write( + "op.context[#T.DefaultChecksumAlgorithm] = #S", + RuntimeTypes.HttpClient.Operation.HttpOperationContext, + "CRC32", + ) + } +} + +/** + * Adds interceptor to handle flexible checksum request calculation + */ +private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { + override val name: String = "flexibleChecksumsRequestMiddleware" + + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + requestChecksumsConfigured(ctx, op) - val requestAlgorithmMemberName = ctx.symbolProvider.toMemberName(requestAlgorithmMember) - val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val httpChecksumTrait = op.getTrait()!! + val requestChecksumRequired = httpChecksumTrait.isRequestChecksumRequired + val requestAlgorithmMember = ctx.model.expectShape(op.input.get()) + .members() + .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } + val requestAlgorithmMemberName = ctx.symbolProvider.toMemberName(requestAlgorithmMember) - writer.withBlock( - "op.interceptors.add(#T<#T>(", - "))", - RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor, - inputSymbol, - ) { - writer.write("#L,", requestChecksumRequired) - writer.write("config.requestChecksumCalculation,") - writer.write("input.#L?.value,", requestAlgorithmMemberName) - } + writer.withBlock( + "op.interceptors.add(#T(", + "))", + RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor, + ) { + writer.write("#L,", requestChecksumRequired) + writer.write("config.requestChecksumCalculation,") + writer.write("input.#L?.value,", requestAlgorithmMemberName) } } } + +/** + * Determines if an operation is set up to send flexible request checksums + */ +private fun requestChecksumsConfigured(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { + val httpChecksumTrait = op.getTrait() + val inputShape = op.input.getOrNull()?.let { ctx.model.expectShape(it) } + + return ( + (httpChecksumTrait != null) && + (httpChecksumTrait.requestAlgorithmMember?.getOrNull() != null) && + (inputShape?.memberNames?.any { it == httpChecksumTrait.requestAlgorithmMember.get() } == true) + ) +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index 9767d2a9718..c5164d40652 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -19,15 +19,15 @@ import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.StructureShape /** - * Adds a middleware which validates checksums returned in responses if the user has opted-in. + * Handles flexible checksum responses */ class FlexibleChecksumsResponse : KotlinIntegration { - override fun enabledForService(model: Model, settings: KotlinSettings) = model - .shapes() - .any { it.hasTrait() } + override fun enabledForService(model: Model, settings: KotlinSettings) = + model.isTraitApplied(HttpChecksumTrait::class.java) override fun additionalServiceConfigProps(ctx: CodegenContext): List = listOf( + // Allows flexible checksum response configuration ConfigProperty { name = "responseChecksumValidation" symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption @@ -39,59 +39,66 @@ class FlexibleChecksumsResponse : KotlinIntegration { ) override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + flexibleChecksumsResponseMiddleware + configBusinessMetrics + resolved + flexibleChecksumsResponseMiddleware + responseChecksumValidationBusinessMetric +} - private val configBusinessMetrics = object : ProtocolMiddleware { - override val name: String = "responseChecksumValidationBusinessMetric" +/** + * Emits business metric based on `responseChecksumValidation` client config + */ +private val responseChecksumValidationBusinessMetric = object : ProtocolMiddleware { + override val name: String = "responseChecksumValidationBusinessMetric" - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - writer.withBlock("when(config.responseChecksumValidation) {", "}") { - writer.write( - "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, - RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, - RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, - ) - writer.write( - "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, - RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, - RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, - ) - } + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + writer.withBlock("when(config.responseChecksumValidation) {", "}") { + // Supported + writer.write( + "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED)", + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) + // Required + writer.write( + "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED)", + RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, + RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, + ) } } +} - private val flexibleChecksumsResponseMiddleware = object : ProtocolMiddleware { - override val name: String = "FlexibleChecksumsResponse" - - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { - val httpChecksumTrait = op.getTrait() - val input = op.input.getOrNull()?.let { ctx.model.expectShape(it) } +/** + * Adds interceptor to handle flexible checksum response validation + */ +private val flexibleChecksumsResponseMiddleware = object : ProtocolMiddleware { + override val name: String = "flexibleChecksumsResponseMiddleware" - return (httpChecksumTrait != null) && - (httpChecksumTrait.requestValidationModeMember?.getOrNull() != null) && - (input?.memberNames?.any { it == httpChecksumTrait.requestValidationModeMember.get() } == true) - } + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { + val httpChecksumTrait = op.getTrait() + val inputShape = op.input.getOrNull()?.let { ctx.model.expectShape(it) } - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) + return (httpChecksumTrait != null) && + (httpChecksumTrait.requestValidationModeMember?.getOrNull() != null) && + (inputShape?.memberNames?.any { it == httpChecksumTrait.requestValidationModeMember.get() } == true) + } - val httpChecksumTrait = op.getTrait()!! - val requestValidationModeMember = ctx.model.expectShape(op.input.get()) - .members() - .first { it.memberName == httpChecksumTrait.requestValidationModeMember.get() } - val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) + val httpChecksumTrait = op.getTrait()!! + val requestValidationModeMember = ctx.model.expectShape(op.input.get()) + .members() + .first { it.memberName == httpChecksumTrait.requestValidationModeMember.get() } + val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) - writer.withBlock( - "op.interceptors.add(#T<#T>(", - "))", - RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, - inputSymbol, - ) { - writer.write("input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) - writer.write("config.responseChecksumValidation,") - } + writer.withBlock( // TODO: ADD DIFFERENT INTERCEPTORS DEPENDING ON IF THE SERVICE HAS COMPOSITE CHECKSUMS ! + "op.interceptors.add(#T<#T>(", + "))", + RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, + inputSymbol, + ) { + writer.write("input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) + writer.write("config.responseChecksumValidation,") } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 067da2da40f..086534f3777 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -16,7 +16,6 @@ import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerato import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType -import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.* import software.amazon.smithy.model.traits.* @@ -26,7 +25,7 @@ import software.amazon.smithy.model.transform.ModelTransformer * An integration which handles codegen for S3 Express, such as: * 1. Configure auth scheme by applying a synthetic shape and trait * 2. Add ExpressClient and Bucket to execution context - * 3. Disable all checksums for s3:UploadPart + * 3. Configuring the default checksum algorithm */ class S3ExpressIntegration : KotlinIntegration { companion object { @@ -97,7 +96,7 @@ class S3ExpressIntegration : KotlinIntegration { resolved + listOf( addClientToExecutionContext, addBucketToExecutionContext, - uploadPartDisableChecksum, + s3ExpressDefaultChecksumAlgorithm, ) private val s3AttributesSymbol = buildSymbol { @@ -130,33 +129,23 @@ class S3ExpressIntegration : KotlinIntegration { } /** - * Disable all checksums for s3:UploadPart + * Re-configures the default checksum algorithm for S3 Express. */ - private val uploadPartDisableChecksum = object : ProtocolMiddleware { - override val name: String = "UploadPartDisableChecksum" + private val s3ExpressDefaultChecksumAlgorithm = object : ProtocolMiddleware { + override val name: String = "s3ExpressDefaultChecksumAlgorithm" + override val order: Byte = -1 // After setting the modeled default (-2) and before calculating the checksum (0) override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - op.isS3UploadPart && op.hasTrait() + op.hasTrait() || op.hasTrait() override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val httpChecksumTrait = op.getTrait()!! - - val requestAlgorithmMemberName = httpChecksumTrait.requestAlgorithmMember?.getOrNull()?.let { - val requestAlgorithmMemberShape = ctx.model.expectShape(op.input.get()) - .members() - .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } - ctx.symbolProvider.toMemberName(requestAlgorithmMemberShape) - } - - val interceptorSymbol = buildSymbol { - namespace = "aws.sdk.kotlin.services.s3.express" - name = "S3ExpressDisableChecksumInterceptor" - } - writer.addImport(interceptorSymbol) writer.write( - "op.interceptors.add(#T(input.#L?.value != null))", - interceptorSymbol, - requestAlgorithmMemberName, + "op.interceptors.add(#T(#L))", + buildSymbol { + namespace = "aws.sdk.kotlin.services.s3.express" + name = "S3ExpressDefaultChecksumAlgorithm" + }, + op.isS3UploadPart ) } } diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt new file mode 100644 index 00000000000..862e96c1bf9 --- /dev/null +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.s3.express + +import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.collections.AttributeKey +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.operation.ExecutionContext + +internal const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" +internal const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" + +/** + * Re-configures the default checksum algorithm for S3 Express + * NOTE: Default checksums are disabled for s3:UploadPart. + */ +internal class S3ExpressDefaultChecksumAlgorithm( + private val isS3UploadPart: Boolean, +) : HttpInterceptor { + override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { + if (usingS3Express(context.executionContext)) { + if (isS3UploadPart) { + context.executionContext.remove(HttpOperationContext.DefaultChecksumAlgorithm) + } else { + context.executionContext[HttpOperationContext.DefaultChecksumAlgorithm] = "CRC32" + } + } + return super.modifyBeforeSigning(context) + } +} + +private fun usingS3Express(executionContext: ExecutionContext): Boolean = + executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt deleted file mode 100644 index cd84c2bfc76..00000000000 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.services.s3.express - -import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext -import aws.smithy.kotlin.runtime.collections.AttributeKey -import aws.smithy.kotlin.runtime.http.DeferredHeadersBuilder -import aws.smithy.kotlin.runtime.http.HeadersBuilder -import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor -import aws.smithy.kotlin.runtime.http.request.HttpRequest -import aws.smithy.kotlin.runtime.http.request.toBuilder -import aws.smithy.kotlin.runtime.telemetry.logging.warn -import kotlin.coroutines.coroutineContext - -private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" -private const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" -private const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" - -/** - * Disables checksums for s3:UploadPart requests that use S3 Express. - */ -internal class S3ExpressDisableChecksumInterceptor( - private val userConfiguredChecksum: Boolean, -) : HttpInterceptor { - override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { - if (context.executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE) { - return context.protocolRequest - } - - if (userConfiguredChecksum) { - coroutineContext.warn { - "Checksums must not be sent with S3 Express UploadPart operation, removing checksum(s)" - } - } - - val request = context.protocolRequest.toBuilder() - - request.headers.removeChecksumHeaders() - request.trailingHeaders.removeChecksumTrailingHeaders() - request.headers.removeChecksumTrailingHeadersFromXAmzTrailer() - - return request.build() - } -} - -/** - * Removes any checksums sent in the request's headers - */ -internal fun HeadersBuilder.removeChecksumHeaders(): Unit = - names().forEach { name -> - if (name.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { - remove(name) - } - } - -/** - * Removes any checksums sent in the request's trailing headers - */ -internal fun DeferredHeadersBuilder.removeChecksumTrailingHeaders(): Unit = - names().forEach { name -> - if (name.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { - remove(name) - } - } - -/** - * Removes any checksums sent in the request's trailing headers from `x-amz-trailer` - */ -internal fun HeadersBuilder.removeChecksumTrailingHeadersFromXAmzTrailer() { - this.getAll("x-amz-trailer")?.forEach { trailingHeader -> - if (trailingHeader.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true)) { - this.remove("x-amz-trailer", trailingHeader) - } - } -} From 41ad742e63456bc0f0f6a148cc46c535ac758280 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 21:38:29 -0500 Subject: [PATCH 56/72] Fix composite checksums --- .../S3CompositeChecksumsInterceptor.kt | 47 ------------------- .../S3FlexibleChecksumResponseInterceptor.kt | 27 +++++++++++ codegen/aws-sdk-codegen/build.gradle.kts | 1 + .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 2 +- .../sdk/kotlin/codegen/PresignerGenerator.kt | 3 +- .../FlexibleChecksumsResponse.kt | 17 +++++-- .../s3/S3CompositeChecksumsIntegration.kt | 44 ----------------- ...tlin.codegen.integration.KotlinIntegration | 1 - services/s3/e2eTest/src/S3ChecksumTest.kt | 1 - .../event-stream-model-template.smithy | 4 +- 10 files changed, 45 insertions(+), 102 deletions(-) delete mode 100644 aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt create mode 100644 aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt delete mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt deleted file mode 100644 index 71ee8107ccd..00000000000 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor.kt +++ /dev/null @@ -1,47 +0,0 @@ -package aws.sdk.kotlin.runtime.http.interceptors - -import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext -import aws.smithy.kotlin.runtime.http.HeadersBuilder -import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor -import aws.smithy.kotlin.runtime.http.request.HttpRequest -import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.http.response.toBuilder -import aws.smithy.kotlin.runtime.telemetry.logging.Logger -import aws.smithy.kotlin.runtime.telemetry.logging.logger -import kotlin.coroutines.coroutineContext - -private const val CHECKSUM_HEADER_PREFIX = "x-amz-checksum-" - -/** - * Removes any checksum headers that contain composite checksums from an S3 response. - */ -public class S3CompositeChecksumsInterceptor : HttpInterceptor { - override suspend fun modifyBeforeDeserialization(context: ProtocolResponseInterceptorContext): HttpResponse { - val response = context.protocolResponse.toBuilder() - val logger = coroutineContext.logger() - - response.headers.removeCompositeChecksums(logger) - - return response.build() - } -} - -/** - * Removes any checksum headers that contain composite checksums. - */ -internal fun HeadersBuilder.removeCompositeChecksums(logger: Logger): Unit = - names().forEach { header -> - if (header.startsWith(CHECKSUM_HEADER_PREFIX, ignoreCase = true) && get(header)!!.isCompositeChecksum()) { - logger.warn { "S3 returned a composite checksum ($header), composite checksums are not supported, removing checksum" } - remove(header) - } - } - -/** - * Verifies if a checksum is composite. - */ -private fun String.isCompositeChecksum(): Boolean { - // Ends with "-#" where "#" is a number - val regex = Regex("-(\\d)+$") - return regex.containsMatchIn(this) -} diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt new file mode 100644 index 00000000000..7845f2f7f61 --- /dev/null +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt @@ -0,0 +1,27 @@ +package aws.sdk.kotlin.runtime.http.interceptors + +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption +import aws.smithy.kotlin.runtime.http.interceptors.FlexibleChecksumsResponseInterceptor + +/** + * S3 variant of the flexible checksum interceptor where composite checksums are not validated + */ +public class S3FlexibleChecksumResponseInterceptor( + responseValidationRequired: Boolean, + responseChecksumValidation: HttpChecksumConfigOption?, +) : FlexibleChecksumsResponseInterceptor( + responseValidationRequired, + responseChecksumValidation, +) { + override fun ignoreChecksum(checksum: String): Boolean = + checksum.isCompositeChecksum() +} + +/** + * Verifies if a checksum is composite. + */ +private fun String.isCompositeChecksum(): Boolean { + // Ends with "-#" where "#" is a number + val regex = Regex("-(\\d)+$") + return regex.containsMatchIn(this) +} diff --git a/codegen/aws-sdk-codegen/build.gradle.kts b/codegen/aws-sdk-codegen/build.gradle.kts index aa404d7a4cb..0348cd5a7f1 100644 --- a/codegen/aws-sdk-codegen/build.gradle.kts +++ b/codegen/aws-sdk-codegen/build.gradle.kts @@ -30,6 +30,7 @@ dependencies { api(libs.smithy.protocol.test.traits) implementation(libs.smithy.aws.endpoints) implementation(libs.smithy.smoke.test.traits) + implementation(libs.smithy.kotlin.runtime.core) testImplementation(libs.junit.jupiter) testImplementation(libs.junit.jupiter.params) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index 70f3a4950ef..0099e436bb1 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -62,7 +62,7 @@ object AwsRuntimeTypes { val UnsupportedSigningAlgorithmInterceptor = symbol("UnsupportedSigningAlgorithmInterceptor") val BusinessMetricsInterceptor = symbol("BusinessMetricsInterceptor") val AwsBusinessMetric = symbol("AwsBusinessMetric") - val S3CompositeChecksumsInterceptor = symbol("S3CompositeChecksumsInterceptor") + val S3FlexibleChecksumResponseInterceptor = symbol("S3FlexibleChecksumResponseInterceptor") } object Retries { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt index 134b11bb3c9..d9b19377c47 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt @@ -5,6 +5,7 @@ package aws.sdk.kotlin.codegen import aws.sdk.kotlin.codegen.model.traits.Presignable +import aws.smithy.kotlin.runtime.hashing.algorithmsSupportedForFlexibleChecksums import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.aws.traits.auth.SigV4Trait import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait @@ -304,7 +305,7 @@ class PresignerGenerator : KotlinIntegration { coroutineContext, warn, "The requested checksum algorithm (\${checksumAlgorithmName}) is not supported for pre-signed URL checksums, sending request without checksum. " + - "Supported checksums for pre-signed URLs include the following: ", + "Supported checksums for pre-signed URLs: $algorithmsSupportedForFlexibleChecksums", ) } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index c5164d40652..3691cc82a26 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -4,6 +4,8 @@ */ package aws.sdk.kotlin.codegen.customization.flexiblechecksums +import aws.sdk.kotlin.codegen.AwsRuntimeTypes +import aws.sdk.kotlin.codegen.customization.s3.isS3 import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.* @@ -16,6 +18,7 @@ import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.model.shapes.StructureShape /** @@ -84,18 +87,22 @@ private val flexibleChecksumsResponseMiddleware = object : ProtocolMiddleware { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) val httpChecksumTrait = op.getTrait()!! val requestValidationModeMember = ctx.model.expectShape(op.input.get()) .members() .first { it.memberName == httpChecksumTrait.requestValidationModeMember.get() } val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) - writer.withBlock( // TODO: ADD DIFFERENT INTERCEPTORS DEPENDING ON IF THE SERVICE HAS COMPOSITE CHECKSUMS ! - "op.interceptors.add(#T<#T>(", + val interceptor = if (ctx.model.expectShape(ctx.settings.service).isS3) { + AwsRuntimeTypes.Http.Interceptors.S3FlexibleChecksumResponseInterceptor + } else { + RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor + } + + writer.withBlock( + "op.interceptors.add(#T(", "))", - RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor, - inputSymbol, + interceptor, ) { writer.write("input.#L?.value == \"ENABLED\",", requestValidationModeMemberName) writer.write("config.responseChecksumValidation,") diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt deleted file mode 100644 index cef54247f72..00000000000 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/s3/S3CompositeChecksumsIntegration.kt +++ /dev/null @@ -1,44 +0,0 @@ -package aws.sdk.kotlin.codegen.customization.flexiblechecksums.s3 - -import aws.sdk.kotlin.codegen.AwsRuntimeTypes.Http.Interceptors.S3CompositeChecksumsInterceptor -import aws.sdk.kotlin.codegen.customization.s3.isS3 -import software.amazon.smithy.aws.traits.HttpChecksumTrait -import software.amazon.smithy.kotlin.codegen.KotlinSettings -import software.amazon.smithy.kotlin.codegen.core.KotlinWriter -import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration -import software.amazon.smithy.kotlin.codegen.model.expectShape -import software.amazon.smithy.kotlin.codegen.model.hasTrait -import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator -import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware -import software.amazon.smithy.model.Model -import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.model.shapes.ServiceShape - -/** - * Removes composite checksums returned by S3 so that flexible checksums won't validate them. - * Composite checksums are used for multipart uploads and end with "-#" where "#" is a number. - */ -class S3CompositeChecksumsIntegration : KotlinIntegration { - override val order: Byte - get() = -128 - - override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = - model.expectShape(settings.service).isS3 - - override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + s3CompositeChecksumsMiddleware -} - -/** - * Adds [S3CompositeChecksumsInterceptor] to interceptors when operation has flexible checksums. - */ -private val s3CompositeChecksumsMiddleware = object : ProtocolMiddleware { - override val name: String = "S3CompositeChecksumsMiddleware" - - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - op.hasTrait() - - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - writer.write("op.interceptors.add(#T())", S3CompositeChecksumsInterceptor) - } -} diff --git a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration index 39e5630c930..60ccb728f63 100644 --- a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +++ b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration @@ -12,7 +12,6 @@ aws.sdk.kotlin.codegen.endpoints.AddAwsEndpointFunctions aws.sdk.kotlin.codegen.PresignerGenerator aws.sdk.kotlin.codegen.customization.flexiblechecksums.FlexibleChecksumsRequest aws.sdk.kotlin.codegen.customization.flexiblechecksums.FlexibleChecksumsResponse -aws.sdk.kotlin.codegen.customization.flexiblechecksums.s3.S3CompositeChecksumsIntegration aws.sdk.kotlin.codegen.customization.AddAwsSpanAttributes aws.sdk.kotlin.codegen.customization.s3.S3OperationErrorHandler aws.sdk.kotlin.codegen.customization.s3.S3SigningConfig diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index 0c271178582..043ae7baae9 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -169,7 +169,6 @@ class S3ChecksumTest { val unsignedPutRequest = PutObjectRequest { bucket = testBucket key = testKey() - checksumCrc32 = "`" } val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds) val contents = "presign-test" diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index dd32f77d8d7..dc87f60a32f 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -4,12 +4,12 @@ use aws.protocols#restJson1 use aws.api#service use aws.auth#sigv4 -@restJson1 +@{protocol-name} @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -@http(method: "POST", uri: "/test-eventstream", code: 200) +{op-traits} operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 5c6c0bf02e36bf00d56e3e60796a426785209e62 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 22:19:52 -0500 Subject: [PATCH 57/72] Make it compile --- aws-runtime/aws-http/api/aws-http.api | 24 ++----------- .../S3CompositeChecksumsInterceptorTest.kt | 35 ------------------- .../FlexibleChecksumsRequest.kt | 6 ++-- .../s3/express/S3ExpressIntegration.kt | 2 +- .../event-stream-model-template.smithy | 4 +-- 5 files changed, 9 insertions(+), 62 deletions(-) delete mode 100644 aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index 0b475eb05fa..e401ba307dd 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -196,27 +196,9 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInter public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } -public final class aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { - public fun ()V - public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeRetryLoop (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun readAfterAttempt (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterDeserialization (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterExecution (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterSerialization (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readAfterSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readAfterTransmit (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V - public fun readBeforeAttempt (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V - public fun readBeforeExecution (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V - public fun readBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V - public fun readBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V +public final class aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor : aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor { + public fun (ZLaws/smithy/kotlin/runtime/client/config/HttpChecksumConfigOption;)V + public fun ignoreChecksum (Ljava/lang/String;)Z } public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { diff --git a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt deleted file mode 100644 index 042dcb5b99e..00000000000 --- a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/interceptors/S3CompositeChecksumsInterceptorTest.kt +++ /dev/null @@ -1,35 +0,0 @@ -package aws.sdk.kotlin.runtime.http.interceptors - -import aws.smithy.kotlin.runtime.http.HeadersBuilder -import aws.smithy.kotlin.runtime.telemetry.logging.logger -import kotlinx.coroutines.test.runTest -import kotlin.coroutines.coroutineContext -import kotlin.test.Test -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class S3CompositeChecksumsInterceptorTest { - @Test - fun compositeChecksumRemoval() = runTest { - val headers = HeadersBuilder() - - headers.append("x-amz-checksum-crc32", "foo-1") - headers.append("x-amz-checksum-sha256", "bar") - - assertTrue( - headers.contains("x-amz-checksum-crc32"), - ) - assertTrue( - headers.contains("x-amz-checksum-sha256"), - ) - - headers.removeCompositeChecksums(coroutineContext.logger()) - - assertFalse( - headers.contains("x-amz-checksum-crc32"), - ) - assertTrue( - headers.contains("x-amz-checksum-sha256"), - ) - } -} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index ec3acddf0a9..767e1d00898 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -131,7 +131,7 @@ private fun requestChecksumsConfigured(ctx: ProtocolGenerator.GenerationContext, return ( (httpChecksumTrait != null) && - (httpChecksumTrait.requestAlgorithmMember?.getOrNull() != null) && - (inputShape?.memberNames?.any { it == httpChecksumTrait.requestAlgorithmMember.get() } == true) - ) + (httpChecksumTrait.requestAlgorithmMember?.getOrNull() != null) && + (inputShape?.memberNames?.any { it == httpChecksumTrait.requestAlgorithmMember.get() } == true) + ) } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 086534f3777..d0a76374bcd 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -145,7 +145,7 @@ class S3ExpressIntegration : KotlinIntegration { namespace = "aws.sdk.kotlin.services.s3.express" name = "S3ExpressDefaultChecksumAlgorithm" }, - op.isS3UploadPart + op.isS3UploadPart, ) } } diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index dc87f60a32f..dd32f77d8d7 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -4,12 +4,12 @@ use aws.protocols#restJson1 use aws.api#service use aws.auth#sigv4 -@{protocol-name} +@restJson1 @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -{op-traits} +@http(method: "POST", uri: "/test-eventstream", code: 200) operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 4d915c65c855afcd0701152ee2096685186c5364 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 22:25:23 -0500 Subject: [PATCH 58/72] Fix smithy model template --- .../commonTest/resources/event-stream-model-template.smithy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index dd32f77d8d7..8150dcec2db 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -1,15 +1,15 @@ namespace aws.sdk.kotlin.test -use aws.protocols#restJson1 +use aws.protocols#{protocol-name} use aws.api#service use aws.auth#sigv4 -@restJson1 +@{protocol-name} @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -@http(method: "POST", uri: "/test-eventstream", code: 200) +{op-traits} operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 609226c8be776b5f09fda0314a581f7306bc1869 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 22:29:40 -0500 Subject: [PATCH 59/72] Fix model smithy template again? --- .../commonTest/resources/event-stream-model-template.smithy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index 8150dcec2db..dd32f77d8d7 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -1,15 +1,15 @@ namespace aws.sdk.kotlin.test -use aws.protocols#{protocol-name} +use aws.protocols#restJson1 use aws.api#service use aws.auth#sigv4 -@{protocol-name} +@restJson1 @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -{op-traits} +@http(method: "POST", uri: "/test-eventstream", code: 200) operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 80ba07013bc186bdc6f21c06f03d4333d96b7cb1 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 18 Dec 2024 22:31:02 -0500 Subject: [PATCH 60/72] Fix model again --- .../commonTest/resources/event-stream-model-template.smithy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index dd32f77d8d7..8150dcec2db 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -1,15 +1,15 @@ namespace aws.sdk.kotlin.test -use aws.protocols#restJson1 +use aws.protocols#{protocol-name} use aws.api#service use aws.auth#sigv4 -@restJson1 +@{protocol-name} @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -@http(method: "POST", uri: "/test-eventstream", code: 200) +{op-traits} operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 77f6ea5be9e899d19a0ca7fefce803be8c7556b1 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 19 Dec 2024 09:42:30 -0500 Subject: [PATCH 61/72] Refactor rules engine codegen tests --- tests/codegen/rules-engine/build.gradle.kts | 135 ++---------------- .../operation-context-params.smithy | 0 2 files changed, 13 insertions(+), 122 deletions(-) rename tests/codegen/rules-engine/{ => src/commonTest/resources}/operation-context-params.smithy (100%) diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index ef35526b380..b6242e9f17a 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -3,64 +3,37 @@ * SPDX-License-Identifier: Apache-2.0 */ -import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir +import aws.sdk.kotlin.shared.CodegenTest +import aws.sdk.kotlin.shared.Model -plugins { - alias(libs.plugins.kotlin.jvm) - alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) -} - -description = "Smithy rules engine codegen integration test suite" - -data class Test( - val projectionName: String, - val protocolName: String, - val modelTemplate: File, -) { - val model: File - get() = layout.buildDirectory.file("$projectionName/model.smithy").get().asFile -} +description = "AWS SDK for Kotlin's rules engine codegen integration test suite" val tests = listOf( - Test("operationContextParams", "operationContextParams", file("operation-context-params.smithy")), + CodegenTest( + "operationContextParams", + Model("operation-context-params.smithy"), + "aws.sdk.kotlin.test#TestService", + ), ) -fun fillInModel(output: File, protocolName: String, template: File) { - val input = template.readText() - val opTraits = when (protocolName) { - "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" - else -> "" - } - val replaced = input - .replace("{protocol-name}", protocolName) - .replace("{op-traits}", opTraits) - - output.parentFile.mkdirs() - output.writeText(replaced) -} - -val testServiceShapeId = "aws.sdk.kotlin.test#TestService" smithyBuild { tests.forEach { test -> - - projections.register(test.projectionName) { - imports = listOf(test.model.absolutePath) + projections.register(test.name) { + imports = listOf(layout.projectDirectory.file(test.model.path + test.model.fileName).asFile.absolutePath) transforms = listOf( """ { "name": "includeServices", "args": { - "services": ["$testServiceShapeId"] + "services": ["${test.serviceShapeId}"] } } """, ) - smithyKotlinPlugin { - serviceShapeId = testServiceShapeId - packageName = "aws.sdk.kotlin.test.${test.projectionName.lowercase()}" + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.test.${test.name.lowercase()}" packageVersion = "1.0" buildSettings { generateFullProject = false @@ -74,85 +47,3 @@ smithyBuild { } } } - -val codegen by configurations.getting -dependencies { - codegen(project(":codegen:aws-sdk-codegen")) - codegen(libs.smithy.cli) - codegen(libs.smithy.model) -} - -tasks.generateSmithyBuild { - doFirst { - tests.forEach { test -> fillInModel(test.model, test.protocolName, test.modelTemplate) } - } -} - -tasks.generateSmithyProjections { - doFirst { - // ensure the generated tests use the same version of the runtime as the aws aws-runtime - val smithyKotlinRuntimeVersion = libs.versions.smithy.kotlin.runtime.version.get() - System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinRuntimeVersion) - } -} - -val optinAnnotations = listOf( - "kotlin.RequiresOptIn", - "aws.smithy.kotlin.runtime.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi", -) - -kotlin.sourceSets.all { - optinAnnotations.forEach { languageSettings.optIn(it) } -} - -kotlin.sourceSets.getByName("test") { - smithyBuild.projections.forEach { - kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) - } -} - -tasks.withType { - dependsOn(tasks.generateSmithyProjections) - // generated clients have quite a few warnings - kotlinOptions.allWarningsAsErrors = false -} - -tasks.test { - useJUnitPlatform() - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showStackTraces = true - showExceptions = true - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL - } -} - -dependencies { - - implementation(libs.kotlinx.coroutines.core) - - testImplementation(libs.kotlin.test) - testImplementation(libs.kotlin.test.junit5) - testImplementation(libs.kotlinx.coroutines.test) - - testImplementation(libs.smithy.kotlin.smithy.test) - testImplementation(libs.smithy.kotlin.aws.signing.default) - testImplementation(libs.smithy.kotlin.telemetry.api) - - // have to manually add all the dependencies of the generated client(s) - // doing it this way (as opposed to doing what we do for protocol-tests) allows - // the tests to work without a publish to maven-local step at the cost of maintaining - // this set of dependencies manually - // <-- BEGIN GENERATED DEPENDENCY LIST --> - implementation(libs.bundles.smithy.kotlin.service.client) - implementation(libs.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(libs.smithy.kotlin.aws.json.protocols) - implementation(libs.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) - // <-- END GENERATED DEPENDENCY LIST --> -} diff --git a/tests/codegen/rules-engine/operation-context-params.smithy b/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy similarity index 100% rename from tests/codegen/rules-engine/operation-context-params.smithy rename to tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy From 4b61347212cd695091342f5f848f22dc9837683e Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 19 Dec 2024 12:43:32 -0500 Subject: [PATCH 62/72] Run CI (empty commit) From 756f66d6b1e0ad8a3625c14a5ef3500317b0b821 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 23 Dec 2024 18:57:32 -0500 Subject: [PATCH 63/72] PR feedback --- .../config/AbstractAwsSdkClientFactory.kt | 4 +- .../ResolveFlexibleChecksumsConfig.kt | 37 +- aws-runtime/aws-http/api/aws-http.api | 4 +- ...iteFlexibleChecksumResponseInterceptor.kt} | 8 +- .../aws/sdk/kotlin/shared/CodegenTest.kt | 1 - .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 2 +- .../sdk/kotlin/codegen/PresignerGenerator.kt | 6 +- .../FlexibleChecksumsRequest.kt | 20 +- .../FlexibleChecksumsResponse.kt | 13 +- .../S3ExpressDefaultChecksumAlgorithm.kt | 6 +- .../s3/express/ChecksumRemovalTest.kt | 87 ---- tests/codegen/build.gradle.kts | 1 + tests/codegen/checksums/build.gradle.kts | 13 +- .../checksums/ChecksumBusinessMetricsTest.kt | 158 ++----- .../codegen/checksums/ChecksumConfigTest.kt | 389 ++++-------------- .../codegen/checksums/ChecksumRequestTest.kt | 118 ++---- .../codegen/checksums/ChecksumResponseTest.kt | 285 ++----------- .../checksums/utils/ChecksumTestUtils.kt | 89 +++- ...-response-test.smithy => checksums.smithy} | 61 +-- .../commonTest/resources/config-test.smithy | 87 ---- tests/codegen/event-stream/build.gradle.kts | 32 -- .../event-stream-model-template.smithy | 6 +- 22 files changed, 388 insertions(+), 1039 deletions(-) rename aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/{S3FlexibleChecksumResponseInterceptor.kt => IgnoreCompositeFlexibleChecksumResponseInterceptor.kt} (67%) delete mode 100644 services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt rename tests/codegen/checksums/src/commonTest/resources/{request-response-test.smithy => checksums.smithy} (72%) delete mode 100644 tests/codegen/checksums/src/commonTest/resources/config-test.smithy diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt index 190f41a0b12..1ce6f4dc98b 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AbstractAwsSdkClientFactory.kt @@ -24,7 +24,7 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.SigV4aClientConfig import aws.smithy.kotlin.runtime.client.* import aws.smithy.kotlin.runtime.client.config.ClientSettings import aws.smithy.kotlin.runtime.client.config.CompressionClientConfig -import aws.smithy.kotlin.runtime.client.config.HttpChecksumClientConfig +import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfig import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.telemetry.TelemetryConfig import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider @@ -97,7 +97,7 @@ public abstract class AbstractAwsSdkClientFactory< config.sigV4aSigningRegionSet ?: resolveSigV4aSigningRegionSet(platform, profile) } - if (config is HttpChecksumClientConfig.Builder) { + if (config is HttpChecksumConfig.Builder) { config.requestChecksumCalculation = config.requestChecksumCalculation ?: resolveRequestChecksumCalculation(platform, profile) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt index bf7049ef2f3..bbbbd241e28 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt @@ -6,7 +6,8 @@ import aws.sdk.kotlin.runtime.config.AwsSdkSetting import aws.sdk.kotlin.runtime.config.profile.AwsProfile import aws.sdk.kotlin.runtime.config.profile.requestChecksumCalculation import aws.sdk.kotlin.runtime.config.profile.responseChecksumValidation -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.RequestHttpChecksumConfig +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.config.resolve import aws.smithy.kotlin.runtime.util.LazyAsyncValue import aws.smithy.kotlin.runtime.util.PlatformProvider @@ -16,27 +17,43 @@ import aws.smithy.kotlin.runtime.util.PlatformProvider * @return requestChecksumCalculation setting if found, the default value if not. */ @InternalSdkApi -public suspend fun resolveRequestChecksumCalculation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { +public suspend fun resolveRequestChecksumCalculation( + platform: PlatformProvider = PlatformProvider.System, + profile: LazyAsyncValue, +): RequestHttpChecksumConfig { val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation - return parseHttpChecksumConfigOption(unparsedString, "requestChecksumCalculation") + return parseRequestHttpChecksumConfig(unparsedString) } +private fun parseRequestHttpChecksumConfig(unparsedString: String?): RequestHttpChecksumConfig = + when (unparsedString?.uppercase()) { + null -> RequestHttpChecksumConfig.WHEN_SUPPORTED + "WHEN_SUPPORTED" -> RequestHttpChecksumConfig.WHEN_SUPPORTED + "WHEN_REQUIRED" -> RequestHttpChecksumConfig.WHEN_REQUIRED + else -> throw ConfigurationException( + "'$unparsedString' is not a valid value for 'requestChecksumCalculation'. Valid values are: ${RequestHttpChecksumConfig.entries}", + ) + } + /** * Attempts to resolve responseChecksumValidation from the specified sources. * @return responseChecksumValidation setting if found, the default value if not. */ @InternalSdkApi -public suspend fun resolveResponseChecksumValidation(platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue): HttpChecksumConfigOption { +public suspend fun resolveResponseChecksumValidation( + platform: PlatformProvider = PlatformProvider.System, + profile: LazyAsyncValue, +): ResponseHttpChecksumConfig { val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation - return parseHttpChecksumConfigOption(unparsedString, "responseChecksumValidation") + return parseResponseHttpChecksumConfig(unparsedString) } -private fun parseHttpChecksumConfigOption(unparsedString: String?, configOption: String): HttpChecksumConfigOption = +private fun parseResponseHttpChecksumConfig(unparsedString: String?): ResponseHttpChecksumConfig = when (unparsedString?.uppercase()) { - null -> HttpChecksumConfigOption.WHEN_SUPPORTED - "WHEN_SUPPORTED" -> HttpChecksumConfigOption.WHEN_SUPPORTED - "WHEN_REQUIRED" -> HttpChecksumConfigOption.WHEN_REQUIRED + null -> ResponseHttpChecksumConfig.WHEN_SUPPORTED + "WHEN_SUPPORTED" -> ResponseHttpChecksumConfig.WHEN_SUPPORTED + "WHEN_REQUIRED" -> ResponseHttpChecksumConfig.WHEN_REQUIRED else -> throw ConfigurationException( - "'$unparsedString' is not a valid value for $configOption. Valid values are: ${HttpChecksumConfigOption.entries}", + "'$unparsedString' is not a valid value for 'responseChecksumValidation'. Valid values are: ${ResponseHttpChecksumConfig.entries}", ) } diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index e401ba307dd..c9aadd34375 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -196,8 +196,8 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInter public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } -public final class aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor : aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor { - public fun (ZLaws/smithy/kotlin/runtime/client/config/HttpChecksumConfigOption;)V +public final class aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor : aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor { + public fun (ZLaws/smithy/kotlin/runtime/client/config/ResponseHttpChecksumConfig;)V public fun ignoreChecksum (Ljava/lang/String;)Z } diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt similarity index 67% rename from aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt rename to aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt index 7845f2f7f61..834924d5081 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3FlexibleChecksumResponseInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt @@ -1,14 +1,14 @@ package aws.sdk.kotlin.runtime.http.interceptors -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.http.interceptors.FlexibleChecksumsResponseInterceptor /** - * S3 variant of the flexible checksum interceptor where composite checksums are not validated + * Variant of the [FlexibleChecksumsResponseInterceptor] where composite checksums are not validated */ -public class S3FlexibleChecksumResponseInterceptor( +public class IgnoreCompositeFlexibleChecksumResponseInterceptor( responseValidationRequired: Boolean, - responseChecksumValidation: HttpChecksumConfigOption?, + responseChecksumValidation: ResponseHttpChecksumConfig?, ) : FlexibleChecksumsResponseInterceptor( responseValidationRequired, responseChecksumValidation, diff --git a/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt b/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt index 4ee9580629c..6b72149fcff 100644 --- a/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt @@ -7,7 +7,6 @@ data class CodegenTest( val name: String, val model: Model, val serviceShapeId: String, - val protocolName: String? = null, ) /** diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index 0099e436bb1..a0b3f834c17 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -62,7 +62,7 @@ object AwsRuntimeTypes { val UnsupportedSigningAlgorithmInterceptor = symbol("UnsupportedSigningAlgorithmInterceptor") val BusinessMetricsInterceptor = symbol("BusinessMetricsInterceptor") val AwsBusinessMetric = symbol("AwsBusinessMetric") - val S3FlexibleChecksumResponseInterceptor = symbol("S3FlexibleChecksumResponseInterceptor") + val IgnoreCompositeFlexibleChecksumResponseInterceptor = symbol("IgnoreCompositeFlexibleChecksumResponseInterceptor") } object Retries { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt index d9b19377c47..fdbd102c406 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt @@ -5,7 +5,6 @@ package aws.sdk.kotlin.codegen import aws.sdk.kotlin.codegen.model.traits.Presignable -import aws.smithy.kotlin.runtime.hashing.algorithmsSupportedForFlexibleChecksums import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.aws.traits.auth.SigV4Trait import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait @@ -14,10 +13,10 @@ import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.isSupportedForFlexibleChecksums import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.toHashFunctionOrThrow import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.Encoding.encodeBase64String -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Utils.runBlocking import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.HttpBody import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.readAll import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.KotlinCoroutines.coroutineContext +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.KotlinxCoroutines.runBlocking import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Observability.TelemetryApi.warn import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.integration.SectionId @@ -304,8 +303,7 @@ class PresignerGenerator : KotlinIntegration { "#T.#T { #S }", coroutineContext, warn, - "The requested checksum algorithm (\${checksumAlgorithmName}) is not supported for pre-signed URL checksums, sending request without checksum. " + - "Supported checksums for pre-signed URLs: $algorithmsSupportedForFlexibleChecksums", + "The requested checksum algorithm (\${checksumAlgorithmName}) is not supported for pre-signed URL checksums, sending request without checksum.", ) } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 767e1d00898..51b5eb72c74 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -33,11 +33,11 @@ class FlexibleChecksumsRequest : KotlinIntegration { // Allows flexible checksum request configuration ConfigProperty { name = "requestChecksumCalculation" - symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption - baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig + symbol = RuntimeTypes.SmithyClient.Config.RequestHttpChecksumConfig + baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumConfig useNestedBuilderBaseClass() documentation = "Configures request checksum calculation" - propertyType = ConfigPropertyType.RequiredWithDefault("HttpChecksumConfigOption.WHEN_SUPPORTED") + propertyType = ConfigPropertyType.RequiredWithDefault("RequestHttpChecksumConfig.WHEN_SUPPORTED") }, ) @@ -59,14 +59,14 @@ private val requestChecksumCalculationBusinessMetric = object : ProtocolMiddlewa // Supported writer.write( "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.RequestHttpChecksumConfig, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) // Required writer.write( "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.RequestHttpChecksumConfig, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) @@ -82,7 +82,7 @@ private val httpChecksumDefaultAlgorithmMiddleware = object : ProtocolMiddleware override val order: Byte = -2 // Before S3 Express (possibly) changes the default (-1) and before calculating checksum (0) override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - requestChecksumsConfigured(ctx, op) + op.hasRequestAlgorithmMember(ctx) override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { writer.write( @@ -100,7 +100,7 @@ private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { override val name: String = "flexibleChecksumsRequestMiddleware" override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - requestChecksumsConfigured(ctx, op) + op.hasRequestAlgorithmMember(ctx) override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { val httpChecksumTrait = op.getTrait()!! @@ -125,9 +125,9 @@ private val flexibleChecksumsRequestMiddleware = object : ProtocolMiddleware { /** * Determines if an operation is set up to send flexible request checksums */ -private fun requestChecksumsConfigured(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { - val httpChecksumTrait = op.getTrait() - val inputShape = op.input.getOrNull()?.let { ctx.model.expectShape(it) } +private fun OperationShape.hasRequestAlgorithmMember(ctx: ProtocolGenerator.GenerationContext): Boolean { + val httpChecksumTrait = this.getTrait() + val inputShape = this.input.getOrNull()?.let { ctx.model.expectShape(it) } return ( (httpChecksumTrait != null) && diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt index 3691cc82a26..ae625d3991b 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsResponse.kt @@ -33,11 +33,11 @@ class FlexibleChecksumsResponse : KotlinIntegration { // Allows flexible checksum response configuration ConfigProperty { name = "responseChecksumValidation" - symbol = RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption - baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumClientConfig + symbol = RuntimeTypes.SmithyClient.Config.ResponseHttpChecksumConfig + baseClass = RuntimeTypes.SmithyClient.Config.HttpChecksumConfig useNestedBuilderBaseClass() documentation = "Configures response checksum validation" - propertyType = ConfigPropertyType.RequiredWithDefault("HttpChecksumConfigOption.WHEN_SUPPORTED") + propertyType = ConfigPropertyType.RequiredWithDefault("ResponseHttpChecksumConfig.WHEN_SUPPORTED") }, ) @@ -56,14 +56,14 @@ private val responseChecksumValidationBusinessMetric = object : ProtocolMiddlewa // Supported writer.write( "#T.WHEN_SUPPORTED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.ResponseHttpChecksumConfig, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) // Required writer.write( "#T.WHEN_REQUIRED -> op.context.#T(#T.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED)", - RuntimeTypes.SmithyClient.Config.HttpChecksumConfigOption, + RuntimeTypes.SmithyClient.Config.ResponseHttpChecksumConfig, RuntimeTypes.Core.BusinessMetrics.emitBusinessMetric, RuntimeTypes.Core.BusinessMetrics.SmithyBusinessMetric, ) @@ -94,7 +94,8 @@ private val flexibleChecksumsResponseMiddleware = object : ProtocolMiddleware { val requestValidationModeMemberName = ctx.symbolProvider.toMemberName(requestValidationModeMember) val interceptor = if (ctx.model.expectShape(ctx.settings.service).isS3) { - AwsRuntimeTypes.Http.Interceptors.S3FlexibleChecksumResponseInterceptor + // S3 needs a custom interceptor because it can send composite checksums, which should be ignored + AwsRuntimeTypes.Http.Interceptors.IgnoreCompositeFlexibleChecksumResponseInterceptor } else { RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsResponseInterceptor } diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt index 862e96c1bf9..075524fb6f2 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDefaultChecksumAlgorithm.kt @@ -22,7 +22,7 @@ internal class S3ExpressDefaultChecksumAlgorithm( private val isS3UploadPart: Boolean, ) : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { - if (usingS3Express(context.executionContext)) { + if (context.executionContext.usingS3Express()) { if (isS3UploadPart) { context.executionContext.remove(HttpOperationContext.DefaultChecksumAlgorithm) } else { @@ -33,5 +33,5 @@ internal class S3ExpressDefaultChecksumAlgorithm( } } -private fun usingS3Express(executionContext: ExecutionContext): Boolean = - executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE +private fun ExecutionContext.usingS3Express(): Boolean = + this.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt deleted file mode 100644 index c00f78e821f..00000000000 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/ChecksumRemovalTest.kt +++ /dev/null @@ -1,87 +0,0 @@ -package aws.sdk.kotlin.services.s3.express - -import aws.smithy.kotlin.runtime.http.DeferredHeadersBuilder -import aws.smithy.kotlin.runtime.http.HeadersBuilder -import kotlin.test.Test -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class ChecksumRemovalTest { - // Header capitalization shouldn't matter - private val crc32Header = "x-aMz-cHeCkSum-cRC32" - private val sha256Header = "X-Amz-ChEcKsum-sHa256" - - @Test - fun removeChecksumHeaders() { - val headers = HeadersBuilder() - - headers.append(crc32Header, "foo") - headers.append(sha256Header, "bar") - - assertTrue( - headers.contains(crc32Header), - ) - assertTrue( - headers.contains(sha256Header), - ) - - headers.removeChecksumHeaders() - - assertFalse( - headers.contains(crc32Header), - ) - assertFalse( - headers.contains(sha256Header), - ) - } - - @Test - fun removeChecksumTrailingHeaders() { - val trailingHeaders = DeferredHeadersBuilder() - - trailingHeaders.add(crc32Header, "foo") - trailingHeaders.add(sha256Header, "bar") - - assertTrue( - trailingHeaders.contains(crc32Header), - ) - assertTrue( - trailingHeaders.contains(sha256Header), - ) - - trailingHeaders.removeChecksumTrailingHeaders() - - assertFalse( - trailingHeaders.contains(crc32Header), - ) - assertFalse( - trailingHeaders.contains(sha256Header), - ) - } - - @Test - fun removeChecksumTrailingHeadersFromXAmzTrailer() { - val headers = HeadersBuilder() - - headers.append("x-amz-trailer", crc32Header) - headers.append("x-amz-trailer", "x-amz-trailing-header") - - val xAmzTrailer = headers.getAll("x-amz-trailer") - - assertTrue( - xAmzTrailer?.contains(crc32Header) ?: false, - ) - assertTrue( - xAmzTrailer?.contains("x-amz-trailing-header") ?: false, - ) - - headers.removeChecksumTrailingHeadersFromXAmzTrailer() - - assertFalse( - xAmzTrailer?.contains(crc32Header) ?: false, - ) - assertTrue( - xAmzTrailer?.contains("x-amz-trailing-header") ?: false, - ) - } -} diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts index 696c3c3104a..f6288dac03a 100644 --- a/tests/codegen/build.gradle.kts +++ b/tests/codegen/build.gradle.kts @@ -76,6 +76,7 @@ subprojects { implementation(libraries.smithy.kotlin.smithy.test) implementation(libraries.smithy.kotlin.aws.signing.default) implementation(libraries.smithy.kotlin.telemetry.api) + implementation(libraries.smithy.kotlin.http.test) } } jvmTest { diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index 6867921e4a2..866844297b8 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -6,19 +6,8 @@ import aws.sdk.kotlin.shared.Model description = "AWS SDK for Kotlin's checksums codegen test suite" -kotlin { - sourceSets { - commonTest { - dependencies { - implementation(libs.smithy.kotlin.http.test) - } - } - } -} - val tests = listOf( - CodegenTest("checksums", Model("request-response-test.smithy"), "aws.sdk.kotlin.test#TestService"), - CodegenTest("clientConfig", Model("config-test.smithy"), "aws.sdk.kotlin.test#ClientConfigTestService"), + CodegenTest("checksums", Model("checksums.smithy"), "aws.sdk.kotlin.test#TestService"), ) smithyBuild { diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt index 065d5f83b76..8c743763011 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt @@ -5,166 +5,86 @@ package aws.sdk.kotlin.tests.codegen.checksums -import aws.sdk.kotlin.test.checksums.TestClient -import aws.sdk.kotlin.test.checksums.httpChecksumOperation import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm import aws.sdk.kotlin.tests.codegen.checksums.utils.BusinessMetricsReader +import aws.sdk.kotlin.tests.codegen.checksums.utils.runChecksumTest import aws.smithy.kotlin.runtime.businessmetrics.SmithyBusinessMetric -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption -import aws.smithy.kotlin.runtime.httptest.TestEngine -import kotlinx.coroutines.runBlocking +import aws.smithy.kotlin.runtime.client.config.RequestHttpChecksumConfig +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import kotlin.test.Test -import kotlin.test.assertTrue class ChecksumBusinessMetricsTest { @Test - fun defaultConfigBusinessMetrics(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun defaultConfigBusinessMetrics() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED, SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + ) @Test - fun whenSupportedBusinessMetrics(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun whenSupportedBusinessMetrics() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED, SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED - responseChecksumValidation = HttpChecksumConfigOption.WHEN_SUPPORTED - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_SUPPORTED, + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_SUPPORTED, + ) @Test - fun whenRequiredBusinessMetrics(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun whenRequiredBusinessMetrics() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED, SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED - responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_REQUIRED, + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_REQUIRED, + ) @Test - fun crc32(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun crc32() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_CRC32, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Crc32 - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Crc32, + ) @Test - fun crc32c(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun crc32c() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_CRC32C, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Crc32C - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Crc32C, + ) @Test - fun sha1(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun sha1() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_SHA1, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Sha1 - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha1, + ) @Test - fun sha256(): Unit = runBlocking { - val businessMetricsReader = BusinessMetricsReader( + fun sha256() = runChecksumTest( + businessMetricsReader = BusinessMetricsReader( expectedBusinessMetrics = setOf( SmithyBusinessMetric.FLEXIBLE_CHECKSUMS_REQ_SHA256, ), - ) - - TestClient { - httpClient = TestEngine() - interceptors = mutableListOf(businessMetricsReader) - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue(businessMetricsReader.containsExpectedBusinessMetrics) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha256, + ) } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt index e7fe9933394..c734d1f8d18 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt @@ -5,373 +5,166 @@ package aws.sdk.kotlin.tests.codegen.checksums -import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.clientconfig.* -import aws.sdk.kotlin.test.clientconfig.model.ChecksumAlgorithm -import aws.sdk.kotlin.test.clientconfig.model.ValidationMode +import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.sdk.kotlin.test.checksums.model.ValidationMode import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderReader import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderSetter -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.client.config.HttpChecksumConfigOption +import aws.sdk.kotlin.tests.codegen.checksums.utils.runChecksumTest +import aws.smithy.kotlin.runtime.client.config.RequestHttpChecksumConfig +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.http.* -import aws.smithy.kotlin.runtime.http.Headers -import aws.smithy.kotlin.runtime.http.HttpStatusCode import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException -import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.httptest.TestEngine -import aws.smithy.kotlin.runtime.io.SdkSource -import aws.smithy.kotlin.runtime.io.source -import aws.smithy.kotlin.runtime.time.Instant -import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertFailsWith -import kotlin.test.assertFalse -import kotlin.test.assertTrue /** - * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` when set to **true**. + * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` param when set to **true**. */ class RequestChecksumRequired { @Test - fun requestChecksumRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { - val headerReader = HeaderReader( + fun requestChecksumCalculationWhenSupported() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-crc32" to null), - ) - - ClientConfigTestClient { - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_SUPPORTED, + ) @Test - fun requestChecksumRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { - val headerReader = HeaderReader( + fun requestChecksumCalculationWhenRequired() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-crc32" to null), - ) - - ClientConfigTestClient { - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_REQUIRED, + ) } /** - * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` when set to **false**. + * Tests the `aws.protocols#httpChecksum` trait's `requestChecksumRequired` param when set to **false**. */ class RequestChecksumNotRequired { @Test - fun requestChecksumNotRequiredRequestChecksumCalculationWhenSupported(): Unit = runBlocking { - val headerReader = HeaderReader( + fun requestChecksumCalculationWhenSupported() = runChecksumTest( + requestChecksumRequired = false, + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-crc32" to null), - ) - - ClientConfigTestClient { - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_SUPPORTED - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_SUPPORTED, + ) @Test - fun requestChecksumNotRequiredRequestChecksumCalculationWhenRequired(): Unit = runBlocking { - val headerReader = HeaderReader( - expectedHeaders = mapOf("x-amz-checksum-crc32" to null), - ) - - ClientConfigTestClient { - requestChecksumCalculation = HttpChecksumConfigOption.WHEN_REQUIRED - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsNotRequiredOperation { - body = "Hello World!" - } - } - - assertFalse( - headerReader.containsExpectedHeaders, - ) - } + fun requestChecksumCalculationWhenRequired() = runChecksumTest( + requestChecksumRequired = false, + headerReader = HeaderReader( + forbiddenHeaders = mapOf("x-amz-checksum-crc32" to null), + ), + requestChecksumCalculationValue = RequestHttpChecksumConfig.WHEN_REQUIRED, + ) } /** - * Tests the `aws.protocols#httpChecksum` trait's `requestAlgorithmMember`. + * Tests user selected checksum **algorithm** */ class UserSelectedChecksumAlgorithm { @Test - fun userSelectedChecksumAlgorithmIsUsed(): Unit = runBlocking { - val headerReader = HeaderReader( + fun userSelectedChecksumAlgorithmIsUsed() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-sha256" to null), - ) - - ClientConfigTestClient { - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha256, + ) } /** - * Tests user provided checksum calculations. + * Tests user provided checksum **calculation** */ class UserProvidedChecksumHeader { @Test - fun userProvidedChecksumIsUsed(): Unit = runBlocking { - val headerSetter = HeaderSetter( + fun userProvidedChecksumIsUsed() = runChecksumTest( + headerSetter = HeaderSetter( mapOf("x-amz-checksum-crc64nvme" to "foo"), - ) - val headerReader = HeaderReader( + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha256, + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-crc64nvme" to "foo"), - forbiddenHeaders = mapOf("x-amz-checksum-sha256" to "foo"), - ) - - ClientConfigTestClient { - interceptors = mutableListOf(headerSetter, headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - - assertFalse( - headerReader.containsForbiddenHeaders, - ) - } + forbiddenHeaders = mapOf( + "x-amz-checksum-sha256" to "foo", // Should be ignored since header checksum has priority + "x-amz-checksum-crc32" to "foo", // Default checksum shouldn't be used + ), + ), + ) @Test - fun unmodeledChecksumIsUsed(): Unit = runBlocking { - val headerSetter = HeaderSetter( + fun newChecksumAlgorithmIsUsed() = runChecksumTest( + headerSetter = HeaderSetter( mapOf("x-amz-checksum-some-future-algorithm" to "foo"), - ) - val headerReader = HeaderReader( + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha256, + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-some-future-algorithm" to "foo"), forbiddenHeaders = mapOf( - "x-amz-checksum-crc32" to "foo", "x-amz-checksum-sha256" to "foo", + "x-amz-checksum-crc32" to "foo", ), - ) - - ClientConfigTestClient { - interceptors = mutableListOf(headerSetter, headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + ) @Test - fun userProvidedMd5IsNotUsed(): Unit = runBlocking { - val headerSetter = HeaderSetter( + fun md5IsNotUsed() = runChecksumTest( + headerSetter = HeaderSetter( mapOf("x-amz-checksum-md5" to "foo"), - ) - val headerReader = HeaderReader( + ), + headerReader = HeaderReader( expectedHeaders = mapOf("x-amz-checksum-crc32" to null), - forbiddenHeaders = mapOf( - "x-amz-checksum-md5" to "foo", - ), - ) - - ClientConfigTestClient { - interceptors = mutableListOf(headerSetter, headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello World!" - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - - assertFalse( - headerReader.containsForbiddenHeaders, - ) - } + forbiddenHeaders = mapOf("x-amz-checksum-md5" to "foo"), // MD5 is not supported for flexible checksums + ), + ) } /** * Tests the `aws.protocols#httpChecksum` trait's `requestValidationModeMember`. */ class ResponseChecksumValidation { - private val responseBody = "Hello world" + private val incorrectChecksumValue = "Kaboom!" @Test - fun responseChecksumValidationWhenSupported(): Unit = runBlocking { + fun whenRequiredAndNotEnabled() = runChecksumTest( + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_REQUIRED, + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = incorrectChecksumValue, + ) + + @Test + fun whenSupportedAndNotEnabled() { assertFailsWith { - ClientConfigTestClient { - responseChecksumValidation = HttpChecksumConfigOption.WHEN_SUPPORTED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello" - } - } + runChecksumTest( + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_SUPPORTED, + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = incorrectChecksumValue, + ) } } @Test - fun responseChecksumValidationWhenRequired(): Unit = runBlocking { - ClientConfigTestClient { - responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") - }, - "World!".toHttpBody(), - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), + fun whenRequiredAndEnabled() { + assertFailsWith { + runChecksumTest( + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_REQUIRED, + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = incorrectChecksumValue, + validationModeValue = ValidationMode.Enabled, ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello" - } } } @Test - fun responseChecksumValidationWhenRequiredWithRequestValidationModeEnabled(): Unit = runBlocking { + fun whenSupportedAndEnabled() { assertFailsWith { - ClientConfigTestClient { - responseChecksumValidation = HttpChecksumConfigOption.WHEN_REQUIRED - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "I will trigger `ChecksumMismatchException` if read!") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.checksumsRequiredOperation { - body = "Hello" - validationMode = ValidationMode.Enabled - } - } + runChecksumTest( + responseChecksumValidationValue = ResponseHttpChecksumConfig.WHEN_SUPPORTED, + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = incorrectChecksumValue, + validationModeValue = ValidationMode.Enabled, + ) } } } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt index d64794a136d..619402eba65 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt @@ -5,126 +5,56 @@ package aws.sdk.kotlin.tests.codegen.checksums -import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.checksums.* import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm import aws.sdk.kotlin.tests.codegen.checksums.utils.HeaderReader -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.httptest.TestEngine -import kotlinx.coroutines.runBlocking +import aws.sdk.kotlin.tests.codegen.checksums.utils.runChecksumTest import kotlin.test.Test -import kotlin.test.assertTrue +/** + * Tests headers match the configured checksum algorithm + */ class ChecksumRequestTest { @Test - fun crc32(): Unit = runBlocking { - val headerReader = HeaderReader( + fun crc32() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf( "x-amz-request-algorithm" to "CRC32", "x-amz-checksum-crc32" to "i9aeUg==", ), - ) - - TestClient { - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Crc32 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Crc32, + ) @Test - fun crc32c(): Unit = runBlocking { - val headerReader = HeaderReader( + fun crc32c() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf( "x-amz-request-algorithm" to "CRC32C", "x-amz-checksum-crc32c" to "crUfeA==", ), - ) - - TestClient { - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Crc32C - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Crc32C, + ) @Test - fun sha1(): Unit = runBlocking { - val headerReader = HeaderReader( + fun sha1() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf( "x-amz-request-algorithm" to "SHA1", "x-amz-checksum-sha1" to "e1AsOh9IyGCa4hLN+2Od7jlnP14=", ), - ) - - TestClient { - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Sha1 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha1, + ) @Test - fun sha256(): Unit = runBlocking { - val headerReader = HeaderReader( + fun sha256() = runChecksumTest( + headerReader = HeaderReader( expectedHeaders = mapOf( "x-amz-request-algorithm" to "SHA256", "x-amz-checksum-sha256" to "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", ), - ) - - TestClient { - interceptors = mutableListOf(headerReader) - httpClient = TestEngine() - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - checksumAlgorithm = ChecksumAlgorithm.Sha256 - } - } - - assertTrue( - headerReader.containsExpectedHeaders, - ) - } + ), + checksumAlgorithmValue = ChecksumAlgorithm.Sha256, + ) } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt index f477a3f1129..5bc5e96a999 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt @@ -5,278 +5,83 @@ package aws.sdk.kotlin.tests.codegen.checksums -import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.test.checksums.* -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.http.* +import aws.sdk.kotlin.tests.codegen.checksums.utils.runChecksumTest import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException -import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.httptest.TestEngine -import aws.smithy.kotlin.runtime.io.SdkSource -import aws.smithy.kotlin.runtime.io.source -import aws.smithy.kotlin.runtime.time.Instant -import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertFailsWith -private val responseBody = "Hello world" - +/** + * Test the SDK validates correct checksum values + */ class SuccessfulChecksumResponseTest { @Test - fun crc32(): Unit = runBlocking { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "i9aeUg==") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - } + fun crc32() = runChecksumTest( + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = "i9aeUg==", + ) @Test - fun crc32c(): Unit = runBlocking { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32c", "crUfeA==") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - } + fun crc32c() = runChecksumTest( + responseChecksumHeader = "x-amz-checksum-crc32c", + responseChecksumValue = "crUfeA==", + ) @Test - fun sha1(): Unit = runBlocking { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-sha1", "e1AsOh9IyGCa4hLN+2Od7jlnP14=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - } + fun sha1() = runChecksumTest( + responseChecksumHeader = "x-amz-checksum-sha1", + responseChecksumValue = "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ) @Test - fun sha256(): Unit = runBlocking { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-sha256", "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } - } + fun sha256() = runChecksumTest( + responseChecksumHeader = "x-amz-checksum-sha256", + responseChecksumValue = "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ) } +/** + * Test the SDK throws exception on incorrect checksum values + */ class FailedChecksumResponseTest { + private val incorrectChecksumValue = "Kaboom!" + @Test - fun crc32(): Unit = runBlocking { + fun crc32() { assertFailsWith { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32", "bm90LWEtY2hlY2tzdW0=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } + runChecksumTest( + responseChecksumHeader = "x-amz-checksum-crc32", + responseChecksumValue = incorrectChecksumValue, + ) } } @Test - fun crc32c(): Unit = runBlocking { + fun crc32c() { assertFailsWith { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-crc32c", "bm90LWEtY2hlY2tzdW0=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } + runChecksumTest( + responseChecksumHeader = "x-amz-checksum-crc32c", + responseChecksumValue = incorrectChecksumValue, + ) } } @Test - fun sha1(): Unit = runBlocking { + fun sha1() { assertFailsWith { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-sha1", "bm90LWEtY2hlY2tzdW0=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } + runChecksumTest( + responseChecksumHeader = "x-amz-checksum-sha1", + responseChecksumValue = incorrectChecksumValue, + ) } } @Test - fun sha256(): Unit = runBlocking { + fun sha256() { assertFailsWith { - TestClient { - httpClient = TestEngine( - roundTripImpl = { _, request -> - val resp = HttpResponse( - HttpStatusCode.OK, - Headers { - append("x-amz-checksum-sha256", "bm90LWEtY2hlY2tzdW0=") - }, - object : HttpBody.SourceContent() { - override val isOneShot: Boolean = false - override val contentLength: Long? = responseBody.length.toLong() - override fun readFrom(): SdkSource = responseBody.toByteArray().source() - }, - ) - val now = Instant.now() - HttpCall(request, resp, now, now) - }, - ) - credentialsProvider = StaticCredentialsProvider( - Credentials("accessKeyID", "secretAccessKey"), - ) - region = "us-east-1" - }.use { client -> - client.httpChecksumOperation { - body = "Hello world".encodeToByteArray() - } - } + runChecksumTest( + responseChecksumHeader = "x-amz-checksum-sha256", + responseChecksumValue = incorrectChecksumValue, + ) } } } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt index c88263bfe70..41fd4faa527 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt @@ -5,19 +5,38 @@ package aws.sdk.kotlin.tests.codegen.checksums.utils +import aws.sdk.kotlin.test.checksums.TestClient +import aws.sdk.kotlin.test.checksums.httpChecksumOperation +import aws.sdk.kotlin.test.checksums.httpChecksumRequestChecksumsNotRequiredOperation +import aws.sdk.kotlin.test.checksums.model.ChecksumAlgorithm +import aws.sdk.kotlin.test.checksums.model.ValidationMode import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetric import aws.smithy.kotlin.runtime.businessmetrics.BusinessMetrics import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.client.config.RequestHttpChecksumConfig +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.collections.get +import aws.smithy.kotlin.runtime.http.Headers +import aws.smithy.kotlin.runtime.http.HttpBody +import aws.smithy.kotlin.runtime.http.HttpCall +import aws.smithy.kotlin.runtime.http.HttpStatusCode import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.toBuilder +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.httptest.TestEngine +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.io.source +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.runBlocking +import kotlin.test.assertFalse +import kotlin.test.assertTrue /** * Checks if the specified headers are set in an HTTP request. */ internal class HeaderReader( - private val expectedHeaders: Map, + private val expectedHeaders: Map = emptyMap(), private val forbiddenHeaders: Map = emptyMap(), ) : HttpInterceptor { var containsExpectedHeaders = false @@ -73,3 +92,71 @@ internal class BusinessMetricsReader( containsExpectedBusinessMetrics = context.executionContext[BusinessMetrics].containsAll(expectedBusinessMetrics) } } + +/** + * Runs a checksum test + */ +internal fun runChecksumTest( + // Interceptors + headerReader: HeaderReader? = null, + headerSetter: HeaderSetter? = null, + businessMetricsReader: BusinessMetricsReader? = null, + // Config + requestChecksumCalculationValue: RequestHttpChecksumConfig? = null, + responseChecksumValidationValue: ResponseHttpChecksumConfig? = null, + checksumAlgorithmValue: ChecksumAlgorithm? = null, + validationModeValue: ValidationMode? = null, + // Request/Response + responseChecksumHeader: String? = null, + responseChecksumValue: String? = null, + requestBody: String = "Hello world", + responseBody: String = "Hello world", + // Test type + requestChecksumRequired: Boolean = true, +): Unit = runBlocking { + val interceptorsList = listOfNotNull(headerReader, headerSetter, businessMetricsReader) + + TestClient { + httpClient = TestEngine() + interceptors = interceptorsList.toMutableList() + requestChecksumCalculation = requestChecksumCalculationValue ?: requestChecksumCalculation + responseChecksumValidation = responseChecksumValidationValue ?: responseChecksumValidation + httpClient = TestEngine( + roundTripImpl = { _, request -> + val resp = HttpResponse( + HttpStatusCode.OK, + Headers { + append(responseChecksumHeader ?: "", responseChecksumValue ?: "") + }, + object : HttpBody.SourceContent() { + override val isOneShot: Boolean = false + override val contentLength: Long? = responseBody.length.toLong() + override fun readFrom(): SdkSource = responseBody.toByteArray().source() + }, + ) + val now = Instant.now() + HttpCall(request, resp, now, now) + }, + ) + }.use { client -> + if (requestChecksumRequired) { + client.httpChecksumOperation { + body = requestBody.encodeToByteArray() + checksumAlgorithm = checksumAlgorithmValue ?: checksumAlgorithm + validationMode = validationModeValue ?: validationMode + } + } else { + client.httpChecksumRequestChecksumsNotRequiredOperation { + body = requestBody.encodeToByteArray() + checksumAlgorithm = checksumAlgorithmValue ?: checksumAlgorithm + validationMode = validationModeValue ?: validationMode + } + } + } + + businessMetricsReader?.let { assertTrue(it.containsExpectedBusinessMetrics) } + headerReader?.let { + assertTrue(it.containsExpectedHeaders) + assertFalse(it.containsForbiddenHeaders) + } +} diff --git a/tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy b/tests/codegen/checksums/src/commonTest/resources/checksums.smithy similarity index 72% rename from tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy rename to tests/codegen/checksums/src/commonTest/resources/checksums.smithy index aa5dc5bc9af..a8b7a89008d 100644 --- a/tests/codegen/checksums/src/commonTest/resources/request-response-test.smithy +++ b/tests/codegen/checksums/src/commonTest/resources/checksums.smithy @@ -20,7 +20,7 @@ use smithy.rules#endpointRuleSet }) service TestService { version: "2023-01-01", - operations: [HttpChecksumOperation, HttpChecksumStreamingOperation] + operations: [HttpChecksumOperation, HttpChecksumRequestChecksumsNotRequiredOperation] } @http(uri: "/HttpChecksumOperation", method: "POST") @@ -36,6 +36,19 @@ operation HttpChecksumOperation { output: SomeOutput } +@http(uri: "/HttpChecksumRequestChecksumsNotRequiredOperation", method: "POST") +@optionalAuth +@httpChecksum( + requestChecksumRequired: false, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] +) +operation HttpChecksumRequestChecksumsNotRequiredOperation { + input: SomeOtherInput, + output: SomeOtherOutput +} + @input structure SomeInput { @httpHeader("x-amz-request-algorithm") @@ -67,40 +80,42 @@ structure SomeInput { body: Blob } -@output -structure SomeOutput {} - -@http(uri: "/HttpChecksumStreamingOperation", method: "POST") -@auth([sigv4]) -@httpChecksum( - requestChecksumRequired: true, - requestAlgorithmMember: "checksumAlgorithm", - requestValidationModeMember: "validationMode", - responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] -) -operation HttpChecksumStreamingOperation { - input: SomeStreamingInput, - output: SomeStreamingOutput -} - -@streaming -blob StreamingBlob - @input -structure SomeStreamingInput { +structure SomeOtherInput { @httpHeader("x-amz-request-algorithm") checksumAlgorithm: ChecksumAlgorithm @httpHeader("x-amz-response-validation-mode") validationMode: ValidationMode + @httpHeader("x-amz-checksum-crc32") + ChecksumCRC32: String + + @httpHeader("x-amz-checksum-crc32c") + ChecksumCRC32C: String + + @httpHeader("x-amz-checksum-crc64nvme") + ChecksumCRC64Nvme: String + + @httpHeader("x-amz-checksum-sha1") + ChecksumSHA1: String + + @httpHeader("x-amz-checksum-sha256") + ChecksumSHA256: String + + @httpHeader("x-amz-checksum-foo") + ChecksumFoo: String + @httpPayload @required - body: StreamingBlob + body: Blob } @output -structure SomeStreamingOutput {} +structure SomeOutput {} + +@output +structure SomeOtherOutput {} enum ChecksumAlgorithm { CRC32 diff --git a/tests/codegen/checksums/src/commonTest/resources/config-test.smithy b/tests/codegen/checksums/src/commonTest/resources/config-test.smithy deleted file mode 100644 index e1d1f24e21d..00000000000 --- a/tests/codegen/checksums/src/commonTest/resources/config-test.smithy +++ /dev/null @@ -1,87 +0,0 @@ -$version: "2" -namespace aws.sdk.kotlin.test - -use aws.api#service -use aws.auth#sigv4 -use aws.protocols#httpChecksum -use aws.protocols#restJson1 -use smithy.rules#endpointRuleSet - -@service(sdkId: "dontcare") -@restJson1 -@sigv4(name: "dontcare") -@auth([sigv4]) -@endpointRuleSet({ - "version": "1.0", - "rules": [{ "type": "endpoint", "conditions": [], "endpoint": { "url": "https://example.com" } }], - "parameters": { - "Region": { "required": false, "type": "String", "builtIn": "AWS::Region" }, - } -}) -service ClientConfigTestService { - version: "2023-01-01", - operations: [ChecksumsNotRequiredOperation, ChecksumsRequiredOperation] -} - -@httpChecksum( - requestChecksumRequired: false, - requestAlgorithmMember: "checksumAlgorithm", -) -@http(method: "POST", uri: "/test-checksums", code: 200) -operation ChecksumsNotRequiredOperation { - input: SomeInput, - output: SomeOutput -} - -@input -structure SomeInput { - @httpHeader("x-amz-request-algorithm") - checksumAlgorithm: ChecksumAlgorithm - - @httpPayload - @required - body: String -} - -@output -structure SomeOutput {} - -@httpChecksum( - requestChecksumRequired: true, - requestAlgorithmMember: "checksumAlgorithm", - requestValidationModeMember: "validationMode", - responseAlgorithms: ["CRC32"] -) -@http(method: "POST", uri: "/test-checksums-2", code: 200) -operation ChecksumsRequiredOperation { - input: AnotherInput, - output: AnotherOutput -} - -@input -structure AnotherInput { - @httpHeader("x-amz-request-algorithm") - checksumAlgorithm: ChecksumAlgorithm - - @httpHeader("x-amz-response-validation-mode") - validationMode: ValidationMode - - @httpPayload - @required - body: String -} - -@output -structure AnotherOutput {} - -enum ChecksumAlgorithm { - CRC32 - CRC32C - CRC64NVME - SHA1 - SHA256 -} - -enum ValidationMode { - ENABLED -} diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 0de5288432e..ed102cf3e47 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -15,13 +15,11 @@ val tests = listOf( "restJson1", Model("event-stream-model-template.smithy"), "aws.sdk.kotlin.test#TestService", - "restJson1", ), CodegenTest( "awsJson11", Model("event-stream-initial-request-response.smithy"), "aws.sdk.kotlin.test#TestService", - "awsJson1_1", ), ) @@ -65,33 +63,3 @@ kotlin { } } } - -tasks.generateSmithyBuild { - doFirst { - tests.forEach { test -> fillInModel(test) } - } -} - -fun fillInModel(test: CodegenTest) { - val modelFile = layout.projectDirectory.file(test.model.path + test.model.fileName).asFile - val model = modelFile.readText() - - val opTraits = - when (test.protocolName) { - "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" - else -> "" - } - - val replaced = model - .replace( - "{protocol-name}", - test.protocolName ?: throw IllegalStateException("Please specify a protocol name for the codegen test"), - ) - .replace( - "{op-traits}", - opTraits, - ) - - modelFile.parentFile.mkdirs() - modelFile.writeText(replaced) -} diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy index 8150dcec2db..dd32f77d8d7 100644 --- a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy +++ b/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy @@ -1,15 +1,15 @@ namespace aws.sdk.kotlin.test -use aws.protocols#{protocol-name} +use aws.protocols#restJson1 use aws.api#service use aws.auth#sigv4 -@{protocol-name} +@restJson1 @sigv4(name: "event-stream-test") @service(sdkId: "EventStreamTest") service TestService { version: "123", operations: [TestStreamOp] } -{op-traits} +@http(method: "POST", uri: "/test-eventstream", code: 200) operation TestStreamOp { input: TestStreamInputOutput, output: TestStreamInputOutput, From 3ed4d917535a480afbde1cbac5353ddf1bdc4156 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 24 Dec 2024 10:47:54 -0500 Subject: [PATCH 64/72] Change JVM version --- codegen/aws-sdk-codegen/build.gradle.kts | 6 +++--- services/s3/e2eTest/src/S3ExpressTest.kt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/codegen/aws-sdk-codegen/build.gradle.kts b/codegen/aws-sdk-codegen/build.gradle.kts index 0348cd5a7f1..9a67ee36c09 100644 --- a/codegen/aws-sdk-codegen/build.gradle.kts +++ b/codegen/aws-sdk-codegen/build.gradle.kts @@ -59,14 +59,14 @@ val generateSdkRuntimeVersion by tasks.registering { tasks.withType { compilerOptions { - jvmTarget.set(JvmTarget.JVM_17) + jvmTarget.set(JvmTarget.JVM_1_8) } dependsOn(generateSdkRuntimeVersion) } tasks.withType { - sourceCompatibility = JavaVersion.VERSION_17.toString() - targetCompatibility = JavaVersion.VERSION_17.toString() + sourceCompatibility = JavaVersion.VERSION_1_8.toString() + targetCompatibility = JavaVersion.VERSION_1_8.toString() } // Reusable license copySpec diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt index c97c58fc2ce..f831583b38c 100644 --- a/services/s3/e2eTest/src/S3ExpressTest.kt +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -136,7 +136,7 @@ class S3ExpressTest { } @Test - fun testUploadPartContainsNoChecksums() = runTest { + fun testUploadPartContainsNoDefaultChecksum() = runTest { val testBucket = testBuckets.first() val testObject = "I-will-be-uploaded-in-parts-!" From 34a815332ab98f2a1edddbbf55a6a5ab4b97e75e Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 30 Dec 2024 16:18:46 -0500 Subject: [PATCH 65/72] Clean up --- .../sdk/kotlin/codegen/PresignerGenerator.kt | 70 ------------------- services/s3/e2eTest/src/S3ChecksumTest.kt | 13 ++-- .../checksums/utils/ChecksumTestUtils.kt | 4 +- 3 files changed, 7 insertions(+), 80 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt index fdbd102c406..f8c33308a42 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/PresignerGenerator.kt @@ -5,19 +5,10 @@ package aws.sdk.kotlin.codegen import aws.sdk.kotlin.codegen.model.traits.Presignable -import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.aws.traits.auth.SigV4Trait import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait import software.amazon.smithy.codegen.core.Symbol import software.amazon.smithy.kotlin.codegen.core.* -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.isSupportedForFlexibleChecksums -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Hashing.toHashFunctionOrThrow -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Core.Text.Encoding.encodeBase64String -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.HttpBody -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Http.readAll -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.KotlinCoroutines.coroutineContext -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.KotlinxCoroutines.runBlocking -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Observability.TelemetryApi.warn import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.integration.SectionId import software.amazon.smithy.kotlin.codegen.integration.SectionKey @@ -28,7 +19,6 @@ import software.amazon.smithy.kotlin.codegen.rendering.endpoints.EndpointResolve import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpBindingProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpBindingResolver import software.amazon.smithy.kotlin.codegen.rendering.serde.serializerName -import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.knowledge.TopDownIndex import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ServiceShape @@ -157,7 +147,6 @@ class PresignerGenerator : KotlinIntegration { requestSymbol, serializerSymbol, contextMap, - op, ) } } @@ -202,7 +191,6 @@ class PresignerGenerator : KotlinIntegration { requestSymbol: Symbol, serializerSymbol: Symbol, contextMap: Map, Any>, - op: OperationShape, ) = writer.apply { dokka { write("Presign a [#T] using the configuration of this [#T].", requestSymbol, serviceSymbol) @@ -275,44 +263,6 @@ class PresignerGenerator : KotlinIntegration { ) } - checksumAlgorithmMember(op, ctx)?.let { checksumAlgorithmMember -> - withBlock("input.#L?.value?.let { checksumAlgorithmName ->", "}", checksumAlgorithmMember) { - withBlock("when (unsignedRequest.body) {", "}") { - withBlock("is #1T.Bytes, is #1T.Empty -> {", "}", HttpBody) { - write("val checksumAlgorithm = checksumAlgorithmName.#T()", toHashFunctionOrThrow) - withInlineBlock( - "if (checksumAlgorithm.#T) {", - "}", - isSupportedForFlexibleChecksums, - ) { - withBlock("#T {", "}", runBlocking) { - withBlock("checksumAlgorithm.update(", ")") { - write("unsignedRequest.body.#T() ?: byteArrayOf()", readAll) - } - } - write( - "checksum = #S.lowercase() to checksumAlgorithm.digest().#T()", - "x-amz-checksum-\${checksumAlgorithmName}", - encodeBase64String, - ) - } - withBlock(" else {", "}") { - withBlock("#T {", "}", runBlocking) { - write("class Presigner") - write( - "#T.#T { #S }", - coroutineContext, - warn, - "The requested checksum algorithm (\${checksumAlgorithmName}) is not supported for pre-signed URL checksums, sending request without checksum.", - ) - } - } - } - write("else -> throw IllegalStateException(#S)", "HTTP body type unsupported for pre-signed URL checksums.") - } - } - } - declareSection(SigningConfigCustomizationSection) write("configBlock()") @@ -335,24 +285,4 @@ class PresignerGenerator : KotlinIntegration { * > "my-object/example/photo.user". This is an incorrect path for that object. */ private fun normalizeUriPath(service: ServiceShape) = service.sdkId != "S3" - - /** - * Gets the checksum algorithm member if configured on a request, otherwise null - */ - private fun checksumAlgorithmMember( - operationShape: OperationShape, - ctx: CodegenContext, - ): String? { - operationShape.getTrait()?.let { httpChecksumTrait -> - httpChecksumTrait.requestAlgorithmMember.getOrNull()?.let { requestAlgorithmMember -> - val memberShape = ctx.model - .expectShape(operationShape.input.get()) - .members() - .first { it.memberName == requestAlgorithmMember } - - return ctx.symbolProvider.toMemberName(memberShape) - } - } - return null - } } diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index 043ae7baae9..48f6894cc64 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -16,7 +16,6 @@ import org.junit.jupiter.api.* import java.io.File import java.io.FileInputStream import java.util.* -import kotlin.test.Ignore import kotlin.test.assertEquals import kotlin.test.assertFalse import kotlin.test.assertTrue @@ -166,28 +165,28 @@ class S3ChecksumTest { @Test fun testPresignedUrlNoDefault() = runBlocking { + val contents = "presign-test" + val unsignedPutRequest = PutObjectRequest { bucket = testBucket key = testKey() } val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds) - val contents = "presign-test" assertFalse(presignedPutRequest.url.toString().contains("x-amz-checksum-crc32")) assertTrue(responseCodeFromPut(presignedPutRequest, contents) in 200..299) } - // FIXME: Sending checksum via query params should work @Test - @Ignore - fun testPresignedUrlContainsChecksum() = runBlocking { + fun testPresignedUrlChecksumValue() = runBlocking { + val contents = "presign-test" + val unsignedPutRequest = PutObjectRequest { bucket = testBucket key = testKey() - checksumAlgorithm = ChecksumAlgorithm.Crc32 + checksumCrc32 = "dBBx+Q==" } val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds) - val contents = "presign-test" assertTrue(presignedPutRequest.url.toString().contains("x-amz-checksum-crc32")) assertTrue(responseCodeFromPut(presignedPutRequest, contents) in 200..299) diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt index 41fd4faa527..1ed897e2605 100644 --- a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt +++ b/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt @@ -114,11 +114,9 @@ internal fun runChecksumTest( // Test type requestChecksumRequired: Boolean = true, ): Unit = runBlocking { - val interceptorsList = listOfNotNull(headerReader, headerSetter, businessMetricsReader) - TestClient { httpClient = TestEngine() - interceptors = interceptorsList.toMutableList() + interceptors = listOfNotNull(headerReader, headerSetter, businessMetricsReader).toMutableList() requestChecksumCalculation = requestChecksumCalculationValue ?: requestChecksumCalculation responseChecksumValidation = responseChecksumValidationValue ?: responseChecksumValidation httpClient = TestEngine( From 6a7ec2f78a00295dd6877fafd45d41bdc7e25059 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 8 Jan 2025 13:59:48 -0500 Subject: [PATCH 66/72] misc: revert toList/JVM compatibility changes --- codegen/aws-sdk-codegen/build.gradle.kts | 6 +++--- tests/codegen/smoke-tests/build.gradle.kts | 6 +++--- .../sdk/kotlin/test/codegen/smoketest/SmokeTestE2ETest.kt} | 0 .../resources/smoke-tests-exception.smithy | 0 .../resources/smoke-tests-failure.smithy | 0 .../resources/smoke-tests-success.smithy | 0 6 files changed, 6 insertions(+), 6 deletions(-) rename tests/codegen/smoke-tests/src/{commonTest/kotlin/aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt => jvmTest/kotlin/aws/sdk/kotlin/test/codegen/smoketest/SmokeTestE2ETest.kt} (100%) rename tests/codegen/smoke-tests/src/{commonTest => jvmTest}/resources/smoke-tests-exception.smithy (100%) rename tests/codegen/smoke-tests/src/{commonTest => jvmTest}/resources/smoke-tests-failure.smithy (100%) rename tests/codegen/smoke-tests/src/{commonTest => jvmTest}/resources/smoke-tests-success.smithy (100%) diff --git a/codegen/aws-sdk-codegen/build.gradle.kts b/codegen/aws-sdk-codegen/build.gradle.kts index 9a67ee36c09..0348cd5a7f1 100644 --- a/codegen/aws-sdk-codegen/build.gradle.kts +++ b/codegen/aws-sdk-codegen/build.gradle.kts @@ -59,14 +59,14 @@ val generateSdkRuntimeVersion by tasks.registering { tasks.withType { compilerOptions { - jvmTarget.set(JvmTarget.JVM_1_8) + jvmTarget.set(JvmTarget.JVM_17) } dependsOn(generateSdkRuntimeVersion) } tasks.withType { - sourceCompatibility = JavaVersion.VERSION_1_8.toString() - targetCompatibility = JavaVersion.VERSION_1_8.toString() + sourceCompatibility = JavaVersion.VERSION_17.toString() + targetCompatibility = JavaVersion.VERSION_17.toString() } // Reusable license copySpec diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index 86422a486f7..c25598ba414 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -16,9 +16,9 @@ dependencies { } val tests = listOf( - CodegenTest("successService", Model("smoke-tests-success.smithy"), "smithy.kotlin.traits#SuccessService"), - CodegenTest("failureService", Model("smoke-tests-failure.smithy"), "smithy.kotlin.traits#FailureService"), - CodegenTest("exceptionService", Model("smoke-tests-exception.smithy"), "smithy.kotlin.traits#ExceptionService"), + CodegenTest("successService", Model("smoke-tests-success.smithy", "src/jvmTest/resources/"), "smithy.kotlin.traits#SuccessService"), + CodegenTest("failureService", Model("smoke-tests-failure.smithy", "src/jvmTest/resources/"), "smithy.kotlin.traits#FailureService"), + CodegenTest("exceptionService", Model("smoke-tests-exception.smithy", "src/jvmTest/resources/"), "smithy.kotlin.traits#ExceptionService"), ) configureProjections() diff --git a/tests/codegen/smoke-tests/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt b/tests/codegen/smoke-tests/src/jvmTest/kotlin/aws/sdk/kotlin/test/codegen/smoketest/SmokeTestE2ETest.kt similarity index 100% rename from tests/codegen/smoke-tests/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/smoketests/SmokeTestE2ETest.kt.kt rename to tests/codegen/smoke-tests/src/jvmTest/kotlin/aws/sdk/kotlin/test/codegen/smoketest/SmokeTestE2ETest.kt diff --git a/tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-exception.smithy b/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-exception.smithy rename to tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy diff --git a/tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-failure.smithy b/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-failure.smithy rename to tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy diff --git a/tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-success.smithy b/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/commonTest/resources/smoke-tests-success.smithy rename to tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy From 77ec52ccd32cb3a549e96350be7977297908234e Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 8 Jan 2025 17:30:14 -0500 Subject: [PATCH 67/72] fix: everything works except smoke-tests:services --- .../aws/sdk/kotlin/shared/CodegenTest.kt | 2 +- tests/codegen/build.gradle.kts | 90 ++++++++----------- tests/codegen/checksums/build.gradle.kts | 10 +-- .../checksums/ChecksumBusinessMetricsTest.kt | 0 .../codegen/checksums/ChecksumConfigTest.kt | 0 .../codegen/checksums/ChecksumRequestTest.kt | 0 .../codegen/checksums/ChecksumResponseTest.kt | 0 .../checksums/utils/ChecksumTestUtils.kt | 0 .../resources/checksums.smithy | 0 tests/codegen/event-stream/build.gradle.kts | 10 +-- .../eventstream/HttpEventStreamTests.kt | 0 .../eventstream/RpcEventStreamTests.kt | 0 ...ent-stream-initial-request-response.smithy | 0 .../event-stream-model-template.smithy | 0 .../resources/operation-context-params.smithy | 0 tests/codegen/smoke-tests/build.gradle.kts | 8 +- .../smoke-tests/services/build.gradle.kts | 12 --- .../codegen/smoketest/SmokeTestE2ETest.kt | 0 .../resources/smoke-tests-exception.smithy | 0 .../resources/smoke-tests-failure.smithy | 0 .../resources/smoke-tests-success.smithy | 0 21 files changed, 48 insertions(+), 84 deletions(-) rename tests/codegen/checksums/src/{commonTest => test}/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt (100%) rename tests/codegen/checksums/src/{commonTest => test}/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt (100%) rename tests/codegen/checksums/src/{commonTest => test}/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt (100%) rename tests/codegen/checksums/src/{commonTest => test}/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt (100%) rename tests/codegen/checksums/src/{commonTest => test}/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt (100%) rename tests/codegen/checksums/src/{commonTest => test}/resources/checksums.smithy (100%) rename tests/codegen/event-stream/src/{commonTest => test}/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt (100%) rename tests/codegen/event-stream/src/{commonTest => test}/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt (100%) rename tests/codegen/event-stream/src/{commonTest => test}/resources/event-stream-initial-request-response.smithy (100%) rename tests/codegen/event-stream/src/{commonTest => test}/resources/event-stream-model-template.smithy (100%) rename tests/codegen/rules-engine/src/{commonTest => test}/resources/operation-context-params.smithy (100%) rename tests/codegen/smoke-tests/src/{jvmTest => test}/kotlin/aws/sdk/kotlin/test/codegen/smoketest/SmokeTestE2ETest.kt (100%) rename tests/codegen/smoke-tests/src/{jvmTest => test}/resources/smoke-tests-exception.smithy (100%) rename tests/codegen/smoke-tests/src/{jvmTest => test}/resources/smoke-tests-failure.smithy (100%) rename tests/codegen/smoke-tests/src/{jvmTest => test}/resources/smoke-tests-success.smithy (100%) diff --git a/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt b/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt index 6b72149fcff..55b5c91364f 100644 --- a/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt @@ -14,5 +14,5 @@ data class CodegenTest( */ data class Model( val fileName: String, - val path: String = "src/commonTest/resources/", + val path: String = "src/test/resources/", ) diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts index f6288dac03a..c321430d17e 100644 --- a/tests/codegen/build.gradle.kts +++ b/tests/codegen/build.gradle.kts @@ -2,18 +2,14 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections plugins { alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) - alias(libs.plugins.kotlin.multiplatform) -} - -kotlin { - jvm() + alias(libs.plugins.kotlin.jvm) } val libraries = libs subprojects { apply(plugin = libraries.plugins.aws.kotlin.repo.tools.smithybuild.get().pluginId) - apply(plugin = libraries.plugins.kotlin.multiplatform.get().pluginId) + apply(plugin = libraries.plugins.kotlin.jvm.get().pluginId) val optinAnnotations = listOf( "aws.smithy.kotlin.runtime.InternalApi", @@ -43,54 +39,42 @@ subprojects { codegen(libraries.smithy.model) } - kotlin { - jvm { - compilations.all { - kotlinOptions.jvmTarget = "1.8" - } - } - sourceSets { - commonMain { - dependencies { - implementation(project(":codegen:aws-sdk-codegen")) - implementation(libraries.smithy.kotlin.codegen) + val implementation by configurations + val api by configurations + val testImplementation by configurations + dependencies { + implementation(project(":codegen:aws-sdk-codegen")) + implementation(libraries.smithy.kotlin.codegen) + + /* We have to manually add all the dependencies of the generated client(s). + Doing it this way (as opposed to doing what we do for protocol-tests) allows the tests to work without a + publish to maven-local step at the cost of maintaining this set of dependencies manually. */ + implementation(libraries.kotlinx.coroutines.core) + implementation(libraries.bundles.smithy.kotlin.service.client) + implementation(libraries.smithy.kotlin.aws.event.stream) + implementation(project(":aws-runtime:aws-http")) + implementation(libraries.smithy.kotlin.aws.json.protocols) + implementation(libraries.smithy.kotlin.serde.json) + api(project(":aws-runtime:aws-config")) + api(project(":aws-runtime:aws-core")) + api(project(":aws-runtime:aws-endpoint")) + + testImplementation(libraries.kotlin.test) + testImplementation(libraries.kotlinx.coroutines.test) + testImplementation(libraries.smithy.kotlin.smithy.test) + testImplementation(libraries.smithy.kotlin.aws.signing.default) + testImplementation(libraries.smithy.kotlin.telemetry.api) + testImplementation(libraries.smithy.kotlin.http.test) + } - /* We have to manually add all the dependencies of the generated client(s). - Doing it this way (as opposed to doing what we do for protocol-tests) allows the tests to work without a - publish to maven-local step at the cost of maintaining this set of dependencies manually. */ - implementation(libraries.kotlinx.coroutines.core) - implementation(libraries.bundles.smithy.kotlin.service.client) - implementation(libraries.smithy.kotlin.aws.event.stream) - implementation(project(":aws-runtime:aws-http")) - implementation(libraries.smithy.kotlin.aws.json.protocols) - implementation(libraries.smithy.kotlin.serde.json) - api(project(":aws-runtime:aws-config")) - api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:aws-endpoint")) - } - } - commonTest { - dependencies { - implementation(libraries.kotlin.test) - implementation(libraries.kotlinx.coroutines.test) - implementation(libraries.smithy.kotlin.smithy.test) - implementation(libraries.smithy.kotlin.aws.signing.default) - implementation(libraries.smithy.kotlin.telemetry.api) - implementation(libraries.smithy.kotlin.http.test) - } - } - jvmTest { - tasks.withType { - useJUnitPlatform() - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showStackTraces = true - showExceptions = true - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL - } - } - } + tasks.withType { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL } } } diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index 866844297b8..e20a5cc4122 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -31,12 +31,8 @@ smithyBuild { } } -kotlin { - sourceSets { - commonTest { - smithyBuild.projections.forEach { - kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) - } - } +kotlin.sourceSets.getByName("test") { + smithyBuild.projections.forEach { + kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) } } diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt b/tests/codegen/checksums/src/test/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt similarity index 100% rename from tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt rename to tests/codegen/checksums/src/test/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumBusinessMetricsTest.kt diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt b/tests/codegen/checksums/src/test/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt similarity index 100% rename from tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt rename to tests/codegen/checksums/src/test/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumConfigTest.kt diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt b/tests/codegen/checksums/src/test/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt similarity index 100% rename from tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt rename to tests/codegen/checksums/src/test/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumRequestTest.kt diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt b/tests/codegen/checksums/src/test/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt similarity index 100% rename from tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt rename to tests/codegen/checksums/src/test/kotlin/aws/sdk/kotlin/tests/codegen/checksums/ChecksumResponseTest.kt diff --git a/tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt b/tests/codegen/checksums/src/test/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt similarity index 100% rename from tests/codegen/checksums/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt rename to tests/codegen/checksums/src/test/kotlin/aws/sdk/kotlin/tests/codegen/checksums/utils/ChecksumTestUtils.kt diff --git a/tests/codegen/checksums/src/commonTest/resources/checksums.smithy b/tests/codegen/checksums/src/test/resources/checksums.smithy similarity index 100% rename from tests/codegen/checksums/src/commonTest/resources/checksums.smithy rename to tests/codegen/checksums/src/test/resources/checksums.smithy diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index ed102cf3e47..6e0fe6e660b 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -54,12 +54,8 @@ smithyBuild { } } -kotlin { - sourceSets { - commonTest { - smithyBuild.projections.forEach { - kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) - } - } +kotlin.sourceSets.getByName("test") { + smithyBuild.projections.forEach { + kotlin.srcDir(smithyBuild.smithyKotlinProjectionSrcDir(it.name)) } } diff --git a/tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt b/tests/codegen/event-stream/src/test/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt similarity index 100% rename from tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt rename to tests/codegen/event-stream/src/test/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/HttpEventStreamTests.kt diff --git a/tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt b/tests/codegen/event-stream/src/test/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt similarity index 100% rename from tests/codegen/event-stream/src/commonTest/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt rename to tests/codegen/event-stream/src/test/kotlin/aws/sdk/kotlin/tests/codegen/eventstream/RpcEventStreamTests.kt diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-initial-request-response.smithy b/tests/codegen/event-stream/src/test/resources/event-stream-initial-request-response.smithy similarity index 100% rename from tests/codegen/event-stream/src/commonTest/resources/event-stream-initial-request-response.smithy rename to tests/codegen/event-stream/src/test/resources/event-stream-initial-request-response.smithy diff --git a/tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy b/tests/codegen/event-stream/src/test/resources/event-stream-model-template.smithy similarity index 100% rename from tests/codegen/event-stream/src/commonTest/resources/event-stream-model-template.smithy rename to tests/codegen/event-stream/src/test/resources/event-stream-model-template.smithy diff --git a/tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy b/tests/codegen/rules-engine/src/test/resources/operation-context-params.smithy similarity index 100% rename from tests/codegen/rules-engine/src/commonTest/resources/operation-context-params.smithy rename to tests/codegen/rules-engine/src/test/resources/operation-context-params.smithy diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index c25598ba414..cb3add53dce 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -12,13 +12,13 @@ import aws.sdk.kotlin.shared.Model description = "AWS SDK for Kotlin's smoke test codegen test suite" dependencies { - jvmTestImplementation(gradleTestKit()) + testImplementation(gradleTestKit()) } val tests = listOf( - CodegenTest("successService", Model("smoke-tests-success.smithy", "src/jvmTest/resources/"), "smithy.kotlin.traits#SuccessService"), - CodegenTest("failureService", Model("smoke-tests-failure.smithy", "src/jvmTest/resources/"), "smithy.kotlin.traits#FailureService"), - CodegenTest("exceptionService", Model("smoke-tests-exception.smithy", "src/jvmTest/resources/"), "smithy.kotlin.traits#ExceptionService"), + CodegenTest("successService", Model("smoke-tests-success.smithy"), "smithy.kotlin.traits#SuccessService"), + CodegenTest("failureService", Model("smoke-tests-failure.smithy"), "smithy.kotlin.traits#FailureService"), + CodegenTest("exceptionService", Model("smoke-tests-exception.smithy"), "smithy.kotlin.traits#ExceptionService"), ) configureProjections() diff --git a/tests/codegen/smoke-tests/services/build.gradle.kts b/tests/codegen/smoke-tests/services/build.gradle.kts index f48b8b7b5dd..e69de29bb2d 100644 --- a/tests/codegen/smoke-tests/services/build.gradle.kts +++ b/tests/codegen/smoke-tests/services/build.gradle.kts @@ -1,12 +0,0 @@ -subprojects { - kotlin { - sourceSets { - commonMain { - kotlin.srcDir("generated-src/main/kotlin") - } - commonTest { - kotlin.srcDir("generated-src/test/kotlin") - } - } - } -} diff --git a/tests/codegen/smoke-tests/src/jvmTest/kotlin/aws/sdk/kotlin/test/codegen/smoketest/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/test/kotlin/aws/sdk/kotlin/test/codegen/smoketest/SmokeTestE2ETest.kt similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/kotlin/aws/sdk/kotlin/test/codegen/smoketest/SmokeTestE2ETest.kt rename to tests/codegen/smoke-tests/src/test/kotlin/aws/sdk/kotlin/test/codegen/smoketest/SmokeTestE2ETest.kt diff --git a/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy b/tests/codegen/smoke-tests/src/test/resources/smoke-tests-exception.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-exception.smithy rename to tests/codegen/smoke-tests/src/test/resources/smoke-tests-exception.smithy diff --git a/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy b/tests/codegen/smoke-tests/src/test/resources/smoke-tests-failure.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-failure.smithy rename to tests/codegen/smoke-tests/src/test/resources/smoke-tests-failure.smithy diff --git a/tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy b/tests/codegen/smoke-tests/src/test/resources/smoke-tests-success.smithy similarity index 100% rename from tests/codegen/smoke-tests/src/jvmTest/resources/smoke-tests-success.smithy rename to tests/codegen/smoke-tests/src/test/resources/smoke-tests-success.smithy From 9a46dbc468f0d5412aee558ae0be1ade68848718 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 8 Jan 2025 18:30:32 -0500 Subject: [PATCH 68/72] fix: finally fix this whole mess --- tests/codegen/build.gradle.kts | 37 ++++++++++--------- .../smoke-tests/services/build.gradle.kts | 32 ++++++++++++++++ 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/tests/codegen/build.gradle.kts b/tests/codegen/build.gradle.kts index c321430d17e..1b6c585df92 100644 --- a/tests/codegen/build.gradle.kts +++ b/tests/codegen/build.gradle.kts @@ -8,6 +8,23 @@ plugins { val libraries = libs subprojects { + tasks.withType { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } + } + + /* + Don't apply the rest of the configuration to the code generated smoke test services! + Those use the KMP plugin not JVM. + */ + if (project.path.startsWith(":tests:codegen:smoke-tests:services")) return@subprojects + apply(plugin = libraries.plugins.aws.kotlin.repo.tools.smithybuild.get().pluginId) apply(plugin = libraries.plugins.kotlin.jvm.get().pluginId) @@ -32,17 +49,14 @@ subprojects { } } - val codegen by configurations + val implementation by configurations + val api by configurations + val testImplementation by configurations dependencies { codegen(project(":codegen:aws-sdk-codegen")) codegen(libraries.smithy.cli) codegen(libraries.smithy.model) - } - val implementation by configurations - val api by configurations - val testImplementation by configurations - dependencies { implementation(project(":codegen:aws-sdk-codegen")) implementation(libraries.smithy.kotlin.codegen) @@ -66,15 +80,4 @@ subprojects { testImplementation(libraries.smithy.kotlin.telemetry.api) testImplementation(libraries.smithy.kotlin.http.test) } - - tasks.withType { - useJUnitPlatform() - testLogging { - events("passed", "skipped", "failed") - showStandardStreams = true - showStackTraces = true - showExceptions = true - exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL - } - } } diff --git a/tests/codegen/smoke-tests/services/build.gradle.kts b/tests/codegen/smoke-tests/services/build.gradle.kts index e69de29bb2d..a7a52c11934 100644 --- a/tests/codegen/smoke-tests/services/build.gradle.kts +++ b/tests/codegen/smoke-tests/services/build.gradle.kts @@ -0,0 +1,32 @@ +plugins { + alias(libs.plugins.kotlin.multiplatform) +} + +val libraries = libs +subprojects { + apply(plugin = libraries.plugins.kotlin.multiplatform.get().pluginId) + + val optinAnnotations = listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + "kotlin.RequiresOptIn", + ) + kotlin.sourceSets.all { + optinAnnotations.forEach { languageSettings.optIn(it) } + } + + kotlin { + sourceSets { + commonMain { + kotlin.srcDir("generated-src/main/kotlin") + + dependencies { + implementation(libraries.kotlin.test) + } + } + commonTest { + kotlin.srcDir("generated-src/test/kotlin") + } + } + } +} From 7cbe93d33b9652080a4b5b361401c44dcc2128c2 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 9 Jan 2025 15:22:24 -0500 Subject: [PATCH 69/72] fix: refactor buildSrc package name --- .../aws/sdk/kotlin/{shared => tests/codegen}/CodegenTest.kt | 2 +- tests/codegen/checksums/build.gradle.kts | 4 ++-- tests/codegen/event-stream/build.gradle.kts | 4 ++-- tests/codegen/rules-engine/build.gradle.kts | 4 ++-- tests/codegen/smoke-tests/build.gradle.kts | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) rename buildSrc/src/main/kotlin/aws/sdk/kotlin/{shared => tests/codegen}/CodegenTest.kt (88%) diff --git a/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt b/buildSrc/src/main/kotlin/aws/sdk/kotlin/tests/codegen/CodegenTest.kt similarity index 88% rename from buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt rename to buildSrc/src/main/kotlin/aws/sdk/kotlin/tests/codegen/CodegenTest.kt index 55b5c91364f..764e0abc44b 100644 --- a/buildSrc/src/main/kotlin/aws/sdk/kotlin/shared/CodegenTest.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/kotlin/tests/codegen/CodegenTest.kt @@ -1,4 +1,4 @@ -package aws.sdk.kotlin.shared +package aws.sdk.kotlin.tests.codegen /** * An AWS SDK for Kotlin codegen test diff --git a/tests/codegen/checksums/build.gradle.kts b/tests/codegen/checksums/build.gradle.kts index e20a5cc4122..6e7795fe4a6 100644 --- a/tests/codegen/checksums/build.gradle.kts +++ b/tests/codegen/checksums/build.gradle.kts @@ -1,8 +1,8 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -import aws.sdk.kotlin.shared.CodegenTest -import aws.sdk.kotlin.shared.Model +import aws.sdk.kotlin.tests.codegen.CodegenTest +import aws.sdk.kotlin.tests.codegen.Model description = "AWS SDK for Kotlin's checksums codegen test suite" diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts index 6e0fe6e660b..7f19ecd15ab 100644 --- a/tests/codegen/event-stream/build.gradle.kts +++ b/tests/codegen/event-stream/build.gradle.kts @@ -5,8 +5,8 @@ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionSrcDir -import aws.sdk.kotlin.shared.CodegenTest -import aws.sdk.kotlin.shared.Model +import aws.sdk.kotlin.tests.codegen.CodegenTest +import aws.sdk.kotlin.tests.codegen.Model description = "AWS SDK for Kotlin's event stream codegen test suite" diff --git a/tests/codegen/rules-engine/build.gradle.kts b/tests/codegen/rules-engine/build.gradle.kts index b6242e9f17a..3d4cb78932f 100644 --- a/tests/codegen/rules-engine/build.gradle.kts +++ b/tests/codegen/rules-engine/build.gradle.kts @@ -4,8 +4,8 @@ */ import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin -import aws.sdk.kotlin.shared.CodegenTest -import aws.sdk.kotlin.shared.Model +import aws.sdk.kotlin.tests.codegen.CodegenTest +import aws.sdk.kotlin.tests.codegen.Model description = "AWS SDK for Kotlin's rules engine codegen integration test suite" diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts index cb3add53dce..e6298681c91 100644 --- a/tests/codegen/smoke-tests/build.gradle.kts +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -6,8 +6,8 @@ import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath -import aws.sdk.kotlin.shared.CodegenTest -import aws.sdk.kotlin.shared.Model +import aws.sdk.kotlin.tests.codegen.CodegenTest +import aws.sdk.kotlin.tests.codegen.Model description = "AWS SDK for Kotlin's smoke test codegen test suite" From 8f65c72d836825d278aa7a860f25adbd44567259 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 9 Jan 2025 18:33:26 -0500 Subject: [PATCH 70/72] fix: pr feedback --- aws-runtime/aws-config/api/aws-config.api | 4 +-- .../kotlin/runtime/config/AwsSdkSetting.kt | 10 +++--- .../ResolveFlexibleChecksumsConfig.kt | 33 +++-------------- .../runtime/config/profile/AwsProfile.kt | 36 ++++++++++++++++--- aws-runtime/aws-http/api/aws-http.api | 2 +- ...siteFlexibleChecksumResponseInterceptor.kt | 7 ++-- 6 files changed, 50 insertions(+), 42 deletions(-) diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index 1d5c135a3ae..5f95b5c120d 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -476,9 +476,9 @@ public final class aws/sdk/kotlin/runtime/config/profile/AwsProfileKt { public static synthetic fun getLongOrNull$default (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Ljava/lang/Long; public static final fun getMaxAttempts (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/Integer; public static final fun getRegion (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; - public static final fun getRequestChecksumCalculation (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; + public static final fun getRequestChecksumCalculation (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Laws/smithy/kotlin/runtime/client/config/RequestHttpChecksumConfig; public static final fun getRequestMinCompressionSizeBytes (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/Long; - public static final fun getResponseChecksumValidation (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; + public static final fun getResponseChecksumValidation (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Laws/smithy/kotlin/runtime/client/config/ResponseHttpChecksumConfig; public static final fun getRetryMode (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Laws/smithy/kotlin/runtime/client/config/RetryMode; public static final fun getRoleArn (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; public static final fun getSdkUserAgentAppId (Laws/sdk/kotlin/runtime/config/profile/ConfigSection;)Ljava/lang/String; diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt index edbfe7cd92c..cbab4aeccfd 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsSdkSetting.kt @@ -9,6 +9,8 @@ import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.config.endpoints.AccountIdEndpointMode import aws.sdk.kotlin.runtime.http.AWS_APP_ID_ENV import aws.sdk.kotlin.runtime.http.AWS_APP_ID_PROP +import aws.smithy.kotlin.runtime.client.config.RequestHttpChecksumConfig +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.client.config.RetryMode import aws.smithy.kotlin.runtime.config.* import aws.smithy.kotlin.runtime.net.url.Url @@ -212,14 +214,14 @@ public object AwsSdkSetting { /** * Configures request checksum calculation */ - public val AwsRequestChecksumCalculation: EnvironmentSetting = - strEnvSetting("aws.requestChecksumCalculation", "AWS_REQUEST_CHECKSUM_CALCULATION") + public val AwsRequestChecksumCalculation: EnvironmentSetting = + enumEnvSetting("aws.requestChecksumCalculation", "AWS_REQUEST_CHECKSUM_CALCULATION") /** * Configures response checksum validation */ - public val AwsResponseChecksumValidation: EnvironmentSetting = - strEnvSetting("aws.responseChecksumValidation", "AWS_RESPONSE_CHECKSUM_VALIDATION") + public val AwsResponseChecksumValidation: EnvironmentSetting = + enumEnvSetting("aws.responseChecksumValidation", "AWS_RESPONSE_CHECKSUM_VALIDATION") } /** diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt index bbbbd241e28..5925fb858f0 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/checksums/ResolveFlexibleChecksumsConfig.kt @@ -1,6 +1,5 @@ package aws.sdk.kotlin.runtime.config.checksums -import aws.sdk.kotlin.runtime.ConfigurationException import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.config.AwsSdkSetting import aws.sdk.kotlin.runtime.config.profile.AwsProfile @@ -20,20 +19,8 @@ import aws.smithy.kotlin.runtime.util.PlatformProvider public suspend fun resolveRequestChecksumCalculation( platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue, -): RequestHttpChecksumConfig { - val unparsedString = AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation - return parseRequestHttpChecksumConfig(unparsedString) -} - -private fun parseRequestHttpChecksumConfig(unparsedString: String?): RequestHttpChecksumConfig = - when (unparsedString?.uppercase()) { - null -> RequestHttpChecksumConfig.WHEN_SUPPORTED - "WHEN_SUPPORTED" -> RequestHttpChecksumConfig.WHEN_SUPPORTED - "WHEN_REQUIRED" -> RequestHttpChecksumConfig.WHEN_REQUIRED - else -> throw ConfigurationException( - "'$unparsedString' is not a valid value for 'requestChecksumCalculation'. Valid values are: ${RequestHttpChecksumConfig.entries}", - ) - } +): RequestHttpChecksumConfig = + AwsSdkSetting.AwsRequestChecksumCalculation.resolve(platform) ?: profile.get().requestChecksumCalculation ?: RequestHttpChecksumConfig.WHEN_SUPPORTED /** * Attempts to resolve responseChecksumValidation from the specified sources. @@ -43,17 +30,5 @@ private fun parseRequestHttpChecksumConfig(unparsedString: String?): RequestHttp public suspend fun resolveResponseChecksumValidation( platform: PlatformProvider = PlatformProvider.System, profile: LazyAsyncValue, -): ResponseHttpChecksumConfig { - val unparsedString = AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation - return parseResponseHttpChecksumConfig(unparsedString) -} - -private fun parseResponseHttpChecksumConfig(unparsedString: String?): ResponseHttpChecksumConfig = - when (unparsedString?.uppercase()) { - null -> ResponseHttpChecksumConfig.WHEN_SUPPORTED - "WHEN_SUPPORTED" -> ResponseHttpChecksumConfig.WHEN_SUPPORTED - "WHEN_REQUIRED" -> ResponseHttpChecksumConfig.WHEN_REQUIRED - else -> throw ConfigurationException( - "'$unparsedString' is not a valid value for 'responseChecksumValidation'. Valid values are: ${ResponseHttpChecksumConfig.entries}", - ) - } +): ResponseHttpChecksumConfig = + AwsSdkSetting.AwsResponseChecksumValidation.resolve(platform) ?: profile.get().responseChecksumValidation ?: ResponseHttpChecksumConfig.WHEN_SUPPORTED diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index 119420a09e3..e0cdf445f1a 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -8,6 +8,8 @@ package aws.sdk.kotlin.runtime.config.profile import aws.sdk.kotlin.runtime.ConfigurationException import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.config.endpoints.AccountIdEndpointMode +import aws.smithy.kotlin.runtime.client.config.RequestHttpChecksumConfig +import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.client.config.RetryMode import aws.smithy.kotlin.runtime.net.url.Url @@ -171,15 +173,15 @@ public val AwsProfile.sigV4aSigningRegionSet: String? * Configures request checksum calculation */ @InternalSdkApi -public val AwsProfile.requestChecksumCalculation: String? - get() = getOrNull("request_checksum_calculation") +public val AwsProfile.requestChecksumCalculation: RequestHttpChecksumConfig? + get() = getOrNull("request_checksum_calculation")?.parseRequestHttpChecksumConfig() /** * Configures response checksum validation */ @InternalSdkApi -public val AwsProfile.responseChecksumValidation: String? - get() = getOrNull("response_checksum_validation") +public val AwsProfile.responseChecksumValidation: ResponseHttpChecksumConfig? + get() = getOrNull("response_checksum_validation")?.parseResponseHttpChecksumConfig() /** * Parse a config value as a boolean, ignoring case. @@ -214,6 +216,32 @@ public fun AwsProfile.getLongOrNull(key: String, subKey: String? = null): Long? ) } +/** + * Parse a string value as [ResponseHttpChecksumConfig] + */ +private fun String?.parseResponseHttpChecksumConfig(): ResponseHttpChecksumConfig? = + when (this?.uppercase()) { + null -> null + "WHEN_SUPPORTED" -> ResponseHttpChecksumConfig.WHEN_SUPPORTED + "WHEN_REQUIRED" -> ResponseHttpChecksumConfig.WHEN_REQUIRED + else -> throw ConfigurationException( + "'$this' is not a valid value for 'response_checksum_validation'. Valid values are: ${ResponseHttpChecksumConfig.entries}", + ) + } + +/** + * Parse a string value as [RequestHttpChecksumConfig] + */ +private fun String?.parseRequestHttpChecksumConfig(): RequestHttpChecksumConfig? = + when (this?.uppercase()) { + null -> null + "WHEN_SUPPORTED" -> RequestHttpChecksumConfig.WHEN_SUPPORTED + "WHEN_REQUIRED" -> RequestHttpChecksumConfig.WHEN_REQUIRED + else -> throw ConfigurationException( + "'$this' is not a valid value for 'request_checksum_calculation'. Valid values are: ${RequestHttpChecksumConfig.entries}", + ) + } + internal fun AwsProfile.getUrlOrNull(key: String, subKey: String? = null): Url? = getOrNull(key, subKey)?.let { try { diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index c9aadd34375..d7f994bc485 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -198,7 +198,7 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInter public final class aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor : aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor { public fun (ZLaws/smithy/kotlin/runtime/client/config/ResponseHttpChecksumConfig;)V - public fun ignoreChecksum (Ljava/lang/String;)Z + public fun ignoreChecksum (Ljava/lang/String;Laws/smithy/kotlin/runtime/telemetry/logging/Logger;)Z } public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt index 834924d5081..4b6dcd5f563 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt @@ -2,6 +2,7 @@ package aws.sdk.kotlin.runtime.http.interceptors import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.http.interceptors.FlexibleChecksumsResponseInterceptor +import aws.smithy.kotlin.runtime.telemetry.logging.Logger /** * Variant of the [FlexibleChecksumsResponseInterceptor] where composite checksums are not validated @@ -13,8 +14,10 @@ public class IgnoreCompositeFlexibleChecksumResponseInterceptor( responseValidationRequired, responseChecksumValidation, ) { - override fun ignoreChecksum(checksum: String): Boolean = - checksum.isCompositeChecksum() + override fun ignoreChecksum(checksum: String, logger: Logger): Boolean = + checksum.isCompositeChecksum().also { compositeChecksum -> + if (compositeChecksum) logger.info { "Checksum validation was skipped because it was a composite checksum" } + } } /** From 6c2a39b8b8b61c00cd7265ee859e2b4415dd7965 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 14 Jan 2025 21:09:36 -0500 Subject: [PATCH 71/72] fix: changelog and last bit of feedback --- .../3339e5cc-978c-4941-a975-6c0c82d6426f.json | 5 ++ .../runtime/config/profile/AwsProfile.kt | 46 ++++++++----------- aws-runtime/aws-http/api/aws-http.api | 2 +- ...siteFlexibleChecksumResponseInterceptor.kt | 16 +++++-- 4 files changed, 37 insertions(+), 32 deletions(-) create mode 100644 .changes/3339e5cc-978c-4941-a975-6c0c82d6426f.json diff --git a/.changes/3339e5cc-978c-4941-a975-6c0c82d6426f.json b/.changes/3339e5cc-978c-4941-a975-6c0c82d6426f.json new file mode 100644 index 00000000000..0200ffccbb8 --- /dev/null +++ b/.changes/3339e5cc-978c-4941-a975-6c0c82d6426f.json @@ -0,0 +1,5 @@ +{ + "id": "3339e5cc-978c-4941-a975-6c0c82d6426f", + "type": "feature", + "description": "S3 client behavior is updated to always calculate a checksum by default for operations that support it (such as PutObject or UploadPart), or require it (such as DeleteObjects). The checksum algorithm used by default varies by SDK (CRC32 or CRC64, CRC32 for the Kotlin SDK). Checksum calculation behavior can be configured using `when_supported` and `when_required` options - in code using requestChecksumCalculation, in shared config using request_checksum_calculation, or as env variable using AWS_REQUEST_CHECKSUM_CALCULATION. The S3 client attempts to validate response checksums for all S3 API operations that support checksums. However, if the SDK has not implemented the specified checksum algorithm then this validation is skipped. Checksum validation behavior can be configured using `when_supported` and `when_required` options - in code using responseChecksumValidation, in shared config using response_checksum_validation, or as env variable using AWS_RESPONSE_CHECKSUM_VALIDATION." +} \ No newline at end of file diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index e0cdf445f1a..b5eb254683a 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -174,14 +174,30 @@ public val AwsProfile.sigV4aSigningRegionSet: String? */ @InternalSdkApi public val AwsProfile.requestChecksumCalculation: RequestHttpChecksumConfig? - get() = getOrNull("request_checksum_calculation")?.parseRequestHttpChecksumConfig() + get() = getOrNull("request_checksum_calculation")?.run { + RequestHttpChecksumConfig + .values() + .firstOrNull { it.name.equals(this, ignoreCase = true) } + ?: throw ConfigurationException( + "request_checksum_calculation $this is not supported, should be one of: " + + RequestHttpChecksumConfig.values().joinToString(", ") { it.name.lowercase() }, + ) + } /** * Configures response checksum validation */ @InternalSdkApi public val AwsProfile.responseChecksumValidation: ResponseHttpChecksumConfig? - get() = getOrNull("response_checksum_validation")?.parseResponseHttpChecksumConfig() + get() = getOrNull("response_checksum_validation")?.run { + ResponseHttpChecksumConfig + .values() + .firstOrNull { it.name.equals(this, ignoreCase = true) } + ?: throw ConfigurationException( + "response_checksum_validation $this is not supported, should be one of: " + + ResponseHttpChecksumConfig.values().joinToString(", ") { it.name.lowercase() }, + ) + } /** * Parse a config value as a boolean, ignoring case. @@ -216,32 +232,6 @@ public fun AwsProfile.getLongOrNull(key: String, subKey: String? = null): Long? ) } -/** - * Parse a string value as [ResponseHttpChecksumConfig] - */ -private fun String?.parseResponseHttpChecksumConfig(): ResponseHttpChecksumConfig? = - when (this?.uppercase()) { - null -> null - "WHEN_SUPPORTED" -> ResponseHttpChecksumConfig.WHEN_SUPPORTED - "WHEN_REQUIRED" -> ResponseHttpChecksumConfig.WHEN_REQUIRED - else -> throw ConfigurationException( - "'$this' is not a valid value for 'response_checksum_validation'. Valid values are: ${ResponseHttpChecksumConfig.entries}", - ) - } - -/** - * Parse a string value as [RequestHttpChecksumConfig] - */ -private fun String?.parseRequestHttpChecksumConfig(): RequestHttpChecksumConfig? = - when (this?.uppercase()) { - null -> null - "WHEN_SUPPORTED" -> RequestHttpChecksumConfig.WHEN_SUPPORTED - "WHEN_REQUIRED" -> RequestHttpChecksumConfig.WHEN_REQUIRED - else -> throw ConfigurationException( - "'$this' is not a valid value for 'request_checksum_calculation'. Valid values are: ${RequestHttpChecksumConfig.entries}", - ) - } - internal fun AwsProfile.getUrlOrNull(key: String, subKey: String? = null): Url? = getOrNull(key, subKey)?.let { try { diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index d7f994bc485..384a802e37f 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -198,7 +198,7 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/BusinessMetricsInter public final class aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor : aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor { public fun (ZLaws/smithy/kotlin/runtime/client/config/ResponseHttpChecksumConfig;)V - public fun ignoreChecksum (Ljava/lang/String;Laws/smithy/kotlin/runtime/telemetry/logging/Logger;)Z + public fun ignoreChecksum (Ljava/lang/String;Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)Z } public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt index 4b6dcd5f563..75c70913894 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor.kt @@ -1,8 +1,11 @@ package aws.sdk.kotlin.runtime.http.interceptors +import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext import aws.smithy.kotlin.runtime.client.config.ResponseHttpChecksumConfig import aws.smithy.kotlin.runtime.http.interceptors.FlexibleChecksumsResponseInterceptor -import aws.smithy.kotlin.runtime.telemetry.logging.Logger +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.telemetry.logging.info /** * Variant of the [FlexibleChecksumsResponseInterceptor] where composite checksums are not validated @@ -14,9 +17,16 @@ public class IgnoreCompositeFlexibleChecksumResponseInterceptor( responseValidationRequired, responseChecksumValidation, ) { - override fun ignoreChecksum(checksum: String, logger: Logger): Boolean = + override fun ignoreChecksum( + checksum: String, + context: ProtocolResponseInterceptorContext, + ): Boolean = checksum.isCompositeChecksum().also { compositeChecksum -> - if (compositeChecksum) logger.info { "Checksum validation was skipped because it was a composite checksum" } + if (compositeChecksum) { + context.executionContext.coroutineContext.info { + "Checksum validation was skipped because it was a composite checksum" + } + } } } From 594901ef053b01f32071c03157c14c67a2362f7e Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 15 Jan 2025 14:31:21 -0500 Subject: [PATCH 72/72] fix: apiDump & changelog fix --- .changes/3339e5cc-978c-4941-a975-6c0c82d6426f.json | 3 ++- aws-runtime/aws-http/api/aws-http.api | 11 +++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.changes/3339e5cc-978c-4941-a975-6c0c82d6426f.json b/.changes/3339e5cc-978c-4941-a975-6c0c82d6426f.json index 0200ffccbb8..7e1b3ba7e17 100644 --- a/.changes/3339e5cc-978c-4941-a975-6c0c82d6426f.json +++ b/.changes/3339e5cc-978c-4941-a975-6c0c82d6426f.json @@ -1,5 +1,6 @@ { "id": "3339e5cc-978c-4941-a975-6c0c82d6426f", "type": "feature", - "description": "S3 client behavior is updated to always calculate a checksum by default for operations that support it (such as PutObject or UploadPart), or require it (such as DeleteObjects). The checksum algorithm used by default varies by SDK (CRC32 or CRC64, CRC32 for the Kotlin SDK). Checksum calculation behavior can be configured using `when_supported` and `when_required` options - in code using requestChecksumCalculation, in shared config using request_checksum_calculation, or as env variable using AWS_REQUEST_CHECKSUM_CALCULATION. The S3 client attempts to validate response checksums for all S3 API operations that support checksums. However, if the SDK has not implemented the specified checksum algorithm then this validation is skipped. Checksum validation behavior can be configured using `when_supported` and `when_required` options - in code using responseChecksumValidation, in shared config using response_checksum_validation, or as env variable using AWS_RESPONSE_CHECKSUM_VALIDATION." + "description": "⚠\uFE0F **IMPORTANT**: S3 client behavior is updated to always calculate a checksum by default for operations that support it (such as PutObject or UploadPart), or require it (such as DeleteObjects). The checksum algorithm used by default varies by SDK (CRC32 or CRC64, CRC32 for the Kotlin SDK). Checksum calculation behavior can be configured using `when_supported` and `when_required` options - in code using requestChecksumCalculation, in shared config using request_checksum_calculation, or as env variable using AWS_REQUEST_CHECKSUM_CALCULATION. The S3 client attempts to validate response checksums for all S3 API operations that support checksums. However, if the SDK has not implemented the specified checksum algorithm then this validation is skipped. Checksum validation behavior can be configured using `when_supported` and `when_required` options - in code using responseChecksumValidation, in shared config using response_checksum_validation, or as env variable using AWS_RESPONSE_CHECKSUM_VALIDATION.", + "requiresMinorVersionBump": true } \ No newline at end of file diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index af2ba0a318d..1b6ced4a550 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -163,6 +163,11 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/AwsSpanInterceptor : public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } +public final class aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor : aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor { + public fun (ZLaws/smithy/kotlin/runtime/client/config/ResponseHttpChecksumConfig;)V + public fun ignoreChecksum (Ljava/lang/String;Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)Z +} + public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { public fun ()V public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; @@ -186,12 +191,6 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAl public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } -public final class aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexibleChecksumResponseInterceptor : aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor { - public fun (ZLaws/smithy/kotlin/runtime/client/config/ResponseHttpChecksumConfig;)V - public fun ignoreChecksum (Ljava/lang/String;Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)Z -} - -public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { public final class aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric : java/lang/Enum, aws/smithy/kotlin/runtime/businessmetrics/BusinessMetric { public static final field DDB_MAPPER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static final field S3_EXPRESS_BUCKET Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric;