diff --git a/.github/workflows/generator-test-sdk.yaml b/.github/workflows/generator-test-sdk.yaml index e332c22a0..4551457c3 100644 --- a/.github/workflows/generator-test-sdk.yaml +++ b/.github/workflows/generator-test-sdk.yaml @@ -27,36 +27,7 @@ on: value: ${{ jobs.sdk-metadata.outputs.version }} jobs: - download-specs: - uses: ./.github/workflows/generator-download-specs.yaml - with: - url: ${{ inputs.specs_url }} - - transform-specs: - needs: [ download-specs ] - uses: ./.github/workflows/selfserve-transform-specs.yaml - with: - specs_key: 'raw-specs' - transformations: -th -te ${{ inputs.endpoint_prefix }} --operationIdsToTags - - upload-templates: - uses: ./.github/workflows/selfserve-checkout-and-upload.yaml - with: - artifact_key: 'templates' - artifact_path: 'generator/openapi/src/main/resources/templates/expediagroup-sdk' - repository: 'ExpediaGroup/expediagroup-java-sdk' - ref: "${{ github.head_ref || github.ref_name }}" - - upload-repo: - uses: ./.github/workflows/selfserve-checkout-and-upload.yaml - with: - artifact_key: 'sdk-repo' - artifact_path: '' - repository: 'ExpediaGroup/expediagroup-java-sdk' - ref: "${{ github.head_ref || github.ref_name }}" - generate-sdk: - needs: [ transform-specs, upload-repo ] uses: ./.github/workflows/selfserve-generate-sdk.yaml with: name: ${{ inputs.namespace }} diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 7633198c4..e276d6f16 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -17,14 +17,16 @@ on: default: '' sdk_version: description: 'Run tests based on SDK' - required: false type: string default: 'LATEST' sdk_namespace: description: 'SDK to test' - required: true type: string default: 'rapid' + sdk_artifact_id: + description: 'SDK artifactId' + type: string + default: 'rapid-sdk' endpoint_prefix: description: 'Endpoint to prepend specs paths with' required: true @@ -42,7 +44,7 @@ on: workflow_call: inputs: source: - description: 'Source of tests' + description: 'Source of tests. Can be `sdk` or `specs`' required: true type: string default: 'sdk' @@ -58,9 +60,12 @@ on: default: 'LATEST' sdk_namespace: description: 'SDK to test' - required: true type: string default: 'rapid' + sdk_artifact_id: + description: 'SDK artifactId' + type: string + default: 'rapid-sdk' endpoint_prefix: description: 'Endpoint to prepend specs paths with' required: true @@ -80,9 +85,45 @@ jobs: print('::error::Invalid SDK version: ${{ inputs.sdk_version }}') exit(1) + download-specs: + uses: ./.github/workflows/generator-download-specs.yaml + with: + url: ${{ inputs.specs_url }} + + transform-and-upload-specs: + needs: [ download-specs ] + uses: ./.github/workflows/selfserve-transform-specs.yaml + with: + specs_key: 'raw-specs' + transformations: -th -te ${{ inputs.endpoint_prefix }} --operationIdsToTags + + upload-templates: + uses: ./.github/workflows/selfserve-checkout-and-upload.yaml + with: + artifact_key: 'templates' + artifact_path: 'generator/openapi/src/main/resources/templates/expediagroup-sdk' + repository: 'ExpediaGroup/expediagroup-java-sdk' + ref: "${{ github.head_ref || github.ref_name }}" + + upload-test-templates: + uses: ./.github/workflows/selfserve-checkout-and-upload.yaml + with: + artifact_key: 'test-templates' + artifact_path: 'sdk-test/src/main/resources/templates/expediagroup-sdk' + repository: 'ExpediaGroup/expediagroup-java-sdk' + ref: "${{ github.head_ref || github.ref_name }}" + + upload-repo: + uses: ./.github/workflows/selfserve-checkout-and-upload.yaml + with: + artifact_key: 'sdk-repo' + artifact_path: '' + repository: 'ExpediaGroup/expediagroup-java-sdk' + ref: "${{ github.head_ref || github.ref_name }}" + generate-test-sdk: if: inputs.source == 'specs' - needs: [ inputs-validation ] + needs: [ inputs-validation, transform-and-upload-specs, upload-templates, upload-repo ] uses: ./.github/workflows/generator-test-sdk.yaml with: version: ${{ inputs.sdk_version }} @@ -90,16 +131,45 @@ jobs: endpoint_prefix: ${{ inputs.endpoint_prefix }} specs_url: ${{ inputs.specs_url }} secrets: inherit + + generate-contract-tests: + needs: [ inputs-validation, generate-test-sdk ] + uses: ./.github/workflows/selfserve-generate-contract-test-sources.yaml + with: + artifact_key: 'sdk-test' + repo_key: 'sdk-repo' + templates_key: 'test-templates' + specs_key: 'transformedSpecs' + namespace: ${{ inputs.sdk_namespace }} + generation_options: '--version 0.0.1 --target-group com.expediagroup --target-artifact ${{ inputs.sdk_artifact_id }} --target-version ${{ inputs.sdk_version }} -m 5' # TODO: remove -m param + run-rapid-examples: - strategy: - matrix: - jdk: [8, 11, 17, 21] if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && inputs.sdk_namespace == 'rapid' needs: [ generate-test-sdk ] uses: "ExpediaGroup/rapid-java-sdk/.github/workflows/run-examples.yaml@main" with: + source: 'specs' sdk_version: ${{ needs.generate-test-sdk.outputs.version }} jdk: ${{ matrix.jdk }} secrets: KEY: ${{ secrets.RAPID_KEY }} SECRET: ${{ secrets.RAPID_SECRET }} + strategy: + matrix: + jdk: [ 8, 11, 17, 21 ] + + run-rapid-contract-tests: + if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && inputs.sdk_namespace == 'rapid' + needs: [ generate-contract-tests ] + uses: ./.github/workflows/selfserve-run-mocked-service-tests.yaml + with: + specs_key: 'transformedSpecs' + test_sources_key: 'sdk-test' + sdk_key: 'sdk' + jdk: ${{ matrix.jdk }} + secrets: + KEY: ${{ secrets.RAPID_KEY }} + SECRET: ${{ secrets.RAPID_SECRET }} + strategy: + matrix: + jdk: [ 8, 11, 17, 21 ] diff --git a/.github/workflows/selfserve-generate-contract-test-sources.yaml b/.github/workflows/selfserve-generate-contract-test-sources.yaml new file mode 100644 index 000000000..5ef16b139 --- /dev/null +++ b/.github/workflows/selfserve-generate-contract-test-sources.yaml @@ -0,0 +1,69 @@ +name: Generate & Upload Tests Sources + +on: + workflow_call: + inputs: + artifact_key: + description: 'Generated output artifact key. Defaults to `sdk-test`' + type: string + default: 'sdk-test' + repo_key: + description: 'SDK repository artifact key' + type: string + templates_key: + description: 'Templates artifact key' + type: string + required: true + specs_key: + description: 'Specs artifact key' + type: string + required: true + namespace: + description: 'Namespace of the product' + type: string + required: true + generation_options: + description: 'Options to pass to the generator. Includes `--version` and `--max-test-combinations`' + type: string + required: true + +jobs: + generate-and-upload-contract-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: 'corretto' + + - name: Download Repository + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.repo_key }} + path: sdk-repo + + - name: Download Templates + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.templates_key }} + path: templates + + - name: Download Specs + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.specs_key }} + path: specs + + - name: Install SDK Generator Module + working-directory: sdk-repo/generator/openapi + run: mvn clean install + continue-on-error: true + + - name: Generate Test Sources + run: mvn -f sdk-repo/sdk-test clean install exec:java -Dexec.args="--input-spec specs/${{ inputs.specs_key }}.yaml --templates-dir templates --namespace ${{ inputs.namespace }} ${{ inputs.generation_options }}" + + - name: Upload Generated Sources + uses: actions/upload-artifact@v4 + with: + path: target/sdk/*-sdk-test/** + name: ${{ inputs.artifact_key }} diff --git a/.github/workflows/selfserve-run-mocked-service-tests.yaml b/.github/workflows/selfserve-run-mocked-service-tests.yaml new file mode 100644 index 000000000..57fb72e7f --- /dev/null +++ b/.github/workflows/selfserve-run-mocked-service-tests.yaml @@ -0,0 +1,99 @@ +name: SelfServe Run Service Mocked Tests +on: + workflow_call: + inputs: + specs_key: + description: 'Specs artifact key. Defaults to `specs`' + type: string + default: 'transformedSpecs' + required: true + test_sources_key: + description: 'Test sources artifact key. Defaults to `sdk-test`' + type: string + default: 'sdk-test' + required: true + sdk_key: + description: 'SDK artifact key to download' + type: string + required: false + default: '' + jdk: + description: 'JDK version to use' + type: string + default: '21' + secrets: + KEY: + description: 'API Key' + required: true + SECRET: + description: 'API Key' + required: true + +jobs: + run-tests-with-mocked-service: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: 'corretto' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 18.18.2 + + - name: Download Specs + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.specs_key }} + path: specs + + - name: Transform Specs + working-directory: specs + run: npx --yes -p @expediagroup/spec-transformer cli -i ${{ inputs.specs_key }}.yaml -o ${{ inputs.specs_key }}.yaml -to + + - name: Download Test Sources Artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.test_sources_key }} + path: test + + - name: Download SDK Artifact + if: inputs.sdk_key != '' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.sdk_key }} + path: sdk + + - name: Install SDK into Local Repository + if: inputs.sdk_key != '' + working-directory: sdk + run: mvn clean install + + - name: Download Latest Specmatic Executable + working-directory: specs + run: | + curl -s https://api.github.com/repos/znsio/specmatic/releases/latest \ + | grep "browser_download_url.*specmatic.jar" \ + | cut -d : -f 2,3 \ + | tr -d \" \ + | xargs curl -L -o specmatic.jar + + - name: Run Mock Server + working-directory: specs + run: java -jar specmatic.jar stub ${{ inputs.specs_key }}.yaml --port 8080 & disown + + - uses: actions/setup-java@v4 + with: + java-version: ${{ inputs.jdk }} + distribution: 'corretto' + + - name: Wait for Mock Server to Start + run: sleep 10 + + - name: Run Tests + working-directory: test + run: | + cd $(find . -type d -name "*-sdk-test" -print -quit) + mvn clean install exec:java -Dsdk.key="${{ secrets.KEY }}" -Dsdk.secret="${{ secrets.SECRET }}" diff --git a/generator/openapi/src/main/kotlin/com/expediagroup/sdk/generators/openapi/OpenApiSdkGenerator.kt b/generator/openapi/src/main/kotlin/com/expediagroup/sdk/generators/openapi/OpenApiSdkGenerator.kt index fd6e1b217..ea4239d21 100644 --- a/generator/openapi/src/main/kotlin/com/expediagroup/sdk/generators/openapi/OpenApiSdkGenerator.kt +++ b/generator/openapi/src/main/kotlin/com/expediagroup/sdk/generators/openapi/OpenApiSdkGenerator.kt @@ -126,6 +126,8 @@ class OpenApiSdkGenerator { if (ProductFamily.isRapid(product.namespace)) { rapidHelpers.forEach { (name, function) -> addAdditionalProperty(name, function()) } } + + addTypeMapping("java.io.File", "java.io.InputStream") } val generatorInput = diff --git a/generator/openapi/src/main/resources/templates/expediagroup-sdk/operation_params.mustache b/generator/openapi/src/main/resources/templates/expediagroup-sdk/operation_params.mustache index 5b69282f8..dd8097a3b 100644 --- a/generator/openapi/src/main/resources/templates/expediagroup-sdk/operation_params.mustache +++ b/generator/openapi/src/main/resources/templates/expediagroup-sdk/operation_params.mustache @@ -93,6 +93,7 @@ ) { {{#allowableValues}} {{#enumVars}} + @JsonProperty({{{value}}}) {{name}}({{{value}}}) {{^-last}},{{/-last}} {{/enumVars}} diff --git a/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/CLI.kt b/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/CLI.kt index 7b0b55729..0aa418499 100644 --- a/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/CLI.kt +++ b/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/CLI.kt @@ -27,6 +27,7 @@ package com.expediagroup.sdk.test import com.expediagroup.sdk.test.contract.ContractTestsGenerator import com.expediagroup.sdk.test.contract.MAX_TEST_REQUEST_PER_SCENARIO +import com.expediagroup.sdk.test.openapi.ArtifactMetadata import com.expediagroup.sdk.test.openapi.SdkTestGenerator import com.expediagroup.sdk.test.openapi.ProductTest import com.github.rvesse.airline.SingleCommand @@ -54,6 +55,15 @@ class CLI { @Option(name = ["-t", "--templates-dir"]) private var templatesDir: File = File("src/main/resources/templates/expediagroup-sdk") + @Option(name=["-tg", "--target-group"]) + private lateinit var targetGroup: String + + @Option(name = ["-ta", "--target-artifact"]) + private lateinit var targetArtifact: String + + @Option(name = ["-tv", "--target-version"]) + private lateinit var targetVersion: String + private lateinit var sdkTestGenerator: SdkTestGenerator private lateinit var contractTestsGenerator: ContractTestsGenerator @@ -81,6 +91,11 @@ class CLI { val sdkTestsOutputDirectory = File(outputDir, productTest.artifactId) val contractTestsOutputDirectory = File("target/sdk/${productTest.artifactId}/src/main/", "resources/contract-tests/").also { it.mkdirs() } + val artifact = ArtifactMetadata( + artifactId = targetArtifact, + version = targetVersion, + groupId = targetGroup, + ) contractTestsGenerator = ContractTestsGenerator( spec = spec, @@ -96,6 +111,7 @@ class CLI { productTest = productTest, templatesDir = templatesDir, outputDir = sdkTestsOutputDirectory, + targetArtifact = artifact, ).also { it.generate() } } } diff --git a/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/openapi/ArtifactMetadata.kt b/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/openapi/ArtifactMetadata.kt new file mode 100644 index 000000000..26592820c --- /dev/null +++ b/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/openapi/ArtifactMetadata.kt @@ -0,0 +1,7 @@ +package com.expediagroup.sdk.test.openapi + +data class ArtifactMetadata( + val artifactId: String, + val groupId: String, + val version: String, +) diff --git a/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/openapi/SdkTestGenerator.kt b/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/openapi/SdkTestGenerator.kt index 21766e4e3..29c1d966b 100644 --- a/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/openapi/SdkTestGenerator.kt +++ b/sdk-test/src/main/kotlin/com/expediagroup/sdk/test/openapi/SdkTestGenerator.kt @@ -45,6 +45,7 @@ import java.io.File class SdkTestGenerator( private val productTest: ProductTest, private val spec: File, + private val targetArtifact: ArtifactMetadata, private val version: String = "1.0.0", private val templatesDir: File = File("src/main/resources/templates/expediagroup-sdk"), private val outputDir: File = File("target/sdk") @@ -86,6 +87,10 @@ class SdkTestGenerator( addAdditionalProperty("namespace", productTest.namespace) addAdditionalProperty("clientClassname", productTest.namespace.pascalCase()) + addAdditionalProperty("targetSdkArtifactId", targetArtifact.artifactId) + addAdditionalProperty("targetSdkGroupId", targetArtifact.groupId) + addAdditionalProperty("targetSdkVersion", targetArtifact.version) + // Mustache Helpers mustacheHelpers.forEach { (name, func) -> addAdditionalProperty(name, func) diff --git a/sdk-test/src/main/resources/templates/expediagroup-sdk/executor.mustache b/sdk-test/src/main/resources/templates/expediagroup-sdk/executor.mustache index e2ca9a5a0..54cc6bdeb 100644 --- a/sdk-test/src/main/resources/templates/expediagroup-sdk/executor.mustache +++ b/sdk-test/src/main/resources/templates/expediagroup-sdk/executor.mustache @@ -3,21 +3,35 @@ package com.expediagroup.sdk.{{{namespace}}}.test; import com.expediagroup.sdk.core.model.Operation import com.expediagroup.sdk.core.model.Response import com.expediagroup.sdk.core.model.exception.service.ExpediaGroupServiceException -import com.expediagroup.sdk.rapid.client.RapidClient -import com.expediagroup.sdk.rapid.operations.* +import com.expediagroup.sdk.{{{namespace}}}.client.* +import com.expediagroup.sdk.{{{namespace}}}.operations.* +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.module.kotlin.KotlinFeature import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule class {{{clientClassname}}}Executor { companion object { - val mapper = jacksonObjectMapper() + @JvmStatic + val mapper = jacksonObjectMapper { + disable(KotlinFeature.StrictNullChecks) + }.apply { + disable(DeserializationFeature.FAIL_ON_MISSING_CREATOR_PROPERTIES) + disable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) + disable(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY) + disable(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES) + disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + disable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES) + registerModule(JavaTimeModule()) + } } private val client = {{{clientClassname}}}Client.builder() - .key(System.getProperty("com.expediagroup.sdk.apikey", "API_KEY")) - .secret(System.getProperty("com.expediagroup.sdk.apisecret", "API_SECRET")) - .endpoint("localhost:8080") + .key(System.getProperty("sdk.key", "API_KEY")) + .secret(System.getProperty("sdk.secret", "API_SECRET")) + .endpoint("http://localhost:8080") .requestTimeout(10000) .build() @@ -26,7 +40,45 @@ class {{{clientClassname}}}Executor { val requestType: Class<*> = metadata[key]!!.requestType - val operation = mapper.readValue(request.asText(), requestType) as Operation<*> + val operation: Operation<*> = try { + val params = + mapper.createObjectNode().apply { + listOf("pathParams", "queryParams", "headers", "params").forEach { + request.get(it)?.let { node -> + node.fields()?.forEachRemaining { (key, value) -> + set(key, value) + } + } + } + } + + val body = + mapper.createObjectNode().apply { + request.get("body")?.let { node -> + node.fields()?.forEachRemaining { (key, value) -> + set(key, value) + } + } + } + + val operationNode = + mapper.createObjectNode().apply { + takeIf { params.fields().hasNext() }?.let { + set("params", params) + } + + takeIf { body.fields().hasNext() }?.let { + set("body", body) + } + } + + mapper.readValue(operationNode.toPrettyString(), requestType) as Operation<*> + } catch (e: Exception) { + if (expectedResponse["status"].asInt() !in 200..299) { + return + } + throw e + } if (expectedResponse["status"].asInt() in 200..299) { val actual = mapper.convertValue(execute(operation), JsonNode::class.java) diff --git a/sdk-test/src/main/resources/templates/expediagroup-sdk/main.mustache b/sdk-test/src/main/resources/templates/expediagroup-sdk/main.mustache index 23d4fd40f..ec95935a8 100644 --- a/sdk-test/src/main/resources/templates/expediagroup-sdk/main.mustache +++ b/sdk-test/src/main/resources/templates/expediagroup-sdk/main.mustache @@ -2,13 +2,17 @@ package com.expediagroup.sdk.{{{namespace}}}.test; import com.fasterxml.jackson.module.kotlin.* import com.fasterxml.jackson.databind.node.ObjectNode +import com.google.common.io.Resources import java.io.File; import com.github.rvesse.airline.annotations.Command; @Command(name = "cli", description = "Command Line Interface for SDK Test") class Main { companion object { + @JvmStatic val mapper = jacksonObjectMapper() + + @JvmStatic val executor = {{{clientClassname}}}Executor() @JvmStatic @@ -18,7 +22,7 @@ class Main { } fun run() { - File("src/main/resources/contract-tests").listFiles().forEach { file -> + File(Resources.getResource("contract-tests")!!.file).listFiles()!!.forEach { file -> mapper.readTree(file).let { test -> val request = test["request"] as ObjectNode val response = test["response"] as ObjectNode diff --git a/sdk-test/src/main/resources/templates/expediagroup-sdk/pom.mustache b/sdk-test/src/main/resources/templates/expediagroup-sdk/pom.mustache index 6bec061b9..c2837b11f 100644 --- a/sdk-test/src/main/resources/templates/expediagroup-sdk/pom.mustache +++ b/sdk-test/src/main/resources/templates/expediagroup-sdk/pom.mustache @@ -11,7 +11,8 @@ 1.8 1.8 1.8 - 4.3.0 + key + secret 2.17.2 3.2.0 @@ -30,6 +31,18 @@ + + {{{targetSdkGroupId}}} + {{{targetSdkArtifactId}}} + {{{targetSdkVersion}}} + + + + com.google.guava + guava + 33.3.1-jre + + com.fasterxml.jackson.module jackson-module-kotlin @@ -45,11 +58,10 @@ jackson-databind ${jackson.version} - - com.expediagroup - rapid-sdk - ${rapid-java-sdk.sdk.version} + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} @@ -58,13 +70,6 @@ 2.23.1 - - - org.mapdb - mapdb - 3.1.0 - - org.apache.logging.log4j log4j-slf4j2-impl