diff --git a/.github/workflows/smithy-dafny-conversion.yml b/.github/workflows/smithy-dafny-conversion.yml index e835e5a7a5..fe2a55574e 100644 --- a/.github/workflows/smithy-dafny-conversion.yml +++ b/.github/workflows/smithy-dafny-conversion.yml @@ -11,6 +11,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + with: + submodules: recursive + - uses: actions/setup-java@v3 with: distribution: "corretto" diff --git a/.github/workflows/smithy-polymorph.yml b/.github/workflows/smithy-polymorph.yml index 842ee98c03..d7527ab945 100644 --- a/.github/workflows/smithy-polymorph.yml +++ b/.github/workflows/smithy-polymorph.yml @@ -11,6 +11,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + with: + submodules: recursive + - uses: actions/setup-java@v3 with: distribution: "corretto" @@ -44,6 +47,14 @@ jobs: arguments: :smithy-dafny-codegen:publishToMavenLocal build-root-directory: codegen + # Required for building Smithy-Dafny + # TODO: This step can and should be removed once Smithy-Python is on Maven central + - name: Locally cache smithy-python-codegen + uses: gradle/gradle-build-action@v2 + with: + arguments: :smithy-python-codegen:publishToMavenLocal + build-root-directory: submodules/smithy-python/codegen + - name: Sanity-check SQS test model via plugin uses: gradle/gradle-build-action@v2 with: diff --git a/.github/workflows/test_models_dafny_verification.yml b/.github/workflows/test_models_dafny_verification.yml index 9f78415d6d..5d798b82ee 100644 --- a/.github/workflows/test_models_dafny_verification.yml +++ b/.github/workflows/test_models_dafny_verification.yml @@ -79,7 +79,14 @@ jobs: run: | git config --global core.longpaths true - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + with: + submodules: recursive + + - uses: actions/setup-java@v3 + with: + distribution: 'corretto' + java-version: '17' - name: Setup Dafny uses: dafny-lang/setup-dafny-action@v1.7.0 diff --git a/.github/workflows/test_models_java_tests.yml b/.github/workflows/test_models_java_tests.yml index 9fbdcf590a..88381806e3 100644 --- a/.github/workflows/test_models_java_tests.yml +++ b/.github/workflows/test_models_java_tests.yml @@ -68,6 +68,8 @@ jobs: role-session-name: JavaTests - uses: actions/checkout@v3 + with: + submodules: recursive - name: Setup Dafny uses: dafny-lang/setup-dafny-action@v1.7.0 diff --git a/.github/workflows/test_models_net_tests.yml b/.github/workflows/test_models_net_tests.yml index 70d1b81cd5..2997c86856 100644 --- a/.github/workflows/test_models_net_tests.yml +++ b/.github/workflows/test_models_net_tests.yml @@ -79,6 +79,8 @@ jobs: role-session-name: NetTests - uses: actions/checkout@v3 + with: + submodules: recursive - name: Setup Dafny uses: dafny-lang/setup-dafny-action@v1.7.0 diff --git a/.github/workflows/test_models_python_tests.yml b/.github/workflows/test_models_python_tests.yml new file mode 100644 index 0000000000..34f737736b --- /dev/null +++ b/.github/workflows/test_models_python_tests.yml @@ -0,0 +1,130 @@ +# This workflow performs tests in Python. +name: Library Python tests + +on: + pull_request: + push: + branches: + - main-1.x + workflow_dispatch: + # Manual trigger for this workflow, either the normal version + # or the nightly build that uses the latest Dafny prerelease + # (accordingly to the "nightly" parameter). + inputs: + nightly: + description: 'Run the nightly build' + required: false + type: boolean + schedule: + # Nightly build against Dafny's nightly prereleases, + # for early warning of verification issues or regressions. + # Timing chosen to be adequately after Dafny's own nightly build, + # but this might need to be tweaked: + # https://github.com/dafny-lang/dafny/blob/master/.github/workflows/deep-tests.yml#L16 + - cron: "30 16 * * *" + +jobs: + testPython: + # Don't run the nightly build on forks + if: github.event_name != 'schedule' || github.repository_owner == 'smithy-lang' + strategy: + fail-fast: false # at least for development; see all errors + matrix: + dafny-version: [ 4.4.0 ] + library: [ + TestModels/dafny-dependencies/StandardLibrary, + TestModels/Aggregate, + # TestModels/AggregateReferences, + TestModels/Constraints, + TestModels/Constructor, + TestModels/Dependencies, + TestModels/Errors, + TestModels/Extendable, + TestModels/Extern, + TestModels/LanguageSpecificLogic, + TestModels/LocalService, + TestModels/MultipleModels, + TestModels/Refinement, + TestModels/Resource, + # TestModels/SimpleTypes/BigDecimal, + # TestModels/SimpleTypes/BigInteger, + TestModels/SimpleTypes/SimpleBlob, + TestModels/SimpleTypes/SimpleBoolean, + # TestModels/SimpleTypes/SimpleByte, + TestModels/SimpleTypes/SimpleDouble, + TestModels/SimpleTypes/SimpleEnum, + TestModels/SimpleTypes/SimpleEnumV2, + # TestModels/SimpleTypes/SimpleFloat, + TestModels/SimpleTypes/SimpleInteger, + TestModels/SimpleTypes/SimpleLong, + # TestModels/SimpleTypes/SimpleShort, + TestModels/SimpleTypes/SimpleString, + # TestModels/SimpleTypes/SimpleTimestamp, + TestModels/Union, + TestModels/aws-sdks/ddb, + TestModels/aws-sdks/kms, + ] + runs-on: "ubuntu-latest" + permissions: + id-token: write + contents: read + env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_NOLOGO: 1 + steps: + - name: Support longpaths on Git checkout + run: | + git config --global core.longpaths true + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-PolymorphTestModels-Role-us-west-2 + role-session-name: PythonTests + + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup Dafny + uses: dafny-lang/setup-dafny-action@v1.7.0 + with: + dafny-version: ${{ matrix.dafny-version }} + + - name: Setup Python for running tests + uses: actions/setup-python@v4 + with: + python-version: 3.11 + architecture: x64 + - run: | + python -m pip install --upgrade pip + pip install --upgrade tox + pip install poetry + + - name: Setup Java 17 for codegen + uses: actions/setup-java@v3 + with: + distribution: 'corretto' + java-version: '17' + + - name: Generate Polymorph Dafny and Python code + shell: bash + working-directory: ./${{ matrix.library }} + run: | + make polymorph_dafny DAFNY_VERSION_OPTION="--dafny-version $DAFNY_VERSION" + make polymorph_python DAFNY_VERSION_OPTION="--dafny-version $DAFNY_VERSION" + + - name: Compile ${{ matrix.library }} implementation + shell: bash + working-directory: ./${{ matrix.library }} + run: | + # This works because `node` is installed by default on GHA runners + CORES=$(node -e 'console.log(os.cpus().length)') + make transpile_python CORES=$CORES + + - name: Test ${{ matrix.library }} + working-directory: ./${{ matrix.library }} + shell: bash + run: | + make test_python diff --git a/.gitignore b/.gitignore index 4dfaf7fadb..0699a3b9a2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ /.idea /.history /.smithy.lsp.log +*.pyc +*/**/.idea */**/.idea /codegen/smithy-dafny-codegen/bin diff --git a/SmithyDafnyMakefile.mk b/SmithyDafnyMakefile.mk index 19e84a8e87..62683b2a8f 100644 --- a/SmithyDafnyMakefile.mk +++ b/SmithyDafnyMakefile.mk @@ -54,6 +54,14 @@ SMITHY_MODEL_ROOT := $(LIBRARY_ROOT)/Model CODEGEN_CLI_ROOT := $(SMITHY_DAFNY_ROOT)/codegen/smithy-dafny-codegen-cli GRADLEW := $(SMITHY_DAFNY_ROOT)/codegen/gradlew +# On macOS, sed requires an extra parameter of "" +OS := $(shell uname) +ifeq ($(OS), Darwin) + SED_PARAMETER := "" +else + SED_PARAMETER := +endif + ########################## Dafny targets # TODO: This target will not work for projects that use `replaceable` @@ -173,6 +181,12 @@ _transpile_implementation_all: transpile_implementation # If this variable is not provided, assume the project does not have `replaceable` modules, # and look for `Index.dfy` in the `src/` directory. transpile_implementation: SRC_INDEX_TRANSPILE=$(if $(SRC_INDEX),$(SRC_INDEX),src) +transpile_implementation: + ifeq ($(TARGET), py) + COMPILE_SUFFIX_OPTION := -compileSuffix:0 + else + COMPILE_SUFFIX_OPTION := -compileSuffix:1 + endif # At this time it is *significatly* faster # to give Dafny a single file # that includes everything @@ -189,7 +203,7 @@ transpile_implementation: -spillTargetCode:3 \ -compile:0 \ -optimizeErasableDatatypeWrapper:0 \ - -compileSuffix:1 \ + $(COMPILE_SUFFIX_OPTION) \ -unicodeChar:0 \ -functionSyntax:3 \ -useRuntimeLib \ @@ -219,6 +233,12 @@ _transpile_test_all: TRANSPILE_DEPENDENCIES=$(if ${DIR_STRUCTURE_V2}, $(patsubst _transpile_test_all: transpile_test # `find` looks for tests in either V1 or V2-styled project directories (single vs. multiple model files). +transpile_test: + ifeq ($(TARGET), py) + COMPILE_SUFFIX_OPTION := -compileSuffix:0 + else + COMPILE_SUFFIX_OPTION := -compileSuffix:1 + endif transpile_test: find ./dafny/**/$(TEST_INDEX_TRANSPILE) ./$(TEST_INDEX_TRANSPILE) -name "*.dfy" -name '*.dfy' | sed -e 's/^/include "/' -e 's/$$/"/' | dafny \ -stdin \ @@ -229,7 +249,7 @@ transpile_test: -runAllTests:1 \ -compile:0 \ -optimizeErasableDatatypeWrapper:0 \ - -compileSuffix:1 \ + $(COMPILE_SUFFIX_OPTION) \ -unicodeChar:0 \ -functionSyntax:3 \ -useRuntimeLib \ @@ -266,10 +286,13 @@ _polymorph: $(OUTPUT_DAFNY) \ $(OUTPUT_JAVA) \ $(OUTPUT_DOTNET) \ + $(OUTPUT_PYTHON) \ + $(MODULE_NAME) \ $(OUTPUT_RUST) \ --model $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(SMITHY_MODEL_ROOT)) \ --dependent-model $(PROJECT_ROOT)/$(SMITHY_DEPS) \ $(patsubst %, --dependent-model $(PROJECT_ROOT)/%/Model, $($(service_deps_var))) \ + $(DEPENDENCY_MODULE_NAMES) \ --namespace $($(namespace_var)) \ $(OUTPUT_LOCAL_SERVICE_$(SERVICE)) \ $(AWS_SDK_CMD) \ @@ -286,9 +309,12 @@ _polymorph_wrapped: $(OUTPUT_DAFNY_WRAPPED) \ $(OUTPUT_DOTNET_WRAPPED) \ $(OUTPUT_JAVA_WRAPPED) \ + $(OUTPUT_PYTHON_WRAPPED) \ + $(MODULE_NAME) \ --model $(if $(DIR_STRUCTURE_V2),$(LIBRARY_ROOT)/dafny/$(SERVICE)/Model,$(LIBRARY_ROOT)/Model) \ --dependent-model $(PROJECT_ROOT)/$(SMITHY_DEPS) \ $(patsubst %, --dependent-model $(PROJECT_ROOT)/%/Model, $($(service_deps_var))) \ + $(DEPENDENCY_MODULE_NAMES) \ --namespace $($(namespace_var)) \ --local-service-test \ $(AWS_SDK_CMD) \ @@ -375,6 +401,32 @@ polymorph_java: _polymorph_java: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated _polymorph_java: _polymorph +# Generates python code for all namespaces in this project +.PHONY: polymorph_python +polymorph_python: POLYMORPH_LANGUAGE_TARGET=python +polymorph_python: _polymorph_dependencies +polymorph_python: + set -e; for service in $(PROJECT_SERVICES) ; do \ + export service_deps_var=SERVICE_DEPS_$${service} ; \ + export namespace_var=SERVICE_NAMESPACE_$${service} ; \ + export SERVICE=$${service} ; \ + $(MAKE) _polymorph_python ; \ + done + +_polymorph_python: OUTPUT_PYTHON=--output-python $(LIBRARY_ROOT)/runtimes/python/src/$(PYTHON_MODULE_NAME)/smithygenerated +_polymorph_python: MODULE_NAME=--module-name $(PYTHON_MODULE_NAME) +# Python codegen MUST know dependencies' module names... +# This greps each service dependency's Makefile for two strings: +# 1. "SERVICE_NAMESPACE_$(dependency)" +# 2. "PYTHON_MODULE_NAME" +# , then assembles them together as +# "SERVICE_NAMESPACE_$(dependency)"="PYTHON_MODULE_NAME" +# , creating a map from a service namespace to its wrapping module name. +# We plan to move this information into Dafny project files. +# This is unfortunately one long line that breaks when I split it up... +_polymorph_python: DEPENDENCY_MODULE_NAMES=$(PYTHON_DEPENDENCY_MODULE_NAMES) +_polymorph_python: _polymorph + # Dependency for formatting generating Java code setup_prettier: npm i --no-save prettier@3 prettier-plugin-java@2.5 @@ -543,6 +595,118 @@ _clean: clean: _clean +########################## Python targets + +transpile_python: _python_underscore_dependency_extern_names +transpile_python: _python_underscore_extern_names +transpile_python: transpile_implementation_python +transpile_python: transpile_test_python +transpile_python: transpile_dependencies_python +transpile_python: _python_revert_underscore_extern_names +transpile_python: _python_revert_underscore_dependency_extern_names +transpile_python: _mv_internaldafny_python +# transpile_python: _remove_src_module_python +transpile_python: _rename_test_main_python + +transpile_implementation_python: TARGET=py +transpile_implementation_python: OUT=runtimes/python/dafny_src +transpile_implementation_python: COMPILE_SUFFIX_OPTION= +transpile_implementation_python: SRC_INDEX=$(PYTHON_SRC_INDEX) +transpile_implementation_python: _transpile_implementation_all +transpile_implementation_python: transpile_dependencies_python +transpile_implementation_python: transpile_src_python +transpile_implementation_python: transpile_test_python +transpile_implementation_python: _mv_internaldafny_python +transpile_implementation_python: _remove_src_module_python + +transpile_src_python: TARGET=py +transpile_src_python: OUT=runtimes/python/dafny_src +transpile_src_python: COMPILE_SUFFIX_OPTION= +transpile_src_python: SRC_INDEX=$(PYTHON_SRC_INDEX) +transpile_src_python: _transpile_implementation_all + +transpile_test_python: TARGET=py +transpile_test_python: OUT=runtimes/python/__main__ +transpile_test_python: COMPILE_SUFFIX_OPTION= +transpile_test_python: SRC_INDEX=$(PYTHON_SRC_INDEX) +transpile_test_python: TEST_INDEX=$(PYTHON_TEST_INDEX) +transpile_test_python: _transpile_test_all + +# Hacky workaround until Dafny supports per-language extern names. +# Replaces `.`s with `_`s in strings like `{:extern ".*"}`. +# This is flawed logic and should be removed, but is a reasonable band-aid for now. +# TODO-Python BLOCKING: Once Dafny supports per-language extern names, remove and replace with Pythonic extern names. +# This is tracked in https://github.com/dafny-lang/dafny/issues/4322. +# This may require new Smithy-Dafny logic to generate Pythonic extern names. +_python_underscore_extern_names: + find $(if ${DIR_STRUCTURE_V2},dafny/**/src,src) -regex ".*\.dfy" -type f -exec sed -i $(SED_PARAMETER) '/.*{:extern \".*\".*/s/\./_/g' {} \; + find $(if ${DIR_STRUCTURE_V2},dafny/**/Model,Model) -regex ".*\.dfy" -type f -exec sed -i $(SED_PARAMETER) '/.*{:extern \".*\.*"/s/\./_/g' {} \; + find $(if ${DIR_STRUCTURE_V2},dafny/**/test,test) -regex ".*\.dfy" -type f -exec sed -i $(SED_PARAMETER) '/.*{:extern \".*\".*/s/\./_/g' {} \; + +_python_underscore_dependency_extern_names: + $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) _python_underscore_extern_names + @$(foreach dependency, \ + $(PROJECT_DEPENDENCIES), \ + $(MAKE) -C $(PROJECT_ROOT)/$(dependency) _python_underscore_extern_names; \ + ) + +_python_revert_underscore_extern_names: + find $(if ${DIR_STRUCTURE_V2},dafny/**/src,src) -regex ".*\.dfy" -type f -exec sed -i $(SED_PARAMETER) '/.*{:extern \".*\".*/s/_/\./g' {} \; + find $(if ${DIR_STRUCTURE_V2},dafny/**/Model,Model) -regex ".*\.dfy" -type f -exec sed -i $(SED_PARAMETER) '/.*{:extern \".*\".*/s/_/\./g' {} \; 2>/dev/null + find $(if ${DIR_STRUCTURE_V2},dafny/**/test,test) -regex ".*\.dfy" -type f -exec sed -i $(SED_PARAMETER) '/.*{:extern \".*\".*/s/_/\./g' {} \; + +_python_revert_underscore_dependency_extern_names: + $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) _python_revert_underscore_extern_names + @$(foreach dependency, \ + $(PROJECT_DEPENDENCIES), \ + $(MAKE) -C $(PROJECT_ROOT)/$(dependency) _python_revert_underscore_extern_names; \ + ) + +# Move Dafny-generated code into its expected location in the Python module +_mv_internaldafny_python: + # Remove any previously generated Dafny code in src/, then copy in newly-generated code + rm -rf runtimes/python/src/$(PYTHON_MODULE_NAME)/internaldafny/generated/ + mkdir -p runtimes/python/src/$(PYTHON_MODULE_NAME)/internaldafny/generated/ + mv runtimes/python/dafny_src-py/*.py runtimes/python/src/$(PYTHON_MODULE_NAME)/internaldafny/generated + rm -rf runtimes/python/dafny_src-py + # Remove any previously generated Dafny code in test/, then copy in newly-generated code + rm -rf runtimes/python/test/internaldafny/generated + mkdir -p runtimes/python/test/internaldafny/generated + mv runtimes/python/__main__-py/*.py runtimes/python/test/internaldafny/generated + rm -rf runtimes/python/__main__-py + +# Versions of Dafny as of ~9/28 seem to ALWAYS write output to __main__.py, +# regardless of the OUT parameter...? +# We should figure out what happened and get a workaround +# For now, always write OUT to __main__, then manually rename the primary file... +# TODO-Python BLOCKING: Resolve this before releasing libraries +# Note the name internaldafny_test_executor is specifically chosen +# so as to not be picked up by pytest, +# which finds test_*.py or *_test.py files. +# This is neither, and will not be picked up by pytest. +# This file SHOULD not be run from a context that has not imported the wrapping shim, +# otherwise the tests will fail. +# We write an extern which WILL be picked up by pytest. +# This extern will import the wrapping shim, then import this `internaldafny_test_executor` to run the tests. +_rename_test_main_python: + mv runtimes/python/test/internaldafny/generated/__main__.py runtimes/python/test/internaldafny/generated/internaldafny_test_executor.py + +_remove_src_module_python: + # Remove the src/ `module_.py` file. + # There is a race condition between the src/ and test/ installation of this file. + # The file that is installed least recently is overwritten by the file installed most recently. + # The test/ file contains code to execute tests. The src/ file is largely empty. + # If the src/ file is installed most recently, tests will fail to run. + # By removing the src/ file, we ensure the test/ file is always the installed file. + rm runtimes/python/src/$(PYTHON_MODULE_NAME)/internaldafny/generated/module_.py + +transpile_dependencies_python: LANG=python +transpile_dependencies_python: transpile_dependencies + +test_python: + rm -rf runtimes/python/.tox + tox -c runtimes/python --verbose + ########################## local testing targets # These targets are added as a convenience for local development. diff --git a/TestModels/.gitignore b/TestModels/.gitignore index b453f0f8ed..5b10a56406 100644 --- a/TestModels/.gitignore +++ b/TestModels/.gitignore @@ -9,6 +9,10 @@ **/runtimes/java/src/main/dafny-generated/ **/runtimes/java/src/test/dafny-generated/ +# Dafny Generated Python +**/runtimes/python/src/**/internaldafny/generated/*.py +**/runtimes/python/test/internaldafny/generated/*.py + # Dafny Generated Rust # (Rust code generation is incomplete so we're patching and checking in for now) #**/runtimes/rust @@ -25,10 +29,21 @@ # Polymorph Generated Java **/runtimes/java/src/main/smithy-generated/ +# Polymorph Generated Python +**/runtimes/python/src/**/smithygenerated/ + # .NET Artifacts **/bin **/obj +# Python Artifacts +**/runtimes/python/src/**.egg-info/ +**/runtimes/python/.pytest_cache +**/runtimes/python/.tox +**/runtimes/python/build +**/runtimes/python/poetry.lock +**/runtimes/python/**/poetry.lock + # JetBrains **/.idea/ **/Folder.DotSettings.user diff --git a/TestModels/Aggregate/Makefile b/TestModels/Aggregate/Makefile index 35b2c325ed..6caa46d135 100644 --- a/TestModels/Aggregate/Makefile +++ b/TestModels/Aggregate/Makefile @@ -5,6 +5,10 @@ CORES=2 include ../SharedMakefile.mk + +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_aggregate + PROJECT_SERVICES := \ SimpleAggregate diff --git a/TestModels/Aggregate/runtimes/python/pyproject.toml b/TestModels/Aggregate/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..a427131bb9 --- /dev/null +++ b/TestModels/Aggregate/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-aggregate" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_aggregate", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +DafnyRuntimePython = "^4.4.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/Aggregate/runtimes/python/src/simple_aggregate/__init__.py b/TestModels/Aggregate/runtimes/python/src/simple_aggregate/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/Aggregate/runtimes/python/src/simple_aggregate/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/Aggregate/runtimes/python/test/internaldafny/extern/wrapped_simple_aggregate.py b/TestModels/Aggregate/runtimes/python/test/internaldafny/extern/wrapped_simple_aggregate.py new file mode 100644 index 0000000000..108cd62c8e --- /dev/null +++ b/TestModels/Aggregate/runtimes/python/test/internaldafny/extern/wrapped_simple_aggregate.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_aggregate_internaldafny_wrapped +from simple_aggregate.smithygenerated.simple_aggregate.client import SimpleAggregate +from simple_aggregate.smithygenerated.simple_aggregate.shim import SimpleAggregateShim +from simple_aggregate.smithygenerated.simple_aggregate.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_aggregate_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleAggregate(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleAggregate(wrapped_config) + wrapped_client = SimpleAggregateShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_aggregate_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/Aggregate/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/Aggregate/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..6587ffa3b6 --- /dev/null +++ b/TestModels/Aggregate/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_aggregate +import wrapped_simple_aggregate + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/Aggregate/runtimes/python/tox.ini b/TestModels/Aggregate/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/Aggregate/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/Constraints/Makefile b/TestModels/Constraints/Makefile index fa31cc5419..ba8eae2a39 100644 --- a/TestModels/Constraints/Makefile +++ b/TestModels/Constraints/Makefile @@ -5,6 +5,9 @@ CORES=2 include ../SharedMakefile.mk +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_constraints + PROJECT_SERVICES := \ SimpleConstraints diff --git a/TestModels/Constraints/runtimes/java/src/main/java/simple/constraints/internaldafny/types/__default.java b/TestModels/Constraints/runtimes/java/src/main/java/simple/constraints/internaldafny/types/__default.java index 1100632e4b..6edcc5fc2d 100644 --- a/TestModels/Constraints/runtimes/java/src/main/java/simple/constraints/internaldafny/types/__default.java +++ b/TestModels/Constraints/runtimes/java/src/main/java/simple/constraints/internaldafny/types/__default.java @@ -1,5 +1,5 @@ // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package simple.constraints.internaldafny.types; +package simple_constraints_internaldafny_types; public class __default extends _ExternBase___default {} diff --git a/TestModels/Constraints/runtimes/python/pyproject.toml b/TestModels/Constraints/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..30b81dec94 --- /dev/null +++ b/TestModels/Constraints/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-constraints" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_constraints", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/**/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/Constraints/runtimes/python/src/simple_constraints/__init__.py b/TestModels/Constraints/runtimes/python/src/simple_constraints/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/Constraints/runtimes/python/src/simple_constraints/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/Constraints/runtimes/python/test/internaldafny/extern/wrapped_simple_constraints.py b/TestModels/Constraints/runtimes/python/test/internaldafny/extern/wrapped_simple_constraints.py new file mode 100644 index 0000000000..de22914118 --- /dev/null +++ b/TestModels/Constraints/runtimes/python/test/internaldafny/extern/wrapped_simple_constraints.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_constraints_internaldafny_wrapped +from simple_constraints.smithygenerated.simple_constraints.client import SimpleConstraints +from simple_constraints.smithygenerated.simple_constraints.shim import SimpleConstraintsShim +from simple_constraints.smithygenerated.simple_constraints.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_constraints_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleConstraints(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleConstraints(wrapped_config) + wrapped_client = SimpleConstraintsShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_constraints_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/Constraints/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/Constraints/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..d9d20d752b --- /dev/null +++ b/TestModels/Constraints/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_constraints +import wrapped_simple_constraints + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/Constraints/runtimes/python/tox.ini b/TestModels/Constraints/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/Constraints/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/Constructor/Makefile b/TestModels/Constructor/Makefile index 2197110e56..4cd12da173 100644 --- a/TestModels/Constructor/Makefile +++ b/TestModels/Constructor/Makefile @@ -5,6 +5,9 @@ CORES=2 include ../SharedMakefile.mk +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_constructor + PROJECT_SERVICES := \ SimpleConstructor @@ -14,6 +17,7 @@ SERVICE_DEPS_SimpleConstructor := SMITHY_DEPS=dafny-dependencies/Model/traits.smithy + # This project has no dependencies # DEPENDENT-MODELS:= diff --git a/TestModels/Constructor/runtimes/python/pyproject.toml b/TestModels/Constructor/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..268de6b193 --- /dev/null +++ b/TestModels/Constructor/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-constructor" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_constructor", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +DafnyRuntimePython = "^4.4.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = { path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/Constructor/runtimes/python/src/simple_constructor/__init__.py b/TestModels/Constructor/runtimes/python/src/simple_constructor/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/Constructor/runtimes/python/src/simple_constructor/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/Constructor/runtimes/python/test/internaldafny/extern/wrapped_simple_constructor.py b/TestModels/Constructor/runtimes/python/test/internaldafny/extern/wrapped_simple_constructor.py new file mode 100644 index 0000000000..70a1dc69c8 --- /dev/null +++ b/TestModels/Constructor/runtimes/python/test/internaldafny/extern/wrapped_simple_constructor.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_constructor_internaldafny_wrapped +from simple_constructor.smithygenerated.simple_constructor.client import SimpleConstructor +from simple_constructor.smithygenerated.simple_constructor.shim import SimpleConstructorShim +from simple_constructor.smithygenerated.simple_constructor.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_constructor_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleConstructor(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleConstructor(wrapped_config) + wrapped_client = SimpleConstructorShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_constructor_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/Constructor/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/Constructor/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..49ce294332 --- /dev/null +++ b/TestModels/Constructor/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_constructor +import wrapped_simple_constructor + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/Constructor/runtimes/python/tox.ini b/TestModels/Constructor/runtimes/python/tox.ini new file mode 100644 index 0000000000..8333ab1863 --- /dev/null +++ b/TestModels/Constructor/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ -p test --import-mode importlib \ No newline at end of file diff --git a/TestModels/Dependencies/Makefile b/TestModels/Dependencies/Makefile index 5186198bcb..8ef6232859 100644 --- a/TestModels/Dependencies/Makefile +++ b/TestModels/Dependencies/Makefile @@ -5,6 +5,9 @@ CORES=2 include ../SharedMakefile.mk +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_dependencies + PROJECT_SERVICES := \ SimpleDependencies diff --git a/TestModels/Dependencies/runtimes/python/pyproject.toml b/TestModels/Dependencies/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..034254e271 --- /dev/null +++ b/TestModels/Dependencies/runtimes/python/pyproject.toml @@ -0,0 +1,30 @@ +[tool.poetry] +name = "simple-dependencies" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_dependencies", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +simple-resources = { path = "../../../Resource/runtimes/python", develop = false} +simple-errors = { path = "../../../Errors/runtimes/python", develop = false} +simple-extendable-resources = { path = "../../../Extendable/runtimes/python", develop = false} +simple-constraints = { path = "../../../Constraints/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/Dependencies/runtimes/python/src/simple_dependencies/__init__.py b/TestModels/Dependencies/runtimes/python/src/simple_dependencies/__init__.py new file mode 100644 index 0000000000..9356d4a805 --- /dev/null +++ b/TestModels/Dependencies/runtimes/python/src/simple_dependencies/__init__.py @@ -0,0 +1,18 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") + + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies IN ORDER THEY MUST BE LOADED. +import standard_library +import simple_errors +import simple_resources +import simple_constraints +import simple_extendable_resources diff --git a/TestModels/Dependencies/runtimes/python/test/internaldafny/extern/wrapped_simple_dependencies.py b/TestModels/Dependencies/runtimes/python/test/internaldafny/extern/wrapped_simple_dependencies.py new file mode 100644 index 0000000000..e83d93c02c --- /dev/null +++ b/TestModels/Dependencies/runtimes/python/test/internaldafny/extern/wrapped_simple_dependencies.py @@ -0,0 +1,19 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import simple_dependencies_internaldafny +import simple_dependencies_internaldafny_wrapped +from simple_dependencies.smithygenerated.simple_dependencies.client import SimpleDependencies +from simple_dependencies.smithygenerated.simple_dependencies.config import dafny_config_to_smithy_config +from simple_dependencies.smithygenerated.simple_dependencies.shim import SimpleDependenciesShim +import Wrappers + +@staticmethod +def WrappedSimpleDependencies(config): + # Use an alternate internal-Dafny constructor to create the Dafny client, then pass that directly into the Smithy client + dafny_client = simple_dependencies_internaldafny.default__.SimpleDependencies(config).value + impl = SimpleDependencies(dafny_client=dafny_client) + wrapped_client = SimpleDependenciesShim(impl) + return Wrappers.Result_Success(wrapped_client) + +simple_dependencies_internaldafny_wrapped.default__.WrappedSimpleDependencies = WrappedSimpleDependencies diff --git a/TestModels/Dependencies/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/Dependencies/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..b15f3a80bd --- /dev/null +++ b/TestModels/Dependencies/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,30 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_dependencies +import wrapped_simple_dependencies + + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/Dependencies/runtimes/python/tox.ini b/TestModels/Dependencies/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/Dependencies/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/Dependencies/src/Index.dfy b/TestModels/Dependencies/src/Index.dfy index 5f4ee12b53..e37128574b 100644 --- a/TestModels/Dependencies/src/Index.dfy +++ b/TestModels/Dependencies/src/Index.dfy @@ -3,7 +3,7 @@ include "../Model/SimpleDependenciesTypes.dfy" include "SimpleDependenciesImpl.dfy" -module {:extern "simple.dependencies.internaldafny" } SimpleDependencies refines AbstractSimpleDependenciesService { +module {:extern "simple_dependencies_internaldafny" } SimpleDependencies refines AbstractSimpleDependenciesService { import Operations = SimpleDependenciesImpl import SimpleResourcesTypes import SimpleConstraints diff --git a/TestModels/Dependencies/src/WrappedSimpleDependenciesImpl.dfy b/TestModels/Dependencies/src/WrappedSimpleDependenciesImpl.dfy index 3a22649ea6..e5897e49d7 100644 --- a/TestModels/Dependencies/src/WrappedSimpleDependenciesImpl.dfy +++ b/TestModels/Dependencies/src/WrappedSimpleDependenciesImpl.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 include "../Model/SimpleDependenciesTypesWrapped.dfy" -module {:extern "simple.dependencies.internaldafny.wrapped"} WrappedSimpleDependenciesService refines WrappedAbstractSimpleDependenciesService { +module {:extern "simple_dependencies_internaldafny_wrapped"} WrappedSimpleDependenciesService refines WrappedAbstractSimpleDependenciesService { import WrappedService = SimpleDependencies function method WrappedDefaultSimpleDependenciesConfig(): SimpleDependenciesConfig { SimpleDependenciesConfig( diff --git a/TestModels/Errors/Makefile b/TestModels/Errors/Makefile index 21cda8f8df..4497f48088 100644 --- a/TestModels/Errors/Makefile +++ b/TestModels/Errors/Makefile @@ -5,6 +5,9 @@ CORES=2 include ../SharedMakefile.mk +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_errors + PROJECT_SERVICES := \ SimpleErrors diff --git a/TestModels/Errors/Model/Errors.smithy b/TestModels/Errors/Model/Errors.smithy index 7ddd74bd59..9cc7e0702a 100644 --- a/TestModels/Errors/Model/Errors.smithy +++ b/TestModels/Errors/Model/Errors.smithy @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 namespace simple.errors +use aws.polymorph#localService + @aws.polymorph#localService( sdkId: "SimpleErrors", config: SimpleErrorsConfig, diff --git a/TestModels/Errors/runtimes/python/pyproject.toml b/TestModels/Errors/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..7210ada002 --- /dev/null +++ b/TestModels/Errors/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-errors" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_errors", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/Errors/runtimes/python/src/simple_errors/__init__.py b/TestModels/Errors/runtimes/python/src/simple_errors/__init__.py new file mode 100644 index 0000000000..0b615f63a7 --- /dev/null +++ b/TestModels/Errors/runtimes/python/src/simple_errors/__init__.py @@ -0,0 +1,16 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") + diff --git a/TestModels/Errors/runtimes/python/test/internaldafny/extern/wrapped_simple_errors.py b/TestModels/Errors/runtimes/python/test/internaldafny/extern/wrapped_simple_errors.py new file mode 100644 index 0000000000..2a7f942064 --- /dev/null +++ b/TestModels/Errors/runtimes/python/test/internaldafny/extern/wrapped_simple_errors.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_errors_internaldafny_wrapped +from simple_errors.smithygenerated.simple_errors.client import SimpleErrors +from simple_errors.smithygenerated.simple_errors.shim import SimpleErrorsShim +from simple_errors.smithygenerated.simple_errors.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_errors_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleErrors(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleErrors(wrapped_config) + wrapped_client = SimpleErrorsShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_errors_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/Errors/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/Errors/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..ff4b3bebea --- /dev/null +++ b/TestModels/Errors/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_errors +import wrapped_simple_errors + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/Errors/runtimes/python/tox.ini b/TestModels/Errors/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/Errors/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/Errors/src/Index.dfy b/TestModels/Errors/src/Index.dfy index 4588871b96..f090786a50 100644 --- a/TestModels/Errors/src/Index.dfy +++ b/TestModels/Errors/src/Index.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 include "SimpleErrorsImpl.dfy" -module {:extern "simple.errors.internaldafny" } SimpleErrors refines AbstractSimpleErrorsService { +module {:extern "simple_errors_internaldafny" } SimpleErrors refines AbstractSimpleErrorsService { import Operations = SimpleErrorsImpl function method DefaultSimpleErrorsConfig(): SimpleErrorsConfig { diff --git a/TestModels/Errors/src/WrappedSimpleErrorsImpl.dfy b/TestModels/Errors/src/WrappedSimpleErrorsImpl.dfy index 35fb0eaba1..cf9efaad19 100644 --- a/TestModels/Errors/src/WrappedSimpleErrorsImpl.dfy +++ b/TestModels/Errors/src/WrappedSimpleErrorsImpl.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 include "../Model/SimpleErrorsTypesWrapped.dfy" -module {:extern "simple.errors.internaldafny.wrapped"} WrappedSimpleErrorsService refines WrappedAbstractSimpleErrorsService { +module {:extern "simple_errors_internaldafny_wrapped"} WrappedSimpleErrorsService refines WrappedAbstractSimpleErrorsService { import WrappedService = SimpleErrors function method WrappedDefaultSimpleErrorsConfig(): SimpleErrorsConfig { SimpleErrorsConfig diff --git a/TestModels/Extendable/Makefile b/TestModels/Extendable/Makefile index 00777e807a..96317754fa 100644 --- a/TestModels/Extendable/Makefile +++ b/TestModels/Extendable/Makefile @@ -7,6 +7,9 @@ include ../SharedMakefile.mk NAMESPACE=simple.extendable.resources +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_extendable_resources + PROJECT_SERVICES := \ SimpleExtendableResources diff --git a/TestModels/Extendable/runtimes/python/pyproject.toml b/TestModels/Extendable/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..f308b54218 --- /dev/null +++ b/TestModels/Extendable/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-extendable-resources" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_extendable_resources", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/Extendable/runtimes/python/src/simple_extendable_resources/__init__.py b/TestModels/Extendable/runtimes/python/src/simple_extendable_resources/__init__.py new file mode 100644 index 0000000000..6f129a7666 --- /dev/null +++ b/TestModels/Extendable/runtimes/python/src/simple_extendable_resources/__init__.py @@ -0,0 +1,19 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") + + +# TODO-Python-PYTHONPATH: Remove +import native_resource \ No newline at end of file diff --git a/TestModels/Extendable/runtimes/python/src/simple_extendable_resources/internaldafny/extern/native_resource.py b/TestModels/Extendable/runtimes/python/src/simple_extendable_resources/internaldafny/extern/native_resource.py new file mode 100644 index 0000000000..6c2eccd80a --- /dev/null +++ b/TestModels/Extendable/runtimes/python/src/simple_extendable_resources/internaldafny/extern/native_resource.py @@ -0,0 +1,49 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import ExtendableResource +from simple_extendable_resources.smithygenerated.simple_extendable_resources.references import IExtendableResource + +class default__(IExtendableResource): + # Importing the type at top-level results in circular dependency import issue + from simple_extendable_resources_internaldafny_types import IExtendableResource as DafnyIExtendableResource + _impl: DafnyIExtendableResource + + def __init__(self, _impl): + self._impl = _impl + + def GetExtendableResourceData(self, nativeInput): + return self._impl.GetExtendableResourceData(nativeInput) + + def AlwaysModeledError(self, nativeInput): + return self._impl.AlwaysModeledError(nativeInput) + + def AlwaysMultipleErrors(self, nativeInput): + return self._impl.AlwaysMultipleErrors(nativeInput) + + def AlwaysOpaqueError(self, nativeInput): + if nativeInput.value == None: + raise Exception("Python Hard Coded Exception") + return self._impl.AlwaysOpaqueError(nativeInput) + +class NativeResourceFactory: + + @staticmethod + def DafnyFactory(): + dafny_resource = ExtendableResource.ExtendableResource() + dafny_resource.ctor__() + native_resource = default__(dafny_resource) + return native_resource + + +# TODO-Python-PYTHONPATH: Remove import and try/catch, only needed for path issues +# Problem: simple_extendable_resources_internaldafny_nativeresourcefactory is only built in tests, +# but SimpleDependencies relies on this current file, +# and ExtendableResource's tests are not built for another file's src. +# Workaround: Only import if able. +# Fix: Remove PYTHONPATH workaround. +try: + import simple_extendable_resources_internaldafny_nativeresourcefactory + simple_extendable_resources_internaldafny_nativeresourcefactory.default__ = NativeResourceFactory +except ModuleNotFoundError: + pass diff --git a/TestModels/Extendable/runtimes/python/test/internaldafny/extern/wrapped_simple_extendable_resources.py b/TestModels/Extendable/runtimes/python/test/internaldafny/extern/wrapped_simple_extendable_resources.py new file mode 100644 index 0000000000..db830ea864 --- /dev/null +++ b/TestModels/Extendable/runtimes/python/test/internaldafny/extern/wrapped_simple_extendable_resources.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_extendable_resources_internaldafny_wrapped +from simple_extendable_resources.smithygenerated.simple_extendable_resources.client import SimpleExtendableResources +from simple_extendable_resources.smithygenerated.simple_extendable_resources.shim import SimpleExtendableResourcesShim +from simple_extendable_resources.smithygenerated.simple_extendable_resources.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_extendable_resources_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleExtendableResources(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleExtendableResources(wrapped_config) + wrapped_client = SimpleExtendableResourcesShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_extendable_resources_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/Extendable/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/Extendable/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..734a27c8e9 --- /dev/null +++ b/TestModels/Extendable/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_extendable_resources +import wrapped_simple_extendable_resources + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/Extendable/runtimes/python/tox.ini b/TestModels/Extendable/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/Extendable/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/Extendable/smithy-build.json b/TestModels/Extendable/smithy-build.json new file mode 100644 index 0000000000..a15a70582d --- /dev/null +++ b/TestModels/Extendable/smithy-build.json @@ -0,0 +1,11 @@ +{ + "version": "1.0", + "plugins": { + "dafny-client-codegen": { + "service": "simple.extendable.resources#SimpleExtendableResources", + "module": "simple_extendable_resources", + "moduleVersion": "0.0.1", + "protocol": "aws.polymorph#localService" + } + } +} diff --git a/TestModels/Extendable/src/Index.dfy b/TestModels/Extendable/src/Index.dfy index d3ccc91d8e..ac9f6500c9 100644 --- a/TestModels/Extendable/src/Index.dfy +++ b/TestModels/Extendable/src/Index.dfy @@ -5,7 +5,7 @@ include "../Model/SimpleExtendableResourcesTypes.dfy" include "./SimpleExtendableResourcesOperations.dfy" module - {:extern "simple.extendable.resources.internaldafny"} + {:extern "simple_extendable_resources_internaldafny"} SimpleExtendableResources refines AbstractSimpleExtendableResourcesService { import Operations = SimpleExtendableResourcesOperations diff --git a/TestModels/Extendable/src/WrappedIndex.dfy b/TestModels/Extendable/src/WrappedIndex.dfy index cba052b7aa..eeb5787b40 100644 --- a/TestModels/Extendable/src/WrappedIndex.dfy +++ b/TestModels/Extendable/src/WrappedIndex.dfy @@ -4,7 +4,7 @@ include "../Model/SimpleExtendableResourcesTypesWrapped.dfy" module - {:extern "simple.extendable.resources.internaldafny.wrapped"} + {:extern "simple_extendable_resources_internaldafny_wrapped"} WrappedSimpleExtendableResources refines WrappedAbstractSimpleExtendableResourcesService { import WrappedService = SimpleExtendableResources diff --git a/TestModels/Extendable/test/NativeResourceFactory.dfy b/TestModels/Extendable/test/NativeResourceFactory.dfy index 1bb787b477..85874932e6 100644 --- a/TestModels/Extendable/test/NativeResourceFactory.dfy +++ b/TestModels/Extendable/test/NativeResourceFactory.dfy @@ -4,11 +4,12 @@ include "../src/Index.dfy" module - {:extern "simple.extendable.resources.internaldafny.nativeresourcefactory"} + {:extern "simple_extendable_resources_internaldafny_nativeresourcefactory"} NativeResourceFactory { import Types = SimpleExtendableResourcesTypes - method {:extern "DafnyFactory"} DafnyFactory() returns (output: Types.IExtendableResource) + method {:extern "DafnyFactory"} DafnyFactory() + returns (output: Types.IExtendableResource) ensures output.ValidState() && fresh(output.History) && fresh(output.Modifies) } diff --git a/TestModels/Extern/Makefile b/TestModels/Extern/Makefile index a8a0c1fccd..74d727b923 100644 --- a/TestModels/Extern/Makefile +++ b/TestModels/Extern/Makefile @@ -5,6 +5,9 @@ CORES=2 include ../SharedMakefile.mk +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_dafnyextern + PROJECT_SERVICES := \ SimpleExtern diff --git a/TestModels/Extern/runtimes/python/pyproject.toml b/TestModels/Extern/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..e4eae1721b --- /dev/null +++ b/TestModels/Extern/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-extern" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_dafnyextern", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/Extern/runtimes/python/src/simple_dafnyextern/__init__.py b/TestModels/Extern/runtimes/python/src/simple_dafnyextern/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/Extern/runtimes/python/src/simple_dafnyextern/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/Extern/runtimes/python/src/simple_dafnyextern/internaldafny/extern/ExternConstructor.py b/TestModels/Extern/runtimes/python/src/simple_dafnyextern/internaldafny/extern/ExternConstructor.py new file mode 100644 index 0000000000..0cf1449dbe --- /dev/null +++ b/TestModels/Extern/runtimes/python/src/simple_dafnyextern/internaldafny/extern/ExternConstructor.py @@ -0,0 +1,23 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from simple_dafnyextern.internaldafny.generated.ExternConstructor import * +import simple_dafnyextern_internaldafny_types +import Wrappers +import _dafny + +class ExternConstructorClass: + + def __init__(self, input): + if (_dafny.string_of(input) == "Error"): + raise Exception("Python Constructor Exception") + self.inputVal = input + + def GetValue(self): + return Wrappers.Result_Success(self.inputVal) + + @staticmethod + def Build(input): + try: + return Wrappers.Result_Success(ExternConstructorClass(input)) + except Exception as e: + return Wrappers.Result_Failure(simple_dafnyextern_internaldafny_types.Error_Opaque(e)) \ No newline at end of file diff --git a/TestModels/Extern/runtimes/python/src/simple_dafnyextern/internaldafny/extern/SimpleExternImpl.py b/TestModels/Extern/runtimes/python/src/simple_dafnyextern/internaldafny/extern/SimpleExternImpl.py new file mode 100644 index 0000000000..6b5e8cf161 --- /dev/null +++ b/TestModels/Extern/runtimes/python/src/simple_dafnyextern/internaldafny/extern/SimpleExternImpl.py @@ -0,0 +1,24 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from simple_dafnyextern.internaldafny.generated.SimpleExternImpl import * +import simple_dafnyextern_internaldafny_types +import Wrappers + +@staticmethod +def GetExtern(config, input): + out = simple_dafnyextern_internaldafny_types.GetExternOutput_GetExternOutput( + blobValue = input.blobValue, + booleanValue = input.booleanValue, + stringValue = input.stringValue, + integerValue = input.integerValue, + longValue = input.longValue + ) + return Wrappers.Result_Success(out) + +@staticmethod +def ExternMustError(config, input): + exception = Exception(input) + return Wrappers.Result_Failure(simple_dafnyextern_internaldafny_types.Error_Opaque(exception)) + +default__.GetExtern = GetExtern +default__.ExternMustError = ExternMustError \ No newline at end of file diff --git a/TestModels/Extern/runtimes/python/test/internaldafny/extern/wrapped_simple_extern.py b/TestModels/Extern/runtimes/python/test/internaldafny/extern/wrapped_simple_extern.py new file mode 100644 index 0000000000..fa6ff6b512 --- /dev/null +++ b/TestModels/Extern/runtimes/python/test/internaldafny/extern/wrapped_simple_extern.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_dafnyextern_internaldafny_wrapped +from simple_dafnyextern.smithygenerated.simple_dafnyextern.client import SimpleExtern +from simple_dafnyextern.smithygenerated.simple_dafnyextern.shim import SimpleExternShim +from simple_dafnyextern.smithygenerated.simple_dafnyextern.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_dafnyextern_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleExtern(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleExtern(wrapped_config) + wrapped_client = SimpleExternShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_dafnyextern_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/Extern/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/Extern/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..4ff9c59400 --- /dev/null +++ b/TestModels/Extern/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_dafnyextern +import wrapped_simple_extern + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/Extern/runtimes/python/tox.ini b/TestModels/Extern/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/Extern/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/LanguageSpecificLogic/Makefile b/TestModels/LanguageSpecificLogic/Makefile index b32098635b..6b9889c4a7 100644 --- a/TestModels/LanguageSpecificLogic/Makefile +++ b/TestModels/LanguageSpecificLogic/Makefile @@ -5,6 +5,9 @@ CORES=2 include ../SharedMakefile.mk +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=language_specific_logic + PROJECT_SERVICES := \ LanguageSpecificLogic @@ -17,6 +20,8 @@ SMITHY_DEPS=dafny-dependencies/Model/traits.smithy # Projects using replaceable modules must explicitly define these values. NET_SRC_INDEX=src/replaces/net NET_TEST_INDEX=test/replaces/net +PYTHON_SRC_INDEX=src/replaces/python +PYTHON_TEST_INDEX=test/replaces/python # This project has no dependencies # DEPENDENT-MODELS:= diff --git a/TestModels/LanguageSpecificLogic/runtimes/net/Extern/ExternOperations.cs b/TestModels/LanguageSpecificLogic/runtimes/net/Extern/ExternOperations.cs index 5efcd3f6e5..dc8d2e5b7e 100644 --- a/TestModels/LanguageSpecificLogic/runtimes/net/Extern/ExternOperations.cs +++ b/TestModels/LanguageSpecificLogic/runtimes/net/Extern/ExternOperations.cs @@ -6,7 +6,7 @@ using Language.Specific.Logic; using Wrappers_Compile; -namespace NetLanguageSpecificLogicImpl_Compile +namespace LanguageSpecificLogicImpl_Compile { public partial class __default { @@ -16,7 +16,7 @@ public static language.specific.logic.internaldafny.types._IError > GetNetRuntimeVersion ( - NetLanguageSpecificLogicImpl_Compile._IConfig config + LanguageSpecificLogicImpl_Compile._IConfig config ) { return Wrappers_Compile.Result< Dafny.ISequence, diff --git a/TestModels/LanguageSpecificLogic/runtimes/python/pyproject.toml b/TestModels/LanguageSpecificLogic/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..c6b75b9d8f --- /dev/null +++ b/TestModels/LanguageSpecificLogic/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "language-specific-logic" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "language_specific_logic", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/LanguageSpecificLogic/runtimes/python/src/language_specific_logic/__init__.py b/TestModels/LanguageSpecificLogic/runtimes/python/src/language_specific_logic/__init__.py new file mode 100644 index 0000000000..8d6f9b8d1c --- /dev/null +++ b/TestModels/LanguageSpecificLogic/runtimes/python/src/language_specific_logic/__init__.py @@ -0,0 +1,17 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") + +import language_specific_logic_externs \ No newline at end of file diff --git a/TestModels/LanguageSpecificLogic/runtimes/python/src/language_specific_logic/internaldafny/extern/language_specific_logic_externs.py b/TestModels/LanguageSpecificLogic/runtimes/python/src/language_specific_logic/internaldafny/extern/language_specific_logic_externs.py new file mode 100644 index 0000000000..b066c6f5e1 --- /dev/null +++ b/TestModels/LanguageSpecificLogic/runtimes/python/src/language_specific_logic/internaldafny/extern/language_specific_logic_externs.py @@ -0,0 +1,13 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import LanguageSpecificLogicImpl +import sys +import Wrappers +import _dafny + +class default__(LanguageSpecificLogicImpl.default__): + @staticmethod + def GetPythonRuntimeVersion(config): + return Wrappers.Result_Success(_dafny.Seq(str(sys.version))) + +LanguageSpecificLogicImpl.default__ = default__ \ No newline at end of file diff --git a/TestModels/LanguageSpecificLogic/runtimes/python/test/internaldafny/extern/wrapped_language_specific_logic.py b/TestModels/LanguageSpecificLogic/runtimes/python/test/internaldafny/extern/wrapped_language_specific_logic.py new file mode 100644 index 0000000000..e463cd5977 --- /dev/null +++ b/TestModels/LanguageSpecificLogic/runtimes/python/test/internaldafny/extern/wrapped_language_specific_logic.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import language_specific_logic_internaldafny_wrapped +from language_specific_logic.smithygenerated.language_specific_logic.client import LanguageSpecificLogic +from language_specific_logic.smithygenerated.language_specific_logic.shim import LanguageSpecificLogicShim +from language_specific_logic.smithygenerated.language_specific_logic.config import dafny_config_to_smithy_config +import Wrappers + +class default__(language_specific_logic_internaldafny_wrapped.default__): + + @staticmethod + def WrappedLanguageSpecificLogic(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = LanguageSpecificLogic(wrapped_config) + wrapped_client = LanguageSpecificLogicShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +language_specific_logic_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/LanguageSpecificLogic/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/LanguageSpecificLogic/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..84135955d3 --- /dev/null +++ b/TestModels/LanguageSpecificLogic/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import language_specific_logic +import wrapped_language_specific_logic + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/LanguageSpecificLogic/runtimes/python/tox.ini b/TestModels/LanguageSpecificLogic/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/LanguageSpecificLogic/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/LanguageSpecificLogic/src/replaces/python/Index.dfy b/TestModels/LanguageSpecificLogic/src/replaces/python/Index.dfy new file mode 100644 index 0000000000..3cf232c6eb --- /dev/null +++ b/TestModels/LanguageSpecificLogic/src/replaces/python/Index.dfy @@ -0,0 +1,8 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Include generic Index, which is the root of generic Dafny code +include "../../Index.dfy" + +// Include Python-specific replacements +include "LanguageSpecificLogicImpl.dfy" diff --git a/TestModels/LanguageSpecificLogic/src/replaces/python/LanguageSpecificLogicImpl.dfy b/TestModels/LanguageSpecificLogic/src/replaces/python/LanguageSpecificLogicImpl.dfy new file mode 100644 index 0000000000..c9f8886f1b --- /dev/null +++ b/TestModels/LanguageSpecificLogic/src/replaces/python/LanguageSpecificLogicImpl.dfy @@ -0,0 +1,32 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +include "../../LanguageSpecificLogicImpl.dfy" + +module PythonLanguageSpecificLogicImpl replaces LanguageSpecificLogicImpl { + // This method is listed as an operation on the service in the Smithy model. + // Smithy-Dafny will write code to call this operation. + // Internally, the generated Dafny will call the extern. + // This provides a consistent interface for users. + method GetRuntimeInformation(config: InternalConfig) + returns (output: Result) + ensures output.Success? ==> + && output.value.language == "Python" + && output.value.runtime != "" + { + var runtimeInfo :- expect GetRuntimeInformationPythonExternMethod(config); + var getRuntimeInformationOutput := GetRuntimeInformationOutput( + language := "Python", + runtime := runtimeInfo + ); + return Success(getRuntimeInformationOutput); + } + + // This method is NOT listed as an operation on the service in the Smithy model. + // Since this is an extern method with a different name per language, we can't define + // the interface for this method on the Smithy model. + // Instead, we define the `AllRuntimesMethod` which IS a Smithy operation + // and call this method from there. + method {:extern "GetPythonRuntimeVersion" } GetRuntimeInformationPythonExternMethod(config: InternalConfig) + returns (output: Result) + ensures output.Success? ==> output.value != "" +} diff --git a/TestModels/LanguageSpecificLogic/test/replaces/python/Index.dfy b/TestModels/LanguageSpecificLogic/test/replaces/python/Index.dfy new file mode 100644 index 0000000000..55e5654a8e --- /dev/null +++ b/TestModels/LanguageSpecificLogic/test/replaces/python/Index.dfy @@ -0,0 +1,9 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// No generic test Index to include + +// Include Python-specific replacements +include "../../../src/WrappedLanguageSpecificLogicImpl.dfy" +include "LanguageSpecificLogicImplTest.dfy" +include "WrappedLanguageSpecificLogicImplTest.dfy" \ No newline at end of file diff --git a/TestModels/LanguageSpecificLogic/test/replaces/python/LanguageSpecificLogicImplTest.dfy b/TestModels/LanguageSpecificLogic/test/replaces/python/LanguageSpecificLogicImplTest.dfy new file mode 100644 index 0000000000..86c6d811e3 --- /dev/null +++ b/TestModels/LanguageSpecificLogic/test/replaces/python/LanguageSpecificLogicImplTest.dfy @@ -0,0 +1,28 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +include "Index.dfy" + +// Note that by replacing a `replaceable` module, this file will also run tests from that module. +module PythonLanguageSpecificLogicImplTest replaces LanguageSpecificLogicImplTest { + + method{:test} PythonSpecificTests() { + var client :- expect LanguageSpecificLogic.LanguageSpecificLogic(); + TestPythonClient(client); + } + + method TestPythonClient(client: ILanguageSpecificLogicClient) + requires client.ValidState() + modifies client.Modifies + ensures client.ValidState() + { + var output := client.GetRuntimeInformation(); + expect output.Success?; + // For Python-only tests, we can assert the output language is Python + expect output.value.language == "Python"; + // We could also assert some result on the extern's result (i.e. runtime version), but won't + + // We should ONLY see printed values like "Python language: Python". + // We should ALSO see printed values like "Generic language: Python" from the `replaceable` tests. + print"Python language: ", output.value.language, "; Python runtime: ", output.value.runtime; + } +} \ No newline at end of file diff --git a/TestModels/LanguageSpecificLogic/test/replaces/python/WrappedLanguageSpecificLogicImplTest.dfy b/TestModels/LanguageSpecificLogic/test/replaces/python/WrappedLanguageSpecificLogicImplTest.dfy new file mode 100644 index 0000000000..c4885ced25 --- /dev/null +++ b/TestModels/LanguageSpecificLogic/test/replaces/python/WrappedLanguageSpecificLogicImplTest.dfy @@ -0,0 +1,15 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +include "../../../src/WrappedLanguageSpecificLogicImpl.dfy" +include "../../WrappedLanguageSpecificLogicImplTest.dfy" +include "LanguageSpecificLogicImplTest.dfy" + +// Note that by replacing a `replaceable` module, this file will also run tests from that module. +module PythonWrappedLanguageSpecificLogicTest replaces WrappedLanguageSpecificLogicTest { + import PythonLanguageSpecificLogicImplTest + + method{:test} WrappedPythonOnlyTests() { + var client :- expect WrappedLanguageSpecificLogicService.WrappedLanguageSpecificLogic(); + PythonLanguageSpecificLogicImplTest.TestPythonClient(client); + } +} diff --git a/TestModels/LocalService/Makefile b/TestModels/LocalService/Makefile index c08fad2316..bde4baed2c 100644 --- a/TestModels/LocalService/Makefile +++ b/TestModels/LocalService/Makefile @@ -7,6 +7,9 @@ include ../SharedMakefile.mk NAMESPACE=simple.localService +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_localservice + PROJECT_SERVICES := \ SimpleLocalService diff --git a/TestModels/LocalService/runtimes/python/pyproject.toml b/TestModels/LocalService/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..877fd77024 --- /dev/null +++ b/TestModels/LocalService/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-localservice" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_localservice", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/LocalService/runtimes/python/src/simple_localservice/__init__.py b/TestModels/LocalService/runtimes/python/src/simple_localservice/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/LocalService/runtimes/python/src/simple_localservice/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/LocalService/runtimes/python/test/internaldafny/extern/wrapped_simple_localservice.py b/TestModels/LocalService/runtimes/python/test/internaldafny/extern/wrapped_simple_localservice.py new file mode 100644 index 0000000000..f8e1d2f909 --- /dev/null +++ b/TestModels/LocalService/runtimes/python/test/internaldafny/extern/wrapped_simple_localservice.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_localservice_internaldafny_wrapped +from simple_localservice.smithygenerated.simple_localservice.client import SimpleLocalService +from simple_localservice.smithygenerated.simple_localservice.shim import SimpleLocalServiceShim +from simple_localservice.smithygenerated.simple_localservice.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_localservice_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleLocalService(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleLocalService(wrapped_config) + wrapped_client = SimpleLocalServiceShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_localservice_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/LocalService/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/LocalService/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..85a69e22f0 --- /dev/null +++ b/TestModels/LocalService/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_localservice +import wrapped_simple_localservice + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/LocalService/runtimes/python/tox.ini b/TestModels/LocalService/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/LocalService/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/MultipleModels/Makefile b/TestModels/MultipleModels/Makefile index 4a22f49ea2..81c55d4b3a 100644 --- a/TestModels/MultipleModels/Makefile +++ b/TestModels/MultipleModels/Makefile @@ -26,4 +26,7 @@ SERVICE_DEPS_PrimaryProject := \ SMITHY_DEPS=dafny-dependencies/Model/traits.smithy +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=multiple_models + # DEPENDENT-MODELS:= diff --git a/TestModels/MultipleModels/runtimes/python/pyproject.toml b/TestModels/MultipleModels/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..cae53fdbd8 --- /dev/null +++ b/TestModels/MultipleModels/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-refinement" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "multiple_models", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/MultipleModels/runtimes/python/src/multiple_models/__init__.py b/TestModels/MultipleModels/runtimes/python/src/multiple_models/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/MultipleModels/runtimes/python/src/multiple_models/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/MultipleModels/runtimes/python/test/internaldafny/extern/wrapped_multiple_models.py b/TestModels/MultipleModels/runtimes/python/test/internaldafny/extern/wrapped_multiple_models.py new file mode 100644 index 0000000000..017d7493b4 --- /dev/null +++ b/TestModels/MultipleModels/runtimes/python/test/internaldafny/extern/wrapped_multiple_models.py @@ -0,0 +1,39 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_multiplemodels_dependencyproject_internaldafny_wrapped +from multiple_models.smithygenerated.simple_multiplemodels_dependencyproject.client import DependencyProject +from multiple_models.smithygenerated.simple_multiplemodels_dependencyproject.shim import DependencyProjectShim +from multiple_models.smithygenerated.simple_multiplemodels_dependencyproject.config import dafny_config_to_smithy_config as dependency_dafny_config_to_smithy_config + +import simple_multiplemodels_primaryproject_internaldafny_wrapped +from multiple_models.smithygenerated.simple_multiplemodels_primaryproject.client import PrimaryProject +from multiple_models.smithygenerated.simple_multiplemodels_primaryproject.shim import PrimaryProjectShim +from multiple_models.smithygenerated.simple_multiplemodels_primaryproject.config import dafny_config_to_smithy_config as primary_dafny_config_to_smithy_config + +import Wrappers + +class dependency_default__(simple_multiplemodels_dependencyproject_internaldafny_wrapped.default__): + + @staticmethod + def WrappedDependencyProject(config): + wrapped_config = dependency_dafny_config_to_smithy_config(config) + impl = DependencyProject(wrapped_config) + wrapped_client = DependencyProjectShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# TODO-Python-PYTHONPATH: Remove +simple_multiplemodels_dependencyproject_internaldafny_wrapped.default__ = dependency_default__ + +class primary_default__(simple_multiplemodels_primaryproject_internaldafny_wrapped.default__): + + @staticmethod + def WrappedPrimaryProject(config): + wrapped_config = primary_dafny_config_to_smithy_config(config) + impl = PrimaryProject(wrapped_config) + wrapped_client = PrimaryProjectShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# TODO-Python-PYTHONPATH: Remove +simple_multiplemodels_primaryproject_internaldafny_wrapped.default__ = primary_default__ diff --git a/TestModels/MultipleModels/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/MultipleModels/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..c6eec8bd4f --- /dev/null +++ b/TestModels/MultipleModels/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import multiple_models +import wrapped_multiple_models + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/MultipleModels/runtimes/python/tox.ini b/TestModels/MultipleModels/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/MultipleModels/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/Refinement/Makefile b/TestModels/Refinement/Makefile index 4b0cf60dc9..9a622c4564 100644 --- a/TestModels/Refinement/Makefile +++ b/TestModels/Refinement/Makefile @@ -7,6 +7,9 @@ include ../SharedMakefile.mk NAMESPACE=simple.refinement +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_refinement + PROJECT_SERVICES := \ SimpleRefinement diff --git a/TestModels/Refinement/runtimes/python/pyproject.toml b/TestModels/Refinement/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..366b8ed997 --- /dev/null +++ b/TestModels/Refinement/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-refinement" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_refinement", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/Refinement/runtimes/python/src/simple_refinement/__init__.py b/TestModels/Refinement/runtimes/python/src/simple_refinement/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/Refinement/runtimes/python/src/simple_refinement/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/Refinement/runtimes/python/test/internaldafny/extern/wrapped_simple_refinement.py b/TestModels/Refinement/runtimes/python/test/internaldafny/extern/wrapped_simple_refinement.py new file mode 100644 index 0000000000..5cdb712e8b --- /dev/null +++ b/TestModels/Refinement/runtimes/python/test/internaldafny/extern/wrapped_simple_refinement.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_refinement_internaldafny_wrapped +from simple_refinement.smithygenerated.simple_refinement.client import SimpleRefinement +from simple_refinement.smithygenerated.simple_refinement.shim import SimpleRefinementShim +from simple_refinement.smithygenerated.simple_refinement.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_refinement_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleRefinement(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleRefinement(wrapped_config) + wrapped_client = SimpleRefinementShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_refinement_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/Refinement/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/Refinement/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..ffd10696f5 --- /dev/null +++ b/TestModels/Refinement/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_refinement +import wrapped_simple_refinement + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/Refinement/runtimes/python/tox.ini b/TestModels/Refinement/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/Refinement/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/Resource/Makefile b/TestModels/Resource/Makefile index a462982dad..16c1558f9c 100644 --- a/TestModels/Resource/Makefile +++ b/TestModels/Resource/Makefile @@ -7,6 +7,9 @@ include ../SharedMakefile.mk NAMESPACE=simple.resources +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_resources + PROJECT_SERVICES := \ SimpleResources diff --git a/TestModels/Resource/runtimes/python/pyproject.toml b/TestModels/Resource/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..ddcb96842e --- /dev/null +++ b/TestModels/Resource/runtimes/python/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "simple-resources" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_resources", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/Resource/runtimes/python/src/simple_resources/__init__.py b/TestModels/Resource/runtimes/python/src/simple_resources/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/Resource/runtimes/python/src/simple_resources/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/Resource/runtimes/python/test/internaldafny/extern/wrapped_simple_resources.py b/TestModels/Resource/runtimes/python/test/internaldafny/extern/wrapped_simple_resources.py new file mode 100644 index 0000000000..ca04587068 --- /dev/null +++ b/TestModels/Resource/runtimes/python/test/internaldafny/extern/wrapped_simple_resources.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_resources_internaldafny_wrapped +from simple_resources.smithygenerated.simple_resources.client import SimpleResources +from simple_resources.smithygenerated.simple_resources.shim import SimpleResourcesShim +from simple_resources.smithygenerated.simple_resources.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_resources_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleResources(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleResources(wrapped_config) + wrapped_client = SimpleResourcesShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_resources_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/Resource/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/Resource/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..6040b2d96f --- /dev/null +++ b/TestModels/Resource/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_resources +import wrapped_simple_resources + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/Resource/runtimes/python/tox.ini b/TestModels/Resource/runtimes/python/tox.ini new file mode 100644 index 0000000000..6fde4a5cbe --- /dev/null +++ b/TestModels/Resource/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest test/ -s -v --import-mode importlib \ No newline at end of file diff --git a/TestModels/Resource/src/Index.dfy b/TestModels/Resource/src/Index.dfy index 6df0f8d083..6fe9608c8b 100644 --- a/TestModels/Resource/src/Index.dfy +++ b/TestModels/Resource/src/Index.dfy @@ -5,7 +5,7 @@ include "../Model/SimpleResourcesTypes.dfy" include "./SimpleResourcesOperations.dfy" module - {:extern "simple.resources.internaldafny"} + {:extern "simple_resources_internaldafny"} SimpleResources refines AbstractSimpleResourcesService { import Operations = SimpleResourcesOperations diff --git a/TestModels/Resource/src/WrappedIndex.dfy b/TestModels/Resource/src/WrappedIndex.dfy index 1df003572a..44e51e0f04 100644 --- a/TestModels/Resource/src/WrappedIndex.dfy +++ b/TestModels/Resource/src/WrappedIndex.dfy @@ -4,7 +4,7 @@ include "../Model/SimpleResourcesTypesWrapped.dfy" module - {:extern "simple.resources.internaldafny.wrapped"} + {:extern "simple_resources_internaldafny_wrapped"} WrappedSimpleResources refines WrappedAbstractSimpleResourcesService { import WrappedService = SimpleResources diff --git a/TestModels/SharedMakefile.mk b/TestModels/SharedMakefile.mk index 0a2a721f76..ad4cfd83a2 100644 --- a/TestModels/SharedMakefile.mk +++ b/TestModels/SharedMakefile.mk @@ -64,4 +64,21 @@ _polymorph_dotnet: OUTPUT_DOTNET_WRAPPED=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped) _polymorph_dotnet: _polymorph_wrapped - +_polymorph_python: OUTPUT_PYTHON=--output-python $(LIBRARY_ROOT)/runtimes/python/src/$(PYTHON_MODULE_NAME)/smithygenerated +_polymorph_python: MODULE_NAME=--module-name $(PYTHON_MODULE_NAME) +# Python codegen MUST know dependencies' module names... +# This greps each service dependency's Makefile for two strings: +# 1. "SERVICE_NAMESPACE_$(dependency)" +# 2. "PYTHON_MODULE_NAME" +# , then assembles them together as +# "SERVICE_NAMESPACE_$(dependency)"="PYTHON_MODULE_NAME" +# , creating a map from a service namespace to its wrapping module name. +# _polymorph_python: DEPENDENCY_MODULE_NAMES=$(foreach dependency, \ +# $($(service_deps_var)), \ +# --dependency-module-name=$(shell cat $(if $(DIR_STRUCTURE_V2),$(PROJECT_ROOT)/$(dependency)/../../Makefile,$(PROJECT_ROOT)/$(dependency)/Makefile) | grep ^SERVICE_NAMESPACE_$(if $(DIR_STRUCTURE_V2),$(shell echo $(dependency) | cut -d "/" -f 3),$(shell echo $($(dependency)))) | cut -d "=" -f 2)=$(shell cat $(if $(DIR_STRUCTURE_V2),$(PROJECT_ROOT)/$(dependency)/../../Makefile,$(PROJECT_ROOT)/$(dependency)/Makefile) | grep ^PYTHON_MODULE_NAME | cut -d "=" -f 2)\ +# ) +_polymorph_python: _polymorph +_polymorph_python: OUTPUT_PYTHON_WRAPPED=--output-python $(LIBRARY_ROOT)/runtimes/python/src/$(PYTHON_MODULE_NAME)/smithygenerated +_polymorph_python: _polymorph_wrapped +_polymorph_python: POLYMORPH_LANGUAGE_TARGET=python +_polymorph_python: _polymorph_dependencies \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleBlob/Makefile b/TestModels/SimpleTypes/SimpleBlob/Makefile index d197826d50..1aac7d9a0d 100644 --- a/TestModels/SimpleTypes/SimpleBlob/Makefile +++ b/TestModels/SimpleTypes/SimpleBlob/Makefile @@ -7,6 +7,9 @@ include ../../SharedMakefile.mk NAMESPACE=simple.types.blob +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_types_blob + PROJECT_SERVICES := \ SimpleBlob diff --git a/TestModels/SimpleTypes/SimpleBlob/runtimes/python/pyproject.toml b/TestModels/SimpleTypes/SimpleBlob/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..db7638ba44 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleBlob/runtimes/python/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "simple-types-blob" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_types_blob", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/SimpleTypes/SimpleBlob/runtimes/python/src/simple_types_blob/__init__.py b/TestModels/SimpleTypes/SimpleBlob/runtimes/python/src/simple_types_blob/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleBlob/runtimes/python/src/simple_types_blob/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/SimpleTypes/SimpleBlob/runtimes/python/test/internaldafny/extern/wrapped_simple_blob.py b/TestModels/SimpleTypes/SimpleBlob/runtimes/python/test/internaldafny/extern/wrapped_simple_blob.py new file mode 100644 index 0000000000..2fc391adeb --- /dev/null +++ b/TestModels/SimpleTypes/SimpleBlob/runtimes/python/test/internaldafny/extern/wrapped_simple_blob.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_types_blob_internaldafny_wrapped +from simple_types_blob.smithygenerated.simple_types_blob.client import SimpleTypesBlob +from simple_types_blob.smithygenerated.simple_types_blob.shim import SimpleBlobShim +from simple_types_blob.smithygenerated.simple_types_blob.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_types_blob_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleBlob(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleTypesBlob(wrapped_config) + wrapped_client = SimpleBlobShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_types_blob_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/SimpleTypes/SimpleBlob/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/SimpleTypes/SimpleBlob/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..fd2c05e1bf --- /dev/null +++ b/TestModels/SimpleTypes/SimpleBlob/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_types_blob +import wrapped_simple_blob + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleBlob/runtimes/python/tox.ini b/TestModels/SimpleTypes/SimpleBlob/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleBlob/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleBoolean/Makefile b/TestModels/SimpleTypes/SimpleBoolean/Makefile index c1e4b87e14..461cc95e34 100644 --- a/TestModels/SimpleTypes/SimpleBoolean/Makefile +++ b/TestModels/SimpleTypes/SimpleBoolean/Makefile @@ -1,15 +1,15 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - CORES=2 include ../../SharedMakefile.mk NAMESPACE=simple.types.boolean +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_types_boolean + PROJECT_SERVICES := \ SimpleBoolean diff --git a/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/pyproject.toml b/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..c0dfbaf469 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/pyproject.toml @@ -0,0 +1,24 @@ +[tool.poetry] +name = "simple-types-boolean" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_types_boolean", from = "src" } +] +# Include all Smithy- and Dafny-generated code in package distributions, +# even though they are not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/src/simple_types_boolean/__init__.py b/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/src/simple_types_boolean/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/src/simple_types_boolean/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/test/internaldafny/extern/wrapped_simple_boolean.py b/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/test/internaldafny/extern/wrapped_simple_boolean.py new file mode 100644 index 0000000000..7083fb3e17 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/test/internaldafny/extern/wrapped_simple_boolean.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_types_boolean_internaldafny_wrapped +from simple_types_boolean.smithygenerated.simple_types_boolean.client import SimpleTypesBoolean +from simple_types_boolean.smithygenerated.simple_types_boolean.shim import SimpleBooleanShim +from simple_types_boolean.smithygenerated.simple_types_boolean.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_types_boolean_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleBoolean(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleTypesBoolean(wrapped_config) + wrapped_client = SimpleBooleanShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_types_boolean_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..8890967bd8 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_types_boolean +import wrapped_simple_boolean + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/tox.ini b/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleBoolean/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleDouble/Makefile b/TestModels/SimpleTypes/SimpleDouble/Makefile index 4b958ce085..cdb8b327f5 100644 --- a/TestModels/SimpleTypes/SimpleDouble/Makefile +++ b/TestModels/SimpleTypes/SimpleDouble/Makefile @@ -7,6 +7,9 @@ include ../../SharedMakefile.mk NAMESPACE=simple.types.smithyDouble +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_types_smithydouble + PROJECT_SERVICES := \ SimpleDouble diff --git a/TestModels/SimpleTypes/SimpleDouble/runtimes/python/pyproject.toml b/TestModels/SimpleTypes/SimpleDouble/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..d40c6fe2d7 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleDouble/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-types-smithydouble" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_types_smithydouble", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + diff --git a/TestModels/SimpleTypes/SimpleDouble/runtimes/python/src/simple_types_smithydouble/__init__.py b/TestModels/SimpleTypes/SimpleDouble/runtimes/python/src/simple_types_smithydouble/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleDouble/runtimes/python/src/simple_types_smithydouble/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/SimpleTypes/SimpleDouble/runtimes/python/test/internaldafny/extern/wrapped_simple_double.py b/TestModels/SimpleTypes/SimpleDouble/runtimes/python/test/internaldafny/extern/wrapped_simple_double.py new file mode 100644 index 0000000000..5a338648f9 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleDouble/runtimes/python/test/internaldafny/extern/wrapped_simple_double.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_types_smithydouble_internaldafny_wrapped +from simple_types_smithydouble.smithygenerated.simple_types_smithydouble.client import SimpleTypesDouble +from simple_types_smithydouble.smithygenerated.simple_types_smithydouble.shim import SimpleDoubleShim +from simple_types_smithydouble.smithygenerated.simple_types_smithydouble.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_types_smithydouble_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleDouble(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleTypesDouble(wrapped_config) + wrapped_client = SimpleDoubleShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_types_smithydouble_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/SimpleTypes/SimpleDouble/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/SimpleTypes/SimpleDouble/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..6eed544ef7 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleDouble/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_types_smithydouble +import wrapped_simple_double + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleDouble/runtimes/python/tox.ini b/TestModels/SimpleTypes/SimpleDouble/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleDouble/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleEnum/Makefile b/TestModels/SimpleTypes/SimpleEnum/Makefile index 766ae13fba..2351b0b7d4 100644 --- a/TestModels/SimpleTypes/SimpleEnum/Makefile +++ b/TestModels/SimpleTypes/SimpleEnum/Makefile @@ -7,6 +7,9 @@ include ../../SharedMakefile.mk NAMESPACE=simple.types.smithyEnum +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_types_smithyenum + PROJECT_SERVICES := \ SimpleEnum diff --git a/TestModels/SimpleTypes/SimpleEnum/runtimes/python/pyproject.toml b/TestModels/SimpleTypes/SimpleEnum/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..15d2429920 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleEnum/runtimes/python/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "simple-types-smithyenum" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_types_smithyenum", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/SimpleTypes/SimpleEnum/runtimes/python/src/simple_types_smithyenum/__init__.py b/TestModels/SimpleTypes/SimpleEnum/runtimes/python/src/simple_types_smithyenum/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleEnum/runtimes/python/src/simple_types_smithyenum/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/SimpleTypes/SimpleEnum/runtimes/python/test/internaldafny/extern/wrapped_simple_enum.py b/TestModels/SimpleTypes/SimpleEnum/runtimes/python/test/internaldafny/extern/wrapped_simple_enum.py new file mode 100644 index 0000000000..824d9ae81a --- /dev/null +++ b/TestModels/SimpleTypes/SimpleEnum/runtimes/python/test/internaldafny/extern/wrapped_simple_enum.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_types_smithyenum_internaldafny_wrapped +from simple_types_smithyenum.smithygenerated.simple_types_smithyenum.client import SimpleTypesEnum +from simple_types_smithyenum.smithygenerated.simple_types_smithyenum.shim import SimpleEnumShim +from simple_types_smithyenum.smithygenerated.simple_types_smithyenum.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_types_smithyenum_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleEnum(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleTypesEnum(wrapped_config) + wrapped_client = SimpleEnumShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_types_smithyenum_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/SimpleTypes/SimpleEnum/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/SimpleTypes/SimpleEnum/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..309d821050 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleEnum/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_types_smithyenum +import wrapped_simple_enum + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleEnum/runtimes/python/tox.ini b/TestModels/SimpleTypes/SimpleEnum/runtimes/python/tox.ini new file mode 100644 index 0000000000..7c68ba7488 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleEnum/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleEnumV2/Makefile b/TestModels/SimpleTypes/SimpleEnumV2/Makefile index c759952d66..e6b2673079 100644 --- a/TestModels/SimpleTypes/SimpleEnumV2/Makefile +++ b/TestModels/SimpleTypes/SimpleEnumV2/Makefile @@ -7,6 +7,9 @@ include ../../SharedMakefile.mk NAMESPACE=simple.types.enumV2 +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_types_smithyenumv2 + PROJECT_SERVICES := \ SimpleTypesEnumV2 diff --git a/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/pyproject.toml b/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..3720c6078f --- /dev/null +++ b/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "simple-types-smithyenum-v2" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_types_smithyenumv2", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/src/simple_types_smithyenumv2/__init__.py b/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/src/simple_types_smithyenumv2/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/src/simple_types_smithyenumv2/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/test/internaldafny/extern/wrapped_simple_enum.py b/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/test/internaldafny/extern/wrapped_simple_enum.py new file mode 100644 index 0000000000..36d6e95fbf --- /dev/null +++ b/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/test/internaldafny/extern/wrapped_simple_enum.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_types_enumv2_internaldafny_wrapped +from simple_types_smithyenumv2.smithygenerated.simple_types_enumv2.client import SimpleTypesEnumV2 +from simple_types_smithyenumv2.smithygenerated.simple_types_enumv2.shim import SimpleEnumV2Shim +from simple_types_smithyenumv2.smithygenerated.simple_types_enumv2.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_types_enumv2_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleEnumV2(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleTypesEnumV2(wrapped_config) + wrapped_client = SimpleEnumV2Shim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_types_enumv2_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..7ba8bdaec1 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_types_smithyenumv2 +import wrapped_simple_enum + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor diff --git a/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/tox.ini b/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/tox.ini new file mode 100644 index 0000000000..6638b94595 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleEnumV2/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib diff --git a/TestModels/SimpleTypes/SimpleInteger/Makefile b/TestModels/SimpleTypes/SimpleInteger/Makefile index 7fd31ac8ab..a3c3705524 100644 --- a/TestModels/SimpleTypes/SimpleInteger/Makefile +++ b/TestModels/SimpleTypes/SimpleInteger/Makefile @@ -7,6 +7,9 @@ include ../../SharedMakefile.mk NAMESPACE=simple.types.integer +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_types_integer + PROJECT_SERVICES := \ SimpleInteger diff --git a/TestModels/SimpleTypes/SimpleInteger/runtimes/python/pyproject.toml b/TestModels/SimpleTypes/SimpleInteger/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..374d18a8d1 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleInteger/runtimes/python/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "simple-types-integer" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_types_integer", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/SimpleTypes/SimpleInteger/runtimes/python/src/simple_types_integer/__init__.py b/TestModels/SimpleTypes/SimpleInteger/runtimes/python/src/simple_types_integer/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleInteger/runtimes/python/src/simple_types_integer/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/SimpleTypes/SimpleInteger/runtimes/python/test/internaldafny/extern/wrapped_simple_integer.py b/TestModels/SimpleTypes/SimpleInteger/runtimes/python/test/internaldafny/extern/wrapped_simple_integer.py new file mode 100644 index 0000000000..93e83dcd95 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleInteger/runtimes/python/test/internaldafny/extern/wrapped_simple_integer.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_types_integer_internaldafny_wrapped +from simple_types_integer.smithygenerated.simple_types_integer.client import SimpleTypesInteger +from simple_types_integer.smithygenerated.simple_types_integer.shim import SimpleIntegerShim +from simple_types_integer.smithygenerated.simple_types_integer.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_types_integer_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleInteger(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleTypesInteger(wrapped_config) + wrapped_client = SimpleIntegerShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_types_integer_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/SimpleTypes/SimpleInteger/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/SimpleTypes/SimpleInteger/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..2fe06bdcda --- /dev/null +++ b/TestModels/SimpleTypes/SimpleInteger/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_types_integer +import wrapped_simple_integer + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleInteger/runtimes/python/tox.ini b/TestModels/SimpleTypes/SimpleInteger/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleInteger/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleLong/Makefile b/TestModels/SimpleTypes/SimpleLong/Makefile index 4740385153..578523cb27 100644 --- a/TestModels/SimpleTypes/SimpleLong/Makefile +++ b/TestModels/SimpleTypes/SimpleLong/Makefile @@ -7,6 +7,9 @@ include ../../SharedMakefile.mk NAMESPACE=simple.types.smithyLong +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_types_smithylong + PROJECT_SERVICES := \ SimpleLong diff --git a/TestModels/SimpleTypes/SimpleLong/runtimes/python/pyproject.toml b/TestModels/SimpleTypes/SimpleLong/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..cced1bc4fb --- /dev/null +++ b/TestModels/SimpleTypes/SimpleLong/runtimes/python/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "simple-types-smithylong" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_types_smithylong", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/SimpleTypes/SimpleLong/runtimes/python/src/simple_types_smithylong/__init__.py b/TestModels/SimpleTypes/SimpleLong/runtimes/python/src/simple_types_smithylong/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleLong/runtimes/python/src/simple_types_smithylong/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/SimpleTypes/SimpleLong/runtimes/python/test/internaldafny/extern/wrapped_simple_long.py b/TestModels/SimpleTypes/SimpleLong/runtimes/python/test/internaldafny/extern/wrapped_simple_long.py new file mode 100644 index 0000000000..27f9423f1d --- /dev/null +++ b/TestModels/SimpleTypes/SimpleLong/runtimes/python/test/internaldafny/extern/wrapped_simple_long.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_types_smithylong_internaldafny_wrapped +from simple_types_smithylong.smithygenerated.simple_types_smithylong.client import SimpleTypesLong +from simple_types_smithylong.smithygenerated.simple_types_smithylong.shim import SimpleLongShim +from simple_types_smithylong.smithygenerated.simple_types_smithylong.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_types_smithylong_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleLong(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleTypesLong(wrapped_config) + wrapped_client = SimpleLongShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_types_smithylong_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/SimpleTypes/SimpleLong/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/SimpleTypes/SimpleLong/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..6bdea5a1ce --- /dev/null +++ b/TestModels/SimpleTypes/SimpleLong/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_types_smithylong +import wrapped_simple_long + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleLong/runtimes/python/tox.ini b/TestModels/SimpleTypes/SimpleLong/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleLong/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleString/Makefile b/TestModels/SimpleTypes/SimpleString/Makefile index 3a00b6af42..7c84e0bcfa 100644 --- a/TestModels/SimpleTypes/SimpleString/Makefile +++ b/TestModels/SimpleTypes/SimpleString/Makefile @@ -7,6 +7,9 @@ include ../../SharedMakefile.mk NAMESPACE=simple.types.smithyString +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_types_smithystring + PROJECT_SERVICES := \ SimpleString diff --git a/TestModels/SimpleTypes/SimpleString/runtimes/python/pyproject.toml b/TestModels/SimpleTypes/SimpleString/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..c97385d3ea --- /dev/null +++ b/TestModels/SimpleTypes/SimpleString/runtimes/python/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "simple-types-smithystring" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_types_smithystring", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/SimpleTypes/SimpleString/runtimes/python/src/simple_types_smithystring/__init__.py b/TestModels/SimpleTypes/SimpleString/runtimes/python/src/simple_types_smithystring/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleString/runtimes/python/src/simple_types_smithystring/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/SimpleTypes/SimpleString/runtimes/python/test/internaldafny/extern/wrapped_simple_string.py b/TestModels/SimpleTypes/SimpleString/runtimes/python/test/internaldafny/extern/wrapped_simple_string.py new file mode 100644 index 0000000000..d6e10b5a83 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleString/runtimes/python/test/internaldafny/extern/wrapped_simple_string.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_types_smithystring_internaldafny_wrapped +from simple_types_smithystring.smithygenerated.simple_types_smithystring.client import SimpleTypesString +from simple_types_smithystring.smithygenerated.simple_types_smithystring.shim import SimpleStringShim +from simple_types_smithystring.smithygenerated.simple_types_smithystring.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_types_smithystring_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleString(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleTypesString(wrapped_config) + wrapped_client = SimpleStringShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_types_smithystring_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/SimpleTypes/SimpleString/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/SimpleTypes/SimpleString/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..c49e8628b5 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleString/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_types_smithystring +import wrapped_simple_string + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/SimpleTypes/SimpleString/runtimes/python/tox.ini b/TestModels/SimpleTypes/SimpleString/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/SimpleTypes/SimpleString/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/Union/Makefile b/TestModels/Union/Makefile index 3a2b168383..5b63f90793 100644 --- a/TestModels/Union/Makefile +++ b/TestModels/Union/Makefile @@ -7,6 +7,9 @@ include ../SharedMakefile.mk NAMESPACE=simple.union +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=simple_union + PROJECT_SERVICES := \ SimpleUnion diff --git a/TestModels/Union/runtimes/python/pyproject.toml b/TestModels/Union/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..43a0f709f6 --- /dev/null +++ b/TestModels/Union/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "simple-union" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "simple_union", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} + +standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/Union/runtimes/python/src/simple_union/__init__.py b/TestModels/Union/runtimes/python/src/simple_union/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/Union/runtimes/python/src/simple_union/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/Union/runtimes/python/test/internaldafny/extern/wrapped_simple_union.py b/TestModels/Union/runtimes/python/test/internaldafny/extern/wrapped_simple_union.py new file mode 100644 index 0000000000..40ecda4ac8 --- /dev/null +++ b/TestModels/Union/runtimes/python/test/internaldafny/extern/wrapped_simple_union.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# TODO-Python-PYTHONPATH: Qualify imports +import simple_union_internaldafny_wrapped +from simple_union.smithygenerated.simple_union.client import SimpleUnion +from simple_union.smithygenerated.simple_union.shim import SimpleUnionShim +from simple_union.smithygenerated.simple_union.config import dafny_config_to_smithy_config +import Wrappers + +class default__(simple_union_internaldafny_wrapped.default__): + + @staticmethod + def WrappedSimpleUnion(config): + wrapped_config = dafny_config_to_smithy_config(config) + impl = SimpleUnion(wrapped_config) + wrapped_client = SimpleUnionShim(impl) + return Wrappers.Result_Success(wrapped_client) + +# (TODO-Python-PYTHONPATH: Remove) +simple_union_internaldafny_wrapped.default__ = default__ diff --git a/TestModels/Union/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/Union/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..76e89123ad --- /dev/null +++ b/TestModels/Union/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import simple_union +import wrapped_simple_union + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/Union/runtimes/python/tox.ini b/TestModels/Union/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/Union/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/aws-sdks/ddb/Makefile b/TestModels/aws-sdks/ddb/Makefile index 6f39f37957..cc4e3758e4 100644 --- a/TestModels/aws-sdks/ddb/Makefile +++ b/TestModels/aws-sdks/ddb/Makefile @@ -5,6 +5,9 @@ CORES=2 include ../../SharedMakefile.mk +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=com_amazonaws_dynamodb + PROJECT_SERVICES := \ ComAmazonawsDynamodb \ @@ -25,6 +28,7 @@ _polymorph_wrapped: ; _polymorph_wrapped_dafny: ; _polymorph_wrapped_net: ; _polymorph_wrapped_java: ; +_polymorph_wrapped_python: ; format_net: pushd runtimes/net && dotnet format && popd diff --git a/TestModels/aws-sdks/ddb/runtimes/python/pyproject.toml b/TestModels/aws-sdks/ddb/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..862e739d79 --- /dev/null +++ b/TestModels/aws-sdks/ddb/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "amazon-cryptography-internal-dafny-shim-ddb" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "com_amazonaws_dynamodb", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/dafnygenerated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +boto3 = "^1.28.38" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/aws-sdks/ddb/runtimes/python/src/com_amazonaws_dynamodb/__init__.py b/TestModels/aws-sdks/ddb/runtimes/python/src/com_amazonaws_dynamodb/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/aws-sdks/ddb/runtimes/python/src/com_amazonaws_dynamodb/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/aws-sdks/ddb/runtimes/python/src/com_amazonaws_dynamodb/internaldafny/extern/software_amazon_cryptography_services_dynamodb_internaldafny.py b/TestModels/aws-sdks/ddb/runtimes/python/src/com_amazonaws_dynamodb/internaldafny/extern/software_amazon_cryptography_services_dynamodb_internaldafny.py new file mode 100644 index 0000000000..023d8beed2 --- /dev/null +++ b/TestModels/aws-sdks/ddb/runtimes/python/src/com_amazonaws_dynamodb/internaldafny/extern/software_amazon_cryptography_services_dynamodb_internaldafny.py @@ -0,0 +1,49 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import module_ +from Wrappers import Option_None, Option_Some +from com_amazonaws_dynamodb.smithygenerated.com_amazonaws_dynamodb.shim import DynamoDBClientShim +from com_amazonaws_dynamodb.internaldafny.generated.software_amazon_cryptography_services_dynamodb_internaldafny import * +import com_amazonaws_dynamodb.internaldafny.generated.software_amazon_cryptography_services_dynamodb_internaldafny +import _dafny + +import boto3 +from botocore.config import Config + +class default__(com_amazonaws_dynamodb.internaldafny.generated.software_amazon_cryptography_services_dynamodb_internaldafny.default__): + @staticmethod + def DynamoDBClient(region=None): + if region is not None: + boto_config = Config( + region_name=region + ) + impl = boto3.client("dynamodb", config=boto_config) + else: + impl = boto3.client("dynamodb") + region = boto3.session.Session().region_name + wrapped_client = DynamoDBClientShim(impl, region) + return Wrappers.Result_Success(wrapped_client) + + @staticmethod + def DDBClientForRegion(region: _dafny.Seq): + return default__.DynamoDBClient(_dafny.string_of(region)) + + @staticmethod + def RegionMatch(client, region): + # We should never be passing anything other than Shim as the 'client'. + # If this assertion fails, that indicates that there is something wrong with + # our code generation. + try: + assert isinstance(client, DynamoDBClientShim) + except assertionError: + raise TypeError("Client provided to RegionMatch is not a DynamoDBClientShim: " + client) + + # Since client is a DynamoDBClientShim, we can reach into its _impl, which is a boto3 client + client_region_name = client._impl.meta.region_name + return Option_Some(region.VerbatimString(False) == client_region_name) + +import software_amazon_cryptography_services_dynamodb_internaldafny +software_amazon_cryptography_services_dynamodb_internaldafny.default__ = default__ +com_amazonaws_dynamodb.internaldafny.generated.software_amazon_cryptography_services_dynamodb_internaldafny.default__ = default__ +com_amazonaws_dynamodb.internaldafny.generated.software_amazon_cryptography_services_dynamodb_internaldafny.DynamoDBClient = default__.DynamoDBClient +com_amazonaws_dynamodb.internaldafny.generated.software_amazon_cryptography_services_dynamodb_internaldafny.DDBClientForRegion = default__.DDBClientForRegion \ No newline at end of file diff --git a/TestModels/aws-sdks/ddb/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/aws-sdks/ddb/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..820b5a551e --- /dev/null +++ b/TestModels/aws-sdks/ddb/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import standard_library +import com_amazonaws_dynamodb + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/aws-sdks/ddb/runtimes/python/tox.ini b/TestModels/aws-sdks/ddb/runtimes/python/tox.ini new file mode 100644 index 0000000000..34552e9320 --- /dev/null +++ b/TestModels/aws-sdks/ddb/runtimes/python/tox.ini @@ -0,0 +1,14 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +passenv = AWS_* +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/aws-sdks/ddb/test/TestDDBv2.dfy b/TestModels/aws-sdks/ddb/test/TestDDBv2.dfy index b563766af8..f577d4356f 100644 --- a/TestModels/aws-sdks/ddb/test/TestDDBv2.dfy +++ b/TestModels/aws-sdks/ddb/test/TestDDBv2.dfy @@ -4,7 +4,7 @@ include "../src/Index.dfy" module TestDDBv2 { - import DDB = Com.Amazonaws.Dynamodb + import DDB = Com_Amazonaws_Dynamodb import opened StandardLibrary.UInt import opened Wrappers diff --git a/TestModels/aws-sdks/kms/Makefile b/TestModels/aws-sdks/kms/Makefile index 9f58da919e..777f1523b7 100644 --- a/TestModels/aws-sdks/kms/Makefile +++ b/TestModels/aws-sdks/kms/Makefile @@ -5,7 +5,8 @@ CORES=2 include ../../SharedMakefile.mk -NAMESPACE=com.amazonaws.kms +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=com_amazonaws_kms PROJECT_SERVICES := \ ComAmazonawsKms\ @@ -27,6 +28,7 @@ _polymorph_wrapped: ; _polymorph_wrapped_dafny: ; _polymorph_wrapped_net: ; _polymorph_wrapped_java: ; +_polymorph_wrapped_python: ; clean: _clean rm -rf $(LIBRARY_ROOT)/runtimes/java/src/main/dafny-generated diff --git a/TestModels/aws-sdks/kms/runtimes/python/pyproject.toml b/TestModels/aws-sdks/kms/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..b0927f374e --- /dev/null +++ b/TestModels/aws-sdks/kms/runtimes/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "amazon-cryptography-internal-dafny-shim-kms" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "com_amazonaws_kms", from = "src" } +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/dafnygenerated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +boto3 = "^1.28.38" +# TODO: Depend on PyPi once Smithy-Python publishes their Python package there +smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} +standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/aws-sdks/kms/runtimes/python/src/com_amazonaws_kms/__init__.py b/TestModels/aws-sdks/kms/runtimes/python/src/com_amazonaws_kms/__init__.py new file mode 100644 index 0000000000..7ded469fd7 --- /dev/null +++ b/TestModels/aws-sdks/kms/runtimes/python/src/com_amazonaws_kms/__init__.py @@ -0,0 +1,15 @@ +# __init__.py for a Smithy-Dafny generated Python project + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +# Import project dependencies. +# TODO-Python-PYTHONPATH: Remove dependency imports to initialize PYTHONPATH with their modules + +import standard_library + +# Add internaldafny and smithygenerated code to PYTHONPATH (TODO-Python-PYTHONPATH: Remove) +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") diff --git a/TestModels/aws-sdks/kms/runtimes/python/src/com_amazonaws_kms/internaldafny/extern/software_amazon_cryptography_services_kms_internaldafny.py b/TestModels/aws-sdks/kms/runtimes/python/src/com_amazonaws_kms/internaldafny/extern/software_amazon_cryptography_services_kms_internaldafny.py new file mode 100644 index 0000000000..fe42852f06 --- /dev/null +++ b/TestModels/aws-sdks/kms/runtimes/python/src/com_amazonaws_kms/internaldafny/extern/software_amazon_cryptography_services_kms_internaldafny.py @@ -0,0 +1,50 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import module_ +from Wrappers import Option_None, Option_Some +from com_amazonaws_kms.smithygenerated.com_amazonaws_kms.shim import KMSClientShim +from com_amazonaws_kms.internaldafny.generated.software_amazon_cryptography_services_kms_internaldafny import * +import com_amazonaws_kms.internaldafny.generated.software_amazon_cryptography_services_kms_internaldafny +import _dafny + +import boto3 +from botocore.config import Config + +class default__(com_amazonaws_kms.internaldafny.generated.software_amazon_cryptography_services_kms_internaldafny.default__): + @staticmethod + def KMSClient(boto_client = None, region=None): + if boto_client is None: + if region is not None: + boto_config = Config( + region_name=region + ) + boto_client = boto3.client("kms", config=boto_config) + else: + boto_client = boto3.client("kms") + region = boto3.session.Session().region_name + wrapped_client = KMSClientShim(boto_client, region) + return Wrappers.Result_Success(wrapped_client) + + @staticmethod + def KMSClientForRegion(region: _dafny.Seq): + region_string = _dafny.string_of(region) + return default__.KMSClient(region=region_string) + + @staticmethod + def RegionMatch(client, region): + # We should never be passing anything other than Shim as the 'client'. + # If this assertion fails, that indicates that there is something wrong with + # our code generation. + try: + assert isinstance(client, KMSClientShim) + except AssertionError: + raise TypeError("Client provided to RegionMatch is not a KMSClientShim: " + client) + + # Since client is a TrentServiceShim, we can reach into its _impl, which is a boto3 client + client_region_name = client._impl.meta.region_name + return Option_Some(region.VerbatimString(False) == client_region_name) + +import software_amazon_cryptography_services_kms_internaldafny +software_amazon_cryptography_services_kms_internaldafny.default__ = default__ +com_amazonaws_kms.internaldafny.generated.software_amazon_cryptography_services_kms_internaldafny.default__ = default__ +com_amazonaws_kms.internaldafny.generated.software_amazon_cryptography_services_kms_internaldafny.TrentServiceClient = default__.KMSClient \ No newline at end of file diff --git a/TestModels/aws-sdks/kms/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/aws-sdks/kms/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..dbdac622d7 --- /dev/null +++ b/TestModels/aws-sdks/kms/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,29 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +# TODO-Python-PYTHONPATH: Remove all sys.path.append logic from this file +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. +# These are only imported to populate the PYTHONPATH. +# This can be removed once PYTHONPATH workaround is removed, +# and all Dafny-generated imports are fully qualified. +# TODO-Python-PYTHONPATH: Remove imports to initialize modules' PYTHONPATHs from this file +import standard_library +import com_amazonaws_kms + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + # TODO-Python-PYTHONPATH: Qualify import + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/aws-sdks/kms/runtimes/python/tox.ini b/TestModels/aws-sdks/kms/runtimes/python/tox.ini new file mode 100644 index 0000000000..34552e9320 --- /dev/null +++ b/TestModels/aws-sdks/kms/runtimes/python/tox.ini @@ -0,0 +1,14 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +passenv = AWS_* +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/TestModels/aws-sdks/kms/smithy-build.json b/TestModels/aws-sdks/kms/smithy-build.json new file mode 100644 index 0000000000..fe3a2f3afe --- /dev/null +++ b/TestModels/aws-sdks/kms/smithy-build.json @@ -0,0 +1,11 @@ +{ + "version": "1.0", + "plugins": { + "dafny-client-codegen": { + "service": "com.amazonaws.kms#TrentService", + "module": "trent_service", + "moduleVersion": "0.0.1", + "protocol": "aws.polymorph#localService" + } + } +} diff --git a/TestModels/aws-sdks/kms/src/Index.dfy b/TestModels/aws-sdks/kms/src/Index.dfy index 1fccebe351..2f5db99f42 100644 --- a/TestModels/aws-sdks/kms/src/Index.dfy +++ b/TestModels/aws-sdks/kms/src/Index.dfy @@ -3,7 +3,7 @@ include "../Model/ComAmazonawsKmsTypes.dfy" -module {:extern "software.amazon.cryptography.services.kms.internaldafny"} Com.Amazonaws.Kms refines AbstractComAmazonawsKmsService { +module {:extern "software_amazon_cryptography_services_kms_internaldafny"} Com_Amazonaws_Kms refines AbstractComAmazonawsKmsService { function method DefaultKMSClientConfigType() : KMSClientConfigType { KMSClientConfigType diff --git a/TestModels/aws-sdks/kms/test/TestComAmazonawsKms.dfy b/TestModels/aws-sdks/kms/test/TestComAmazonawsKms.dfy index 6666611399..df4b67e805 100644 --- a/TestModels/aws-sdks/kms/test/TestComAmazonawsKms.dfy +++ b/TestModels/aws-sdks/kms/test/TestComAmazonawsKms.dfy @@ -4,7 +4,7 @@ include "../src/Index.dfy" module TestComAmazonawsKms { - import Com.Amazonaws.Kms + import Kms = Com_Amazonaws_Kms import opened StandardLibrary.UInt import opened Wrappers diff --git a/TestModels/dafny-dependencies/Model/traits.smithy b/TestModels/dafny-dependencies/Model/traits.smithy index 1841af1395..82204341b2 100644 --- a/TestModels/dafny-dependencies/Model/traits.smithy +++ b/TestModels/dafny-dependencies/Model/traits.smithy @@ -24,6 +24,7 @@ list ServiceList { // A trait for explicitly modeling the configuration options that should be // available in the generated methods for creating clients. @trait(selector: "service") +@protocolDefinition structure localService { @required sdkId: String, diff --git a/TestModels/dafny-dependencies/StandardLibrary/Makefile b/TestModels/dafny-dependencies/StandardLibrary/Makefile index 6a36dd0066..6b3e5a34a3 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/Makefile +++ b/TestModels/dafny-dependencies/StandardLibrary/Makefile @@ -6,6 +6,9 @@ CORES=2 include ../../SharedMakefile.mk MAX_RESOURCE_COUNT=500000000 +# TODO-Python-PYTHONPATH: Move into dfyproject.toml +PYTHON_MODULE_NAME=standard_library +LIBRARIES := # define standard colors ifneq (,$(findstring xterm,${TERM})) @@ -36,6 +39,9 @@ polymorph_dotnet : polymorph_java : echo "Skipping polymorph_java for StandardLibrary" +polymorph_python : + echo "Skipping polymorph_python for StandardLibrary" + # Using this target for the side-effect of maintaining patches, # Since Dafny Rust code generation is incomplete and also needs to be patched. # That only works if you run transpile_rust first. @@ -63,6 +69,26 @@ transpile_net: | transpile_implementation_net transpile_test_net # StandardLibrary as a dependency transpile_java: | transpile_implementation_java transpile_test_java +# Override SharedMakefile's transpile_python to not transpile +# StandardLibrary as a dependency +transpile_python: | transpile_implementation_python transpile_test_python + +# Hacky workaround until Dafny supports per-language extern names. +# Replaces `.`s with `_`s in strings like `{:extern ".*"}`. +# This is flawed logic and should be removed, but is a reasonable band-aid for now. +# TODO: Once Dafny supports per-language extern names, remove and replace with Pythonic extern names. +# This may require new Smithy-Dafny logic to generate Pythonic extern names. +_python_underscore_extern_names: + find src -regex ".*\.dfy" -type f -exec sed -i $(SED_PARAMETER) '/module {:extern \".*\"/s/\./_/g' {} \; + echo "Skipping Model modification for StandardLibrary" + find test -regex ".*\.dfy" -type f -exec sed -i $(SED_PARAMETER) '/module {:extern \".*\"/s/\./_/g' {} \; + +_python_revert_underscore_extern_names: + find src -regex ".*\.dfy" -type f -exec sed -i $(SED_PARAMETER) '/module {:extern \".*\"/s/_/\./g' {} \; + echo "Skipping Model modification for StandardLibrary" + find test -regex ".*\.dfy" -type f -exec sed -i $(SED_PARAMETER) '/module {:extern \".*\"/s/_/\./g' {} \; + +########################## Rust targets # Override SharedMakefile's transpile_java to not transpile # StandardLibrary as a dependency transpile_rust: | transpile_implementation_rust @@ -97,12 +123,22 @@ transpile_implementation: -spillTargetCode:3 \ -compile:0 \ -optimizeErasableDatatypeWrapper:0 \ - -compileSuffix:1 \ + -compileSuffix:0 \ -unicodeChar:0 \ -functionSyntax:3 \ -useRuntimeLib \ -out $(OUT) +build_implementation: + dafny build \ + -t:$(TARGET) \ + ./src/Index.dfy \ + -o $(OUT) \ + --quantifier-syntax:3 \ + --function-syntax:3 \ + --optimize-erasable-datatype-wrapper:false \ + ./src/Index.dfy + # Override SharedMakefile's build_java to not install # StandardLibrary as a dependency build_java: transpile_java diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/.gitignore b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/.gitignore new file mode 100644 index 0000000000..3c31a7deb8 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/.gitignore @@ -0,0 +1,8 @@ +# Python Artifacts +src/**.egg-info/ +.pytest_cache +.tox +build +poetry.lock +**/poetry.lock +dist \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/pyproject.toml b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/pyproject.toml new file mode 100644 index 0000000000..54e590772f --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/pyproject.toml @@ -0,0 +1,22 @@ +[tool.poetry] +name = "standard-library" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "standard_library", from = "src" }, +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +DafnyRuntimePython = "^4.4.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/__init__.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/__init__.py new file mode 100644 index 0000000000..0f7c380977 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/__init__.py @@ -0,0 +1,7 @@ +import sys + +# TODO-Python: Remove PYTHONPATH workaround, use fully-qualified module names via dfyproject.toml. +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir + "/internaldafny/extern") +sys.path.append(module_root_dir + "/internaldafny/generated") \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/__init__.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/__init__.py new file mode 100644 index 0000000000..b626871a3f --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/__init__.py @@ -0,0 +1,7 @@ +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +# extern MUST come first in path +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/DafnyLibraries.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/DafnyLibraries.py new file mode 100644 index 0000000000..e5551c9c2a --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/DafnyLibraries.py @@ -0,0 +1,76 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from pathlib import Path + +from standard_library.internaldafny.generated.DafnyLibraries import * +import standard_library.internaldafny.generated.DafnyLibraries +import _dafny + +class MutableMap(standard_library.internaldafny.generated.DafnyLibraries.MutableMap): + def __init__(self): + super().__init__() + self.m = {} + + def content(self): + return _dafny.Map(self.m) + + def Put(self, k, v): + self.m[k] = v + + def Keys(self): + return _dafny.Set(self.m.keys()) + + def HasKey(self, k): + return k in self.m + + def Values(self): + return _dafny.Set(self.m.values()) + + def Items(self): + return _dafny.Set(self.m.items()) + + def Select(self, k): + return self.m[k] + + def Remove(self, k): + del self.m[k] + + def Size(self): + return len(self.m) + +class FileIO: + @staticmethod + def INTERNAL_WriteBytesToFile(dafny_path, dafny_bytes): + try: + native_path = FileIO.dafny_string_to_path(dafny_path) + print(f"write {native_path =}") + FileIO.create_parent_dirs(native_path) + native_bytes = bytes(dafny_bytes.Elements) + native_path.write_bytes(native_bytes) + return False, _dafny.Seq([]) + except Exception as e: + return True, _dafny.Seq(str(e)) + + @staticmethod + def INTERNAL_ReadBytesFromFile(dafny_path): + try: + native_path = FileIO.dafny_string_to_path(dafny_path) + print(f"read {native_path =}") + native_bytes = native_path.read_bytes() + dafny_bytes = _dafny.Seq(native_bytes) + return False, dafny_bytes, _dafny.Seq([]) + except Exception as e: + return True, _dafny.Seq([]), _dafny.Seq(str(e)) + + @staticmethod + def dafny_string_to_path(path_as_dafny_string): + return Path("../../" + _dafny.string_of(path_as_dafny_string)) + + @staticmethod + def create_parent_dirs(native_path): + parent = native_path.parent + parent_path = Path(parent) + parent_path.mkdir(parents=True, exist_ok=True) + +standard_library.internaldafny.generated.DafnyLibraries.FileIO = FileIO +standard_library.internaldafny.generated.DafnyLibraries.MutableMap = MutableMap \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/SortedSets.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/SortedSets.py new file mode 100644 index 0000000000..26a2e1a4ac --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/SortedSets.py @@ -0,0 +1,13 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from standard_library.internaldafny.generated.SortedSets import * +import standard_library.internaldafny.generated.SortedSets +import _dafny + +class default__: + + @staticmethod + def SetToSequence(input_set): + return _dafny.Seq(input_set.Elements) + +standard_library.internaldafny.generated.SortedSets.default__ = default__ \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/Time.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/Time.py new file mode 100644 index 0000000000..1f5922b88a --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/Time.py @@ -0,0 +1,20 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import standard_library.internaldafny.generated.Time +import datetime +import pytz +import Wrappers +import _dafny + +class default__: + def CurrentRelativeTime(): + return datetime.datetime.now(tz = pytz.UTC).timestamp() * 1000 + + def GetCurrentTimeStamp(): + try: + d = datetime.datetime.now(tz = pytz.UTC).strftime("%Y-%m-%d'T'%H:%M:%S:%f'Z'") + return Wrappers.Result_Success(_dafny.Seq(_dafny.string_of(d))) + except Exception as e: + return Wrappers.Result_Failure(_dafny.string_of("Could not generate a timestamp in ISO8601: " + e)) + +standard_library.internaldafny.generated.Time.default__ = default__ \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/UTF8.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/UTF8.py new file mode 100644 index 0000000000..cb8707c05a --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/standard_library/internaldafny/extern/UTF8.py @@ -0,0 +1,100 @@ +import standard_library.internaldafny.generated.UTF8 + + +from standard_library.internaldafny.generated.UTF8 import * +import _dafny +import struct + +''' +Extern UTF8 encode and decode methods. + +Note: +Python's `.encode('utf-8')` does not handle surrogates. +However, these encode/decode methods are expected to handle surrogates (e.g. "\uD808\uDC00"). +To work around this, we encode Dafny strings into UTF-16-LE (little endian) +and decode them before encoding into UTF-8 (`_strict_tostring`). +To decode, we reverse the encode step. (`_reverse_strict_tostring`) +''' + +# Extend the Dafny-generated class with our extern methods +class default__(standard_library.internaldafny.generated.UTF8.default__): + + @staticmethod + def Encode(s): + print(f"{s.Elements=}") + try: + return Wrappers.Result_Success(_dafny.Seq( + default__._strict_tostring(s).encode('utf-8') + )) + # Catch both UnicodeEncodeError and UnicodeDecodeError. + # The `try` block involves both encoding and decoding. + # OverflowError is possibly raised at `_strict_tostring`'s `ord(c).to_bytes` + # if the char `c` is not valid. + except (UnicodeDecodeError, UnicodeEncodeError, OverflowError): + return Wrappers.Result_Failure(_dafny.Seq("Could not encode input to Dafny Bytes.")) + + @staticmethod + def _strict_tostring(dafny_ascii_string): + ''' + Converts a Dafny Seq of unicode-escaped ASCII characters + into a string that can be encoded with Python's built-in `.encode('utf-8')`. + + This encoding-decoding allows subsequent UTF8 encodings + to handle surrogates as expected by Dafny code. + + This is exactly the `_dafny.string_from_utf_16` method from the DafnyRuntime, except with + `errors = 'strict'` here, + instead of + `errors = 'replace'` in the `_dafny.string_from_utf_16` function. + `strict` will throw an exception for invalid encodings, allowing us + to detect invalid encodings and raise exceptions, + while `replace` will fail silently. + :param s: + :return: + ''' + return b''.join([c.to_bytes(2, 'little') if isinstance(c, int) else ord(c).to_bytes(2, 'little') for c in dafny_ascii_string]).decode("utf-16-le", errors = 'strict') + + @staticmethod + def Decode(s): + try: + utf8_str = bytes(s).decode('utf-8') + # out = [] + # for a in s: + # out.append(a.to_bytes(2, "little")) + # out2 = [] + # for a in out: + # out2.append(a.decode("utf-8")) + # utf8_str = ''.join(out2) + unicode_escaped_utf8_str = default__._reverse_strict_tostring(utf8_str) + return Wrappers.Result_Success(unicode_escaped_utf8_str) + # Catch both UnicodeEncodeError and UnicodeDecodeError. + # The `try` block involves both encoding and decoding. + # ValueError and TypeError are possibly raised at `_reverse_strict_tostring`'s `chr()`. + # struct.error is possibly raised at `struct.unpack`. + except (UnicodeDecodeError, UnicodeEncodeError, ValueError, TypeError, struct.error): + return Wrappers.Result_Failure(_dafny.Seq("Could not decode input from Dafny Bytes.")) + + + @staticmethod + def _reverse_strict_tostring(utf8_str): + ''' + Converts a string into a Dafny Seq of unicode-escaped ASCII characters. + This is the inverse of the `_strict_tostring` function in this file. + :param s: + :return: + ''' + utf16_bytes = utf8_str.encode("utf-16-le", errors = "strict") + out = [] + # len(b)/2 is an integer by construction of UTF-16 encoding (2 bytes per encoded character) + for i in range(int(len(utf16_bytes)/2)): + # Take two consecutive bytes; + utf_16_bytepair = utf16_bytes[2*i:2*i+2] + # Unpack them into an ordinal representation; + packed_bytes = struct.unpack('"] +packages = [ + # Globally install all internaldafny modules + # such that a module X can be imported with `import X` + # from anywhere in the Python runtime that installs this module + { include = "*.py", from = "generated" }, +] +# Include all of the following .gitignored files in package distributions, +# even though they are not included in version control. +# Poetry will exclude .gitignored files from package distributions by default. +include = ["generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +# The test extern pyproject.toml +# (or test internaldafny pyproject.toml, if no test extern pyproject.toml is present) +# MUST depend on the src extern pyproject.toml +# (or src internaldafny pyproject.toml, if no src extern pyproject.toml is present) +# to overwrite any src-generated files with the test-generated files (e.g. module_.py). +standard-library-externs = { path = "../../src/standard_library/extern", develop = false} + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/test/internaldafny/test_dafny_wrapper.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/test/internaldafny/test_dafny_wrapper.py new file mode 100644 index 0000000000..47b3a46d98 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/test/internaldafny/test_dafny_wrapper.py @@ -0,0 +1,25 @@ +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +import sys + +internaldafny_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(internaldafny_dir + "/extern") +sys.path.append(internaldafny_dir + "/generated") + +# Import modules required for Dafny-generated tests. +# This is not generated; these must be manually added. + +import standard_library + +# End import modules required for Dafny-generated tests + +def test_dafny(): + # Dafny tests are executed when importing `internaldafny_test_executor` + import internaldafny_test_executor \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/tox.ini b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/tox.ini new file mode 100644 index 0000000000..846f9031b2 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ --import-mode importlib \ No newline at end of file diff --git a/codegen/build.gradle.kts b/codegen/build.gradle.kts index 2e5fa89f50..c24da4b549 100644 --- a/codegen/build.gradle.kts +++ b/codegen/build.gradle.kts @@ -56,179 +56,186 @@ subprojects { * Java * ==================================================== */ - if (subproject.name != "smithy-dafny-codegen-test") { - if (subproject.name == "smithy-dafny-codegen-cli") { - apply(plugin = "application") - } else { - apply(plugin = "java-library") - } - java { - toolchain {languageVersion.set(JavaLanguageVersion.of(17))} - } + when (subproject.name) { + "smithy-dafny-codegen-test" -> print("Skipping publish for smithy-dafny-codegen-test") + "smithy-python-codegen" -> print("Skipping publish for smithy-python-codegen") + else -> { + if (subproject.name == "smithy-dafny-codegen-cli") { + apply(plugin = "application") + } else { + apply(plugin = "java-library") + } - tasks.withType { - options.encoding = "UTF-8" - } + java { + toolchain {languageVersion.set(JavaLanguageVersion.of(17))} + } - // Reusable license copySpec - val licenseSpec = copySpec { - from("${project.rootDir}/LICENSE") - from("${project.rootDir}/NOTICE") - } + tasks.withType { + options.encoding = "UTF-8" + } - // Set up tasks that build source and javadoc jars. - tasks.register("sourcesJar") { - metaInf.with(licenseSpec) - from(sourceSets.main.get().allJava) - archiveClassifier.set("sources") - } + // Reusable license copySpec + val licenseSpec = copySpec { + from("${project.rootDir}/LICENSE") + from("${project.rootDir}/NOTICE") + } - /*tasks.register("javadocJar") { - metaInf.with(licenseSpec) - from(tasks.javadoc) - archiveClassifier.set("javadoc") - }*/ - - // Configure jars to include license related info - tasks.jar { - metaInf.with(licenseSpec) - inputs.property("moduleName", subproject.extra["moduleName"]) - manifest { - attributes["Automatic-Module-Name"] = subproject.extra["moduleName"] + // Set up tasks that build source and javadoc jars. + tasks.register("sourcesJar") { + metaInf.with(licenseSpec) + from(sourceSets.main.get().allJava) + archiveClassifier.set("sources") } - } - // Always run javadoc after build. - /*tasks["build"].finalizedBy(tasks["javadoc"])*/ + /*tasks.register("javadocJar") { + metaInf.with(licenseSpec) + from(tasks.javadoc) + archiveClassifier.set("javadoc") + }*/ + + // Configure jars to include license related info + tasks.jar { + metaInf.with(licenseSpec) + inputs.property("moduleName", subproject.extra["moduleName"]) + manifest { + attributes["Automatic-Module-Name"] = subproject.extra["moduleName"] + } + } - /* - * Maven - * ==================================================== - */ - apply(plugin = "maven-publish") - apply(plugin = "signing") + // Always run javadoc after build. + /*tasks["build"].finalizedBy(tasks["javadoc"])*/ - repositories { - mavenLocal() - mavenCentral() - } + /* + * Maven + * ==================================================== + */ + apply(plugin = "maven-publish") + apply(plugin = "signing") - publishing { repositories { - mavenCentral { - url = uri("https://aws.oss.sonatype.org/service/local/staging/deploy/maven2/") - credentials { - username = sonatypeUser - password = sonatypePassword + mavenLocal() + mavenCentral() + } + + publishing { + repositories { + mavenCentral { + url = uri("https://aws.oss.sonatype.org/service/local/staging/deploy/maven2/") + credentials { + username = sonatypeUser + password = sonatypePassword + } } } - } - publications { - create("mavenJava") { - from(components["java"]) - - // Ship the source and javadoc jars. - artifact(tasks["sourcesJar"]) - /*artifact(tasks["javadocJar"])*/ - - // Include extra information in the POMs. - afterEvaluate { - pom { - name.set(subproject.extra["displayName"].toString()) - description.set(subproject.description) - url.set("https://github.com/smithy-lang/smithy") - licenses { - license { - name.set("Apache License 2.0") - url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") - distribution.set("repo") + publications { + create("mavenJava") { + from(components["java"]) + + // Ship the source and javadoc jars. + artifact(tasks["sourcesJar"]) + /*artifact(tasks["javadocJar"])*/ + + // Include extra information in the POMs. + afterEvaluate { + pom { + name.set(subproject.extra["displayName"].toString()) + description.set(subproject.description) + url.set("https://github.com/smithy-lang/smithy") + licenses { + license { + name.set("Apache License 2.0") + url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") + distribution.set("repo") + } } - } - developers { - developer { - id.set("smithy") - name.set("Smithy") - organization.set("Amazon Web Services") - organizationUrl.set("https://aws.amazon.com") - roles.add("developer") + developers { + developer { + id.set("smithy") + name.set("Smithy") + organization.set("Amazon Web Services") + organizationUrl.set("https://aws.amazon.com") + roles.add("developer") + } + } + scm { + url.set("https://github.com/smithy-lang/smithy.git") } - } - scm { - url.set("https://github.com/smithy-lang/smithy.git") } } } } } - } - // Don't sign the artifacts if we didn't get a key and password to use. - val signingKey: String? by project - val signingPassword: String? by project - if (signingKey != null && signingPassword != null) { - signing { - useInMemoryPgpKeys(signingKey, signingPassword) - sign(publishing.publications["mavenJava"]) + // Don't sign the artifacts if we didn't get a key and password to use. + val signingKey: String? by project + val signingPassword: String? by project + if (signingKey != null && signingPassword != null) { + signing { + useInMemoryPgpKeys(signingKey, signingPassword) + sign(publishing.publications["mavenJava"]) + } } - } - /* - * CheckStyle - * ==================================================== - */ - //apply(plugin = "checkstyle") - - //tasks["checkstyleTest"].enabled = false - - /* - * Tests - * ==================================================== - * - * Configure the running of tests. - */ - // Log on passed, skipped, and failed test events if the `-Plog-tests` property is set. - if (project.hasProperty("log-tests")) { - tasks.test { - testLogging.events("passed", "skipped", "failed") + /* + * CheckStyle + * ==================================================== + */ + //apply(plugin = "checkstyle") + + //tasks["checkstyleTest"].enabled = false + + /* + * Tests + * ==================================================== + * + * Configure the running of tests. + */ + // Log on passed, skipped, and failed test events if the `-Plog-tests` property is set. + if (project.hasProperty("log-tests")) { + tasks.test { + testLogging.events("passed", "skipped", "failed") + } } - } - /* - * Code coverage - * ==================================================== - */ - apply(plugin = "jacoco") - - // Always run the jacoco test report after testing. - tasks["test"].finalizedBy(tasks["jacocoTestReport"]) - - // Configure jacoco to generate an HTML report. - tasks.jacocoTestReport { - reports { - xml.isEnabled = false - csv.isEnabled = false - html.destination = file("$buildDir/reports/jacoco") + /* + * Code coverage + * ==================================================== + */ + apply(plugin = "jacoco") + + // Always run the jacoco test report after testing. + tasks["test"].finalizedBy(tasks["jacocoTestReport"]) + + // Configure jacoco to generate an HTML report. + tasks.jacocoTestReport { + reports { + xml.isEnabled = false + csv.isEnabled = false + html.destination = file("$buildDir/reports/jacoco") + } } + + /* + * Spotbugs + * ==================================================== + */ + //apply(plugin = "com.github.spotbugs") + + // We don't need to lint tests. + //tasks["spotbugsTest"].enabled = false + + // Configure the bug filter for spotbugs. + /*spotbugs { + setEffort("max") + val excludeFile = File("${project.rootDir}/config/spotbugs/filter.xml") + if (excludeFile.exists()) { + excludeFilter.set(excludeFile) + } + }*/ } - /* - * Spotbugs - * ==================================================== - */ - //apply(plugin = "com.github.spotbugs") - - // We don't need to lint tests. - //tasks["spotbugsTest"].enabled = false - - // Configure the bug filter for spotbugs. - /*spotbugs { - setEffort("max") - val excludeFile = File("${project.rootDir}/config/spotbugs/filter.xml") - if (excludeFile.exists()) { - excludeFilter.set(excludeFile) - } - }*/ } + } diff --git a/codegen/settings.gradle.kts b/codegen/settings.gradle.kts index d0f20a2de9..623a2a1699 100644 --- a/codegen/settings.gradle.kts +++ b/codegen/settings.gradle.kts @@ -5,6 +5,9 @@ rootProject.name = "smithy-dafny" include(":smithy-dafny-codegen") include(":smithy-dafny-codegen-cli") //include(":smithy-dafny-codegen-test") +// TODO: Once Smithy-Python is published to Maven, and we do not rely on a fork, use that +include(":smithy-python-codegen") +project(":smithy-python-codegen").projectDir = file("./smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen") pluginManagement { repositories { diff --git a/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java b/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java index 1ea88f811c..ec7e50d95a 100644 --- a/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java +++ b/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java @@ -23,6 +23,14 @@ import software.amazon.smithy.model.Model; import software.amazon.smithy.model.loader.ModelAssembler; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Arrays; +import java.util.stream.Collectors; + public class CodegenCli { private static final Logger LOGGER = LoggerFactory.getLogger( @@ -80,12 +88,16 @@ public static void main(String[] args) { cliArguments.outputRustDir.ifPresent(path -> outputDirs.put(TargetLanguage.RUST, path) ); + cliArguments.outputPythonDir.ifPresent(path -> + outputDirs.put(TargetLanguage.PYTHON, path) + ); final CodegenEngine.Builder engineBuilder = new CodegenEngine.Builder() .withFromSmithyBuildPlugin(false) .withLibraryRoot(cliArguments.libraryRoot) .withServiceModel(serviceModel) .withDependentModelPaths(cliArguments.dependentModelPaths) + .withDependencyModuleNames(cliArguments.dependencyModuleNames) .withNamespace(cliArguments.namespace) .withTargetLangOutputDirs(outputDirs) .withAwsSdkStyle(cliArguments.awsSdkStyle) @@ -99,6 +111,7 @@ public static void main(String[] args) { cliArguments.includeDafnyFile.ifPresent( engineBuilder::withIncludeDafnyFile ); + cliArguments.moduleName.ifPresent(engineBuilder::withModuleName); cliArguments.patchFilesDir.ifPresent(engineBuilder::withPatchFilesDir); final CodegenEngine engine = engineBuilder.build(); engine.run(); @@ -136,6 +149,14 @@ private static Options getCliOptions() { .required() .build() ) + .addOption( + Option + .builder("dmn") + .longOpt("dependency-module-name") + .desc("directory for dependent model file[s] (.smithy format)") + .hasArg() + .build() + ) .addOption( Option .builder("n") @@ -145,6 +166,14 @@ private static Options getCliOptions() { .required() .build() ) + .addOption( + Option + .builder("mn") + .longOpt("module-name") + .desc("if generating for a language that uses modules (go, python), the name of the module") + .hasArg() + .build() + ) .addOption( Option .builder() @@ -169,6 +198,14 @@ private static Options getCliOptions() { .hasArg() .build() ) + .addOption( + Option + .builder() + .longOpt("output-python") + .desc(" output directory for generated Python files") + .hasArg() + .build() + ) .addOption( Option .builder() @@ -255,10 +292,13 @@ private record CliArguments( Path libraryRoot, Path modelPath, Path[] dependentModelPaths, + Map dependencyModuleNames, String namespace, + Optional moduleName, Optional outputDotnetDir, Optional outputJavaDir, Optional outputRustDir, + Optional outputPythonDir, Optional outputDafnyDir, Optional javaAwsSdkVersion, DafnyVersion dafnyVersion, @@ -291,8 +331,21 @@ static Optional parse(String[] args) throws ParseException { .map(Path::of) .toArray(Path[]::new); + // Maps a Smithy namespace to its module name + // ex. `aws.cryptography.materialproviders` -> `aws_cryptographic_materialproviders` + // These values are provided via the command line right now, + // but should eventually be sourced from doo files + final Map dependencyNamespacesToModuleNamesMap = + commandLine.hasOption("dependency-module-name") + ? Arrays.stream(commandLine.getOptionValues("dmn")) + .map(s -> s.split("=")) + .collect(Collectors.toMap(i -> i[0], i -> i[1])) + : new HashMap<>(); + final String namespace = commandLine.getOptionValue('n'); + final Optional moduleName = Optional.ofNullable(commandLine.getOptionValue("module-name")); + Optional outputDafnyDir = Optional .ofNullable(commandLine.getOptionValue("output-dafny")) .map(Paths::get); @@ -314,6 +367,9 @@ static Optional parse(String[] args) throws ParseException { final Optional outputRustDir = Optional .ofNullable(commandLine.getOptionValue("output-rust")) .map(Paths::get); + final Optional outputPythonDir = Optional + .ofNullable(commandLine.getOptionValue("output-python")) + .map(Paths::get); boolean localServiceTest = commandLine.hasOption("local-service-test"); final boolean awsSdkStyle = commandLine.hasOption("aws-sdk"); @@ -367,10 +423,13 @@ static Optional parse(String[] args) throws ParseException { libraryRoot, modelPath, dependentModelPaths, + dependencyNamespacesToModuleNamesMap, namespace, + moduleName, outputDotnetDir, outputJavaDir, outputRustDir, + outputPythonDir, outputDafnyDir, javaAwsSdkVersion, dafnyVersion, diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/.codecov.yml b/codegen/smithy-dafny-codegen-modules/smithy-python/.codecov.yml new file mode 100644 index 0000000000..5cd767e7f8 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/.codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + patch: + default: + target: '100' + project: + default: + target: '88' diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/.coveragerc b/codegen/smithy-dafny-codegen-modules/smithy-python/.coveragerc new file mode 100644 index 0000000000..cb600ca5e9 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/.coveragerc @@ -0,0 +1,8 @@ +[report] +exclude_lines = + # Ignore lines with no cover + pragma: no cover + # Ignore functions marked as crt-callback, coverage can't pick up on + # callbacks the crt calls + pragma: crt-callback + raise NotImplementedError diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/.github/CODEOWNERS b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/CODEOWNERS new file mode 100644 index 0000000000..a67d518bbe --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Add core contributors to all prs by default +* @smithy-lang/aws-sdk-python diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/.github/PULL_REQUEST_TEMPLATE.md b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..6bdaa999fc --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,6 @@ +*Issue #, if available:* + +*Description of changes:* + + +By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/ci.yml b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/ci.yml new file mode 100644 index 0000000000..8fd6f944fb --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/ci.yml @@ -0,0 +1,101 @@ +name: CI + +on: + push: + branches: develop + pull_request: + branches: develop + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + + steps: + - name: Checkout Repository + uses: actions/checkout@v1 + - name: Set up Python for Pants - 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Set up Python for CI - ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Run tests + run: | + ./pants --no-dynamic-ui test --use-coverage :: + - name: Upload coverage + if: matrix.python-version == 3.11 + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./dist/coverage/python/coverage.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: true + + test-codegen: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + + name: Codegen Java 17 / Python ${{ matrix.python-version }} + + steps: + - uses: actions/checkout@v3 + + - name: Download Coretto 17 JDK + run: | + download_url="https://corretto.aws/downloads/latest/amazon-corretto-17-x64-linux-jdk.tar.gz" + wget -O $RUNNER_TEMP/java_package.tar.gz $download_url + + - name: Set up Coretto 17 JDK + uses: actions/setup-java@v3 + with: + distribution: 'jdkfile' + jdkFile: ${{ runner.temp }}/java_package.tar.gz + java-version: 17 + architecture: x64 + + - name: clean and build without python + run: cd codegen && ./gradlew clean build -Plog-tests + + - name: Set up Python for Pants - 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Set Up Python for CI - ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install smithy-python + run: | + ./pants --no-dynamic-ui package :: + python${{ matrix.python-version }} -m pip install dist/*.whl + + - name: clean and build without formatting/linting installed + run: cd codegen && ./gradlew clean build -Plog-tests + + - name: Install black + run: | + python${{ matrix.python-version }} -m pip install --upgrade black + + - name: clean and build without linting installed + run: cd codegen && ./gradlew clean build -Plog-tests + + - name: Install mypy and other libraries necessary for typing + run: | + python${{ matrix.python-version }} -m pip install --upgrade mypy pytest pytest-asyncio + + - name: clean and build with all optional tools installed + run: cd codegen && ./gradlew clean build -Plog-tests diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/lint.yml b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/lint.yml new file mode 100644 index 0000000000..aec6f2337d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/lint.yml @@ -0,0 +1,24 @@ +name: Lint code + +on: + push: + branches: develop + pull_request: + branches: develop + +jobs: + lint: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python for Pants - 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Set up Python for linting - 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Run pre-commit + uses: pre-commit/action@v3.0.0 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/protocol-test.yml b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/protocol-test.yml new file mode 100644 index 0000000000..356a8b9b31 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/protocol-test.yml @@ -0,0 +1,38 @@ +name: Protocol Tests + +on: + push: + branches: develop + pull_request: + branches: develop + +jobs: + protocol-test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + name: RestJson1 + + steps: + - uses: actions/checkout@v3 + + - name: Set up Corretto 17 JDK + uses: actions/setup-java@v3 + with: + distribution: corretto + java-version: 17 + + - name: Set up Python for Pants - 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Set up Python for protocol tests - 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Run protocol tests + run: make test-protocols diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/typecheck.yml b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/typecheck.yml new file mode 100644 index 0000000000..4a489668ca --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/.github/workflows/typecheck.yml @@ -0,0 +1,26 @@ +name: Type Check + +on: + push: + branches: develop + pull_request: + branches: develop + +jobs: + mypy: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repository + uses: actions/checkout@v1 + - name: Set up Python for Pants - 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Set up Python for type checking 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Run type checks + run: | + ./pants --no-dynamic-ui check :: diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/.gitignore b/codegen/smithy-dafny-codegen-modules/smithy-python/.gitignore new file mode 100644 index 0000000000..715279f21d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/.gitignore @@ -0,0 +1,41 @@ +# Eclipse +.classpath +.project +.settings/ + +# Intellij +.idea/ +*.iml +*.iws + +# VS Code +.vscode/ + +# Mac +.DS_Store + +# Maven +target/ +**/dependency-reduced-pom.xml + +# Gradle +/.gradle +build/ +*/out/ +*/*/out/ + +# Python +__pycache__ +*.swp +.mypy_cache +.coverage +coverage.xml +htmlcov +*.egg-info +dist + +# Pants workspace files +/.pants.d/ +/dist/ +/.pids +/.pants.workdir.file_lock* diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/.pre-commit-config.yaml b/codegen/smithy-dafny-codegen-modules/smithy-python/.pre-commit-config.yaml new file mode 100644 index 0000000000..521872b981 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +repos: + - repo: 'https://github.com/pre-commit/pre-commit-hooks' + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + exclude: python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/ + - id: trailing-whitespace + - repo: local + hooks: + - id: pants-fmt + name: pants fmt + description: runs ./pants fmt + entry: './pants --no-dynamic-ui fmt ::' + language: system + pass_filenames: false + - id: pants-lint + name: pants lint + description: runs ./pants lint + entry: './pants --no-dynamic-ui lint ::' + language: system + pass_filenames: false diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/BUILD b/codegen/smithy-dafny-codegen-modules/smithy-python/BUILD new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/CHANGES.md b/codegen/smithy-dafny-codegen-modules/smithy-python/CHANGES.md new file mode 100644 index 0000000000..6722a75678 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/CHANGES.md @@ -0,0 +1,5 @@ +# Change Log + +## Unreleased + +* Description of change. (Issue Number) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/CODE_OF_CONDUCT.md b/codegen/smithy-dafny-codegen-modules/smithy-python/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..5b627cfa60 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/CODE_OF_CONDUCT.md @@ -0,0 +1,4 @@ +## Code of Conduct +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +opensource-codeofconduct@amazon.com with any additional questions or comments. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/CONTRIBUTING.md b/codegen/smithy-dafny-codegen-modules/smithy-python/CONTRIBUTING.md new file mode 100644 index 0000000000..905fa66ebc --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/CONTRIBUTING.md @@ -0,0 +1,60 @@ +# Contributing Guidelines + +Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional +documentation, we greatly value feedback and contributions from our community. + +Please read through this document before submitting any issues or pull requests to ensure we have all the necessary +information to effectively respond to your bug report or contribution. + + +## Reporting Bugs/Feature Requests + +We welcome you to use the GitHub issue tracker to report bugs or suggest features. + +When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already +reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: + +* A reproducible test case or series of steps +* The version of our code being used +* Any modifications you've made relevant to the bug +* Anything unusual about your environment or deployment + + +## Contributing via Pull Requests +Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: + +1. You are working against the latest source on the *develop* branch. +2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. +3. You open an issue to discuss any significant work - we would hate for your time to be wasted. + +To send us a pull request, please: + +1. Fork the repository. +2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. +3. Ensure local tests pass. +4. Run `./pants fmt ::`, `./pants lint ::`, and `./pants check ::` if you've changed any python source. +4. Commit to your fork using clear commit messages. +5. Send us a pull request, answering any default questions in the pull request interface. +6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. + +GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and +[creating a pull request](https://help.github.com/articles/creating-a-pull-request/). + + +## Finding contributions to work on +Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. + + +## Code of Conduct +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +opensource-codeofconduct@amazon.com with any additional questions or comments. + + +## Security issue notifications +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. + + +## Licensing + +See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/LICENSE b/codegen/smithy-dafny-codegen-modules/smithy-python/LICENSE new file mode 100644 index 0000000000..67db858821 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/LICENSE @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/Makefile b/codegen/smithy-dafny-codegen-modules/smithy-python/Makefile new file mode 100644 index 0000000000..fd846da425 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/Makefile @@ -0,0 +1,31 @@ +help: ## Show this help. + @sed -ne '/@sed/!s/## //p' $(MAKEFILE_LIST) + + +install-python-components: ## Builds and installs the python packages. + ./pants package :: + python3 -m pip install dist/*.whl --force-reinstall + + +install-java-components: ## Publishes java packages to maven local. + cd codegen && ./gradlew publishToMavenLocal + + +install-components: install-python-components install-java-components ## Installs java and python components locally. + + +smithy-build: ## Builds the Java code generation packages. + cd codegen && ./gradlew clean build + + +generate-protocol-tests: ## Generates the protocol tests, rebuilding necessary Java packages. + cd codegen && ./gradlew clean :smithy-python-protocol-test:build + + +run-protocol-tests: ## Runs already-generated protocol tests + cd codegen/smithy-python-protocol-test/build/smithyprojections/smithy-python-protocol-test/rest-json-1/python-client-codegen && \ + python3 -m pip install '.[tests]' && \ + python3 -m pytest tests + + +test-protocols: install-python-components generate-protocol-tests run-protocol-tests ## Generates and runs protocol tests. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/README.md b/codegen/smithy-dafny-codegen-modules/smithy-python/README.md new file mode 100644 index 0000000000..67fd5e07f2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/README.md @@ -0,0 +1,287 @@ +## Smithy Python + +**WARNING: All interfaces are subject to change.** + +We are in the early stages of beginning work on low-level Python SDK modules +that aim to provide basic, reusable, and composable interfaces for lower level +SDK tasks. Using these modules customers should be able to generate +asynchronous service client implementations based on services defined using +[Smithy](https://smithy.io/). + +This code generator, and the clients it generates, are unstable and should not +be used in production systems yet. Several features, such as detailed logging, +have not been implemented yet. + +### What is this repository? + +This repository contains two major components: + +1) Smithy code generators for Python +2) Core modules and interfaces for building service clients in Python + +These components facilitate generating clients for any [Smithy](https://smithy.io/) +service. The `codegen` directory contains the source code for generating clients. +The `python-packages` directory contains the source code for the hand-written python +components. + +This repository does *not* contain any generated clients, such as for S3 or other +AWS services. Rather, these are the tools that facilitate the generation of those +clients and non-AWS Smithy clients. + +### How do I use this? + +The first step is to create a Smithy pacakge. If this is your first time working +with Smithy, follow [this quickstart guide](https://smithy.io/2.0/quickstart.html) +to learn the basics and create a simple Smithy model. + +Once you have a service defined in Smithy, you will need to define what protocol +it uses. Currently the only supported protocol is +[restJson1](https://smithy.io/2.0/aws/protocols/aws-restjson1-protocol.html). +This is a protocol based on AWS services, but is broadly applicable to any +service that uses rest bindings with a JSON body type. Simply add the protocol +trait to your service shape and you'll be ready. + +The following is a basic example service model that echoes messages sent to it. +To use this model to generate a client, save it to a file called `main.smithy` +in a folder called `model`. + +```smithy +$version: "2.0" + +namespace com.example + +use aws.protocols#restJson1 + +/// Echoes input +@restJson1 +service EchoService { + version: "2006-03-01" + operations: [EchoMessage] +} + +@http(uri: "/echo", method: "POST") +operation EchoMessage { + input := { + @httpHeader("x-echo-message") + message: String + } + output := { + message: String + } +} +``` + +You also will need a build configuration file named `smithy-build.json`, which +for this example service should look the following json. For more information on +this file, see the +[smithy-build docs](https://smithy.io/2.0/guides/building-models/build-config.html). + +```json +{ + "version": "1.0", + "sources": ["model"], + "maven": { + "dependencies": [ + "software.amazon.smithy:smithy-model:[1.34.0,2.0)", + "software.amazon.smithy:smithy-aws-traits:[1.34.0,2.0)", + "software.amazon.smithy.python:smithy-python-codegen:0.1.0" + ] + }, + "projections": { + "client": { + "plugins": { + "python-client-codegen": { + "service": "com.example#EchoService", + "module": "echo", + "moduleVersion": "0.0.1" + } + } + } + } +} +``` + +The code generator, `smithy-python-codegen`, hasn't been published yet, so +you'll need to build it yourself. To build and run the generator you will need +the following prerequisites: + +* Python 3.11 or newer + * (optional) Install [black](https://black.readthedocs.io/en/stable/) in your + environment to have the generated output be auto-formatted. +* The [Smithy CLI](https://smithy.io/2.0/guides/smithy-cli/cli_installation.html) +* JDK 17 or newer +* make + +Now run `make install-components` from the root of this repository. This will +install the python dependencies in your environment and make the code generator +available locally. For more information on the underlying build process, see the +"Using repository tooling" section. + +Now from your model directory run `smithy build` and you'll have a generated +client! The client can be found in `build/smithy/client/python-client-codegen`. +The following is a snippet showing how you might use it: + +```python +import asyncio + +from echo.client import EchoService +from echo.config import Config +from echo.models import EchoMessageInput + + +async def main() -> None: + client = EchoService(Config(endpoint_uri="https://example.com/")) + response = await client.echo_message(EchoMessageInput(message="spam")) + print(response.message) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +#### Is Java really required? + +Only for now. Once the generator has been published, the Smithy CLI will be able +to run the it without a separate Java installation. Similarly, once the python +helper libraries have been published you won't need to install them manually. + +### Core Modules and Interfaces + +The `smithy-python` package provides the core modules and interfaces required +to build a service client. These basic modules include things like: +an HTTP/1.1 and HTTP/2 client implementation, retry strategies, etc. + +The `aws-smithy-python` package provides implementations of those interfaces +for AWS, such as sigv4 signers. + +### What are the design goals of this project? + +* **Components must be modular** - Most importantly, these building blocks +need to be composable and reusable across a wide variety of use cases, +including use cases beyond an AWS SDK. Interfaces such as credential resolvers, +request signing, data models, serialization, etc. should all be reusable across +many different contexts. + +* **Components should be well documented and publicly exported** - Both AWS and +customers should have a high level of confidence that the building blocks we're +creating are well supported, understood, and maintained. Customers should not +have to hack on internal or undocumented interfaces to achieve their goals. + +* **Components must be typed** - All of the buildings blocks we create must be +typed and usable via `mypy`. Given the nature of gradual typing it's paramount +that foundational components and interfaces be typed to preserve the integrity +of the typing system. + +* **Components should be consistent with other AWS SDKs** - When building +interfaces or libraries that overlap with the required functionality of other +AWS SDKs we should strive to be consistent with other SDKs as our deafult +stance. This project will heavily draw insipiration from the precedents set +by the [smithy-typescript](https://github.com/awslabs/smithy-typescript/) and +[smithy-go](https://github.com/aws/smithy-go) packages. + +### How can I contribute? + +We're currently heavily investing in writing proposals and documenting the +design decisions made. Feedback on the +[proposed designs and interfaces](https://github.com/awslabs/smithy-python/tree/develop/designs) +is extremely helpful at this stage to ensure we're providing functional and +ergonomic interfaces that meet customer expectations. + +### Using repository tooling + +This repository is intended to contain the source for multiple Python and Java +packages, so the process of development may be a bit different than what you're +familiar with. + +#### Java - gradle + +The Java-based code generation uses Gradle, which is a fairly common Java build +tool that natively supports building, testing, and publishing multiple packages +in one place. If you've used Gradle before, then there's nothing in this repo +that will surprise you. + +If you haven't used Gradle before, don't worry - it's pretty easy to use. You +will need to have JDK 17 or newer installed, but that's the only thing you need +to install yourself. We recommend the +[Coretto](https://docs.aws.amazon.com/corretto/latest/corretto-17-ug/downloads-list.html) +distribution, but any JDK that's at least version 17 will work. + +To build and run all the Java packages, simply run `./gradlew clean build` from +the `codegen` directory. If this is the first time you have run this, it will +download Gradle onto your system to run along with any dependencies of the Java +packages. + +For more details on working on the code generator, see the readme in the +`codegen` directory. + +#### Python - pants + +Building multiple python packages in a single repo is a little less common than +it is for Java or some other languages, so even if you're a python expert you +may be unfamiliar with the tooling. The tool we're using is called +[pants](https://www.pantsbuild.org), and you use it pretty similarly to how you +use Gradle. + +Like Gradle, pants provides a wrapper script that downloads its dependencies as +needed. Currently, pants requires python 3.7, 3.8, or 3.9 to run, so one of +those must be available on your path. (It doesn't have to be the version that +is linked to `python` or `python3`, it just needs `python3.9` etc.) It is, +however, fully capable of building and working with code that uses newer python +versions like we do. This repository uses a minimum python version of 3.11 +for all its packages, so you will need that too to work on it. + +Pants provides a number of python commands it calls goals, documented +[here](https://www.pantsbuild.org/docs/python-goals). In short: + +* `./pants fmt ::` - This will run our formatters on all of the python library + code. Use this before you make a commit. +* `./pants lint ::` - This will run our linters on all of the python library + code. You should also use this before you make a commit, and particularly + before you make a pull request. +* `./pants check ::` - This will run mypy on all of the python library code. + This should be used regularly, and must pass for any pull request. +* `./pants test ::` - This will run all of the tests written for the python + library code. Use this as often as you'd run pytest or any other testing + tool. Under the hood, we are using pytest. + +There are other commands as well that you can find in the +[docs](https://www.pantsbuild.org/docs/python-goals), but these are the ones +you'll use the most. + +Important to note is those pairs of colons. These are pants +[targets](https://www.pantsbuild.org/docs/targets#target-addresses). The double +colon is a special target that means "everything". So running exactly what's +listed above will run those goals on every python file or other relevant file. +You can also target just `smithy_python`, for example, with +`./pants check python-packages/smithy-python/smithy_python:source`, or even +individual files with something like +`./pants check python-packages/smithy-python/smithy_python/interfaces/http.py:../source`. +To list what targets are available in a directory, run +`./pants list path/to/dir:`. For more detailed information, see the +[docs](https://www.pantsbuild.org/docs/targets#target-addresses). + +#### Common commands - make + +There is also a `Makefile` that bridges the Python and Java build systems together to +make common workflows simple, single commands. The two most important commands are: + +* `make install-components` which builds and installs the Java generator and the python + packages. The generator is published to maven local and the python packages are + installed into the active python environment. This command is most useful for those + who simply want to run the generator and use a generated client.v +* `make test-protocols` which runs all the protocol tests. It will first (re)install + all necessary components to ensure that the latest is being used. This is most useful + for developers working on the generator and python packages. + +To see what else available, run `make help` or examine the file directly. + +## Security issue notifications + +If you discover a potential security issue in this project we ask that you +notify AWS/Amazon Security via our +[vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). +Please do **not** create a public github issue. + +## License + +This project is licensed under the Apache-2.0 License. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/ROADMAP.md b/codegen/smithy-dafny-codegen-modules/smithy-python/ROADMAP.md new file mode 100644 index 0000000000..1ecde3d6d7 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/ROADMAP.md @@ -0,0 +1,34 @@ +# smithy-python interfaces +* Data Class Generation + * Scalar Values + * Enums + * Structures + * List + * Set + * Union + * Documents + * Exceptions +* HTTP Client Interfaces + * Request / Response data classes + * Mypy Protocol Definitions for HTTP client/session +* Middleware Stack +* Service Client Generation +* Operation Interface Generation +* Paginators +* Waiters +* Event Streams + +# Standalone +* CRT based HTTP client that implements Smithy HTTP Client Interface +* AWS signing package + * SigV4 + * SigV4a +* AWS credential resolution package + +# aws-python-sdk +* Endpoint Resolution +* Marshalling Generation + * Serialization (Query, JSON, RestJSON, RestXML) + * Deserialization (Query, JSON, RestJSON, RestXML) +* Credential Resolution +* Client Configuration diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/.gitignore b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/.gitignore new file mode 100644 index 0000000000..c01141aa45 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/.gitignore @@ -0,0 +1,22 @@ +# Eclipse +.classpath +.project +.settings/ + +# Intellij +.idea/ +*.iml +*.iws + +# Mac +.DS_Store + +# Maven +target/ +**/dependency-reduced-pom.xml + +# Gradle +/.gradle +build/ +*/out/ +*/*/out/ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/README.md b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/README.md new file mode 100644 index 0000000000..0f1444d92c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/README.md @@ -0,0 +1,88 @@ +## Smithy Python + +This project defines the [Smithy](https://smithy.io/) code generators for Python +clients. + +### Prerequisites + +* JDK 17 or newer is required to run the code generator. The + [Coretto](https://docs.aws.amazon.com/corretto/latest/corretto-17-ug/downloads-list.html) + distribution is recommended. + +#### Optional Prerequisites - Python + +* Python 3.11 is required to run the generated code, but not run the generator. + If it is present on the path, the generator will use it for linting and + formatting. +* Use of a [Python virtual environment](https://docs.python.org/3/library/venv.html) + is highly recommended. +* If `black` is installed in the version of python found on the path, it will + be used to format the generated code. +* If `mypy` is installed in the version of python found on the path, it will + be used to check the generated code. For mypy to pass, the `smithy_python` + package will need to be installed. To install those into your active environment, + run `make install-python-components` from the repository root. + +### Building the generator + +The code generator uses the [gradle](https://gradle.org) build system, accessed +via the gradle wrapper `gradlew`. To build the generator, simply run +`./gradlew clean build`. Alternatively, run `make smithy-build` from the repo +root. To run the protocol tests, run `make test-protocols`. + +**WARNING: All interfaces are subject to change.** + +### Where should I get started? + +The best place to start is the [Smithy documentation](https://smithy.io/) to understand +what Smithy is and how this project relates to it. In particular, the [Creating a +Smithy Code Generator](https://smithy.io/2.0/guides/building-codegen/index.html) guide +covers the overall design of Smithy generators. + + +[`PythonClientCodegenPlugin` +](https://github.com/awslabs/smithy-python/blob/develop/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.java), +a [Smithy build plugin +](https://smithy.io/2.0/guides/building-codegen/creating-codegen-repo.html#creating-a-smithy-build-plugin), +is the entry point where this code generator links to the Smithy build process +(see also: [SmithyBuildPlugin javadoc +](https://smithy.io/javadoc/1.26.1/software/amazon/smithy/build/SmithyBuildPlugin.html)). +This class doesn't do much by itself, but everything flows from here. + +Another good place to start is [`DirectedPythonCodegen` +](https://github.com/awslabs/smithy-python/blob/develop/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java). +This is an implementation of Smithy's [directed codegen interface +](https://smithy.io/javadoc/1.26.1/software/amazon/smithy/codegen/core/directed/DirectedCodegen.html) +, which enables us to make use of shared orchestration code and provides a more guided +path to generating a client. This class is the heart of the generator, everything else +follows from here. For example, you could look here to find out how Smithy structures +are generated into Python objects. Most of the code that does that is somewhere else, +but it's called directly from here. This class is constructed by `PythonCodegenPlugin` +and handed off to a [`CodegenDirector` +](https://smithy.io/javadoc/1.26.1/software/amazon/smithy/codegen/core/directed/CodegenDirector.html) +which calls its public methods. + +One more possible starting point is [`SymbolVisitor` +](https://github.com/awslabs/smithy-python/blob/develop/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SymbolVisitor.java). +This class is responsible for taking a Smithy shape and determining what its +name in Python should be, where it should be defined in Python, what dependencies +it has, and attaching any other important properties that are needed for generating +the python types. See the [smithy docs +](https://smithy.io/2.0/guides/building-codegen/decoupling-codegen-with-symbols.html) +for more details on the symbol provider concept. + +Finally, you might look at the [`integration` package +](https://github.com/awslabs/smithy-python/tree/develop/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration). +This package is what provides [plugins +](https://smithy.io/2.0/guides/building-codegen/making-codegen-pluggable.html) +to the python generator. Anything that implements [`PythonIntegration` +](https://github.com/awslabs/smithy-python/blob/develop/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/PythonIntegration.java) +which is present on the Java classpath can hook into the code generation process. +Crucially, this is how protocol implementations are implemented and how default +runtime customizations are added. See +[`RestJsonIntegration`](https://github.com/awslabs/smithy-python/blob/develop/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/RestJsonIntegration.java) +for an example of a protocol implementation. + +## License + +This project is licensed under the Apache-2.0 License. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/build.gradle.kts b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/build.gradle.kts new file mode 100644 index 0000000000..8e43ff05e5 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/build.gradle.kts @@ -0,0 +1,257 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +plugins { + `java-library` + `maven-publish` + signing + checkstyle + jacoco + id("com.github.spotbugs") version "5.0.14" + id("io.codearte.nexus-staging") version "0.30.0" +} + +allprojects { + group = "software.amazon.smithy.python" + version = "0.1.0" +} + +// The root project doesn't produce a JAR. +tasks["jar"].enabled = false + +// Load the Sonatype user/password for use in publishing tasks. +val sonatypeUser: String? by project +val sonatypePassword: String? by project + +/* + * Sonatype Staging Finalization + * ==================================================== + * + * When publishing to Maven Central, we need to close the staging + * repository and release the artifacts after they have been + * validated. This configuration is for the root project because + * it operates at the "group" level. + */ +if (sonatypeUser != null && sonatypePassword != null) { + apply(plugin = "io.codearte.nexus-staging") + + nexusStaging { + packageGroup = "software.amazon" + stagingProfileId = "e789115b6c941" + + username = sonatypeUser + password = sonatypePassword + } +} + +repositories { + mavenLocal() + mavenCentral() +} + +subprojects { + val subproject = this + + /* + * Java + * ==================================================== + */ + if (subproject.name != "smithy-python-codegen-test" && subproject.name != "smithy-python-protocol-test") { + apply(plugin = "java-library") + + java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(17)) + } + } + + tasks.withType { + options.encoding = "UTF-8" + } + + // Use Junit5's test runner. + tasks.withType { + useJUnitPlatform() + } + + // Apply junit 5 and hamcrest test dependencies to all java projects. + dependencies { + testCompileOnly("org.junit.jupiter:junit-jupiter-api:5.4.0") + testImplementation("org.junit.jupiter:junit-jupiter-engine:5.4.0") + testCompileOnly("org.junit.jupiter:junit-jupiter-params:5.4.0") + testCompileOnly("org.hamcrest:hamcrest:2.1") + } + + // Reusable license copySpec + val licenseSpec = copySpec { + from("${project.rootDir}/LICENSE") + from("${project.rootDir}/NOTICE") + } + + // Set up tasks that build source and javadoc jars. + tasks.register("sourcesJar") { + metaInf.with(licenseSpec) + from(sourceSets.main.get().allJava) + archiveClassifier.set("sources") + } + + tasks.register("javadocJar") { + metaInf.with(licenseSpec) + from(tasks.javadoc) + archiveClassifier.set("javadoc") + } + + // Configure jars to include license related info + tasks.jar { + metaInf.with(licenseSpec) + inputs.property("moduleName", subproject.extra["moduleName"]) + manifest { + attributes["Automatic-Module-Name"] = subproject.extra["moduleName"] + } + } + + // Always run javadoc after build. + tasks["build"].finalizedBy(tasks["javadoc"]) + + /* + * Maven + * ==================================================== + */ + apply(plugin = "maven-publish") + apply(plugin = "signing") + + repositories { + mavenLocal() + mavenCentral() + } + + publishing { + repositories { + mavenCentral { + url = uri("https://aws.oss.sonatype.org/service/local/staging/deploy/maven2/") + credentials { + username = sonatypeUser + password = sonatypePassword + } + } + } + + publications { + create("mavenJava") { + from(components["java"]) + + // Ship the source and javadoc jars. + artifact(tasks["sourcesJar"]) + artifact(tasks["javadocJar"]) + + // Include extra information in the POMs. + afterEvaluate { + pom { + name.set(subproject.extra["displayName"].toString()) + description.set(subproject.description) + url.set("https://github.com/awslabs/smithy-python") + licenses { + license { + name.set("Apache License 2.0") + url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") + distribution.set("repo") + } + } + developers { + developer { + id.set("smithy") + name.set("Smithy") + organization.set("Amazon Web Services") + organizationUrl.set("https://aws.amazon.com") + roles.add("developer") + } + } + scm { + url.set("https://github.com/awslabs/smithy-python.git") + } + } + } + } + } + } + + // Don't sign the artifacts if we didn't get a key and password to use. + val signingKey: String? by project + val signingPassword: String? by project + if (signingKey != null && signingPassword != null) { + signing { + useInMemoryPgpKeys(signingKey, signingPassword) + sign(publishing.publications["mavenJava"]) + } + } + + /* + * CheckStyle + * ==================================================== + */ + apply(plugin = "checkstyle") + + tasks["checkstyleTest"].enabled = false + + /* + * Tests + * ==================================================== + * + * Configure the running of tests. + */ + // Log on passed, skipped, and failed test events if the `-Plog-tests` property is set. + if (project.hasProperty("log-tests")) { + tasks.test { + testLogging.events("passed", "skipped", "failed") + } + } + + /* + * Code coverage + * ==================================================== + */ + apply(plugin = "jacoco") + + // Always run the jacoco test report after testing. + tasks["test"].finalizedBy(tasks["jacocoTestReport"]) + + // Configure jacoco to generate an HTML report. + tasks.jacocoTestReport { + reports { + xml.required.set(false) + csv.required.set(false) + html.outputLocation.set(file("$buildDir/reports/jacoco")) + } + } + + /* + * Spotbugs + * ==================================================== + */ + apply(plugin = "com.github.spotbugs") + + // We don't need to lint tests. + tasks["spotbugsTest"].enabled = false + + // Configure the bug filter for spotbugs. + spotbugs { + setEffort("max") + val excludeFile = File("${project.rootDir}/config/spotbugs/filter.xml") + if (excludeFile.exists()) { + excludeFilter.set(excludeFile) + } + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/config/checkstyle/checkstyle.xml b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/config/checkstyle/checkstyle.xml new file mode 100644 index 0000000000..419a3d1a7f --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/config/checkstyle/checkstyle.xml @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/config/checkstyle/suppressions.xml b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/config/checkstyle/suppressions.xml new file mode 100644 index 0000000000..e1564229e1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/config/checkstyle/suppressions.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/config/spotbugs/filter.xml b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/config/spotbugs/filter.xml new file mode 100644 index 0000000000..9290b58a87 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/config/spotbugs/filter.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/gradle.properties b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/gradle.properties new file mode 100644 index 0000000000..a8e2413eaf --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/gradle.properties @@ -0,0 +1,2 @@ +smithyVersion=[1.34.0,2.0) +smithyGradleVersion=0.6.0 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/gradlew b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/gradlew new file mode 100755 index 0000000000..aeb74cbb43 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/gradlew @@ -0,0 +1,245 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/gradlew.bat b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/gradlew.bat new file mode 100644 index 0000000000..93e3f59f13 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/settings.gradle.kts b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/settings.gradle.kts new file mode 100644 index 0000000000..84886a78b0 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/settings.gradle.kts @@ -0,0 +1,27 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +rootProject.name = "smithy-python" +include(":smithy-python-codegen") +include(":smithy-aws-python-codegen") +include(":smithy-python-codegen-test") +include(":smithy-python-protocol-test") + +pluginManagement { + repositories { + mavenLocal() + gradlePluginPortal() + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/README.md b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/README.md new file mode 100644 index 0000000000..24a3b3affe --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/README.md @@ -0,0 +1,31 @@ +## Smithy AWS Python Codegen + +This package implements AWS-specific code generation plugins to the python generator. +Anything that is specific to AWS MUST be implemented here. Examples include most [AWS +protocols](https://smithy.io/2.0/aws/protocols/index.html)(*), +[sigv4](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_aws-signing.html), +and AWS service customizations. Conversely, any features that are +NOT specific to AWS MUST NOT be implemented here. + +*The only exception to this rule is the `RestJson1` protocol implementation, which is +included in the generic generator for now to provide a default supported protocol. + +One very important thing to keep in mind when implementing features and integrations +here is that they MUST NOT be coupled wherever possible. For example, a user +MUST be able to use sigv4 even if they aren't using an AWS protocol or even the +[service trait](https://smithy.io/2.0/aws/aws-core.html#aws-api-service-trait). + +### Why separate AWS components from the core package and each other? + +Smithy is intended to be a generic IDL that can describe a broad range of protocols, +not just AWS protocols. Separating the code generation components forces developers +to provide interfaces capable of achieving that goal and ensures that users only +have to take what they need. + +In the future, these components may be moved to an entirely different repository to +strengthen that divide even more. + +### When should I change this package? + +Any time an AWS component needs to be modified or added. This can include adding +support for new protocols, new auth traits, or new service customizations. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/bin/main/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/bin/main/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration new file mode 100644 index 0000000000..9743a30a03 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/bin/main/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration @@ -0,0 +1,7 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +software.amazon.smithy.aws.python.codegen.AwsAuthIntegration +software.amazon.smithy.aws.python.codegen.AwsProtocolsIntegration diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/bin/main/software/amazon/smithy/aws/python/codegen/AwsAuthIntegration.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/bin/main/software/amazon/smithy/aws/python/codegen/AwsAuthIntegration.class new file mode 100644 index 0000000000..183d1a21bc Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/bin/main/software/amazon/smithy/aws/python/codegen/AwsAuthIntegration.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/bin/main/software/amazon/smithy/aws/python/codegen/AwsProtocolsIntegration.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/bin/main/software/amazon/smithy/aws/python/codegen/AwsProtocolsIntegration.class new file mode 100644 index 0000000000..a74efdd474 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/bin/main/software/amazon/smithy/aws/python/codegen/AwsProtocolsIntegration.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/build.gradle.kts b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/build.gradle.kts new file mode 100644 index 0000000000..14e78d25df --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/build.gradle.kts @@ -0,0 +1,27 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +description = "Generates AWS Python code from Smithy models" +extra["displayName"] = "Smithy :: AWS :: Python :: Codegen" +extra["moduleName"] = "software.amazon.smithy.aws.python.codegen" + +val smithyVersion: String by project + +buildscript { + val smithyVersion: String by project + + repositories { + mavenLocal() + mavenCentral() + } + dependencies { + "classpath"("software.amazon.smithy:smithy-cli:$smithyVersion") + } +} + +dependencies { + implementation(project(":smithy-python-codegen")) + implementation("software.amazon.smithy:smithy-aws-traits:$smithyVersion") +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/src/main/java/software/amazon/smithy/aws/python/codegen/AwsAuthIntegration.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/src/main/java/software/amazon/smithy/aws/python/codegen/AwsAuthIntegration.java new file mode 100644 index 0000000000..93b964b7d7 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/src/main/java/software/amazon/smithy/aws/python/codegen/AwsAuthIntegration.java @@ -0,0 +1,16 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.aws.python.codegen; + +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Adds support for AWS auth traits. + */ +@SmithyInternalApi +public class AwsAuthIntegration implements PythonIntegration { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/src/main/java/software/amazon/smithy/aws/python/codegen/AwsProtocolsIntegration.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/src/main/java/software/amazon/smithy/aws/python/codegen/AwsProtocolsIntegration.java new file mode 100644 index 0000000000..a5e858f807 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/src/main/java/software/amazon/smithy/aws/python/codegen/AwsProtocolsIntegration.java @@ -0,0 +1,22 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.aws.python.codegen; + +import java.util.List; +import software.amazon.smithy.python.codegen.integration.ProtocolGenerator; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Adds AWS protocols to the generator's list of supported protocols. + */ +@SmithyInternalApi +public class AwsProtocolsIntegration implements PythonIntegration { + @Override + public List getProtocolGenerators() { + return List.of(); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration new file mode 100644 index 0000000000..9743a30a03 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-aws-python-codegen/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration @@ -0,0 +1,7 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +software.amazon.smithy.aws.python.codegen.AwsAuthIntegration +software.amazon.smithy.aws.python.codegen.AwsProtocolsIntegration diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/README.md b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/README.md new file mode 100644 index 0000000000..ca1ec75410 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/README.md @@ -0,0 +1,22 @@ +## Smithy Python Codegen Test + +This package is a hand-written Smithy package that generates a python client. The +shapes in this package are intended to exercise as many parts of the generator as +possible to ensure they generate valid code that passes mypy checks. For +example, there are cases that ensure that we're properly escaping shape names that +would collide with built in types (e.g. a shape called `Exception` would collide with +the builtin of the same name if it weren't escaped). + +This package does not contain any actual unit tests. Mypy passing and python not +failing to parse the output is the intended test. For actual unit tests, see the +[`smithy-python-protocol-test` package +](https://github.com/awslabs/smithy-python/tree/develop/codegen/smithy-python-protocol-test). + +### When should I change this package? + +Any time an issue is discovered where the code generator generates invalid code +from a valid model. A similar shape should then be added to this package's Smithy +model. That shape then MUST be connected to the service, for example by adding it to +the output of an operation already attached to the service. If the shape isn't +connected in this way, it will be stripped from the model before code generation and +therefore will not be generated. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/build.gradle.kts b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/build.gradle.kts new file mode 100644 index 0000000000..e584bbdf62 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/build.gradle.kts @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +extra["displayName"] = "Smithy :: Python :: Codegen :: Test" +extra["moduleName"] = "software.amazon.smithy.python.codegen.test" + +tasks["jar"].enabled = false + +val smithyVersion: String by project + +buildscript { + val smithyVersion: String by project + + repositories { + mavenLocal() + mavenCentral() + } + dependencies { + "classpath"("software.amazon.smithy:smithy-cli:$smithyVersion") + } +} + +plugins { + val smithyGradleVersion: String by project + id("software.amazon.smithy").version(smithyGradleVersion) +} + +repositories { + mavenLocal() + mavenCentral() +} + +dependencies { + implementation(project(":smithy-python-codegen")) + implementation("software.amazon.smithy:smithy-waiters:$smithyVersion") + implementation("software.amazon.smithy:smithy-protocol-test-traits:$smithyVersion") +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/model/main.smithy b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/model/main.smithy new file mode 100644 index 0000000000..e1e21f1725 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/model/main.smithy @@ -0,0 +1,612 @@ +$version: "2.0" + +namespace example.weather + +use smithy.test#httpRequestTests +use smithy.test#httpResponseTests +use smithy.waiters#waitable +use aws.protocols#restJson1 + +/// Provides weather forecasts. +@restJson1 +@fakeProtocol +@paginated(inputToken: "nextToken", outputToken: "nextToken", pageSize: "pageSize") +@httpApiKeyAuth(name: "weather-auth", in: "header") +service Weather { + version: "2006-03-01" + resources: [City] + operations: [GetCurrentTime] +} + +resource City { + identifiers: { cityId: CityId } + read: GetCity + list: ListCities + resources: [Forecast, CityImage] + operations: [GetCityAnnouncements] +} + +resource Forecast { + identifiers: { cityId: CityId } + read: GetForecast +} + +resource CityImage { + identifiers: { cityId: CityId } + read: GetCityImage +} + +// "pattern" is a trait. +@pattern("^[A-Za-z0-9 ]+$") +string CityId + +@readonly +@suppress(["WaitableTraitInvalidErrorType"]) +@waitable( + CityExists: { + acceptors: [ + // Fail-fast if the thing transitions to a "failed" state. + { + state: "failure", + matcher: { + errorType: "NoSuchResource" + } + }, + // Fail-fast if the thing transitions to a "failed" state. + { + state: "failure", + matcher: { + errorType: "UnModeledError" + } + }, + // Succeed when the city image value is not empty i.e. enters into a "success" state. + { + state: "success", + matcher: { + success: true + } + }, + // Retry if city id input is of same length as city name in output + { + state: "retry", + matcher: { + inputOutput: { + path: "length(input.cityId) == length(output.name)", + comparator: "booleanEquals", + expected: "true", + } + } + }, + // Success if city name in output is seattle + { + state: "success", + matcher: { + output: { + path: "name", + comparator: "stringEquals", + expected: "seattle", + } + } + } + ] + } +) +@http(method: "GET", uri: "/cities/{cityId}") +operation GetCity { + input := { + // "cityId" provides the identifier for the resource and + // has to be marked as required. + @required + @httpLabel + cityId: CityId + } + output := { + // "required" is used on output to indicate if the service + // will always provide a value for the member. + @required + name: String + + @required + coordinates: CityCoordinates + + city: CitySummary + + cityData: JsonString + binaryCityData: JsonBlob + } + errors: [NoSuchResource, EmptyError] +} + +// Tests that HTTP protocol tests are generated. +apply GetCity @httpRequestTests([ + { + id: "WriteGetCityAssertions", + documentation: "Does something", + protocol: "example.weather#fakeProtocol", + method: "GET", + uri: "/cities/123", + body: "", + params: { + cityId: "123" + } + } +]) + +apply GetCity @httpResponseTests([ + { + id: "WriteGetCityResponseAssertions", + documentation: "Does something", + protocol: "example.weather#fakeProtocol", + code: 200, + body: """ + { + "name": "Seattle", + "coordinates": { + "latitude": 12.34, + "longitude": -56.78 + }, + "city": { + "cityId": "123", + "name": "Seattle", + "number": "One", + "case": "Upper" + } + }""", + bodyMediaType: "application/json", + params: { + name: "Seattle", + coordinates: { + latitude: 12.34, + longitude: -56.78 + }, + city: { + cityId: "123", + name: "Seattle", + number: "One", + case: "Upper" + } + } + } +]) + +@mediaType("application/json") +string JsonString + +@mediaType("application/json") +blob JsonBlob + +// This structure is nested within GetCityOutput. +structure CityCoordinates { + @required + latitude: Float + + @required + longitude: Float +} + +/// Error encountered when no resource could be found. +@error("client") +@httpError(404) +structure NoSuchResource { + /// The type of resource that was not found. + @required + resourceType: String + + message: String +} + +apply NoSuchResource @httpResponseTests([ + { + id: "WriteNoSuchResourceAssertions", + documentation: "Does something", + protocol: "example.weather#fakeProtocol", + code: 404, + body: """ + { + "resourceType": "City", + "message": "Your custom message" + }""", + bodyMediaType: "application/json", + params: { + resourceType: "City", + message: "Your custom message" + } + } +]) + +// This will have a synthetic message member added to it, even +// though it doesn't actually have one. +@error("client") +@httpError(400) +structure EmptyError {} + +// The paginated trait indicates that the operation may +// return truncated results. +@readonly +@paginated(items: "items") +@waitable( + "ListContainsCity": { + acceptors: [ + // failure in case all items returned match to seattle + { + state: "failure", + matcher: { + output: { + path: "items[].name", + comparator: "allStringEquals", + expected: "seattle", + } + } + }, + // success in case any items returned match to NewYork + { + state: "success", + matcher: { + output: { + path: "items[].name", + comparator: "anyStringEquals", + expected: "NewYork", + } + } + } + ] + } +) +@http(method: "GET", uri: "/cities") +operation ListCities { + input := { + @httpQuery("nextToken") + nextToken: String + + @httpQuery("aString") + aString: String + + @httpQuery("someEnum") + someEnum: StringYesNo + + @httpQuery("pageSize") + pageSize: Integer + }, + output := { + nextToken: String + + someEnum: StringYesNo + aString: String + defaults: Defaults + escaping: MemberEscaping + escapeTrue: True + escapeFalse: False + escapeNone: None + + @required + items: CitySummaries + sparseItems: SparseCitySummaries + + mutual: MutuallyRecursiveA + } +} + +apply ListCities @httpRequestTests([ + { + id: "WriteListCitiesAssertions" + documentation: "Does something" + protocol: "example.weather#fakeProtocol" + method: "GET" + uri: "/cities" + body: "" + queryParams: ["pageSize=50"] + forbidQueryParams: ["nextToken"] + params: { + pageSize: 50 + } + } +]) + +structure Defaults { + @required + requiredBool: Boolean + optionalBool: Boolean + defaultTrue: Boolean = true + defaultFalse: Boolean = false + @required + requiredDefaultBool: Boolean = true + + @required + requiredStr: String + optionalStr: String + defaultString: String = "spam" + @required + requiredDefaultStr: String = "eggs" + + @required + requiredInt: Integer + optionalInt: Integer + defaultInt: Integer = 42 + @required + requiredDefaultInt: Integer = 42 + + @required + requiredFloat: Float + optionalFloat: Float + defaultFloat: Float = 4.2 + @required + requiredDefaultFloat: Float = 4.2 + + @required + requiredBlob: Blob + optionalBlob: Blob + defaultBlob: Blob = "c3BhbQ==" + @required + requiredDefaultBlob: Blob = "c3BhbQ==" + + // timestamp + @required + requiredTimestamp: Timestamp + optionalTimestamp: Timestamp + defaultImplicitDateTime: Timestamp = "2011-12-03T10:15:30Z" + defaultImplicitEpochTime: Timestamp = 4.2 + defaultExplicitDateTime: DateTime = "2011-12-03T10:15:30Z" + defaultExplicitEpochTime: EpochSeconds = 4.2 + defaultExplicitHttpTime: HttpDate = "Tue, 3 Jun 2008 11:05:30 GMT" + @required + requiredDefaultTimestamp: Timestamp = 4.2 + + @required + requiredList: StringList + optionalList: StringList + defaultList: StringList = [] + @required + requiredDefaultList: StringList = [] + + @required + requiredMap: StringMap + optionalMap: StringMap + defaultMap: StringMap = {} + @required + requiredDefaultMap: StringMap = {} + + @required + requiredDocument: Document + optionalDocument: Document + defaultNullDocument: Document = null + defaultNumberDocument: Document = 42 + defaultStringDocument: Document = "spam" + defaultBooleanDocument: Document = true + defaultListDocument: Document = [] + defaultMapDocument: Document = {} + @required + requiredDefaultDocument: Document = "eggs" +} + +// This structure has members that need to be escaped. +structure MemberEscaping { + // This first set of member names are all reserved words that are a syntax + // error to use as identifiers. A full list of these can be found here: + // https://docs.python.org/3/reference/lexical_analysis.html#keywords + and: String + as: String + assert: String + async: String + await: String + break: String + class: String + continue: String + def: String + del: String + elif: String + else: String + except: String + finally: String + for: String + from: String + global: String + if: String + import: String + in: String + is: String + lambda: String + nonlocal: String + not: String + or: String + pass: String + raise: String + return: String + try: String + while: String + with: String + yield: String + + // These are built-in types, but not reserved words. They can be shadowed, + // but the shadowing naturally makes it impossible to use them later in + // scope. A listing of these can be found here: + // https://docs.python.org/3/library/stdtypes.html + bool: Boolean + dict: StringMap + float: Float + int: Integer + list: StringList + str: String + bytes: Blob + bytearray: Blob + + // We don't actually use these, but they're here for completeness. + complex: Float + tuple: StringList + range: StringList + memoryview: Blob + set: StringList + frozenset: StringList +} + +// These would result in class names that produce syntax errors since they're +// reserved words. +structure True {} +structure False {} +structure None {} + +@timestampFormat("date-time") +timestamp DateTime + +@timestampFormat("epoch-seconds") +timestamp EpochSeconds + +@timestampFormat("http-date") +timestamp HttpDate + +list StringList { + member: String +} + +structure MutuallyRecursiveA { + mutual: MutuallyRecursiveB +} + +structure MutuallyRecursiveB { + mutual: MutuallyRecursiveA +} + +// CitySummaries is a list of CitySummary structures. +list CitySummaries { + member: CitySummary +} + +// CitySummaries is a sparse list of CitySummary structures. +@sparse +list SparseCitySummaries { + member: CitySummary +} + +// CitySummary contains a reference to a City. +@references([{resource: City}]) +structure CitySummary { + @required + cityId: CityId + + @required + name: String + + number: String + case: String +} + +@readonly +@http(method: "GET", uri: "/current-time") +operation GetCurrentTime { + output := { + @required + time: Timestamp + } +} + +@readonly +@http(method: "GET", uri: "/cities/{cityId}/forecast") +operation GetForecast { + input := { + @required + @httpLabel + cityId: CityId + }, + output := { + chanceOfRain: Float + precipitation: Precipitation + } +} + +union Precipitation { + rain: PrimitiveBoolean + sleet: PrimitiveBoolean + hail: StringMap + snow: StringYesNo + mixed: IntYesNo + other: OtherStructure + blob: Blob + foo: example.weather.nested#Foo + baz: example.weather.nested.more#Baz +} + +structure OtherStructure {} + +enum StringYesNo { + YES + NO +} + +intEnum IntYesNo { + YES = 1 + NO = 2 +} + +map StringMap { + key: String + value: String +} + +@readonly +@suppress(["HttpMethodSemantics"]) +@http(method: "POST", uri: "/cities/{cityId}/image") +operation GetCityImage { + input := { + @required @httpLabel + cityId: CityId + + @required + imageType: ImageType + } + output := { + @httpPayload + @required + image: CityImageData + } + errors: [NoSuchResource] +} + +union ImageType { + raw: Boolean + png: PNGImage +} + +structure PNGImage { + @required + height: Integer + + @required + width: Integer +} + +@streaming +blob CityImageData + +@readonly +@http(method: "GET", uri: "/cities/{cityId}/announcements") +operation GetCityAnnouncements { + input := { + @required + @httpLabel + cityId: CityId + } + output := { + @httpHeader("x-last-updated") + lastUpdated: Timestamp + + @httpPayload + announcements: Announcements + } + errors: [NoSuchResource] +} + +@streaming +union Announcements { + police: Message + fire: Message + health: Message +} + +structure Message { + message: String + author: String +} + +// Define a fake protocol trait for use. +@trait +@protocolDefinition +structure fakeProtocol {} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/model/more-nesting.smithy b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/model/more-nesting.smithy new file mode 100644 index 0000000000..4274dd7507 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/model/more-nesting.smithy @@ -0,0 +1,8 @@ +$version: "2.0" + +namespace example.weather.nested.more + +structure Baz { + baz: String, + bar: String, +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/model/nested.smithy b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/model/nested.smithy new file mode 100644 index 0000000000..1dadfe5db4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/model/nested.smithy @@ -0,0 +1,8 @@ +$version: "2.0" + +namespace example.weather.nested + +structure Foo { + baz: String, + bar: String, +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/smithy-build.json b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/smithy-build.json new file mode 100644 index 0000000000..6249eaf5e1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen-test/smithy-build.json @@ -0,0 +1,10 @@ +{ + "version": "1.0", + "plugins": { + "python-client-codegen": { + "service": "example.weather#Weather", + "module": "weather", + "moduleVersion": "0.0.1" + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/README.md b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/README.md new file mode 100644 index 0000000000..2a64600a1b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/README.md @@ -0,0 +1,23 @@ +## Smithy Python Codegen + +This package implements generic Smithy Python client generation. This includes, but is +not limited to, the following capabilities: + +* Generating Python data types from Smithy shapes. +* Generating fully functional clients for Smithy services. +* Generating serializers and deserializers for generic protocols. +* Providing interfaces for implementing protocols and customizing of all code + generation. + +This package MUST NOT include any components that are only applicable to a particular +organization. For example, [sigv4 +](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_aws-signing.html) +(an AWS HTTP authorization mechanism) support MUST NOT be implemented in this package +since it isn't generally applicable outside of AWS. + +### When should I change this package? + +Any time one of the above capabilities needs to change. For example, if the plugin +mechanism for the code generator is missing some features then [`PythonIntegration` +](https://github.com/awslabs/smithy-python/blob/develop/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/PythonIntegration.java) +likely needs to be changed. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin new file mode 100644 index 0000000000..9df651510d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin @@ -0,0 +1,6 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +software.amazon.smithy.python.codegen.PythonClientCodegenPlugin diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration new file mode 100644 index 0000000000..10665fde4f --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration @@ -0,0 +1,7 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +software.amazon.smithy.python.codegen.integration.RestJsonIntegration +software.amazon.smithy.python.codegen.integration.HttpApiKeyAuth diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ApplicationProtocol.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ApplicationProtocol.class new file mode 100644 index 0000000000..6ef225266c Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ApplicationProtocol.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ClientGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ClientGenerator.class new file mode 100644 index 0000000000..a5691f3a20 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ClientGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/CodegenUtils.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/CodegenUtils.class new file mode 100644 index 0000000000..52f9a5388b Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/CodegenUtils.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/CollectionGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/CollectionGenerator.class new file mode 100644 index 0000000000..d7acd97730 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/CollectionGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ConfigGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ConfigGenerator.class new file mode 100644 index 0000000000..46071d6335 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ConfigGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ConfigProperty.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ConfigProperty.class new file mode 100644 index 0000000000..69fbcd1c1f Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ConfigProperty.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/DerivedProperty.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/DerivedProperty.class new file mode 100644 index 0000000000..41d1a0e284 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/DerivedProperty.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/DirectedPythonCodegen.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/DirectedPythonCodegen.class new file mode 100644 index 0000000000..af2f31305a Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/DirectedPythonCodegen.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/EnumGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/EnumGenerator.class new file mode 100644 index 0000000000..086b92bcc1 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/EnumGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/GenerationContext.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/GenerationContext.class new file mode 100644 index 0000000000..085bd15ec2 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/GenerationContext.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/HttpAuthGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/HttpAuthGenerator.class new file mode 100644 index 0000000000..5584b41121 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/HttpAuthGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.class new file mode 100644 index 0000000000..5c853fb435 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ImportDeclarations.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ImportDeclarations.class new file mode 100644 index 0000000000..f41e9a7325 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/ImportDeclarations.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/IntEnumGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/IntEnumGenerator.class new file mode 100644 index 0000000000..0bd7218913 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/IntEnumGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/MapGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/MapGenerator.class new file mode 100644 index 0000000000..f37532967f Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/MapGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.class new file mode 100644 index 0000000000..a196cfb450 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonDelegator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonDelegator.class new file mode 100644 index 0000000000..5f1dae59cb Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonDelegator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonDependency.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonDependency.class new file mode 100644 index 0000000000..167e5116b6 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonDependency.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonSettings.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonSettings.class new file mode 100644 index 0000000000..d2807066e0 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonSettings.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonWriter.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonWriter.class new file mode 100644 index 0000000000..f05ff39512 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/PythonWriter.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/SetupGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/SetupGenerator.class new file mode 100644 index 0000000000..60422eb4c4 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/SetupGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/SmithyPythonDependency.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/SmithyPythonDependency.class new file mode 100644 index 0000000000..bb4c131fe0 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/SmithyPythonDependency.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/StructureGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/StructureGenerator.class new file mode 100644 index 0000000000..fc3859c9ad Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/StructureGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/SymbolVisitor.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/SymbolVisitor.class new file mode 100644 index 0000000000..ce280bfc56 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/SymbolVisitor.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/UnionGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/UnionGenerator.class new file mode 100644 index 0000000000..228c16af40 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/UnionGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/AuthScheme.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/AuthScheme.class new file mode 100644 index 0000000000..545ea3aaed Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/AuthScheme.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/DocumentMemberDeserVisitor.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/DocumentMemberDeserVisitor.class new file mode 100644 index 0000000000..7086ad945a Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/DocumentMemberDeserVisitor.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/DocumentMemberSerVisitor.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/DocumentMemberSerVisitor.class new file mode 100644 index 0000000000..f9d89b7e6f Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/DocumentMemberSerVisitor.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/HttpApiKeyAuth.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/HttpApiKeyAuth.class new file mode 100644 index 0000000000..a84c1278b1 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/HttpApiKeyAuth.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.class new file mode 100644 index 0000000000..f53fd7cd0b Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.class new file mode 100644 index 0000000000..c80a78b6a5 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonMemberSerVisitor.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonMemberSerVisitor.class new file mode 100644 index 0000000000..9dc86b3e5e Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonMemberSerVisitor.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonPayloadDeserVisitor.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonPayloadDeserVisitor.class new file mode 100644 index 0000000000..74ea143511 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonPayloadDeserVisitor.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonShapeDeserVisitor.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonShapeDeserVisitor.class new file mode 100644 index 0000000000..baabdb9b5e Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonShapeDeserVisitor.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonShapeSerVisitor.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonShapeSerVisitor.class new file mode 100644 index 0000000000..26b6d6bac1 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/JsonShapeSerVisitor.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/ProtocolGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/ProtocolGenerator.class new file mode 100644 index 0000000000..cee176c91f Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/ProtocolGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/PythonIntegration.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/PythonIntegration.class new file mode 100644 index 0000000000..4e53e3a7b8 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/PythonIntegration.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/RestJsonIntegration.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/RestJsonIntegration.class new file mode 100644 index 0000000000..83ad10b47c Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/RestJsonIntegration.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/RestJsonProtocolGenerator.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/RestJsonProtocolGenerator.class new file mode 100644 index 0000000000..f09453db9d Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/RestJsonProtocolGenerator.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/RuntimeClientPlugin.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/RuntimeClientPlugin.class new file mode 100644 index 0000000000..0944bc285c Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/integration/RuntimeClientPlugin.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/reserved-class-names.txt b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/reserved-class-names.txt new file mode 100644 index 0000000000..a0a78e1853 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/reserved-class-names.txt @@ -0,0 +1,15 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Python reserved words for class names +# +# Most of python's reserved words are lower-case, so there isn't much +# here. Furthermore, import aliases can resolve any other conflicts +# with non-reserved builtins or other conflicting classes. +# +# A full list of reserved words can be found here: +# https://docs.python.org/3/reference/lexical_analysis.html#keywords +True +False +None diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/reserved-member-names.txt b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/reserved-member-names.txt new file mode 100644 index 0000000000..97a0cfd515 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/reserved-member-names.txt @@ -0,0 +1,83 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Python reserved words for members. + +# The following are reserved words that can't be used as identifiers at all. +# For example, the following would produce a syntax error: +# +# class Foo: +# pass: int +# +# A full list of these can be found here: +# +# https://docs.python.org/3/reference/lexical_analysis.html#keywords +# +and +as +assert +async +await +break +class +continue +def +del +elif +else +except +False +finally +for +from +global +if +import +in +is +lambda +None +nonlocal +not +or +pass +raise +return +True +try +while +with +yield + +# The following aren't reserved words, but are built-in types / functions that +# would break if you ever tried to refer to the type again in scope. For +# example: +# +# class Foo: +# str: str +# +# def __init__(self, str: str): +# pass +# +# That would have an exception in the definition of __init__ since when you use +# `str` as the type after you've defined `str` in scope, it thinks you're +# referencing `Foo.str` rather than the built-in type (or a type at all). +# +# A listing of these types can be found here: +# https://docs.python.org/3/library/stdtypes.html +# +# Note though that we only need to escape the types we use. +bool +bytes +bytearray +dict +float +int +list +str + +# For the exact same reason as above, these are names of common types +# that are likely imported in the generated code (e.g. datetime) +# We only escape the types we use. +datetime diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ConfigSection.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ConfigSection.class new file mode 100644 index 0000000000..cffd084bad Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ConfigSection.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthParametersSection.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthParametersSection.class new file mode 100644 index 0000000000..562b828b3a Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthParametersSection.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthSchemeResolverSection.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthSchemeResolverSection.class new file mode 100644 index 0000000000..a0158374ea Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthSchemeResolverSection.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/InitializeHttpAuthParametersSection.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/InitializeHttpAuthParametersSection.class new file mode 100644 index 0000000000..ca07f1f9d9 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/InitializeHttpAuthParametersSection.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/PyprojectSection.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/PyprojectSection.class new file mode 100644 index 0000000000..7f2ede97bd Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/PyprojectSection.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ReadmeSection.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ReadmeSection.class new file mode 100644 index 0000000000..0e32cfcf2a Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ReadmeSection.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ResolveEndpointSection.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ResolveEndpointSection.class new file mode 100644 index 0000000000..95b3e326a7 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ResolveEndpointSection.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ResolveIdentitySection.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ResolveIdentitySection.class new file mode 100644 index 0000000000..c7ab5e4e09 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/ResolveIdentitySection.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/SendRequestSection.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/SendRequestSection.class new file mode 100644 index 0000000000..1d9450f4f8 Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/SendRequestSection.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/SignRequestSection.class b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/SignRequestSection.class new file mode 100644 index 0000000000..ce02112c0a Binary files /dev/null and b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/bin/main/software/amazon/smithy/python/codegen/sections/SignRequestSection.class differ diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/build.gradle.kts b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/build.gradle.kts new file mode 100644 index 0000000000..5e134bcea5 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/build.gradle.kts @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +description = "Generates Python code from Smithy models" +extra["displayName"] = "Smithy :: Python :: Codegen" +extra["moduleName"] = "software.amazon.smithy.python.codegen" + +val smithyVersion: String by project + +plugins { + `java-library` +} + +repositories { + mavenLocal() + mavenCentral() +} + +buildscript { + val smithyVersion: String by project + + repositories { + mavenLocal() + mavenCentral() + } + dependencies { + "classpath"("software.amazon.smithy:smithy-cli:$smithyVersion") + } +} + +dependencies { + api("software.amazon.smithy:smithy-codegen-core:$smithyVersion") + implementation("software.amazon.smithy:smithy-waiters:$smithyVersion") + implementation("software.amazon.smithy:smithy-protocol-test-traits:$smithyVersion") + // We have this because we're using RestJson1 as a 'generic' protocol. + implementation("software.amazon.smithy:smithy-aws-traits:$smithyVersion") +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ApplicationProtocol.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ApplicationProtocol.java new file mode 100644 index 0000000000..8d6b3e2694 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ApplicationProtocol.java @@ -0,0 +1,63 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Represents the resolves {@link Symbol}s and references for an + * application protocol (e.g., "http", "mqtt", etc). + */ +@SmithyUnstableApi +public record ApplicationProtocol(String name, SymbolReference requestType, SymbolReference responseType) { + + /** + * Checks if the protocol is an HTTP based protocol. + * + * @return Returns true if it is HTTP based. + */ + public boolean isHttpProtocol() { + return name.startsWith("http"); + } + + /** + * Creates a default HTTP application protocol. + * + * @return Returns the created application protocol. + */ + public static ApplicationProtocol createDefaultHttpApplicationProtocol() { + return new ApplicationProtocol( + "http", + SymbolReference.builder() + .symbol(createHttpSymbol("HTTPRequest")) + .build(), + SymbolReference.builder() + .symbol(createHttpSymbol("HTTPResponse")) + .build() + ); + } + + private static Symbol createHttpSymbol(String symbolName) { + PythonDependency dependency = SmithyPythonDependency.SMITHY_PYTHON; + return Symbol.builder() + .namespace(dependency.packageName() + ".interfaces.http", ".") + .name(symbolName) + .addDependency(dependency) + .build(); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java new file mode 100644 index 0000000000..53c7da60d4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -0,0 +1,658 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.Set; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.model.knowledge.ServiceIndex; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.traits.StringTrait; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.python.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.python.codegen.sections.InitializeHttpAuthParametersSection; +import software.amazon.smithy.python.codegen.sections.ResolveEndpointSection; +import software.amazon.smithy.python.codegen.sections.ResolveIdentitySection; +import software.amazon.smithy.python.codegen.sections.SendRequestSection; +import software.amazon.smithy.python.codegen.sections.SignRequestSection; +import software.amazon.smithy.utils.CodeSection; + +/** + * Generates the actual client and implements operations. + */ +public class ClientGenerator implements Runnable { + + protected final GenerationContext context; + protected final ServiceShape service; + + protected ClientGenerator(GenerationContext context, ServiceShape service) { + this.context = context; + this.service = service; + } + + @Override + public void run() { + context.writerDelegator().useShapeWriter(service, this::generateService); + } + + protected void generateService(PythonWriter writer) { + var serviceSymbol = context.symbolProvider().toSymbol(service); + var configSymbol = CodegenUtils.getConfigSymbol(context.settings()); + var pluginSymbol = CodegenUtils.getPluginSymbol(context.settings()); + + writer.addStdlibImport("typing", "TypeVar"); + writer.write(""" + Input = TypeVar("Input") + Output = TypeVar("Output") + """); + + writer.openBlock("class $L:", "", serviceSymbol.getName(), () -> { + var docs = service.getTrait(DocumentationTrait.class) + .map(StringTrait::getValue) + .orElse("Client for " + serviceSymbol.getName()); + writer.writeDocs(() -> { + writer.write(""" + $L + + :param config: Optional configuration for the client. Here you can set things like the + endpoint for HTTP services or auth credentials. + + :param plugins: A list of callables that modify the configuration dynamically. These + can be used to set defaults, for example.""", docs); + }); + + var defaultPlugins = new LinkedHashSet(); + + for (PythonIntegration integration : context.integrations()) { + for (RuntimeClientPlugin runtimeClientPlugin : integration.getClientPlugins()) { + if (runtimeClientPlugin.matchesService(context.model(), service)) { + runtimeClientPlugin.getPythonPlugin().ifPresent(defaultPlugins::add); + } + } + } + + writer.write(""" + def __init__(self, config: $1T | None = None, plugins: list[$2T] | None = None): + self._config = config or $1T() + + client_plugins: list[$2T] = [ + $3C + ] + if plugins: + client_plugins.extend(plugins) + + for plugin in client_plugins: + plugin(self._config) + """, configSymbol, pluginSymbol, writer.consumer(w -> writeDefaultPlugins(w, defaultPlugins))); + + var topDownIndex = TopDownIndex.of(context.model()); + for (OperationShape operation : topDownIndex.getContainedOperations(service)) { + generateOperation(writer, operation); + } + }); + + if (context.protocolGenerator() != null) { + generateOperationExecutor(writer); + } + } + + protected void generateOperationExecutor(PythonWriter writer) { + var transportRequest = context.applicationProtocol().requestType(); + var transportResponse = context.applicationProtocol().responseType(); + var errorSymbol = CodegenUtils.getServiceError(context.settings()); + var pluginSymbol = CodegenUtils.getPluginSymbol(context.settings()); + var configSymbol = CodegenUtils.getConfigSymbol(context.settings()); + + writer.addStdlibImport("typing", "Callable"); + writer.addStdlibImport("typing", "Awaitable"); + writer.addStdlibImport("typing", "cast"); + writer.addStdlibImport("copy", "deepcopy"); + writer.addStdlibImport("asyncio", "sleep"); + + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.exceptions", "SmithyRetryException"); + writer.addImport("smithy_python.interfaces.interceptor", "Interceptor"); + writer.addImport("smithy_python.interfaces.interceptor", "InterceptorContext"); + writer.addImport("smithy_python.interfaces.retries", "RetryErrorInfo"); + writer.addImport("smithy_python.interfaces.retries", "RetryErrorType"); + + writer.indent(); + writer.write(""" + async def _execute_operation( + self, + input: Input, + plugins: list[$1T], + serialize: Callable[[Input, $5T], Awaitable[$2T]], + deserialize: Callable[[$3T, $5T], Awaitable[Output]], + config: $5T, + operation_name: str, + ) -> Output: + try: + return await self._handle_execution( + input, plugins, serialize, deserialize, config, operation_name + ) + except Exception as e: + # Make sure every exception that we throw is an instance of $4T so + # customers can reliably catch everything we throw. + if not isinstance(e, $4T): + raise $4T(e) from e + raise e + + async def _handle_execution( + self, + input: Input, + plugins: list[$1T], + serialize: Callable[[Input, $5T], Awaitable[$2T]], + deserialize: Callable[[$3T, $5T], Awaitable[Output]], + config: $5T, + operation_name: str, + ) -> Output: + context: InterceptorContext[Input, None, None, None] = InterceptorContext( + request=input, + response=None, + transport_request=None, + transport_response=None, + ) + _client_interceptors = config.interceptors + client_interceptors = cast( + list[Interceptor[Input, Output, $2T, $3T]], _client_interceptors + ) + interceptors = client_interceptors + + try: + # Step 1a: Invoke read_before_execution on client-level interceptors + for interceptor in client_interceptors: + interceptor.read_before_execution(context) + + # Step 1b: Run operation-level plugins + config = deepcopy(config) + for plugin in plugins: + plugin(config) + + _client_interceptors = config.interceptors + interceptors = cast( + list[Interceptor[Input, Output, $2T, $3T]], + _client_interceptors, + ) + + # Step 1c: Invoke the read_before_execution hooks on newly added + # interceptors. + for interceptor in interceptors: + if interceptor not in client_interceptors: + interceptor.read_before_execution(context) + + # Step 2: Invoke the modify_before_serialization hooks + for interceptor in interceptors: + context._request = interceptor.modify_before_serialization(context) + + # Step 3: Invoke the read_before_serialization hooks + for interceptor in interceptors: + interceptor.read_before_serialization(context) + + # Step 4: Serialize the request + context_with_transport_request = cast( + InterceptorContext[Input, None, $2T, None], context + ) + context_with_transport_request._transport_request = await serialize( + context_with_transport_request.request, config + ) + + # Step 5: Invoke read_after_serialization + for interceptor in interceptors: + interceptor.read_after_serialization(context_with_transport_request) + + # Step 6: Invoke modify_before_retry_loop + for interceptor in interceptors: + context_with_transport_request._transport_request = ( + interceptor.modify_before_retry_loop(context_with_transport_request) + ) + + # Step 7: Acquire the retry token. + retry_strategy = config.retry_strategy + retry_token = retry_strategy.acquire_initial_retry_token() + + while True: + # Make an attempt, creating a copy of the context so we don't pass + # around old data. + context_with_response = await self._handle_attempt( + deserialize, + interceptors, + context_with_transport_request.copy(), + config, + operation_name, + ) + + # We perform this type-ignored re-assignment because `context` needs + # to point at the latest context so it can be generically handled + # later on. This is only an issue here because we've created a copy, + # so we're no longer simply pointing at the same object in memory + # with different names and type hints. It is possible to address this + # without having to fall back to the type ignore, but it would impose + # unnecessary runtime costs. + context = context_with_response # type: ignore + + if isinstance(context_with_response.response, Exception): + # Step 7u: Reacquire retry token if the attempt failed + try: + retry_token = retry_strategy.refresh_retry_token_for_retry( + token_to_renew=retry_token, + error_info=RetryErrorInfo( + # TODO: Determine the error type. + error_type=RetryErrorType.CLIENT_ERROR, + ) + ) + except SmithyRetryException: + raise context_with_response.response + await sleep(retry_token.retry_delay) + else: + # Step 8: Invoke record_success + retry_strategy.record_success(token=retry_token) + break + except Exception as e: + if context.response is not None: + # config.logger.exception(f"Exception occurred while handling: {context.response}") + pass + context._response = e + + # At this point, the context's request will have been definitively set, and + # The response will be set either with the modeled output or an exception. The + # transport_request and transport_response may be set or None. + execution_context = cast( + InterceptorContext[Input, Output, $2T | None, $3T | None], context + ) + return await self._finalize_execution(interceptors, execution_context) + + async def _handle_attempt( + self, + deserialize: Callable[[$3T, $5T], Awaitable[Output]], + interceptors: list[Interceptor[Input, Output, $2T, $3T]], + context: InterceptorContext[Input, None, $2T, None], + config: $5T, + operation_name: str, + ) -> InterceptorContext[Input, Output, $2T, $3T | None]: + try: + # assert config.interceptors is not None + # Step 7a: Invoke read_before_attempt + for interceptor in interceptors: + interceptor.read_before_attempt(context) + + """, pluginSymbol, transportRequest, transportResponse, errorSymbol, configSymbol); + + boolean supportsAuth = !ServiceIndex.of(context.model()).getAuthSchemes(service).isEmpty(); + writer.pushState(new ResolveIdentitySection()); + if (context.applicationProtocol().isHttpProtocol() && supportsAuth) { + writer.pushState(new InitializeHttpAuthParametersSection()); + writer.write(""" + # Step 7b: Invoke service_auth_scheme_resolver.resolve_auth_scheme + auth_parameters: $1T = $1T( + operation=operation_name, + ${2C|} + ) + + """, CodegenUtils.getHttpAuthParamsSymbol(context.settings()), + writer.consumer(this::initializeHttpAuthParameters)); + writer.popState(); + + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.interfaces.identity", "Identity"); + writer.addImports("smithy_python.interfaces.auth", Set.of("HTTPSigner", "HTTPAuthOption")); + writer.addStdlibImport("typing", "Any"); + writer.write(""" + auth_options = config.http_auth_scheme_resolver.resolve_auth_scheme( + auth_parameters=auth_parameters + ) + auth_option: HTTPAuthOption | None = None + for option in auth_options: + if option.scheme_id in config.http_auth_schemes: + auth_option = option + + signer: HTTPSigner[Any, Any] | None = None + identity: Identity | None = None + + if auth_option: + auth_scheme = config.http_auth_schemes[auth_option.scheme_id] + + # Step 7c: Invoke auth_scheme.identity_resolver + identity_resolver = auth_scheme.identity_resolver(config=config) + + # Step 7d: Invoke auth_scheme.signer + signer = auth_scheme.signer + + # Step 7e: Invoke identity_resolver.get_identity + identity = await identity_resolver.get_identity( + identity_properties=auth_option.identity_properties + ) + + """); + } + writer.popState(); + + writer.pushState(new ResolveEndpointSection()); + if (context.applicationProtocol().isHttpProtocol()) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + // TODO: implement the endpoints 2.0 spec and remove the hard-coded handling of static params. + writer.addImport("smithy_python._private.http", "StaticEndpointParams"); + writer.addImport("smithy_python._private", "URI"); + writer.write(""" + # Step 7f: Invoke endpoint_resolver.resolve_endpoint + if config.endpoint_resolver is None: + raise $1T( + "No endpoint_resolver found on the operation config." + ) + if config.endpoint_uri is None: + raise $1T( + "No endpoint_uri found on the operation config." + ) + + endpoint = await config.endpoint_resolver.resolve_endpoint( + StaticEndpointParams(uri=config.endpoint_uri) + ) + if not endpoint.uri.path: + path = "" + elif endpoint.uri.path.endswith("/"): + path = endpoint.uri.path[:-1] + else: + path = endpoint.uri.path + if context.transport_request.destination.path: + path += context.transport_request.destination.path + context._transport_request.destination = URI( + scheme=endpoint.uri.scheme, + host=context.transport_request.destination.host + endpoint.uri.host, + path=path, + query=context.transport_request.destination.query, + ) + context._transport_request.fields.extend(endpoint.headers) + + """, errorSymbol); + } + writer.popState(); + + writer.write(""" + # Step 7g: Invoke modify_before_signing + for interceptor in interceptors: + context._transport_request = interceptor.modify_before_signing(context) + + # Step 7h: Invoke read_before_signing + for interceptor in interceptors: + interceptor.read_before_signing(context) + + """); + + writer.pushState(new SignRequestSection()); + if (context.applicationProtocol().isHttpProtocol() && supportsAuth) { + writer.write(""" + # Step 7i: sign the request + if auth_option and signer: + context._transport_request = await signer.sign( + http_request=context.transport_request, + identity=identity, + signing_properties=auth_option.signer_properties, + ) + """); + } + writer.popState(); + + writer.write(""" + # Step 7j: Invoke read_after_signing + for interceptor in interceptors: + interceptor.read_after_signing(context) + + # Step 7k: Invoke modify_before_transmit + for interceptor in interceptors: + context._transport_request = interceptor.modify_before_transmit(context) + + # Step 7l: Invoke read_before_transmit + for interceptor in interceptors: + interceptor.read_before_transmit(context) + + """); + + writer.pushState(new SendRequestSection()); + if (context.applicationProtocol().isHttpProtocol()) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.interfaces.http", "HTTPRequestConfiguration"); + writer.write(""" + # Step 7m: Invoke http_client.send + request_config = config.http_request_config or HTTPRequestConfiguration() + context_with_response = cast( + InterceptorContext[Input, None, $1T, $2T], context + ) + context_with_response._transport_response = await config.http_client.send( + request=context_with_response.transport_request, + request_config=request_config, + ) + + """, transportRequest, transportResponse); + } + writer.popState(); + + writer.write(""" + # Step 7n: Invoke read_after_transmit + for interceptor in interceptors: + interceptor.read_after_transmit(context_with_response) + + # Step 7o: Invoke modify_before_deserialization + for interceptor in interceptors: + context_with_response._transport_response = ( + interceptor.modify_before_deserialization(context_with_response) + ) + + # Step 7p: Invoke read_before_deserialization + for interceptor in interceptors: + interceptor.read_before_deserialization(context_with_response) + + # Step 7q: deserialize + context_with_output = cast( + InterceptorContext[Input, Output, $1T, $2T], + context_with_response, + ) + context_with_output._response = await deserialize( + context_with_output._transport_response, config + ) + + # Step 7r: Invoke read_after_deserialization + for interceptor in interceptors: + interceptor.read_after_deserialization(context_with_output) + except Exception as e: + if context.response is not None: + # config.logger.exception(f"Exception occurred while handling: {context.response}") + pass + context._response = e + + # At this point, the context's request and transport_request have definitively been set, + # the response is either set or an exception, and the transport_resposne is either set or + # None. This will also be true after _finalize_attempt because there is no opportunity + # there to set the transport_response. + attempt_context = cast( + InterceptorContext[Input, Output, $1T, $2T | None], context + ) + return await self._finalize_attempt(interceptors, attempt_context) + + async def _finalize_attempt( + self, + interceptors: list[Interceptor[Input, Output, $1T, $2T]], + context: InterceptorContext[Input, Output, $1T, $2T | None], + ) -> InterceptorContext[Input, Output, $1T, $2T | None]: + # Step 7s: Invoke modify_before_attempt_completion + try: + for interceptor in interceptors: + context._response = interceptor.modify_before_attempt_completion( + context + ) + except Exception as e: + if context.response is not None: + # config.logger.exception(f"Exception occurred while handling: {context.response}") + pass + context._response = e + + # Step 7t: Invoke read_after_attempt + for interceptor in interceptors: + try: + interceptor.read_after_attempt(context) + except Exception as e: + if context.response is not None: + # config.logger.exception(f"Exception occurred while handling: {context.response}") + pass + context._response = e + + return context + + async def _finalize_execution( + self, + interceptors: list[Interceptor[Input, Output, $1T, $2T]], + context: InterceptorContext[Input, Output, $1T | None, $2T | None], + ) -> Output: + try: + # Step 9: Invoke modify_before_completion + for interceptor in interceptors: + context._response = interceptor.modify_before_completion(context) + + # Step 10: Invoke trace_probe.dispatch_events + try: + pass + except Exception as e: + # log and ignore exceptions + # config.logger.exception(f"Exception occurred while dispatching trace events: {e}") + pass + except Exception as e: + if context.response is not None: + # config.logger.exception(f"Exception occurred while handling: {context.response}") + pass + context._response = e + + # Step 11: Invoke read_after_execution + for interceptor in interceptors: + try: + interceptor.read_after_execution(context) + except Exception as e: + if context.response is not None: + # config.logger.exception(f"Exception occurred while handling: {context.response}") + pass + context._response = e + + # Step 12: Return / throw + if isinstance(context.response, Exception): + raise context.response + + # We may want to add some aspects of this context to the output types so we can + # return it to the end-users. + return context.response + """, transportRequest, transportResponse); + writer.dedent(); + } + + protected void initializeHttpAuthParameters(PythonWriter writer) { + var derived = new LinkedHashSet(); + for (PythonIntegration integration : context.integrations()) { + for (RuntimeClientPlugin plugin : integration.getClientPlugins()) { + if (plugin.matchesService(context.model(), service) + && plugin.getAuthScheme().isPresent() + && plugin.getAuthScheme().get().getApplicationProtocol().isHttpProtocol()) { + derived.addAll(plugin.getAuthScheme().get().getAuthProperties()); + } + } + } + + for (DerivedProperty property : derived) { + var source = property.source().scopeLocation(); + if (property.initializationFunction().isPresent()) { + writer.write("$L=$T($L),", property.name(), property.initializationFunction().get(), source); + } else if (property.sourcePropertyName().isPresent()) { + writer.write("$L=$L.$L,", property.name(), source, property.sourcePropertyName().get()); + } + } + } + + protected void writeDefaultPlugins(PythonWriter writer, Collection plugins) { + for (SymbolReference plugin : plugins) { + writer.write("$T,", plugin); + } + } + + /** + * Generates the function for a single operation. + */ + protected void generateOperation(PythonWriter writer, OperationShape operation) { + var operationSymbol = context.symbolProvider().toSymbol(operation); + var pluginSymbol = CodegenUtils.getPluginSymbol(context.settings()); + + var input = context.model().expectShape(operation.getInputShape()); + var inputSymbol = context.symbolProvider().toSymbol(input); + + var output = context.model().expectShape(operation.getOutputShape()); + var outputSymbol = context.symbolProvider().toSymbol(output); + + writer.openBlock("async def $L(self, input: $T, plugins: list[$T] | None = None) -> $T:", "", + operationSymbol.getName(), inputSymbol, pluginSymbol, outputSymbol, () -> { + writer.writeDocs(() -> { + var docs = operation.getTrait(DocumentationTrait.class) + .map(StringTrait::getValue) + .orElse(String.format("Invokes the %s operation.", operation.getId().getName())); + + var inputDocs = input.getTrait(DocumentationTrait.class) + .map(StringTrait::getValue) + .orElse("The operation's input."); + + writer.write(""" + $L + + :param input: $L + + :param plugins: A list of callables that modify the configuration dynamically. + Changes made by these plugins only apply for the duration of the operation + execution and will not affect any other operation invocations.""", docs, inputDocs); + }); + + var defaultPlugins = new LinkedHashSet(); + for (PythonIntegration integration : context.integrations()) { + for (RuntimeClientPlugin runtimeClientPlugin : integration.getClientPlugins()) { + if (runtimeClientPlugin.matchesOperation(context.model(), service, operation)) { + runtimeClientPlugin.getPythonPlugin().ifPresent(defaultPlugins::add); + } + } + } + writer.write(""" + operation_plugins = [ + $C + ] + if plugins: + operation_plugins.extend(plugins) + """, writer.consumer(w -> writeDefaultPlugins(w, defaultPlugins))); + + if (context.protocolGenerator() == null) { + writer.write("raise NotImplementedError()"); + } else { + var protocolGenerator = context.protocolGenerator(); + var serSymbol = protocolGenerator.getSerializationFunction(context, operation); + var deserSymbol = protocolGenerator.getDeserializationFunction(context, operation); + writer.write(""" + return await self._execute_operation( + input=input, + plugins=operation_plugins, + serialize=$T, + deserialize=$T, + config=self._config, + operation_name=$S, + ) + """, serSymbol, deserSymbol, operation.getId().getName()); + } + }); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java new file mode 100644 index 0000000000..b4a8d9f093 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java @@ -0,0 +1,389 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import static java.lang.String.format; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.nio.file.Path; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoField; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Logger; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.NullableIndex; +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.model.traits.TimestampFormatTrait; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import software.amazon.smithy.utils.SetUtils; +import software.amazon.smithy.utils.StringUtils; + +/** + * Utility methods likely to be needed across packages. + */ +public final class CodegenUtils { + + /** + * The maximum preferred line length for generated code. In most cases it won't + * be practical to try to adhere to this in the generator, but we can make some + * amount of effort. Eventually a formatter like black will be run on the output + * to fix any lingering issues. + */ + public static final int MAX_PREFERRED_LINE_LENGTH = 88; + + static final Set ERROR_MESSAGE_MEMBER_NAMES = SetUtils.of( + "errormessage", "error_message", "message"); + + private static final Logger LOGGER = Logger.getLogger(CodegenUtils.class.getName()); + + private CodegenUtils() {} + + /** + * @param settings The client settings, used to account for module configuration. + * @return Returns the client's configuration object symbol. + */ + public static Symbol getConfigSymbol(PythonSettings settings) { + return Symbol.builder() + .name("Config") + .namespace(format("%s.config", settings.getModuleName()), ".") + .definitionFile(format("./%s/config.py", settings.getModuleName())) + .build(); + } + + /** + * @param settings The client settings, used to account for module configuration. + * @return Returns the client's plugin type hint symbol. + */ + public static Symbol getPluginSymbol(PythonSettings settings) { + return Symbol.builder() + .name("Plugin") + .namespace(format("%s.config", settings.getModuleName()), ".") + .definitionFile(format("./%s/config.py", settings.getModuleName())) + .build(); + } + + /** + * Gets the service error symbol. + * + *

This error is the top-level error for the client. Every error surfaced by + * the client MUST be a subclass of this so that customers can reliably catch all + * exceptions it raises. The client implementation will wrap any errors that aren't + * already subclasses. + * + * @param settings The client settings, used to account for module configuration. + * @return Returns the symbol for the client's error class. + */ + public static Symbol getServiceError(PythonSettings settings) { + return Symbol.builder() + .name("ServiceError") + .namespace(format("%s.errors", settings.getModuleName()), ".") + .definitionFile(format("./%s/errors.py", settings.getModuleName())) + .build(); + } + + /** + * Gets the service API error symbol. + * + *

This error is the parent class for all errors returned over the wire by the + * service, including unknown errors. + * + * @param settings The client settings, used to account for module configuration. + * @return Returns the symbol for the client's API error class. + */ + public static Symbol getApiError(PythonSettings settings) { + return Symbol.builder() + .name("ApiError") + .namespace(format("%s.errors", settings.getModuleName()), ".") + .definitionFile(format("./%s/errors.py", settings.getModuleName())) + .build(); + } + + /** + * Gets the unknown API error symbol. + * + *

This error is the parent class for all errors returned over the wire by + * the service which aren't in the model. + * + * @param settings The client settings, used to account for module configuration. + * @return Returns the symbol for unknown API errors. + */ + public static Symbol getUnknownApiError(PythonSettings settings) { + return Symbol.builder() + .name("UnknownApiError") + .namespace(format("%s.errors", settings.getModuleName()), ".") + .definitionFile(format("./%s/errors.py", settings.getModuleName())) + .build(); + } + + /** + * Gets the symbol for the http auth parameters object. + * + * @param settings The client settings, used to account for module configuration. + * @return Returns the symbol for http auth params. + */ + public static Symbol getHttpAuthParamsSymbol(PythonSettings settings) { + return Symbol.builder() + .name("HTTPAuthParams") + .namespace(format("%s.auth", settings.getModuleName()), ".") + .definitionFile(format("./%s/auth.py", settings.getModuleName())) + .build(); + } + + /** + * Gets the symbol for the http auth scheme resolver. + * + * @param settings The client settings, used to account for module configuration. + * @return Returns the http auth scheme resolver symbol. + */ + public static Symbol getHttpAuthSchemeResolverSymbol(PythonSettings settings) { + return Symbol.builder() + .name("HTTPAuthSchemeResolver") + .namespace(format("%s.auth", settings.getModuleName()), ".") + .definitionFile(format("./%s/auth.py", settings.getModuleName())) + .build(); + } + + /** + * Determines whether a given member is probably the main "message" of an error shape. + * + * @param model The whole service model. + * @param shape The member to check. + * @return Returns whether the member is probably the error message. + */ + public static boolean isErrorMessage(Model model, MemberShape shape) { + return ERROR_MESSAGE_MEMBER_NAMES.contains(shape.getMemberName().toLowerCase(Locale.US)) + && model.expectShape(shape.getContainer()).hasTrait(ErrorTrait.class); + } + + /** + * Executes a given shell command in a given directory. + * + * @param command The string command to execute, e.g. "python fmt". + * @param directory The directory to run the command in. + * @return Returns the console output of the command. + */ + public static String runCommand(String command, Path directory) { + String[] finalizedCommand; + if (System.getProperty("os.name").toLowerCase().startsWith("windows")) { + finalizedCommand = new String[]{"cmd.exe", "/c", command}; + } else { + finalizedCommand = new String[]{"sh", "-c", command}; + } + + ProcessBuilder processBuilder = new ProcessBuilder(finalizedCommand) + .redirectErrorStream(true) + .directory(directory.toFile()); + + try { + Process process = processBuilder.start(); + List output = new ArrayList<>(); + + // Capture output for reporting. + try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader( + process.getInputStream(), Charset.defaultCharset()))) { + String line; + while ((line = bufferedReader.readLine()) != null) { + LOGGER.finest(line); + output.add(line); + } + } + + process.waitFor(); + process.destroy(); + + String joinedOutput = String.join(System.lineSeparator(), output); + if (process.exitValue() != 0) { + throw new CodegenException(format( + "Command `%s` failed with output:%n%n%s", command, joinedOutput)); + } + return joinedOutput; + } catch (InterruptedException | IOException e) { + throw new CodegenException(e); + } + } + + /** + * Gets the name under which the given package will be exported by default. + * + * @param packageName The full package name of the exported package. + * @return The name a the package will be imported under by default. + */ + public static String getDefaultPackageImportName(String packageName) { + if (StringUtils.isBlank(packageName) || !packageName.contains("/")) { + return packageName; + } + return packageName.substring(packageName.lastIndexOf('/') + 1); + } + + /** + * Convert a map of k,v to a list of pairwise arrays. + * + *

For example: + * + * 

{@code
+     * Map.of("a", 1, "b", 2) -> List.of({"a", 1}, {"b", 2})
+     * }
+ * + * @param map The map to be converted + * @return The list of arrays + */ + public static List toTuples(Map map) { + return map.entrySet().stream().map((entry) -> + List.of(entry.getKey(), entry.getValue()).toArray()).toList(); + } + + /** + * Generates a Python datetime constructor for the given ZonedDateTime. + * + * @param writer A writer to add dependencies to. + * @param value The ZonedDateTime to convert. + * @return A string containing a Python datetime constructor representing the given ZonedDateTime. + */ + public static String getDatetimeConstructor(PythonWriter writer, ZonedDateTime value) { + writer.addStdlibImport("datetime", "datetime"); + writer.addStdlibImport("datetime", "timezone"); + var timezone = "timezone.utc"; + if (value.getOffset() != ZoneOffset.UTC) { + writer.addStdlibImport("datetime", "timedelta"); + timezone = String.format("timezone(timedelta(seconds=%d))", value.getOffset().getTotalSeconds()); + } + return String.format("datetime(%d, %d, %d, %d, %d, %d, %d, %s)", value.get(ChronoField.YEAR), + value.get(ChronoField.MONTH_OF_YEAR), value.get(ChronoField.DAY_OF_MONTH), + value.get(ChronoField.HOUR_OF_DAY), value.get(ChronoField.MINUTE_OF_HOUR), + value.get(ChronoField.SECOND_OF_MINUTE), value.get(ChronoField.MICRO_OF_SECOND), timezone); + } + + /** + * Parses a timestamp Node. + * + *

This is used to offload modeled timestamp parsing from Python runtime to Java build time. + * + * @param model The model being generated. + * @param shape The shape of the node. + * @param value The node to parse. + * @return A parsed ZonedDateTime representation of the given node. + */ + public static ZonedDateTime parseTimestampNode(Model model, Shape shape, Node value) { + if (value.isNumberNode()) { + return parseEpochTime(value); + } + + Optional trait = shape.getTrait(TimestampFormatTrait.class); + if (shape.isMemberShape()) { + trait = shape.asMemberShape().get().getMemberTrait(model, TimestampFormatTrait.class); + } + var format = Format.DATE_TIME; + if (trait.isPresent()) { + format = trait.get().getFormat(); + } + + return switch (format) { + case DATE_TIME -> parseDateTime(value); + case HTTP_DATE -> parseHttpDate(value); + default -> throw new CodegenException("Unexpected timestamp format: " + format); + }; + } + + private static ZonedDateTime parseEpochTime(Node value) { + Number number = value.expectNumberNode().getValue(); + Instant instant = Instant.ofEpochMilli(Double.valueOf(number.doubleValue() * 1000).longValue()); + return instant.atZone(ZoneId.of("UTC")); + } + + private static ZonedDateTime parseDateTime(Node value) { + Instant instant = Instant.from(DateTimeFormatter.ISO_INSTANT.parse(value.expectStringNode().getValue())); + return instant.atZone(ZoneId.of("UTC")); + } + + private static ZonedDateTime parseHttpDate(Node value) { + Instant instant = Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(value.expectStringNode().getValue())); + return instant.atZone(ZoneId.of("UTC")); + } + + /** + * Writes an accessor for a structure member, handling defaultedness and nullability. + * + * @param context The generation context. + * @param writer The writer to write to. + * @param variableName The python variable name pointing to the structure to be accessed. + * @param member The member to access. + * @param runnable A runnable which uses the member. + */ + public static void accessStructureMember( + GenerationContext context, + PythonWriter writer, + String variableName, + MemberShape member, + Runnable runnable + ) { + accessStructureMember(context, writer, variableName, member, true, runnable); + } + + /** + * Writes an accessor for a structure member, handling defaultedness and nullability. + * + * @param context The generation context. + * @param writer The writer to write to. + * @param variableName The python variable name pointing to the structure to be accessed. + * @param member The member to access. + * @param accessFalsey Whether to access falsey members such as empty strings. + * @param runnable A runnable which uses the member. + */ + public static void accessStructureMember( + GenerationContext context, + PythonWriter writer, + String variableName, + MemberShape member, + boolean accessFalsey, + Runnable runnable + ) { + var shouldDedent = false; + var isNullable = NullableIndex.of(context.model()).isMemberNullable(member); + var memberName = context.symbolProvider().toMemberName(member); + if (!accessFalsey) { + writer.write("if $L.$L:", variableName, memberName); + writer.indent(); + shouldDedent = true; + } else if (isNullable) { + writer.write("if $L.$L is not None:", variableName, memberName); + writer.indent(); + shouldDedent = true; + } + + runnable.run(); + + if (shouldDedent) { + writer.dedent(); + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/CollectionGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/CollectionGenerator.java new file mode 100644 index 0000000000..a838e212af --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/CollectionGenerator.java @@ -0,0 +1,102 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.CollectionShape; +import software.amazon.smithy.model.traits.SparseTrait; + +/** + * Generates private helper methods for collections. + */ +final class CollectionGenerator implements Runnable { + + private final Model model; + private final SymbolProvider symbolProvider; + private final PythonWriter writer; + private final CollectionShape shape; + + CollectionGenerator( + Model model, + SymbolProvider symbolProvider, + PythonWriter writer, + CollectionShape shape + ) { + this.model = model; + this.symbolProvider = symbolProvider; + this.writer = writer; + this.shape = shape; + } + + @Override + public void run() { + var symbol = symbolProvider.toSymbol(shape); + if (symbol.getProperty("asDict").isPresent()) { + writeAsDict(); + } + if (symbol.getProperty("fromDict").isPresent()) { + writeFromDict(); + } + } + + private void writeAsDict() { + var symbol = symbolProvider.toSymbol(shape); + var asDictSymbol = symbol.expectProperty("asDict", Symbol.class); + var target = model.expectShape(shape.getMember().getTarget()); + var targetSymbol = symbolProvider.toSymbol(target); + + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-sparse-trait + var sparseGuard = shape.hasTrait(SparseTrait.class) ? " if v is not None else None" : ""; + + writer.addStdlibImport("typing", "List"); + writer.addStdlibImport("typing", "Any"); + writer.openBlock("def $L(given: $T) -> List[Any]:", "", asDictSymbol.getName(), symbol, () -> { + if (target.isUnionShape() || target.isStructureShape()) { + writer.write("return [v.as_dict()$L for v in given]", sparseGuard); + } else if (target.isMapShape() || target instanceof CollectionShape) { + var targetAsDictSymbol = targetSymbol.expectProperty("asDict", Symbol.class); + writer.write("return [$T(v)$L for v in given]", targetAsDictSymbol, sparseGuard); + } else { + writer.write("return given"); + } + }); + } + + private void writeFromDict() { + var symbol = symbolProvider.toSymbol(shape); + var fromDictSymbol = symbol.expectProperty("fromDict", Symbol.class); + var target = model.expectShape(shape.getMember().getTarget()); + var targetSymbol = symbolProvider.toSymbol(target); + + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-sparse-trait + var sparseGuard = shape.hasTrait(SparseTrait.class) ? " if v is not None else None" : ""; + + writer.addStdlibImport("typing", "List"); + writer.addStdlibImport("typing", "Any"); + writer.openBlock("def $L(given: List[Any]) -> $T:", "", fromDictSymbol.getName(), symbol, () -> { + if (target.isUnionShape() || target.isStructureShape()) { + writer.write("return [$T.from_dict(v)$L for v in given]", targetSymbol, sparseGuard); + } else if (target.isMapShape() || target instanceof CollectionShape) { + var targetFromDictSymbol = targetSymbol.expectProperty("fromDict", Symbol.class); + writer.write("return [$T(v)$L for v in given]", targetFromDictSymbol, sparseGuard); + } else { + writer.write("return given"); + } + }); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ConfigGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ConfigGenerator.java new file mode 100644 index 0000000000..8acbaf1069 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ConfigGenerator.java @@ -0,0 +1,371 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.knowledge.ServiceIndex; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.python.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.python.codegen.sections.ConfigSection; +import software.amazon.smithy.utils.CodeInterceptor; + +/** + * Generates the client's config object. + */ +public class ConfigGenerator implements Runnable { + + // This list contains any properties that should unconditionally be added to every + // config object. This should be as minimal as possible, and importantly should + // not contain any HTTP related config since Smithy is transport agnostic. + private static final List BASE_PROPERTIES = Arrays.asList( + ConfigProperty.builder() + .name("interceptors") + .type(Symbol.builder() + .name("list[_ServiceInterceptor]") + .build()) + .documentation( + "The list of interceptors, which are hooks that are called during the execution of a request.") + .nullable(false) + .initialize(writer -> writer.write("self.interceptors = interceptors or []")) + .build(), + ConfigProperty.builder() + .name("retry_strategy") + .type(Symbol.builder() + .name("RetryStrategy") + .namespace("smithy_python.interfaces.retries", ".") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build()) + .documentation("The retry strategy for issuing retry tokens and computing retry delays.") + .nullable(false) + .initialize(writer -> { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python._private.retries", "SimpleRetryStrategy"); + writer.write("self.retry_strategy = retry_strategy or SimpleRetryStrategy()"); + }) + .build() + ); + + // This list contains any properties that must be added to any http-based + // service client. + private static final List HTTP_PROPERTIES = Arrays.asList( + ConfigProperty.builder() + .name("http_client") + .type(Symbol.builder() + .name("HTTPClient") + .namespace("smithy_python.interfaces.http", ".") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build()) + .documentation("The HTTP client used to make requests.") + .nullable(false) + .initialize(writer -> { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python._private.http.aiohttp_client", "AIOHTTPClient"); + writer.write("self.http_client = http_client or AIOHTTPClient()"); + }) + .build(), + ConfigProperty.builder() + .name("http_request_config") + .type(Symbol.builder() + .name("HTTPRequestConfiguration") + .namespace("smithy_python.interfaces.http", ".") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build()) + .documentation("Configuration for individual HTTP requests.") + .build(), + ConfigProperty.builder() + .name("endpoint_resolver") + .type(Symbol.builder() + .name("EndpointResolver[Any]") + .addReference(Symbol.builder() + .name("Any") + .namespace("typing", ".") + .putProperty("stdlib", true) + .build()) + .addReference(Symbol.builder() + .name("EndpointResolver") + .namespace("smithy_python.interfaces.http", ".") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build()) + .build()) + .documentation(""" + The endpoint resolver used to resolve the final endpoint per-operation based on the \ + configuration.""") + .nullable(false) + .initialize(writer -> { + writer.addImport("smithy_python._private.http", "StaticEndpointResolver"); + writer.write("self.endpoint_resolver = endpoint_resolver or StaticEndpointResolver()"); + }) + .build(), + ConfigProperty.builder() + .name("endpoint_uri") + .type(Symbol.builder() + .name("str | URI") + .addReference(Symbol.builder() + .name("URI") + .namespace("smithy_python.interfaces", ".") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build()) + .build()) + .documentation("A static URI to route requests to.") + .build() + ); + + protected final PythonSettings settings; + protected final GenerationContext context; + + protected ConfigGenerator(PythonSettings settings, GenerationContext context) { + this.context = context; + this.settings = settings; + } + + private static List getHttpAuthProperties(GenerationContext context) { + return List.of( + ConfigProperty.builder() + .name("http_auth_schemes") + .type(Symbol.builder() + .name("dict[str, HTTPAuthScheme[Any, Any, Any, Any]]") + .addReference(Symbol.builder() + .name("HTTPAuthScheme") + .namespace("smithy_python.interfaces.auth", ".") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build()) + .addReference(Symbol.builder() + .name("Any") + .namespace("typing", ".") + .putProperty("stdlib", true) + .build()) + .build()) + .documentation("A map of http auth scheme ids to http auth schemes.") + .nullable(false) + .initialize(writer -> writeDefaultHttpAuthSchemes(context, writer)) + .build(), + ConfigProperty.builder() + .name("http_auth_scheme_resolver") + .type(CodegenUtils.getHttpAuthSchemeResolverSymbol(context.settings())) + .documentation("An http auth scheme resolver that determines the auth scheme for each operation.") + .nullable(false) + .initialize(writer -> writer.write( + "self.http_auth_scheme_resolver = http_auth_scheme_resolver or HTTPAuthSchemeResolver()")) + .build() + ); + } + + private static void writeDefaultHttpAuthSchemes(GenerationContext context, PythonWriter writer) { + var supportedAuthSchemes = new LinkedHashMap(); + var service = context.settings().getService(context.model()); + for (PythonIntegration integration : context.integrations()) { + for (RuntimeClientPlugin plugin : integration.getClientPlugins()) { + if (plugin.matchesService(context.model(), service) + && plugin.getAuthScheme().isPresent() + && plugin.getAuthScheme().get().getApplicationProtocol().isHttpProtocol()) { + var scheme = plugin.getAuthScheme().get(); + supportedAuthSchemes.put(scheme.getAuthTrait().toString(), scheme.getAuthSchemeSymbol(context)); + } + } + } + writer.pushState(); + writer.putContext("authSchemes", supportedAuthSchemes); + writer.write(""" + self.http_auth_schemes = http_auth_schemes or { + ${#authSchemes} + ${key:S}: ${value:T}(), + ${/authSchemes} + } + """); + writer.popState(); + } + + @Override + public void run() { + var config = CodegenUtils.getConfigSymbol(context.settings()); + context.writerDelegator().useFileWriter(config.getDefinitionFile(), config.getNamespace(), writer -> { + writeInterceptorsType(writer); + generateConfig(context, writer); + }); + + // Generate the plugin symbol. This is just a callable. We could do something + // like have a class to implement, but that seems unnecessarily burdensome for + // a single function. + var plugin = CodegenUtils.getPluginSymbol(context.settings()); + context.writerDelegator().useFileWriter(plugin.getDefinitionFile(), plugin.getNamespace(), writer -> { + writer.addStdlibImport("typing", "Callable"); + writer.addStdlibImport("typing", "TypeAlias"); + writer.writeComment("A callable that allows customizing the config object on each request."); + writer.write("$L: TypeAlias = Callable[[$T], None]", plugin.getName(), config); + }); + } + + protected void writeInterceptorsType(PythonWriter writer) { + var symbolProvider = context.symbolProvider(); + var operationShapes = TopDownIndex.of(context.model()) + .getContainedOperations(settings.getService()); + + writer.addStdlibImport("typing", "Union"); + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.getImportContainer().addImport("smithy_python.interfaces.interceptor", "Interceptor", "Interceptor"); + + writer.writeInline("_ServiceInterceptor = Union["); + if (operationShapes.isEmpty()) { + writer.writeInline("None]"); + } else { + var iter = operationShapes.iterator(); + while (iter.hasNext()) { + var operation = iter.next(); + var input = symbolProvider.toSymbol(context.model().expectShape(operation.getInputShape())); + var output = symbolProvider.toSymbol(context.model().expectShape(operation.getOutputShape())); + + // TODO: pull the transport request/response types off of the application protocol + writer.addStdlibImport("typing", "Any"); + writer.writeInline("Interceptor[$T, $T, Any, Any]", input, output); + if (iter.hasNext()) { + writer.writeInline(", "); + } else { + writer.writeInline("]"); + } + } + } + writer.write(""); + } + + protected void generateConfig(GenerationContext context, PythonWriter writer) { + var symbol = CodegenUtils.getConfigSymbol(context.settings()); + + // Initialize the list of config properties with our base properties. Here a new + // list is constructed because that base list is immutable. + var properties = new ArrayList<>(BASE_PROPERTIES); + + // Smithy is transport agnostic, so we don't add http-related properties by default. + // Nevertheless, HTTP is the most common use case so we standardize those settings + // and add them in if the protocol is going to need them. + var serviceIndex = ServiceIndex.of(context.model()); + if (context.applicationProtocol().isHttpProtocol()) { + properties.addAll(HTTP_PROPERTIES); + if (!serviceIndex.getAuthSchemes(settings.getService()).isEmpty()) { + properties.addAll(getHttpAuthProperties(context)); + writer.onSection(new AddAuthHelper()); + } + } + + var model = context.model(); + var service = context.settings().getService(model); + + // Add any relevant config properties from plugins. + for (PythonIntegration integration : context.integrations()) { + for (RuntimeClientPlugin plugin : integration.getClientPlugins()) { + if (plugin.matchesService(model, service)) { + properties.addAll(plugin.getConfigProperties()); + } + } + } + + var finalProperties = List.copyOf(properties); + writer.pushState(new ConfigSection(finalProperties)); + writer.addStdlibImport("dataclasses", "dataclass"); + writer.write(""" + @dataclass(init=False) + class $L: + \"""Configuration for $L.\""" + + ${C|} + + def __init__( + self, + *, + ${C|} + ): + \"""Constructor. + + ${C|} + \""" + ${C|} + """, symbol.getName(), context.settings().getService().getName(), + writer.consumer(w -> writePropertyDeclarations(w, finalProperties)), + writer.consumer(w -> writeInitParams(w, finalProperties)), + writer.consumer(w -> documentProperties(w, finalProperties)), + writer.consumer(w -> initializeProperties(w, finalProperties))); + writer.popState(); + } + + private void writePropertyDeclarations(PythonWriter writer, Collection properties) { + for (ConfigProperty property : properties) { + var formatString = property.isNullable() + ? "$L: $T | None" + : "$L: $T"; + writer.write(formatString, property.name(), property.type()); + } + } + + private void writeInitParams(PythonWriter writer, Collection properties) { + for (ConfigProperty property : properties) { + writer.write("$L: $T | None = None,", property.name(), property.type()); + } + } + + private void documentProperties(PythonWriter writer, Collection properties) { + var iter = properties.iterator(); + while (iter.hasNext()) { + var property = iter.next(); + var docs = writer.formatDocs(String.format(":param %s: %s", property.name(), property.documentation())); + + if (iter.hasNext()) { + docs += "\n"; + } + + writer.write(docs); + } + } + + private void initializeProperties(PythonWriter writer, Collection properties) { + for (ConfigProperty property : properties) { + property.initialize(writer); + } + } + + private static final class AddAuthHelper implements CodeInterceptor { + @Override + public Class sectionType() { + return ConfigSection.class; + } + + @Override + public void write(PythonWriter writer, String previousText, ConfigSection section) { + // First write the previous text, the generated config, back out. The entire + // section would otherwise be erased and replaced with what is written in this + // method. + writer.write(previousText); + + // Add the helper function to the end of the config definition. + // Note that this is indented to keep it at the proper indentation level. + writer.write(""" + + def set_http_auth_scheme(self, scheme: HTTPAuthScheme[Any, Any, Any, Any]) -> None: + \"""Sets the implementation of an auth scheme. + + Using this method ensures the correct key is used. + + :param scheme: The auth scheme to add. + \""" + self.http_auth_schemes[scheme.scheme_id] = scheme + """); + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ConfigProperty.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ConfigProperty.java new file mode 100644 index 0000000000..ef94bb5044 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ConfigProperty.java @@ -0,0 +1,196 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.Objects; +import java.util.function.Consumer; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyBuilder; +import software.amazon.smithy.utils.SmithyUnstableApi; +import software.amazon.smithy.utils.ToSmithyBuilder; + +/** + * Represents a property to be added to the generated client config object. + */ +@SmithyUnstableApi +public final class ConfigProperty implements ToSmithyBuilder { + private final String name; + private final Symbol type; + private final boolean nullable; + private final String documentation; + private final Consumer initialize; + + /** + * Constructor. + */ + private ConfigProperty(Builder builder) { + this.name = Objects.requireNonNull(builder.name); + this.type = Objects.requireNonNull(builder.type); + this.nullable = builder.nullable; + this.documentation = Objects.requireNonNull(builder.documentation); + this.initialize = Objects.requireNonNull(builder.initialize); + } + + /** + * @return Returns the name of the config field. + */ + public String name() { + return name; + } + + /** + * @return Returns the symbol representing the type of the config field. + */ + public Symbol type() { + return type; + } + + /** + * @return Returns whether the property is nullable. + */ + public boolean isNullable() { + return nullable; + } + + /** + * @return Returns the config field's documentation. + */ + public String documentation() { + return documentation; + } + + /** + * Initializes the config field on the config object. + * + *

This will be wrapped in an {@link InitializeConfigPropertySection}. + * + * @param writer The writer to write to. + */ + public void initialize(PythonWriter writer) { + writer.pushState(new InitializeConfigPropertySection(this)); + initialize.accept(writer); + writer.popState(); + } + + /** + * The section that handles initializing a config property inside __init__. + * + * @param property The property being initialized. + */ + public record InitializeConfigPropertySection(ConfigProperty property) implements CodeSection {} + + public static Builder builder() { + return new Builder(); + } + + @Override + public SmithyBuilder toBuilder() { + return builder() + .name(name) + .type(type) + .nullable(nullable) + .documentation(documentation) + .initialize(initialize); + } + + /** + * Builds a {@link ConfigProperty}. + */ + public static final class Builder implements SmithyBuilder { + private String name; + private Symbol type; + private boolean nullable = true; + private String documentation; + private Consumer initialize = writer -> writer.write("self.$1L = $1L", name); + + @Override + public ConfigProperty build() { + return new ConfigProperty(this); + } + + /** + * Sets the name of the config property. + * + * @param name The name to use for the config property. + * @return Returns the builder. + */ + public Builder name(String name) { + this.name = name; + return this; + } + + /** + * Sets the type to use for the config property. + * + *

Properties that are nullable must not have that reflected here. + * Rather, the nullable builder property should be set. The config + * generator will handle adjusting the type hints. + * + * @param type The type of the config property. + * @return Returns the builder. + */ + public Builder type(Symbol type) { + this.type = type; + return this; + } + + /** + * Sets whether the config property is nullable. + * + *

Defaults to true. + * + *

All properties will be optional on the config object's constructor, + * regardless of whether this value is true or false. Properties that + * are not nullable MUST set a default value in the initialize function. + * + * @param nullable Whether the property is nullable. + * @return Returns the builder. + */ + public Builder nullable(boolean nullable) { + this.nullable = nullable; + return this; + } + + /** + * Sets the documentation for the config property. + * + * @param documentation The documentation for the config property. + * @return Returns the builder. + */ + public Builder documentation(String documentation) { + this.documentation = documentation; + return this; + } + + /** + * Sets the initializer function for the config property. + * + *

This will be called when creating the __init__ function for the + * client's Config object. It MUST set the property on "self" based + * on the optional __init__ parameter of the same name. + * + *

By default, this directly sets whatever value was provided. + * + * @param initialize The initializer function for the property. + * @return Returns the builder. + */ + public Builder initialize(Consumer initialize) { + this.initialize = initialize; + return this; + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/DerivedProperty.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/DerivedProperty.java new file mode 100644 index 0000000000..b9b2d42f7e --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/DerivedProperty.java @@ -0,0 +1,231 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.python.codegen; + +import java.util.Objects; +import java.util.Optional; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.utils.SmithyBuilder; +import software.amazon.smithy.utils.ToSmithyBuilder; + +/** + * A property for some generated intermediate configuration. + * + *

This allows for automatically populating the intermediate config. + */ +public final class DerivedProperty implements ToSmithyBuilder { + private final String name; + private final Source source; + private final Symbol type; + private final Symbol initializationFunction; + private final String sourcePropertyName; + + private DerivedProperty(Builder builder) { + this.name = Objects.requireNonNull(builder.name); + this.source = Objects.requireNonNull(builder.source); + this.type = Objects.requireNonNull(builder.type); + this.initializationFunction = builder.initializationFunction; + + if (builder.sourcePropertyName == null && builder.initializationFunction == null) { + this.sourcePropertyName = this.name; + } else { + this.sourcePropertyName = builder.sourcePropertyName; + } + } + + /** + * @return Returns the name of the property on the intermediate config. + */ + public String name() { + return name; + } + + /** + * Gets the source where the property should be derived from. + * + *

If an {@code initializationFunction} is defined, the source will be passed + * to it. Otherwise, the property will be extracted from it based on the + * {@code sourceProperty}. + * + * @return Returns the source of the derived config property. + */ + public Source source() { + return source; + } + + /** + * @return Returns the symbol representing the property's type. + */ + public Symbol type() { + return type; + } + + /** + * Gets the optional function used to initialize the derived property. + * + *

This function will be given the {@code source}. + * + *

This is mutually exclusive with {@code sourcePropertyName}. + * + * @return Optionally returns the symbol for a property initialization function. + */ + public Optional initializationFunction() { + return Optional.ofNullable(initializationFunction); + } + + /** + * Gets the optional property name on the source that maps to this property. + * + *

If present, the derived property will be directly mapped to the named + * source property. + * + *

This is mutually exclusive with {@code initializationFunction}. + * + * @return Returns the name of the property on the source. + */ + public Optional sourcePropertyName() { + return Optional.ofNullable(sourcePropertyName); + } + + @Override + public SmithyBuilder toBuilder() { + var builder = builder() + .name(name) + .source(source); + + initializationFunction().ifPresent(builder::initializationFunction); + sourcePropertyName().ifPresent(builder::sourcePropertyName); + return builder; + } + + /** + * @return Returns a builder for a {@link DerivedProperty}. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * The source that the derived property is derived from. + */ + public enum Source { + + /** + * Indicates that the property is derived from the service config object. + */ + CONFIG { + @Override + public String scopeLocation() { + return "config"; + } + }, + + /** + * Indicates that the property is derived from the operation input. + */ + INPUT { + @Override + public String scopeLocation() { + return "context.input"; + } + }; + + /** + * @return Returns the symbol in scope mapping to the location. + */ + public abstract String scopeLocation(); + } + + /** + * A builder for {@link DerivedProperty}. + */ + public static final class Builder implements SmithyBuilder { + private String name; + private Source source = Source.CONFIG; + private Symbol type; + private Symbol initializationFunction; + private String sourcePropertyName; + + @Override + public DerivedProperty build() { + return new DerivedProperty(this); + } + + /** + * Sets the name of the property on the intermediate config. + * + * @param name The property's name. + * @return Returns the builder. + */ + public Builder name(String name) { + this.name = name; + return this; + } + + /** + * Sets the source where the property should be derived from. + * + *

If an {@code initializationFunction} is defined, the source will be passed + * to it. Otherwise, the property will be extracted from it based on the + * {@code sourceProperty}. + * + * @param source The property's source. + * @return Returns the builder. + */ + public Builder source(Source source) { + this.source = source; + return this; + } + + /** + * Sets the symbol representing the type of the derived property. + * + * @param type The property's type symbol. + * @return Returns the builder. + */ + public Builder type(Symbol type) { + this.type = type; + return this; + } + + /** + * Gets the optional function used to initialize the derived property. + * + *

This function will be given the {@code source}. + * + *

This is mutually exclusive with {@code sourcePropertyName}. If set, + * any value for {@code sourcePropertyName} will be nullified. + * + * @param initializationFunction The property's initialization function. + * @return Returns the builder. + */ + public Builder initializationFunction(Symbol initializationFunction) { + this.sourcePropertyName = null; + this.initializationFunction = initializationFunction; + return this; + } + + /** + * Sets the optional property name on the source that maps to this property. + * + *

If present, the derived property will be directly mapped to the named + * source property. + * + *

This is mutually exclusive with {@code initializationFunction}. If set, + * any value for {@code initializationFunction} will be nullified. + * + *

By default, this value will be set to the value of {@code name}. + * + * @param sourceProperty The property's source property. + * @return Returns the builder. + */ + public Builder sourcePropertyName(String sourceProperty) { + this.initializationFunction = null; + this.sourcePropertyName = sourceProperty; + return this; + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java new file mode 100644 index 0000000000..00d5b30c7a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java @@ -0,0 +1,365 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import static java.lang.String.format; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import software.amazon.smithy.build.FileManifest; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.codegen.core.TopologicalIndex; +import software.amazon.smithy.codegen.core.WriterDelegator; +import software.amazon.smithy.codegen.core.directed.CreateContextDirective; +import software.amazon.smithy.codegen.core.directed.CreateSymbolProviderDirective; +import software.amazon.smithy.codegen.core.directed.CustomizeDirective; +import software.amazon.smithy.codegen.core.directed.DirectedCodegen; +import software.amazon.smithy.codegen.core.directed.GenerateEnumDirective; +import software.amazon.smithy.codegen.core.directed.GenerateErrorDirective; +import software.amazon.smithy.codegen.core.directed.GenerateIntEnumDirective; +import software.amazon.smithy.codegen.core.directed.GenerateResourceDirective; +import software.amazon.smithy.codegen.core.directed.GenerateServiceDirective; +import software.amazon.smithy.codegen.core.directed.GenerateStructureDirective; +import software.amazon.smithy.codegen.core.directed.GenerateUnionDirective; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.ServiceIndex; +import software.amazon.smithy.model.neighbor.Walker; +import software.amazon.smithy.model.shapes.CollectionShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.integration.ProtocolGenerator; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.utils.SmithyUnstableApi; + +@SmithyUnstableApi +public class DirectedPythonCodegen implements DirectedCodegen { + + private static final Logger LOGGER = Logger.getLogger(DirectedPythonCodegen.class.getName()); + + @Override + public SymbolProvider createSymbolProvider(CreateSymbolProviderDirective directive) { + return new SymbolVisitor(directive.model(), directive.settings()); + } + + @Override + public GenerationContext createContext(CreateContextDirective directive) { + return GenerationContext.builder() + .model(directive.model()) + .settings(directive.settings()) + .symbolProvider(directive.symbolProvider()) + .fileManifest(directive.fileManifest()) + .writerDelegator(new PythonDelegator( + directive.fileManifest(), directive.symbolProvider(), directive.settings())) + .integrations(directive.integrations()) + .protocolGenerator(resolveProtocolGenerator( + directive.integrations(), directive.model(), directive.service(), directive.settings())) + .build(); + } + + private ProtocolGenerator resolveProtocolGenerator( + Collection integrations, + Model model, + ServiceShape service, + PythonSettings settings + ) { + // Collect all of the supported protocol generators. + Map generators = new HashMap<>(); + for (PythonIntegration integration : integrations) { + for (ProtocolGenerator generator : integration.getProtocolGenerators()) { + generators.put(generator.getProtocol(), generator); + } + } + + ShapeId protocolName; + try { + protocolName = settings.resolveServiceProtocol(model, service, generators.keySet()); + } catch (CodegenException e) { + LOGGER.warning("Unable to find a protocol generator for " + service.getId() + ": " + e.getMessage()); + protocolName = null; + } + + return protocolName != null ? generators.get(protocolName) : null; + } + + @Override + public void customizeBeforeShapeGeneration(CustomizeDirective directive) { + generateServiceErrors(directive.settings(), directive.context().writerDelegator()); + new ConfigGenerator(directive.settings(), directive.context()).run(); + + var serviceIndex = ServiceIndex.of(directive.model()); + if (directive.context().applicationProtocol().isHttpProtocol() + && !serviceIndex.getAuthSchemes(directive.service()).isEmpty()) { + new HttpAuthGenerator(directive.context(), directive.settings()).run(); + } + } + + @Override + public void generateService(GenerateServiceDirective directive) { + new ClientGenerator(directive.context(), directive.service()).run(); + + var protocolGenerator = directive.context().protocolGenerator(); + if (protocolGenerator == null) { + return; + } + + protocolGenerator.generateSharedSerializerComponents(directive.context()); + protocolGenerator.generateRequestSerializers(directive.context()); + + protocolGenerator.generateSharedDeserializerComponents(directive.context()); + protocolGenerator.generateResponseDeserializers(directive.context()); + + protocolGenerator.generateProtocolTests(directive.context()); + } + + protected void generateServiceErrors(PythonSettings settings, WriterDelegator writers) { + var serviceError = CodegenUtils.getServiceError(settings); + writers.useFileWriter(serviceError.getDefinitionFile(), serviceError.getNamespace(), writer -> { + // TODO: subclass a shared error + writer.openBlock("class $L(Exception):", "", serviceError.getName(), () -> { + writer.writeDocs("Base error for all errors in the service."); + writer.write("pass"); + }); + }); + + var apiError = CodegenUtils.getApiError(settings); + writers.useFileWriter(apiError.getDefinitionFile(), apiError.getNamespace(), writer -> { + writer.addStdlibImport("typing", "Generic"); + writer.addStdlibImport("typing", "TypeVar"); + writer.write("T = TypeVar('T')"); + writer.openBlock("class $L($T, Generic[T]):", "", apiError.getName(), serviceError, () -> { + writer.writeDocs("Base error for all api errors in the service."); + writer.write("code: T"); + writer.openBlock("def __init__(self, message: str):", "", () -> { + writer.write("super().__init__(message)"); + writer.write("self.message = message"); + }); + }); + + var unknownApiError = CodegenUtils.getUnknownApiError(settings); + writer.addStdlibImport("typing", "Literal"); + writer.openBlock("class $L($T[Literal['Unknown']]):", "", unknownApiError.getName(), apiError, () -> { + writer.writeDocs("Error representing any unknown api errors"); + writer.write("code: Literal['Unknown'] = 'Unknown'"); + }); + }); + } + + @Override + public void generateResource(GenerateResourceDirective directive) { + } + + @Override + public void generateStructure(GenerateStructureDirective directive) { + directive.context().writerDelegator().useShapeWriter(directive.shape(), writer -> { + StructureGenerator generator = new StructureGenerator( + directive.model(), + directive.settings(), + directive.symbolProvider(), + writer, + directive.shape(), + TopologicalIndex.of(directive.model()).getRecursiveShapes() + ); + generator.run(); + }); + } + + @Override + public void generateError(GenerateErrorDirective directive) { + directive.context().writerDelegator().useShapeWriter(directive.shape(), writer -> { + StructureGenerator generator = new StructureGenerator( + directive.model(), + directive.settings(), + directive.symbolProvider(), + writer, + directive.shape(), + TopologicalIndex.of(directive.model()).getRecursiveShapes() + ); + generator.run(); + }); + } + + @Override + public void generateUnion(GenerateUnionDirective directive) { + directive.context().writerDelegator().useShapeWriter(directive.shape(), writer -> { + UnionGenerator generator = new UnionGenerator( + directive.model(), + directive.symbolProvider(), + writer, + directive.shape(), + TopologicalIndex.of(directive.model()).getRecursiveShapes() + ); + generator.run(); + }); + } + + @Override + public void generateEnumShape(GenerateEnumDirective directive) { + if (!directive.shape().isEnumShape()) { + return; + } + directive.context().writerDelegator().useShapeWriter(directive.shape(), writer -> { + EnumGenerator generator = new EnumGenerator( + directive.model(), + directive.symbolProvider(), + writer, + directive.shape().asEnumShape().get() + ); + generator.run(); + }); + } + + @Override + public void generateIntEnumShape(GenerateIntEnumDirective directive) { + new IntEnumGenerator(directive).run(); + } + + @Override + public void customizeBeforeIntegrations(CustomizeDirective directive) { + generateInits(directive); + generateDictHelpers(directive); + } + + private void generateDictHelpers(CustomizeDirective directive) { + Iterator shapes = new Walker(directive.model()).iterateShapes(directive.service()); + while (shapes.hasNext()) { + Shape shape = shapes.next(); + if (shape.isListShape()) { + generateCollectionDictHelpers(directive.context(), shape.asListShape().get()); + } else if (shape.isMapShape()) { + generateMapDictHelpers(directive.context(), shape.asMapShape().get()); + } + } + } + + private Void generateCollectionDictHelpers(GenerationContext context, CollectionShape shape) { + SymbolProvider symbolProvider = context.symbolProvider(); + WriterDelegator writers = context.writerDelegator(); + var optionalAsDictSymbol = symbolProvider.toSymbol(shape).getProperty("asDict", Symbol.class); + optionalAsDictSymbol.ifPresent(asDictSymbol -> { + writers.useFileWriter(asDictSymbol.getDefinitionFile(), asDictSymbol.getNamespace(), writer -> { + new CollectionGenerator(context.model(), symbolProvider, writer, shape).run(); + }); + }); + return null; + } + + public Void generateMapDictHelpers(GenerationContext context, MapShape shape) { + SymbolProvider symbolProvider = context.symbolProvider(); + WriterDelegator writers = context.writerDelegator(); + var optionalAsDictSymbol = symbolProvider.toSymbol(shape).getProperty("asDict", Symbol.class); + optionalAsDictSymbol.ifPresent(asDictSymbol -> { + writers.useFileWriter(asDictSymbol.getDefinitionFile(), asDictSymbol.getNamespace(), writer -> { + new MapGenerator(context.model(), symbolProvider, writer, shape).run(); + }); + }); + return null; + } + + /** + * Creates __init__.py files where not already present. + */ + private void generateInits(CustomizeDirective directive) { + var directories = directive.context().writerDelegator().getWriters().keySet().stream() + .map(Paths::get) + .filter(path -> !path.getParent().equals(directive.fileManifest().getBaseDir())) + .collect(Collectors.groupingBy(Path::getParent, Collectors.toSet())); + for (var entry : directories.entrySet()) { + var initPath = entry.getKey().resolve("__init__.py"); + if (!entry.getValue().contains(initPath)) { + directive.fileManifest().writeFile(initPath, + """ + # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + # SPDX-License-Identifier: Apache-2.0 + # Do not modify this file. This file is machine generated, and any changes to it will be overwritten.\n + """); + } + } + } + + @Override + public void customizeAfterIntegrations(CustomizeDirective directive) { + Pattern versionPattern = Pattern.compile("Python \\d\\.(?\\d+)\\.(?\\d+)"); + FileManifest fileManifest = directive.fileManifest(); + SetupGenerator.generateSetup(directive.settings(), directive.context()); + + LOGGER.info("Flushing writers in preparation for formatting and linting."); + directive.context().writerDelegator().flushWriters(); + + String output; + try { + LOGGER.info("Attempting to discover python version"); + output = CodegenUtils.runCommand("python3 --version", fileManifest.getBaseDir()).strip(); + } catch (CodegenException e) { + LOGGER.warning("Unable to find python on the path. Skipping formatting and type checking."); + return; + } + var matcher = versionPattern.matcher(output); + if (!matcher.find()) { + LOGGER.warning("Unable to parse python version string. Skipping formatting and type checking."); + } + int minorVersion = Integer.parseInt(matcher.group("minor")); + if (minorVersion < 11) { + LOGGER.warning(format(""" + Found incompatible python version 3.%s.%s, expected 3.11.0 or greater. \ + Skipping formatting and type checking.""", + matcher.group("minor"), matcher.group("patch"))); + return; + } + LOGGER.info("Verifying python files"); + for (var file : fileManifest.getFiles()) { + var fileName = file.getFileName(); + if (fileName == null || !fileName.endsWith(".py")) { + continue; + } + CodegenUtils.runCommand("python3 " + file, fileManifest.getBaseDir()); + } + formatCode(fileManifest); + runMypy(fileManifest); + } + + private void formatCode(FileManifest fileManifest) { + try { + CodegenUtils.runCommand("python3 -m black -h", fileManifest.getBaseDir()); + } catch (CodegenException e) { + LOGGER.warning("Unable to find the python package black. Skipping formatting."); + return; + } + LOGGER.info("Running code formatter on generated code"); + CodegenUtils.runCommand("python3 -m black . --exclude \"\"", fileManifest.getBaseDir()); + } + + private void runMypy(FileManifest fileManifest) { + try { + CodegenUtils.runCommand("python3 -m mypy -h", fileManifest.getBaseDir()); + } catch (CodegenException e) { + LOGGER.warning("Unable to find the python package mypy. Skipping type checking."); + return; + } + LOGGER.info("Running mypy on generated code"); + CodegenUtils.runCommand("python3 -m mypy .", fileManifest.getBaseDir()); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/EnumGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/EnumGenerator.java new file mode 100644 index 0000000000..dd62b700dc --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/EnumGenerator.java @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.Iterator; +import java.util.Locale; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.EnumShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.traits.EnumValueTrait; + +/** + * Renders enums. + */ +public final class EnumGenerator implements Runnable { + private final Model model; + private final SymbolProvider symbolProvider; + private final PythonWriter writer; + private final EnumShape shape; + + public EnumGenerator(Model model, SymbolProvider symbolProvider, PythonWriter writer, EnumShape enumShape) { + this.model = model; + this.symbolProvider = symbolProvider; + this.writer = writer; + this.shape = enumShape; + } + + @Override + public void run() { + var enumSymbol = symbolProvider.toSymbol(shape).expectProperty("enumSymbol", Symbol.class); + writer.openBlock("class $L:", "", enumSymbol.getName(), () -> { + shape.getTrait(DocumentationTrait.class).ifPresent(trait -> { + writer.writeDocs(writer.formatDocs(trait.getValue())); + }); + + for (MemberShape member: shape.members()) { + member.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeComment(trait.getValue())); + var name = symbolProvider.toMemberName(member); + writer.write("$L = $S\n", name, getEnumValue(member)); + } + + writer.writeComment(""" + This set contains every possible value known at the time this was \ + generated. New values may be added in the future."""); + writer.writeInline("values = frozenset({"); + for (Iterator iter = shape.members().iterator(); iter.hasNext();) { + writer.writeInline("$S", getEnumValue(iter.next())); + if (iter.hasNext()) { + writer.writeInline(", "); + } + } + writer.writeInline("})\n"); + }); + } + + public String getEnumValue(MemberShape member) { + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-enumvalue-trait + return member.getTrait(EnumValueTrait.class) + .flatMap(EnumValueTrait::getStringValue) + .orElseGet(() -> member.getMemberName().toUpperCase(Locale.ENGLISH)); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/GenerationContext.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/GenerationContext.java new file mode 100644 index 0000000000..feecfb72b5 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/GenerationContext.java @@ -0,0 +1,200 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.List; +import software.amazon.smithy.build.FileManifest; +import software.amazon.smithy.codegen.core.CodegenContext; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.codegen.core.WriterDelegator; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.python.codegen.integration.ProtocolGenerator; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.utils.SmithyBuilder; +import software.amazon.smithy.utils.SmithyUnstableApi; +import software.amazon.smithy.utils.ToSmithyBuilder; + +/** + * Holds context related to code generation. + */ +@SmithyUnstableApi +public final class GenerationContext + implements CodegenContext, ToSmithyBuilder { + + private final Model model; + private final PythonSettings settings; + private final SymbolProvider symbolProvider; + private final FileManifest fileManifest; + private final PythonDelegator delegator; + private final List integrations; + private final ProtocolGenerator protocolGenerator; + + private GenerationContext(Builder builder) { + model = builder.model; + settings = builder.settings; + symbolProvider = builder.symbolProvider; + fileManifest = builder.fileManifest; + delegator = builder.delegator; + integrations = builder.integrations; + protocolGenerator = builder.protocolGenerator; + } + + @Override + public Model model() { + return model; + } + + @Override + public PythonSettings settings() { + return settings; + } + + @Override + public SymbolProvider symbolProvider() { + return symbolProvider; + } + + @Override + public FileManifest fileManifest() { + return fileManifest; + } + + @Override + public WriterDelegator writerDelegator() { + return delegator; + } + + @Override + public List integrations() { + return integrations; + } + + /** + * @return Returns the protocol generator to use in code generation. + */ + public ProtocolGenerator protocolGenerator() { + return protocolGenerator; + } + + /** + * Gets the application protocol for the service protocol. + * + * @return Returns the application protocol the protocol makes use of. + */ + public ApplicationProtocol applicationProtocol() { + return protocolGenerator != null + ? protocolGenerator.getApplicationProtocol() + : ApplicationProtocol.createDefaultHttpApplicationProtocol(); + } + + /** + * @return Returns a builder. + */ + public static Builder builder() { + return new Builder(); + } + + @Override + public SmithyBuilder toBuilder() { + return builder() + .model(model) + .settings(settings) + .symbolProvider(symbolProvider) + .fileManifest(fileManifest) + .writerDelegator(delegator); + } + + /** + * Builds {@link GenerationContext}s. + */ + public static final class Builder implements SmithyBuilder { + private Model model; + private PythonSettings settings; + private SymbolProvider symbolProvider; + private FileManifest fileManifest; + private PythonDelegator delegator; + private List integrations; + private ProtocolGenerator protocolGenerator; + + @Override + public GenerationContext build() { + return new GenerationContext(this); + } + + /** + * @param model The model being generated. + * @return Returns the builder. + */ + public Builder model(Model model) { + this.model = model; + return this; + } + + /** + * @param settings The resolved settings for the generator. + * @return Returns the builder. + */ + public Builder settings(PythonSettings settings) { + this.settings = settings; + return this; + } + + /** + * @param symbolProvider The finalized symbol provider for the generator. + * @return Returns the builder. + */ + public Builder symbolProvider(SymbolProvider symbolProvider) { + this.symbolProvider = symbolProvider; + return this; + } + + /** + * @param fileManifest The file manifest being used in the generator. + * @return Returns the builder. + */ + public Builder fileManifest(FileManifest fileManifest) { + this.fileManifest = fileManifest; + return this; + } + + /** + * @param delegator The writer delegator to use in the generator. + * @return Returns the builder. + */ + public Builder writerDelegator(PythonDelegator delegator) { + this.delegator = delegator; + return this; + } + + /** + * @param integrations The integrations to use in the generator. + * @return Returns the builder. + */ + public Builder integrations(List integrations) { + this.integrations = integrations; + return this; + } + + /** + * @param protocolGenerator The resolved protocol generator to use. + * @return Returns the builder. + */ + public Builder protocolGenerator(ProtocolGenerator protocolGenerator) { + this.protocolGenerator = protocolGenerator; + return this; + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpAuthGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpAuthGenerator.java new file mode 100644 index 0000000000..421250b327 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpAuthGenerator.java @@ -0,0 +1,153 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.python.codegen; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.knowledge.ServiceIndex; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.traits.AuthTrait; +import software.amazon.smithy.python.codegen.integration.AuthScheme; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.python.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.python.codegen.sections.GenerateHttpAuthParametersSection; +import software.amazon.smithy.python.codegen.sections.GenerateHttpAuthSchemeResolverSection; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * This class is responsible for generating the http auth scheme resolver and its configuration. + */ +@SmithyInternalApi +final class HttpAuthGenerator implements Runnable { + + private final PythonSettings settings; + private final GenerationContext context; + + HttpAuthGenerator(GenerationContext context, PythonSettings settings) { + this.settings = settings; + this.context = context; + } + + @Override + public void run() { + var supportedAuthSchemes = new HashMap(); + var properties = new ArrayList(); + var service = context.settings().getService(context.model()); + for (PythonIntegration integration : context.integrations()) { + for (RuntimeClientPlugin plugin : integration.getClientPlugins()) { + if (plugin.matchesService(context.model(), service) + && plugin.getAuthScheme().isPresent() + && plugin.getAuthScheme().get().getApplicationProtocol().isHttpProtocol()) { + var scheme = plugin.getAuthScheme().get(); + supportedAuthSchemes.put(scheme.getAuthTrait(), scheme); + properties.addAll(scheme.getAuthProperties()); + } + } + } + + var params = CodegenUtils.getHttpAuthParamsSymbol(settings); + context.writerDelegator().useFileWriter(params.getDefinitionFile(), params.getNamespace(), writer -> { + generateAuthParameters(writer, params, properties); + }); + + var resolver = CodegenUtils.getHttpAuthSchemeResolverSymbol(settings); + context.writerDelegator().useFileWriter(resolver.getDefinitionFile(), resolver.getNamespace(), writer -> { + generateAuthSchemeResolver(writer, params, resolver, supportedAuthSchemes); + }); + } + + private void generateAuthParameters(PythonWriter writer, Symbol symbol, List properties) { + var propertyMap = new LinkedHashMap(); + for (DerivedProperty property : properties) { + propertyMap.put(property.name(), property.type()); + } + writer.pushState(new GenerateHttpAuthParametersSection(Map.copyOf(propertyMap))); + writer.addStdlibImport("dataclasses", "dataclass"); + writer.write(""" + @dataclass + class $L: + operation: str + ${#properties} + ${key:L}: ${value:T} | None + ${/properties} + """, symbol.getName()); + writer.popState(); + } + + private void generateAuthSchemeResolver( + PythonWriter writer, + Symbol paramsSymbol, + Symbol resolverSymbol, + Map supportedAuthSchemes + ) { + var resolvedAuthSchemes = ServiceIndex.of(context.model()) + .getEffectiveAuthSchemes(settings.getService()).keySet().stream() + .filter(supportedAuthSchemes::containsKey) + .map(supportedAuthSchemes::get) + .toList(); + + writer.pushState(new GenerateHttpAuthSchemeResolverSection(resolvedAuthSchemes)); + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.interfaces.auth", "HTTPAuthOption"); + writer.write(""" + class $1L: + def resolve_auth_scheme(self, auth_parameters: $2T) -> list[HTTPAuthOption]: + auth_options: list[HTTPAuthOption] = [] + + ${3C|} + ${4C|} + + """, resolverSymbol.getName(), paramsSymbol, + writer.consumer(w -> writeOperationAuthOptions(w, supportedAuthSchemes)), + writer.consumer(w -> writeAuthOptions(w, resolvedAuthSchemes))); + writer.popState(); + } + + private void writeOperationAuthOptions(PythonWriter writer, Map supportedAuthSchemes) { + var operations = TopDownIndex.of(context.model()).getContainedOperations(settings.getService()); + var serviceIndex = ServiceIndex.of(context.model()); + for (OperationShape operation : operations) { + if (!operation.hasTrait(AuthTrait.class)) { + continue; + } + + var operationAuthSchemes = serviceIndex + .getEffectiveAuthSchemes(settings.getService(), operation).keySet().stream() + .filter(supportedAuthSchemes::containsKey) + .map(supportedAuthSchemes::get) + .toList(); + + writer.write(""" + if auth_parameters.operation == $S: + ${C|} + + """, operation.getId().getName(), writer.consumer(w -> writeAuthOptions(w, operationAuthSchemes))); + } + } + + private void writeAuthOptions(PythonWriter writer, List authSchemes) { + var authOptionInitializers = authSchemes.stream() + .map(scheme -> scheme.getAuthOptionGenerator(context)) + .toList(); + writer.pushState(); + writer.putContext("authOptionInitializers", authOptionInitializers); + writer.write(""" + ${#authOptionInitializers} + if ((option := ${value:T}(auth_parameters)) is not None): + auth_options.append(option) + + ${/authOptionInitializers} + return auth_options + """); + writer.popState(); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java new file mode 100644 index 0000000000..261662fd53 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java @@ -0,0 +1,877 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.BiFunction; +import java.util.function.BiPredicate; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.HttpBinding; +import software.amazon.smithy.model.knowledge.HttpBinding.Location; +import software.amazon.smithy.model.knowledge.HttpBindingIndex; +import software.amazon.smithy.model.knowledge.OperationIndex; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.node.ArrayNode; +import software.amazon.smithy.model.node.BooleanNode; +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.node.NodeVisitor; +import software.amazon.smithy.model.node.NullNode; +import software.amazon.smithy.model.node.NumberNode; +import software.amazon.smithy.model.node.ObjectNode; +import software.amazon.smithy.model.node.StringNode; +import software.amazon.smithy.model.shapes.CollectionShape; +import software.amazon.smithy.model.shapes.DocumentShape; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.StreamingTrait; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import software.amazon.smithy.protocoltests.traits.AppliesTo; +import software.amazon.smithy.protocoltests.traits.HttpMessageTestCase; +import software.amazon.smithy.protocoltests.traits.HttpRequestTestCase; +import software.amazon.smithy.protocoltests.traits.HttpRequestTestsTrait; +import software.amazon.smithy.protocoltests.traits.HttpResponseTestCase; +import software.amazon.smithy.protocoltests.traits.HttpResponseTestsTrait; +import software.amazon.smithy.utils.CaseUtils; +import software.amazon.smithy.utils.Pair; +import software.amazon.smithy.utils.SimpleParser; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Generates protocol tests for a given HTTP protocol. + * + *

This should preferably be instantiated and used within an + * implementation of a `ProtocolGeneration` + * + *

See Also: + * HTTP Protocol Compliance Tests + */ +@SmithyUnstableApi +public final class HttpProtocolTestGenerator implements Runnable { + + private static final Logger LOGGER = Logger.getLogger(HttpProtocolTestGenerator.class.getName()); + private static final Symbol REQUEST_TEST_ASYNC_HTTP_CLIENT_SYMBOL = Symbol.builder() + .name("RequestTestHTTPClient") + .build(); + private static final Symbol RESPONSE_TEST_ASYNC_HTTP_CLIENT_SYMBOL = Symbol.builder() + .name("ResponseTestHTTPClient") + .build(); + private static final Symbol TEST_HTTP_SERVICE_ERR_SYMBOL = Symbol.builder() + .name("TestHttpServiceError") + .build(); + + private final PythonSettings settings; + private final Model model; + private final ShapeId protocol; + private final ServiceShape service; + private final PythonWriter writer; + private final GenerationContext context; + private final BiPredicate testFilter; + + /** + * Constructor. + * + * @param context The code generation context. + * @param protocol The protocol whose tests should be generated. + * @param writer The writer to write to. + * @param testFilter A filter that indicates tests which are expected to fail. + */ + public HttpProtocolTestGenerator( + GenerationContext context, + ShapeId protocol, + PythonWriter writer, + BiPredicate testFilter + ) { + this.settings = context.settings(); + this.model = context.model(); + this.protocol = protocol; + this.service = settings.getService(model); + this.writer = writer; + this.context = context; + this.testFilter = testFilter; + + writer.putFormatter('J', new JavaToPythonFormatter()); + } + + /** + * Generates the HTTP-based protocol tests for the given protocol in the model. + */ + @Override + public void run() { + OperationIndex operationIndex = OperationIndex.of(model); + TopDownIndex topDownIndex = TopDownIndex.of(model); + writer.addDependency(SmithyPythonDependency.PYTEST); + writer.addDependency(SmithyPythonDependency.PYTEST_ASYNCIO); + + // Use a TreeSet to have a fixed ordering of tests. + for (OperationShape operation : new TreeSet<>(topDownIndex.getContainedOperations(service))) { + generateOperationTests(AppliesTo.CLIENT, operation, operationIndex); + } + // Write the testing implementations for various objects + writeUtilStubs(context.symbolProvider().toSymbol(service)); + } + + private void generateOperationTests( + AppliesTo implementation, + OperationShape operation, + OperationIndex operationIndex + ) { + // Request Tests + operation.getTrait(HttpRequestTestsTrait.class).ifPresent(trait -> { + for (HttpRequestTestCase testCase : trait.getTestCasesFor(implementation)) { + onlyIfProtocolMatches(testCase, () -> generateRequestTest(operation, testCase)); + } + }); + + // Response Tests + operation.getTrait(HttpResponseTestsTrait.class).ifPresent(trait -> { + for (HttpResponseTestCase testCase : trait.getTestCasesFor(implementation)) { + onlyIfProtocolMatches(testCase, () -> generateResponseTest(operation, testCase)); + } + }); + + // Error Tests + for (StructureShape error : operationIndex.getErrors(operation, service)) { + if (!error.hasTag("server-only")) { + error.getTrait(HttpResponseTestsTrait.class).ifPresent(trait -> { + for (HttpResponseTestCase testCase : trait.getTestCasesFor(implementation)) { + onlyIfProtocolMatches(testCase, + () -> generateErrorResponseTest(operation, error, testCase)); + } + }); + } + } + } + + // See also: https://smithy.io/2.0/additional-specs/http-protocol-compliance-tests.html#httprequesttests-trait + private void generateRequestTest(OperationShape operation, HttpRequestTestCase testCase) { + writeTestBlock( + testCase, + String.format("%s_request_%s", testCase.getId(), operation.getId().getName()), + testFilter.test(operation, testCase), + () -> { + var hostSplit = testCase.getHost().orElse("example.com").split("/", 2); + var host = hostSplit[0]; + var resolvedHost = testCase.getResolvedHost().map(h -> h.split("/", 2)[0]).orElse(host); + String path; + if (hostSplit.length != 1) { + path = hostSplit[1]; + } else { + path = ""; + } + writer.addImport("smithy_python._private.retries", "SimpleRetryStrategy"); + writeClientBlock(context.symbolProvider().toSymbol(service), testCase, Optional.of(() -> { + writer.write(""" + config = $T( + endpoint_uri="https://$L/$L", + http_client = $T(), + retry_strategy=SimpleRetryStrategy(max_attempts=1), + ) + """, + CodegenUtils.getConfigSymbol(context.settings()), + host, + path, + REQUEST_TEST_ASYNC_HTTP_CLIENT_SYMBOL + ); + })); + + // Generate the input using the expected shape and params + var inputShape = model.expectShape(operation.getInputShape(), StructureShape.class); + writer.write("input_ = $C\n", + (Runnable) () -> testCase.getParams().accept(new ValueNodeVisitor(inputShape)) + ); + + // Execute the command, and catch the expected exception + writer.addImport(SmithyPythonDependency.PYTEST.packageName(), "fail"); + writer.addImport(SmithyPythonDependency.PYTEST.packageName(), "raises"); + writer.addStdlibImport("urllib.parse", "parse_qsl"); + writer.write(""" + try: + await client.$1T(input_) + fail("Expected '$2T' exception to be thrown!") + except $2T as err: + actual = err.request + + assert actual.method == $3S + assert actual.destination.path == $4S + assert actual.destination.host == $5S + + query = actual.destination.query + actual_query_segments: list[str] = query.split("&") if query else [] + expected_query_segments: list[str] = $6J + for expected_query_segment in expected_query_segments: + assert expected_query_segment in actual_query_segments + actual_query_segments.remove(expected_query_segment) + + actual_query_keys: list[str] = [k.lower() for k, v in parse_qsl(query)] + forbidden_query_keys: set[str] = set($7J) + for forbidden_key in forbidden_query_keys: + assert forbidden_key.lower() not in actual_query_keys + + required_query_keys: list[str] = $8J + for required_query_key in required_query_keys: + assert required_query_key.lower() in actual_query_keys + # These are removed because the required list could require more than one + # value. By removing each value after we assert that it's there, we can + # effectively validate that without having to have a more complex comparator. + actual_query_keys.remove(required_query_key) + + expected_headers: list[tuple[str, str]] = [ + ${9C|} + ] + for expected_key, expected_val in expected_headers: + assert expected_val in actual.fields.get_field(expected_key).values + + forbidden_headers: set[str] = set($10J) + for forbidden_key in forbidden_headers: + with raises(KeyError): + actual.fields.get_field(forbidden_key) + + required_headers: list[str] = $11J + for required_key in required_headers: + # Fields.remove_field() raises KeyError if key does not exist + actual.fields.remove_field(required_key) + + ${12C|} + + except Exception as err: + fail(f"Expected '$2L' exception to be thrown, but received {type(err).__name__}: {err}") + """, + context.symbolProvider().toSymbol(operation), + TEST_HTTP_SERVICE_ERR_SYMBOL, + testCase.getMethod(), + testCase.getUri(), + resolvedHost, + testCase.getQueryParams(), + toLowerCase(testCase.getForbidQueryParams()), + toLowerCase(testCase.getRequireQueryParams()), + (Runnable) () -> writeExpectedHeaders(testCase, operation), + toLowerCase(testCase.getForbidHeaders()), + toLowerCase(testCase.getRequireHeaders()), + writer.consumer(w -> writeRequestBodyComparison(testCase, w)) + ); + }); + } + + private List toLowerCase(List given) { + return given.stream().map(str -> str.toLowerCase(Locale.US)).collect(Collectors.toList()); + } + + private void writeExpectedHeaders( + HttpRequestTestCase testCase, + OperationShape operation + ) { + var headerPairs = splitHeaders(testCase, operation); + for (Pair pair : headerPairs) { + writer.write("($S, $S),", pair.getKey().toLowerCase(Locale.US), pair.getValue()); + } + } + + // TODO: upstream this to Smithy itself or update the protocol test traits + private List> splitHeaders( + HttpRequestTestCase testCase, + OperationShape operation + ) { + // Get a map of headers to binding info for headers that are bound to lists. + var listBindings = HttpBindingIndex.of(model) + .getRequestBindings(operation, Location.HEADER) + .stream() + .filter(binding -> model.expectShape(binding.getMember().getTarget()).isListShape()) + .collect(Collectors.toMap(HttpBinding::getLocationName, binding -> binding)); + + // Go through each of the headers on the protocol test and turn them into key-value tuples. + var headerPairs = new ArrayList>(); + for (Map.Entry entry : testCase.getHeaders().entrySet()) { + // If we know a list isn't bound to this header, then we know it's static so we can just + // add it directly. + if (!listBindings.containsKey(entry.getKey())) { + headerPairs.add(Pair.of(entry.getKey(), entry.getValue())); + continue; + } + var binding = listBindings.get(entry.getKey()); + var paramValue = testCase.getParams().expectArrayMember(binding.getMemberName()); + if (paramValue.size() == 1) { + // If it's a single value of a list, we want to keep it as-is. + headerPairs.add(Pair.of(entry.getKey(), entry.getValue())); + } + try { + headerPairs.addAll(splitHeader(binding, entry.getKey(), entry.getValue())); + } catch (Exception e) { + throw new CodegenException( + String.format("Failed to split header in protocol test %s - `%s`: `%s` - %s", + testCase.getId(), entry.getKey(), entry.getValue(), e)); + } + } + + return headerPairs; + } + + private List> splitHeader(HttpBinding binding, String key, String value) { + var values = new ArrayList>(); + var parser = new SimpleParser(value); + + boolean isHttpDateMember = false; + var listShape = model.expectShape(binding.getMember().getTarget(), ListShape.class); + var listMember = model.expectShape(listShape.getMember().getTarget()); + if (listMember.isTimestampShape()) { + var httpIndex = HttpBindingIndex.of(model); + var format = httpIndex.determineTimestampFormat( + binding.getMember(), binding.getLocation(), Format.HTTP_DATE); + isHttpDateMember = format == Format.HTTP_DATE; + } + + // Strip any leading whitespace + parser.ws(); + while (!parser.eof()) { + values.add(Pair.of(key, parseEntry(parser, isHttpDateMember))); + } + return values; + } + + private String parseEntry(SimpleParser parser, boolean skipFirstComma) { + String value; + + // If the first character is a dquote, parse as a quoted string. + if (parser.peek() == '"') { + parser.expect('"'); + var builder = new StringBuilder(); + while (!parser.eof() && parser.peek() != '"') { + if (parser.peek() == '\\') { + parser.skip(); + } + builder.append(parser.peek()); + parser.skip(); + } + // Ensure that the string ends in a dquote + parser.expect('"'); + value = builder.toString(); + } else { + var start = parser.position(); + parser.consumeUntilNoLongerMatches(character -> character != ','); + if (skipFirstComma) { + parser.expect(','); + parser.consumeUntilNoLongerMatches(character -> character != ','); + } + // We can use substring instead of a StringBuilder here because we don't + // need to worry about escaped characters. + value = parser.expression().substring(start, parser.position()).trim(); + } + + // Strip trailing whitespace + parser.ws(); + + // If we're not at the end of the line, assert that we encounter a comma. + if (!parser.eof()) { + parser.expect(','); + parser.ws(); + } + + return value; + } + + private void writeRequestBodyComparison(HttpMessageTestCase testCase, PythonWriter writer) { + if (testCase.getBody().isEmpty()) { + return; + } + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.interfaces.blobs", "AsyncBytesReader"); + writer.write("actual_body_content = await AsyncBytesReader(actual.body).read()"); + writer.write("expected_body_content = b$S", testCase.getBody().get()); + compareMediaBlob(testCase, writer); + } + + private void compareMediaBlob(HttpMessageTestCase testCase, PythonWriter writer) { + var contentType = testCase.getBodyMediaType().orElse("application/octet-stream"); + if (contentType.equals("application/json") || contentType.endsWith("+json")) { + writer.addStdlibImport("json"); + writer.write(""" + actual_body = json.loads(actual_body_content) if actual_body_content else "" + expected_body = json.loads(expected_body_content) + assert actual_body == expected_body + + """); + return; + } + writer.write("assert actual_body_content == expected_body_content\n"); + } + + // See also: https://smithy.io/2.0/additional-specs/http-protocol-compliance-tests.html#httpresponsetests-trait + private void generateResponseTest(OperationShape operation, HttpResponseTestCase testCase) { + writeTestBlock( + testCase, + String.format("%s_response_%s", testCase.getId(), operation.getId().getName()), + testFilter.test(operation, testCase), + () -> { + writeClientBlock(context.symbolProvider().toSymbol(service), testCase, Optional.of(() -> { + writer.write(""" + config = $T( + endpoint_uri="https://example.com", + http_client = $T( + status=$L, + headers=$J, + body=b$S, + ), + ) + """, + CodegenUtils.getConfigSymbol(context.settings()), + RESPONSE_TEST_ASYNC_HTTP_CLIENT_SYMBOL, + testCase.getCode(), + CodegenUtils.toTuples(testCase.getHeaders()), + testCase.getBody().filter(body -> !body.isEmpty()).orElse("") + ); + })); + // Create an empty input object to pass + var inputShape = model.expectShape(operation.getInputShape(), StructureShape.class); + var outputShape = model.expectShape(operation.getOutputShape(), StructureShape.class); + writer.write("input_ = $C\n", + (Runnable) () -> (ObjectNode.builder().build()).accept(new ValueNodeVisitor(inputShape)) + ); + + // Execute the command, fail if unexpected exception + writer.addImport(SmithyPythonDependency.PYTEST.packageName(), "fail", "fail"); + writer.write(""" + try: + actual = await client.$T(input_) + except Exception as err: + fail(f"Expected a valid response, but received: {type(err).__name__}: {err}") + else: + expected = $C + + ${C|} + """, + context.symbolProvider().toSymbol(operation), + (Runnable) () -> testCase.getParams().accept(new ValueNodeVisitor(outputShape)), + (Runnable) () -> assertResponseEqual(testCase, operation) + ); + }); + } + + // See also: https://smithy.io/2.0/additional-specs/http-protocol-compliance-tests.html#httpresponsetests-trait + private void generateErrorResponseTest( + OperationShape operation, + StructureShape error, + HttpResponseTestCase testCase + ) { + writeTestBlock(testCase, + String.format("%s_error_%s", testCase.getId(), operation.getId().getName()), + testFilter.test(error, testCase), + () -> { + writeClientBlock(context.symbolProvider().toSymbol(service), testCase, Optional.of(() -> { + writer.write(""" + config = $T( + endpoint_uri="https://example.com", + http_client = $T( + status=$L, + headers=$J, + body=b$S, + ), + ) + """, + CodegenUtils.getConfigSymbol(context.settings()), + RESPONSE_TEST_ASYNC_HTTP_CLIENT_SYMBOL, + testCase.getCode(), + CodegenUtils.toTuples(testCase.getHeaders()), + testCase.getBody().orElse("") + ); + })); + // Create an empty input object to pass + var inputShape = model.expectShape(operation.getInputShape(), StructureShape.class); + writer.write("input_ = $C\n", + (Runnable) () -> (Node.objectNode()).accept(new ValueNodeVisitor(inputShape)) + ); + // Execute the command, fail if unexpected exception + writer.addImport(SmithyPythonDependency.PYTEST.packageName(), "fail", "fail"); + writer.write(""" + try: + await client.$1T(input_) + fail("Expected '$2L' exception to be thrown!") + except Exception as err: + if type(err).__name__ != $2S: + fail(f"Expected '$2L' exception to be thrown, but received {type(err).__name__}: {err}") + """, + context.symbolProvider().toSymbol(operation), + error.getId().getName() + ); + // TODO: Correctly assert the status code and other values + }); + } + + private void assertResponseEqual(HttpMessageTestCase testCase, Shape operationOrError) { + var index = HttpBindingIndex.of(context.model()); + var streamBinding = index.getResponseBindings(operationOrError, Location.PAYLOAD) + .stream() + .filter(binding -> binding.getMember().getMemberTrait(context.model(), StreamingTrait.class).isPresent()) + .findAny(); + + if (streamBinding.isEmpty()) { + writer.write("assert actual == expected\n"); + return; + } + + StructureShape responseShape = operationOrError.asStructureShape().orElseGet(() -> { + var operation = operationOrError.asOperationShape().get(); + return context.model().expectShape(operation.getOutputShape(), StructureShape.class); + }); + + var streamingMember = streamBinding.get().getMember(); + + for (MemberShape member : responseShape.members()) { + var memberName = context.symbolProvider().toMemberName(member); + if (member.equals(streamingMember)) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.interfaces.blobs", "AsyncByteStream"); + writer.addImport("smithy_python.interfaces.blobs", "AsyncBytesReader"); + writer.write(""" + assert isinstance(actual.$1L, AsyncByteStream) + actual_body_content = await actual.$1L.read() + expected_body_content = await AsyncBytesReader(expected.$1L).read() + """, memberName); + compareMediaBlob(testCase, writer); + continue; + } + writer.write("assert actual.$1L == expected.$1L\n", memberName); + } + } + + // Only generate test cases when protocol matches the target protocol. + private void onlyIfProtocolMatches(T testCase, Runnable runnable) { + if (testCase.getProtocol().equals(protocol)) { + LOGGER.fine(() -> String.format("Generating protocol test case for %s.%s", + service.getId(), + testCase.getId()) + ); + runnable.run(); + } + } + + // write the test block, which may include certain decorators (i.e. `skip`) + private void writeTestBlock( + HttpMessageTestCase testCase, + String testName, + boolean shouldSkip, + Runnable f + ) { + LOGGER.fine(String.format("Writing test block for %s", testName)); + writer.addDependency(SmithyPythonDependency.PYTEST); + + // Skipped tests are still generated, just not run. + if (shouldSkip) { + LOGGER.fine(String.format("Marking test (%s) as skipped.", testName)); + writer.addImport(SmithyPythonDependency.PYTEST.packageName(), "mark", "mark"); + writer.write("@mark.xfail()"); + } + writer.openBlock("async def test_$L() -> None:", "", CaseUtils.toSnakeCase(testName), () -> { + testCase.getDocumentation().ifPresent(writer::writeDocs); + f.run(); + }); + } + + // write the client block, which may have additional configuration that should + // be written when instantiating the client + private void writeClientBlock( + Symbol serviceSymbol, + HttpMessageTestCase testCase, + Optional additionalConfigurator + ) { + LOGGER.fine(String.format("Writing client block for %s in %s", serviceSymbol.getName(), testCase.getId())); + + // Set up the test http client, which is used to "handle" the requests + writer.openBlock("client = $T(", ")\n", serviceSymbol, () -> { + additionalConfigurator.ifPresent(Runnable::run); + }); + } + + private void writeUtilStubs(Symbol serviceSymbol) { + LOGGER.fine(String.format("Writing utility stubs for %s : %s", serviceSymbol.getName(), protocol.getName())); + writer.addStdlibImport("typing", "Any"); + writer.addImport("smithy_python.interfaces", "Fields"); + writer.addImports("smithy_python.interfaces.http", Set.of( + "HTTPRequestConfiguration", "HTTPRequest", "HTTPResponse") + ); + writer.addImport("smithy_python._private", "tuples_to_fields"); + writer.addImport("smithy_python._private.http", "HTTPResponse", "_HTTPResponse"); + writer.addImport("smithy_python.async_utils", "async_list"); + + writer.write(""" + class $1L($2T): + ""\"A test error that subclasses the service-error for protocol tests.""\" + + def __init__(self, request: HTTPRequest): + self.request = request + + + class $3L: + ""\"An asynchronous HTTP client solely for testing purposes.""\" + + async def send( + self, *, request: HTTPRequest, request_config: HTTPRequestConfiguration | None + ) -> HTTPResponse: + # Raise the exception with the request object to bypass actual request handling + raise $1T(request) + + + class $4L: + ""\"An asynchronous HTTP client solely for testing purposes.""\" + + def __init__(self, status: int, headers: list[tuple[str, str]], body: bytes): + self.status = status + self.fields = tuples_to_fields(headers) + self.body = body + + async def send( + self, *, request: HTTPRequest, request_config: HTTPRequestConfiguration | None + ) -> _HTTPResponse: + # Pre-construct the response from the request and return it + return _HTTPResponse( + status=self.status, + fields=self.fields, + body=async_list([self.body]), + ) + """, + TEST_HTTP_SERVICE_ERR_SYMBOL, + CodegenUtils.getServiceError(context.settings()), + REQUEST_TEST_ASYNC_HTTP_CLIENT_SYMBOL, + RESPONSE_TEST_ASYNC_HTTP_CLIENT_SYMBOL + ); + } + + /** + * NodeVisitor implementation for converting node values for + * input shape(s) to proper Python values in the generated code. + */ + private final class ValueNodeVisitor implements NodeVisitor { + private final Shape inputShape; + + private ValueNodeVisitor(Shape inputShape) { + this.inputShape = inputShape; + } + + @Override + public Void arrayNode(ArrayNode node) { + writer.openBlock("[", "]", () -> { + // The target visitor won't change if the input shape is a union + ValueNodeVisitor targetVisitor; + if (inputShape instanceof CollectionShape) { + var target = model.expectShape(((CollectionShape) inputShape).getMember().getTarget()); + targetVisitor = new ValueNodeVisitor(target); + } else { + targetVisitor = this; + } + + node.getElements().forEach(elementNode -> { + writer.write("$C, ", (Runnable) () -> elementNode.accept(targetVisitor)); + }); + }); + return null; + } + + @Override + public Void booleanNode(BooleanNode node) { + writer.writeInline(node.getValue() ? "True" : "False"); + return null; + } + + @Override + public Void nullNode(NullNode node) { + writer.writeInline("None"); + return null; + } + + @Override + public Void numberNode(NumberNode node) { + // TODO: Add support for timestamp, int-enum, and others + if (inputShape.isTimestampShape()) { + var parsed = CodegenUtils.parseTimestampNode(model, inputShape, node); + writer.writeInline(CodegenUtils.getDatetimeConstructor(writer, parsed)); + } else if (inputShape.isFloatShape() || inputShape.isDoubleShape()) { + writer.writeInline("float($L)", node.getValue()); + } else { + writer.writeInline("$L", node.getValue()); + } + return null; + } + + @Override + public Void objectNode(ObjectNode node) { + switch (inputShape.getType()) { + case STRUCTURE -> structureShape((StructureShape) inputShape, node); + case MAP -> mapShape((MapShape) inputShape, node); + case UNION -> unionShape((UnionShape) inputShape, node); + case DOCUMENT -> documentShape((DocumentShape) inputShape, node); + default -> throw new CodegenException("unexpected input shape: " + inputShape.getType()); + } + return null; + } + + @Override + public Void stringNode(StringNode node) { + if (inputShape.isBlobShape()) { + writer.writeInline("b$S", node.getValue()); + } else if (inputShape.isFloatShape() || inputShape.isDoubleShape()) { + var value = switch (node.getValue()) { + case "NaN" -> "nan"; + case "Infinity" -> "inf"; + case "-Infinity" -> "-inf"; + default -> throw new CodegenException("Invalid value: " + node.getValue()); + }; + + writer.writeInline("float($S)", value); + } else { + writer.writeInline("$S", node.getValue()); + } + return null; + } + + private Void structureShape(StructureShape shape, ObjectNode node) { + writer.openBlock("$T(", ")", + context.symbolProvider().toSymbol(shape), + () -> structureMemberShapes(shape, node) + ); + return null; + } + + private Void structureMemberShapes(StructureShape container, ObjectNode node) { + node.getMembers().forEach((keyNode, valueNode) -> { + var memberShape = container.getMember(keyNode.getValue()).orElseThrow(() -> + new CodegenException("unknown memberShape: " + keyNode.getValue()) + ); + var targetShape = model.expectShape(memberShape.getTarget()); + writer.write("$L = $C,", + context.symbolProvider().toMemberName(memberShape), + (Runnable) () -> valueNode.accept(new ValueNodeVisitor(targetShape)) + ); + }); + return null; + } + + private Void mapShape(MapShape shape, ObjectNode node) { + writer.openBlock("{", "}", + () -> node.getMembers().forEach((keyNode, valueNode) -> { + var targetShape = model.expectShape(shape.getValue().getTarget()); + writer.write("$S: $C,", + keyNode.getValue(), + (Runnable) () -> valueNode.accept(new ValueNodeVisitor(targetShape)) + ); + }) + ); + return null; + } + + private Void documentShape(DocumentShape shape, ObjectNode node) { + writer.openBlock("{", "}", + () -> node.getMembers().forEach((keyNode, valueNode) -> { + writer.write("$S: $C,", + keyNode.getValue(), + (Runnable) () -> valueNode.accept(this) + ); + }) + ); + return null; + } + + private Void unionShape(UnionShape shape, ObjectNode node) { + if (node.getMembers().size() == 1) { + node.getMembers().forEach((keyNode, valueNode) -> { + var memberShape = shape.getMember(keyNode.getValue()) + .orElseThrow(() -> new CodegenException("unknown member: " + keyNode.getValue())); + var targetShape = model.expectShape(memberShape.getTarget()); + unionShape(memberShape, targetShape, valueNode); + }); + } else { + throw new CodegenException("exactly 1 named member must be set."); + } + return null; + } + + private Void unionShape(MemberShape memberShape, Shape targetShape, Node node) { + writer.openBlock("$T(", ")", + context.symbolProvider().toSymbol(memberShape), + () -> writer.write("value = $C", + (Runnable) () -> node.accept(new ValueNodeVisitor(targetShape))) + ); + return null; + } + } + + /** + * Implements Python formatting for the {@code J} formatter. + * + *

Convert an Object in Java to a literal python representation. + * The type of Object is constrained to a set of standard types in Java + * + *

For example: + * 

{@code
+     * List.of("a", "b") -> ["a", "b"]
+     * int[] {1, 2, 3} -> (1, 2, 3)
+     * Set.of("a", "b") -> {"a", "b"}
+     * String "abc" -> "abc"
+     * Integer 1 -> 1
+     * }
+ */ + private final class JavaToPythonFormatter implements BiFunction { + @Override + public String apply(Object type, String indent) { + if (type instanceof List) { + return apply(((List) type).stream(), ",", "[", "]", indent); + } else if (type instanceof Object[]) { + return apply(Stream.of((Object[]) type), ",", "(", ")", indent); + } else if (type instanceof Set) { + return apply(((Set) type).stream(), ",", "{", "}", indent); + } else if (type instanceof Map) { + return apply((Map) type, ",", "{", "}", indent); + } else if (type instanceof String) { + return writer.format("$S", type); + } else if (type instanceof Number) { + return writer.format("$L", type); + } else { + throw new CodegenException( + "Invalid type provided to $J: `" + type + "`"); + } + } + + private String apply(Stream stream, String sep, String start, String end, String indent) { + return mapApply(stream, indent).collect(Collectors.joining(sep, start, end)); + } + + private String apply(Map map, String sep, String start, String end, String indent) { + return map.entrySet().stream().map((entry) -> { + return writer.format("$L: $L", apply(entry.getKey(), indent), apply(entry.getValue(), indent)); + }).collect(Collectors.joining(sep, start, end)); + } + + private Stream mapApply(Stream stream, String indent) { + Set test; + return stream.map((item) -> apply(item, indent)); + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ImportDeclarations.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ImportDeclarations.java new file mode 100644 index 0000000000..165d8a887c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ImportDeclarations.java @@ -0,0 +1,178 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.Iterator; +import java.util.Map; +import java.util.TreeMap; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.codegen.core.ImportContainer; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.utils.SmithyInternalApi; +import software.amazon.smithy.utils.StringUtils; + +/** + * Internal class used for aggregating imports of a file. + */ +@SmithyInternalApi +final class ImportDeclarations implements ImportContainer { + private static final String MODULE_IMPORT_TEMPLATE = "import %s%n"; + + private final Map> stdlibImports = new TreeMap<>(); + private final Map> externalImports = new TreeMap<>(); + private final Map> localImports = new TreeMap<>(); + + private final PythonSettings settings; + private final String localNamespace; + + ImportDeclarations(PythonSettings settings, String namespace) { + this.settings = settings; + this.localNamespace = namespace; + } + + @Override + public void importSymbol(Symbol symbol, String alias) { + if (!symbol.getNamespace().isEmpty() && !symbol.getNamespace().equals(localNamespace)) { + if (symbol.getProperty("stdlib", Boolean.class).orElse(false)) { + addStdlibImport(symbol.getNamespace(), symbol.getName(), alias); + } else { + addImport(symbol.getNamespace(), symbol.getName(), alias); + } + } + } + + ImportDeclarations addImport(String namespace, String name, String alias) { + var isTestModule = this.localNamespace.startsWith("tests"); + if (namespace.startsWith(settings.getModuleName())) { + // if the module is for tests, we shouldn't relativize the imports + // as python will complain that the imports are beyond the top-level package + var ns = isTestModule ? namespace : relativize(namespace); + return addImportToMap(ns, name, alias, localImports); + } + return addImportToMap(namespace, name, alias, externalImports); + } + + private String relativize(String namespace) { + if (namespace.startsWith(localNamespace)) { + return "." + namespace.substring(localNamespace.length()); + } + var localParts = localNamespace.split("\\."); + var parts = namespace.split("\\."); + int commonSegments = 0; + for (; commonSegments < Math.min(localParts.length, parts.length); commonSegments++) { + if (!parts[commonSegments].equals(localParts[commonSegments])) { + break; + } + } + var prefix = StringUtils.repeat(".", localParts.length - commonSegments); + return prefix + namespace.split("\\.", commonSegments + 1)[commonSegments]; + } + + ImportDeclarations addStdlibImport(String namespace) { + return addStdlibImport(namespace, "", ""); + } + + ImportDeclarations addStdlibImport(String namespace, String name) { + return addStdlibImport(namespace, name, name); + } + + ImportDeclarations addStdlibImport(String namespace, String name, String alias) { + return addImportToMap(namespace, name, alias, stdlibImports); + } + + private ImportDeclarations addImportToMap( + String namespace, + String name, + String alias, + Map> importMap + ) { + if (name.equals("*")) { + throw new CodegenException("Wildcard imports are forbidden."); + } + Map namespaceImports = importMap.computeIfAbsent(namespace, ns -> new TreeMap<>()); + namespaceImports.put(name, alias); + return this; + } + + @Override + public String toString() { + if (externalImports.isEmpty() && stdlibImports.isEmpty() && localImports.isEmpty()) { + return ""; + } + StringBuilder builder = new StringBuilder(); + if (!stdlibImports.isEmpty()) { + formatImportList(builder, stdlibImports); + } + if (!externalImports.isEmpty()) { + formatImportList(builder, externalImports); + } + if (!localImports.isEmpty()) { + formatImportList(builder, localImports); + } + builder.append("\n"); + return builder.toString(); + } + + private void formatImportList(StringBuilder builder, Map> importMap) { + for (Map.Entry> namespaceEntry: importMap.entrySet()) { + if (namespaceEntry.getValue().remove("") != null) { + builder.append(formatModuleImport(namespaceEntry.getKey())); + } + if (namespaceEntry.getValue().isEmpty()) { + continue; + } + String namespaceImport = formatSingleLineImport(namespaceEntry.getKey(), namespaceEntry.getValue()); + if (namespaceImport.length() > CodegenUtils.MAX_PREFERRED_LINE_LENGTH) { + namespaceImport = formatMultiLineImport(namespaceEntry.getKey(), namespaceEntry.getValue()); + } + builder.append(namespaceImport); + } + builder.append("\n"); + } + + private String formatModuleImport(String namespace) { + return String.format(MODULE_IMPORT_TEMPLATE, namespace); + } + + private String formatSingleLineImport(String namespace, Map names) { + StringBuilder builder = new StringBuilder("from ").append(namespace).append(" import"); + for (Iterator> iter = names.entrySet().iterator(); iter.hasNext();) { + Map.Entry entry = iter.next(); + builder.append(" ").append(entry.getKey()); + if (!entry.getKey().equals(entry.getValue())) { + builder.append(" as ").append(entry.getValue()); + } + if (iter.hasNext()) { + builder.append(","); + } + } + builder.append("\n"); + return builder.toString(); + } + + private String formatMultiLineImport(String namespace, Map names) { + StringBuilder builder = new StringBuilder("from ").append(namespace).append(" import (\n"); + for (Map.Entry entry : names.entrySet()) { + builder.append(" ").append(entry.getKey()); + if (!entry.getKey().equals(entry.getValue())) { + builder.append(" as ").append(entry.getValue()); + } + builder.append(",\n"); + } + builder.append(")\n"); + return builder.toString(); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/IntEnumGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/IntEnumGenerator.java new file mode 100644 index 0000000000..6d3f0fdf5d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/IntEnumGenerator.java @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.directed.GenerateIntEnumDirective; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.traits.EnumValueTrait; + +final class IntEnumGenerator implements Runnable { + + private final GenerateIntEnumDirective directive; + + IntEnumGenerator(GenerateIntEnumDirective directive) { + this.directive = directive; + } + + @Override + public void run() { + var enumSymbol = directive.symbol().expectProperty("enumSymbol", Symbol.class); + directive.context().writerDelegator().useShapeWriter(directive.shape(), writer -> { + writer.addStdlibImport("enum", "IntEnum"); + writer.openBlock("class $L(IntEnum):", "", enumSymbol.getName(), () -> { + directive.shape().getTrait(DocumentationTrait.class).ifPresent(trait -> { + writer.writeDocs(writer.formatDocs(trait.getValue())); + }); + + for (MemberShape member : directive.shape().members()) { + member.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeComment(trait.getValue())); + var name = directive.symbolProvider().toMemberName(member); + var value = member.expectTrait(EnumValueTrait.class).expectIntValue(); + writer.write("$L = $L\n", name, value); + } + }); + }); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/MapGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/MapGenerator.java new file mode 100644 index 0000000000..652d70b114 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/MapGenerator.java @@ -0,0 +1,99 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.CollectionShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.traits.SparseTrait; + +/** + * Generates private helper methods for maps. + */ +final class MapGenerator implements Runnable { + + private final Model model; + private final SymbolProvider symbolProvider; + private final PythonWriter writer; + private final MapShape shape; + + MapGenerator( + Model model, + SymbolProvider symbolProvider, + PythonWriter writer, + MapShape shape + ) { + this.model = model; + this.symbolProvider = symbolProvider; + this.writer = writer; + this.shape = shape; + } + + @Override + public void run() { + var symbol = symbolProvider.toSymbol(shape); + if (symbol.getProperty("asDict").isPresent()) { + writeAsDict(); + } + if (symbol.getProperty("fromDict").isPresent()) { + writeFromDict(); + } + } + + private void writeAsDict() { + var symbol = symbolProvider.toSymbol(shape); + var asDictSymbol = symbol.expectProperty("asDict", Symbol.class); + var target = model.expectShape(shape.getValue().getTarget()); + var targetSymbol = symbolProvider.toSymbol(target); + + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-sparse-trait + var sparseGuard = shape.hasTrait(SparseTrait.class) ? " if v is not None else None" : ""; + + writer.addStdlibImport("typing", "Any"); + writer.addStdlibImport("typing", "Dict"); + writer.openBlock("def $L(given: $T) -> Dict[str, Any]:", "", asDictSymbol.getName(), symbol, () -> { + if (target.isUnionShape() || target.isStructureShape()) { + writer.write("return {k: v.as_dict()$L for k, v in given.items()}", sparseGuard); + } else if (target.isMapShape() || target instanceof CollectionShape) { + var targetAsDictSymbol = targetSymbol.expectProperty("asDict", Symbol.class); + writer.write("return {k: $T(v)$L for k, v in given.items()}", targetAsDictSymbol, sparseGuard); + } + }); + } + + private void writeFromDict() { + var symbol = symbolProvider.toSymbol(shape); + var fromDictSymbol = symbol.expectProperty("fromDict", Symbol.class); + var target = model.expectShape(shape.getValue().getTarget()); + var targetSymbol = symbolProvider.toSymbol(target); + + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-sparse-trait + var sparseGuard = shape.hasTrait(SparseTrait.class) ? " if v is not None else None" : ""; + + writer.addStdlibImport("typing", "Any"); + writer.addStdlibImport("typing", "Dict"); + writer.openBlock("def $L(given: Dict[str, Any]) -> $T:", "", fromDictSymbol.getName(), symbol, () -> { + if (target.isUnionShape() || target.isStructureShape()) { + writer.write("return {k: $T.from_dict(v)$L for k, v in given.items()}", targetSymbol, sparseGuard); + } else if (target.isMapShape() || target instanceof CollectionShape) { + var targetFromDictSymbol = targetSymbol.expectProperty("fromDict", Symbol.class); + writer.write("return {k: $T(v)$L for k, v in given.items()}", targetFromDictSymbol, sparseGuard); + } + }); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.java new file mode 100644 index 0000000000..40a095eaf1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonClientCodegenPlugin.java @@ -0,0 +1,50 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import software.amazon.smithy.build.PluginContext; +import software.amazon.smithy.build.SmithyBuildPlugin; +import software.amazon.smithy.codegen.core.directed.CodegenDirector; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Plugin to trigger Python code generation. + */ +@SmithyUnstableApi +public final class PythonClientCodegenPlugin implements SmithyBuildPlugin { + @Override + public String getName() { + return "python-client-codegen"; + } + + @Override + public void execute(PluginContext context) { + CodegenDirector runner + = new CodegenDirector<>(); + + PythonSettings settings = PythonSettings.from(context.getSettings()); + runner.settings(settings); + runner.directedCodegen(new DirectedPythonCodegen()); + runner.fileManifest(context.getFileManifest()); + runner.service(settings.getService()); + runner.model(context.getModel()); + runner.integrationClass(PythonIntegration.class); + runner.performDefaultCodegenTransforms(); + runner.createDedicatedInputsAndOutputs(); + runner.run(); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonDelegator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonDelegator.java new file mode 100644 index 0000000000..be887add43 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonDelegator.java @@ -0,0 +1,64 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import software.amazon.smithy.build.FileManifest; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.codegen.core.WriterDelegator; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; + +/** + * Manages writers for Python files. + */ +final class PythonDelegator extends WriterDelegator { + + PythonDelegator( + FileManifest fileManifest, + SymbolProvider symbolProvider, + PythonSettings settings + ) { + super( + fileManifest, + new EnumSymbolProviderWrapper(symbolProvider), + new PythonWriter.PythonWriterFactory(settings) + ); + } + + private static final class EnumSymbolProviderWrapper implements SymbolProvider { + + private final SymbolProvider wrapped; + + EnumSymbolProviderWrapper(SymbolProvider wrapped) { + this.wrapped = wrapped; + } + + @Override + public Symbol toSymbol(Shape shape) { + Symbol symbol = wrapped.toSymbol(shape); + if (shape.isEnumShape() || shape.isIntEnumShape()) { + symbol = symbol.expectProperty("enumSymbol", Symbol.class); + } + return symbol; + } + + @Override + public String toMemberName(MemberShape shape) { + return wrapped.toMemberName(shape); + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonDependency.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonDependency.java new file mode 100644 index 0000000000..68040d88b2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonDependency.java @@ -0,0 +1,72 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.Collections; +import java.util.List; +import software.amazon.smithy.codegen.core.SymbolDependency; +import software.amazon.smithy.codegen.core.SymbolDependencyContainer; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * A record of a python package dependency. + */ +@SmithyUnstableApi +public record PythonDependency( + String packageName, String version, Type type, boolean isLink +) implements SymbolDependencyContainer { + + @Override + public List getDependencies() { + return Collections.singletonList(getDependency()); + } + + /** + * @return the SymbolDependency representation of this dependency. + */ + public SymbolDependency getDependency() { + return SymbolDependency.builder() + .dependencyType(type.getType()) + .packageName(packageName) + .version(version) + .putProperty("isLink", isLink) + .build(); + } + + /** + * An enum of valid dependency types. + */ + public enum Type { + /** A normal dependency. */ + DEPENDENCY("dependency"), + + /** A dependency only used for testing purposes. */ + TEST_DEPENDENCY("testDependency"); + + private final String type; + + Type(String type) { + this.type = type; + } + + /** + * @return the string representation of the dependency type. + */ + public String getType() { + return type; + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonSettings.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonSettings.java new file mode 100644 index 0000000000..d524ac0ca4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonSettings.java @@ -0,0 +1,213 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.Arrays; +import java.util.Objects; +import java.util.Set; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.ServiceIndex; +import software.amazon.smithy.model.node.ObjectNode; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Settings used by {@link PythonClientCodegenPlugin}. + * TODO: make this immutable + */ +@SmithyUnstableApi +public final class PythonSettings { + + private static final String SERVICE = "service"; + private static final String MODULE_NAME = "module"; + private static final String MODULE_DESCRIPTION = "moduleDescription"; + private static final String MODULE_VERSION = "moduleVersion"; + + private ShapeId service; + private String moduleName; + private String moduleVersion; + private String moduleDescription = ""; + private ShapeId protocol; + + /** + * Create a settings object from a configuration object node. + * + * @param config Config object to load. + * @return Returns the extracted settings. + */ + public static PythonSettings from(ObjectNode config) { + PythonSettings settings = new PythonSettings(); + config.warnIfAdditionalProperties(Arrays.asList(SERVICE, MODULE_NAME, MODULE_DESCRIPTION, MODULE_VERSION)); + + settings.setService(config.expectStringMember(SERVICE).expectShapeId()); + settings.setModuleName(config.expectStringMember(MODULE_NAME).getValue()); + settings.setModuleVersion(config.expectStringMember(MODULE_VERSION).getValue()); + settings.setModuleDescription(config.getStringMemberOrDefault( + MODULE_DESCRIPTION, settings.getModuleName() + " client")); + + return settings; + } + + /** + * Gets the id of the service that is being generated. + * + * @return Returns the service id. + * @throws NullPointerException if the service has not been set. + */ + public ShapeId getService() { + return Objects.requireNonNull(service, SERVICE + " not set"); + } + + /** + * Gets the corresponding {@link ServiceShape} from a model. + * + * @param model Model to search for the service shape by ID. + * @return Returns the found {@code Service}. + * @throws NullPointerException if the service has not been set. + * @throws CodegenException if the service is invalid or not found. + */ + public ServiceShape getService(Model model) { + return model + .getShape(getService()) + .orElseThrow(() -> new CodegenException("Service shape not found: " + getService())) + .asServiceShape() + .orElseThrow(() -> new CodegenException("Shape is not a Service: " + getService())); + } + + /** + * Sets the service to generate. + * + * @param service The service to generate. + */ + public void setService(ShapeId service) { + this.service = Objects.requireNonNull(service); + } + + /** + * Gets the required module name for the module that will be generated. + * + * @return Returns the module name. + * @throws NullPointerException if the module name has not been set. + */ + public String getModuleName() { + return Objects.requireNonNull(moduleName, MODULE_NAME + " not set"); + } + + /** + * Sets the name of the module to generate. + * + * @param moduleName The name of the module to generate. + */ + public void setModuleName(String moduleName) { + if (moduleName != null && !moduleName.matches("[a-z_\\d]+")) { + throw new CodegenException( + "Python package names may only consist of lowercase letters, numbers, and underscores."); + } + this.moduleName = Objects.requireNonNull(moduleName); + } + + /** + * Gets the required module version for the module that will be generated. + * + * @return The version of the module that will be generated. + * @throws NullPointerException if the module version has not been set. + */ + public String getModuleVersion() { + return Objects.requireNonNull(moduleVersion, MODULE_VERSION + " not set"); + } + + /** + * Sets the required module version for the module that will be generated. + * + * @param moduleVersion The version of the module that will be generated. + */ + public void setModuleVersion(String moduleVersion) { + this.moduleVersion = Objects.requireNonNull(moduleVersion); + } + + /** + * Gets the optional module description for the module that will be generated. + * + * @return Returns the module description. + */ + public String getModuleDescription() { + return moduleDescription; + } + + /** + * Sets the description of the module to generate. + * + * @param moduleDescription The description of the module to generate. + */ + public void setModuleDescription(String moduleDescription) { + this.moduleDescription = Objects.requireNonNull(moduleDescription); + } + + /** + * Gets the configured protocol to generate. + * + * @return Returns the configured protocol. + */ + public ShapeId getProtocol() { + return protocol; + } + + /** + * Resolves the highest priority protocol from a service shape that is + * supported by the generator. + * + * @param model Model to enable finding protocols on the service. + * @param service Service to get the protocols from if "protocols" is not set. + * @param supportedProtocols The set of protocol names supported by the generator. + * @return Returns the resolved protocol name. + * @throws CodegenException if no protocol could be resolved. + */ + public ShapeId resolveServiceProtocol(Model model, ServiceShape service, Set supportedProtocols) { + if (protocol != null) { + return protocol; + } + + ServiceIndex serviceIndex = ServiceIndex.of(model); + Set resolvedProtocols = serviceIndex.getProtocols(service).keySet(); + if (resolvedProtocols.isEmpty()) { + throw new CodegenException( + "Unable to derive the protocol setting of the service `" + service.getId() + "` because no " + + "protocol definition traits were present. You need to set an explicit `protocol` to " + + "generate in smithy-build.json to generate this service."); + } + + protocol = resolvedProtocols.stream() + .filter(supportedProtocols::contains) + .findFirst() + .orElseThrow(() -> new CodegenException(String.format( + "The %s service supports the following unsupported protocols %s. The following protocol " + + "generators were found on the class path: %s", + service.getId(), resolvedProtocols, supportedProtocols))); + + return protocol; + } + + /** + * Sets the protocol to generate. + * + * @param protocol Protocols to generate. + */ + public void setProtocol(ShapeId protocol) { + this.protocol = Objects.requireNonNull(protocol); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonWriter.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonWriter.java new file mode 100644 index 0000000000..a47db23d68 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/PythonWriter.java @@ -0,0 +1,295 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.Set; +import java.util.function.BiFunction; +import java.util.logging.Logger; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.codegen.core.SymbolWriter; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.utils.SmithyUnstableApi; +import software.amazon.smithy.utils.StringUtils; + +/** + * Specialized code writer for managing Python dependencies. + * + *

Use the {@code $T} formatter to refer to {@link Symbol}s. + */ +@SmithyUnstableApi +public final class PythonWriter extends SymbolWriter { + + private static final Logger LOGGER = Logger.getLogger(PythonWriter.class.getName()); + + private final String fullPackageName; + private final boolean addCodegenWarningHeader; + + /** + * Constructs a PythonWriter. + * + * @param settings The python plugin settings. + * @param fullPackageName The fully-qualified name of the package. + */ + public PythonWriter(PythonSettings settings, String fullPackageName) { + this(settings, fullPackageName, true); + } + + /** + * Constructs a PythonWriter. + * + * @param settings The python plugin settings. + * @param fullPackageName The fully-qualified name of the package. + * @param addCodegenWarningHeader Whether to add a header comment warning that the file is code generated. + */ + public PythonWriter(PythonSettings settings, String fullPackageName, boolean addCodegenWarningHeader) { + super(new ImportDeclarations(settings, fullPackageName)); + this.fullPackageName = fullPackageName; + trimBlankLines(); + trimTrailingSpaces(); + putFormatter('T', new PythonSymbolFormatter()); + this.addCodegenWarningHeader = addCodegenWarningHeader; + } + + /** + * A factory class to create {@link PythonWriter}s. + */ + public static final class PythonWriterFactory implements SymbolWriter.Factory { + + private final PythonSettings settings; + + /** + * @param settings The python plugin settings. + */ + public PythonWriterFactory(PythonSettings settings) { + this.settings = settings; + } + + @Override + public PythonWriter apply(String filename, String namespace) { + // Markdown doesn't have comments, so there's no non-intrusive way to + // add the warning. + var addWarningHeader = !filename.endsWith(".md"); + return new PythonWriter(settings, namespace, addWarningHeader); + } + } + + /** + * Writes documentation comments from a runnable. + * + * @param runnable A runnable that writes docs. + * @return Returns the writer. + */ + public PythonWriter writeDocs(Runnable runnable) { + pushState(); + writeInline("\"\"\""); + runnable.run(); + write("\"\"\""); + popState(); + return this; + } + + /** + * Writes documentation comments from a string. + * + * @param docs Documentation to write. + * @return Returns the writer. + */ + public PythonWriter writeDocs(String docs) { + writeDocs(() -> write(formatDocs(docs))); + return this; + } + + /** + * Formats a given Commonmark string and wraps it for use in a doc + * comment. + * + * @param docs Documentation to format. + * @return Formatted documentation. + */ + public String formatDocs(String docs) { + // TODO: write a documentation converter to convert markdown to rst + return StringUtils.wrap(docs, CodegenUtils.MAX_PREFERRED_LINE_LENGTH - 8) + .replace("$", "$$"); + } + + /** + * Opens a block to write comments. + * + * @param runnable Runnable function to execute inside the block. + * @return Returns the writer. + */ + public PythonWriter openComment(Runnable runnable) { + pushState(); + setNewlinePrefix("# "); + runnable.run(); + setNewlinePrefix(""); + popState(); + return this; + } + + /** + * Writes a comment from a string. + * + * @param comment The comment to write. + * @return Returns the writer. + */ + public PythonWriter writeComment(String comment) { + return openComment(() -> write(formatDocs(comment.replace("\n", " ")))); + } + + + /** + * Imports a module from the standard library without an alias. + * + * @param namespace Module to import. + * @return Returns the writer. + */ + public PythonWriter addStdlibImport(String namespace) { + getImportContainer().addStdlibImport(namespace); + return this; + } + + /** + * Imports a type using an alias from the standard library only if necessary. + * + * @param namespace Module to import the type from. + * @param name Type to import. + * @return Returns the writer. + */ + public PythonWriter addStdlibImport(String namespace, String name) { + getImportContainer().addStdlibImport(namespace, name); + return this; + } + + /** + * Imports a type using an alias from the standard library only if necessary. + * + * @param namespace Module to import the type from. + * @param name Type to import. + * @param alias The name to import the type as. + * @return Returns the writer. + */ + public PythonWriter addStdlibImport(String namespace, String name, String alias) { + getImportContainer().addStdlibImport(namespace, name, alias); + return this; + } + + /** + * Imports a type using an alias from a module only if necessary. + * + *

If you use this, you MUST add the dependency manually using {@link PythonWriter#addDependency}. + * + * @param namespace Module to import the type from. + * @param name Type to import. + * @return Returns the writer. + */ + public PythonWriter addImport(String namespace, String name) { + getImportContainer().addImport(namespace, name, name); + return this; + } + + /** + * Imports a type using an alias from a module only if necessary. + * + *

If you use this, you MUST add the dependency manually using {@link PythonWriter#addDependency}. + * + * @param namespace Module to import the type from. + * @param name Type to import. + * @param alias The name to import the type as. + * @return Returns the writer. + */ + public PythonWriter addImport(String namespace, String name, String alias) { + getImportContainer().addImport(namespace, name, alias); + return this; + } + + /** + * Imports a set of types from a module only if necessary. + * + * @param namespace Module to import the type from. + * @param names Set of types to import. + * @return Returns the writer. + */ + public PythonWriter addImports(String namespace, Set names) { + names.forEach((name) -> getImportContainer().addImport(namespace, name, name)); + return this; + } + + /** + * Conditionally write text. + * + *

Useful for short-handing a conditional write when + * you aren't able to use the built-in conditional + * formatting functionality. + * + * @param shouldWrite Whether to write the text or not. + * @param content Content to write. + * @param args String arguments to use for formatting. + * @return Returns self. + */ + public PythonWriter maybeWrite(boolean shouldWrite, Object content, Object... args) { + if (shouldWrite) { + write(content, args); + } + return this; + } + + @Override + public String toString() { + String contents = getImportContainer().toString() + super.toString(); + if (addCodegenWarningHeader) { + String header = + """ + # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + # SPDX-License-Identifier: Apache-2.0 + # Do not modify this file. This file is machine generated, and any changes to it will be overwritten.\n + """; + contents = header + contents; + } + return contents; + } + + /** + * Implements Python symbol formatting for the {@code $T} formatter. + */ + private final class PythonSymbolFormatter implements BiFunction { + @Override + public String apply(Object type, String indent) { + if (type instanceof Symbol) { + Symbol typeSymbol = (Symbol) type; + // Check if the symbol is an operation - we shouldn't add imports for operations, since + // they are methods of the service object and *can't* be imported + if (!isOperationSymbol(typeSymbol)) { + addUseImports(typeSymbol); + } + return typeSymbol.getName(); + } else if (type instanceof SymbolReference) { + SymbolReference typeSymbol = (SymbolReference) type; + addImport(typeSymbol.getSymbol(), typeSymbol.getAlias(), SymbolReference.ContextOption.USE); + return typeSymbol.getAlias(); + } else { + throw new CodegenException( + "Invalid type provided to $T. Expected a Symbol, but found `" + type + "`"); + } + } + } + + private Boolean isOperationSymbol(Symbol typeSymbol) { + return typeSymbol.getProperty("shape", Shape.class).map(Shape::isOperationShape).orElse(false); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SetupGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SetupGenerator.java new file mode 100644 index 0000000000..14dfa9924a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SetupGenerator.java @@ -0,0 +1,180 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import software.amazon.smithy.codegen.core.SymbolDependency; +import software.amazon.smithy.codegen.core.WriterDelegator; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.traits.StringTrait; +import software.amazon.smithy.model.traits.TitleTrait; +import software.amazon.smithy.python.codegen.PythonDependency.Type; +import software.amazon.smithy.python.codegen.sections.PyprojectSection; +import software.amazon.smithy.python.codegen.sections.ReadmeSection; +import software.amazon.smithy.utils.StringUtils; + +/** + * Generates package setup configuration files. + */ +final class SetupGenerator { + + private SetupGenerator() {} + + static void generateSetup( + PythonSettings settings, + GenerationContext context + ) { + var dependencies = SymbolDependency.gatherDependencies(context.writerDelegator().getDependencies().stream()); + writePyproject(settings, context.writerDelegator(), dependencies); + writeReadme(settings, context); + } + + /** + * Write a pyproject.toml file. + * + *

This file format is what the python ecosystem is trying to transition to + * for package configuration. It allows for arbitrary build tools to share + * configuration. + */ + private static void writePyproject( + PythonSettings settings, + WriterDelegator writers, + Map> dependencies + ) { + // TODO: Allow all of these settings to be configured, particularly build system + // The use of interceptors ostensibly allows this, but it would be better to have + // a system that allows end users to configure it without writing code. Just + // giving a generic JSON mapping would work since you can convert that to toml. + // We can use the jsonAdd from Smithy's OpenAPI transforms for inspiration. + writers.useFileWriter("pyproject.toml", "", writer -> { + writer.pushState(new PyprojectSection(dependencies)); + writer.write(""" + [build-system] + requires = ["setuptools", "setuptools-scm", "wheel"] + build-backend = "setuptools.build_meta" + + [project] + name = $1S + version = $2S + description = $3S + readme = "README.md" + requires-python = ">=3.11" + keywords = ["smithy", $1S] + license = {text = "Apache-2.0"} + classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11" + ] + """, settings.getModuleName(), settings.getModuleVersion(), settings.getModuleDescription()); + + Optional.ofNullable(dependencies.get(Type.DEPENDENCY.getType())).ifPresent(deps -> { + writer.openBlock("dependencies = [", "]", () -> writeDependencyList(writer, deps.values())); + }); + + Optional.ofNullable(dependencies.get(Type.TEST_DEPENDENCY.getType())).ifPresent(deps -> { + writer.write("[project.optional-dependencies]"); + writer.openBlock("tests = [", "]", () -> writeDependencyList(writer, deps.values())); + }); + + writer.write(""" + [tool.setuptools.packages.find] + exclude=["tests*"] + + [tool.mypy] + strict = true + warn_unused_configs = true + + [[tool.mypy.overrides]] + module = ["awscrt", "pytest"] + ignore_missing_imports = true + + [tool.black] + target-version = ["py311"] + + [tool.pytest.ini_options] + python_classes = ["!Test"] + asyncio_mode = "auto" + """); + + writer.popState(); + }); + } + + private static void writeDependencyList(PythonWriter writer, Collection dependencies) { + for (var iter = dependencies.iterator(); iter.hasNext();) { + var dependency = iter.next(); + if (dependency.getProperty("isLink", Boolean.class).orElse(false)) { + writer.writeInline("\"$L @ $L\"", dependency.getPackageName(), dependency.getVersion()); + } else { + writer.writeInline("\"$L$L\"", dependency.getPackageName(), dependency.getVersion()); + } + if (iter.hasNext()) { + writer.write(","); + } else { + writer.write(""); + } + } + } + + private static void writeReadme( + PythonSettings settings, + GenerationContext context + ) { + var service = context.model().expectShape(settings.getService()); + + // see: https://smithy.io/2.0/spec/documentation-traits.html#smithy-api-title-trait + var title = service.getTrait(TitleTrait.class) + .map(StringTrait::getValue) + .orElse(StringUtils.capitalize(settings.getModuleName())); + + var description = StringUtils.isBlank(settings.getModuleDescription()) + ? "Generated service client for " + title + : StringUtils.wrap(settings.getModuleDescription(), 80); + + context.writerDelegator().useFileWriter("README.md", writer -> { + writer.pushState(new ReadmeSection()); + writer.write(""" + ## $L Client + + $L + """, title, description); + + service.getTrait(DocumentationTrait.class).map(StringTrait::getValue).ifPresent(documentation -> { + // TODO: make sure this documentation is well-formed + // Existing services in AWS, for example, have a lot of HTML docs. + // HTML nodes *are* valid commonmark technically, so it should be + // fine here. If we were to make this file RST formatted though, + // we'd have a problem. We have to solve that at some point anyway + // since the python code docs are RST format. + writer.write(""" + ### Documentation + + $L + """, documentation); + }); + writer.popState(); + }); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java new file mode 100644 index 0000000000..f3b44e85c7 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java @@ -0,0 +1,70 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.List; +import software.amazon.smithy.codegen.core.SymbolDependency; +import software.amazon.smithy.python.codegen.PythonDependency.Type; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Dependencies used in the smithy python generator. + */ +@SmithyUnstableApi +public final class SmithyPythonDependency { + + /** + * The core smithy-python python package. + * + * While in development this will use the develop branch. + */ + public static final PythonDependency SMITHY_PYTHON = new PythonDependency( + "smithy_python", + // You'll need to locally install this before we publish + "==0.0.1", + Type.DEPENDENCY, + false + ); + + /** + * testing framework used in generated functional tests. + */ + public static final PythonDependency PYTEST = new PythonDependency( + "pytest", + ">=7.2.0,<8.0.0", + Type.TEST_DEPENDENCY, + false + ); + + /** + * testing framework used in generated functional tests. + */ + public static final PythonDependency PYTEST_ASYNCIO = new PythonDependency( + "pytest-asyncio", + ">=0.20.3,<0.21.0", + Type.TEST_DEPENDENCY, + false + ); + + private SmithyPythonDependency() {} + + /** + * @return a list of dependencies that are always needed. + */ + public static List getUnconditionalDependencies() { + return List.of(SMITHY_PYTHON.getDependency()); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/StructureGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/StructureGenerator.java new file mode 100644 index 0000000000..62e9945ccc --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/StructureGenerator.java @@ -0,0 +1,524 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import static java.lang.String.format; +import static software.amazon.smithy.python.codegen.CodegenUtils.isErrorMessage; + +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.NullableIndex; +import software.amazon.smithy.model.knowledge.NullableIndex.CheckMode; +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.*; + + +/** + * Renders structures. + */ +public class StructureGenerator implements Runnable { + + private static final Logger LOGGER = Logger.getLogger(StructureGenerator.class.getName()); + + protected final Model model; + protected final SymbolProvider symbolProvider; + protected final PythonWriter writer; + protected final StructureShape shape; + protected final List requiredMembers; + protected final List optionalMembers; + protected final Set recursiveShapes; + protected final PythonSettings settings; + + public StructureGenerator( + Model model, + PythonSettings settings, + SymbolProvider symbolProvider, + PythonWriter writer, + StructureShape shape, + Set recursiveShapes + ) { + this.model = model; + this.settings = settings; + this.symbolProvider = symbolProvider; + this.writer = writer; + this.shape = shape; + var required = new ArrayList(); + var optional = new ArrayList(); + var index = NullableIndex.of(model); + for (MemberShape member: shape.members()) { + if (index.isMemberNullable(member) || member.hasTrait(DefaultTrait.class)) { + optional.add(member); + } else { + required.add(member); + } + } + this.requiredMembers = filterMessageMembers(required); + this.optionalMembers = filterMessageMembers(optional); + this.recursiveShapes = recursiveShapes; + } + + @Override + public void run() { + if (!shape.hasTrait(ErrorTrait.class)) { + renderStructure(); + } else { + renderError(); + } + } + + /** + * Renders a normal, non-error structure. + */ + protected void renderStructure() { + writer.addStdlibImport("typing", "Dict"); + writer.addStdlibImport("typing", "Any"); + var symbol = symbolProvider.toSymbol(shape); + writer.openBlock("class $L:", "", symbol.getName(), () -> { + writeProperties(false); + writeInit(false); + writeAsDict(false); + writeFromDict(false); + writeRepr(false); + writeEq(false); + }); + writer.write(""); + } + + protected void renderError() { + writer.addStdlibImport("typing", "Dict"); + writer.addStdlibImport("typing", "Any"); + writer.addStdlibImport("typing", "Literal"); + // TODO: Implement protocol-level customization of the error code + var code = shape.getId().getName(); + var symbol = symbolProvider.toSymbol(shape); + var apiError = CodegenUtils.getApiError(settings); + writer.openBlock("class $L($L[Literal[$S]]):", "", symbol.getName(), apiError, code, () -> { + writer.write("code: Literal[$1S] = $1S", code); + writer.write("message: str"); + writeProperties(true); + writeInit(true); + writeAsDict(true); + writeFromDict(true); + writeRepr(true); + writeEq(true); + }); + writer.write(""); + } + + protected void writeProperties(boolean isError) { + for (MemberShape member : shape.members()) { + writePropertyForMember(isError, member); + } + } + + protected void writePropertyForMember(boolean isError, MemberShape member) { + NullableIndex index = NullableIndex.of(model); + + if (isError && isErrorMessage(model, member)) { + return; + } + var memberName = symbolProvider.toMemberName(member); + if (index.isMemberNullable(member)) { + writer.addStdlibImport("typing", "Optional"); + String formatString = format("$L: Optional[%s]", getTargetFormat(member)); + writer.write(formatString, memberName, symbolProvider.toSymbol(member)); + } else { + String formatString = format("$L: %s", getTargetFormat(member)); + writer.write(formatString, memberName, symbolProvider.toSymbol(member)); + } + } + + protected void writeInit(boolean isError) { + if (!isError && shape.members().isEmpty()) { + writeClassDocs(false); + return; + } + + writer.openBlock("def __init__(", "):", () -> { + writer.write("self,"); + if (!shape.members().isEmpty() || isError) { + // Adding this star to the front prevents the use of positional arguments. + writer.write("*,"); + } + if (isError) { + writer.write("message: str,"); + } + for (MemberShape member : requiredMembers) { + writeInitMethodParameterForRequiredMember(isError, member); + } + for (MemberShape member : optionalMembers) { + writeInitMethodParameterForOptionalMember(isError, member); + } + }); + + writer.indent(); + + writeClassDocs(isError); + if (isError) { + writer.write("super().__init__(message)"); + } + + Stream.concat(requiredMembers.stream(), optionalMembers.stream()).forEach(member -> { + String memberName = symbolProvider.toMemberName(member); + if (isOptionalDefault(member)) { + writeInitMethodAssignerForOptionalMember(member, memberName); + } else { + writeInitMethodAssignerForRequiredMember(member, memberName); + } + }); + writer.dedent(); + writer.write(""); + } + + protected void writeInitMethodAssignerForOptionalMember(MemberShape member, String memberName) { + writer.write("self.$1L = $1L if $1L is not None else $2L", + memberName, getDefaultValue(writer, member)); + } + + protected void writeInitMethodAssignerForRequiredMember(MemberShape member, String memberName) { + writer.write("self.$1L = $1L", memberName); + } + + protected void writeInitMethodParameterForRequiredMember(boolean isError, MemberShape member) { + var memberName = symbolProvider.toMemberName(member); + String formatString = format("$L: %s,", getTargetFormat(member)); + writer.write(formatString, memberName, symbolProvider.toSymbol(member)); + } + + protected void writeInitMethodParameterForOptionalMember(boolean isError, MemberShape member) { + var memberName = symbolProvider.toMemberName(member); + var nullableIndex = NullableIndex.of(model); + + if (nullableIndex.isMemberNullable(member)) { + writer.addStdlibImport("typing", "Optional"); + String formatString = format("$L: Optional[%s] = None,", getTargetFormat(member)); + writer.write(formatString, memberName, symbolProvider.toSymbol(member)); + } else if (member.hasTrait(RequiredTrait.class)) { + String formatString = format("$L: %s = $L,", getTargetFormat(member)); + writer.write(formatString, memberName, symbolProvider.toSymbol(member), + getDefaultValue(writer, member)); + } else { + // Shapes that are simple types, lists, or maps can have default values. + // https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-default-trait + var target = model.expectShape(member.getTarget()); + var memberSymbol = symbolProvider.toSymbol(member); + String formatString; + if (target.isDocumentShape() || target.isListShape() || target.isMapShape()) { + // Documents, lists, and maps can have mutable defaults so just use None in the + // constructor. + writer.addStdlibImport("typing", "Optional"); + formatString = format("$L: Optional[%1$s] = None,", getTargetFormat(member)); + writer.write(formatString, memberName, memberSymbol); + } else { + formatString = format("$L: %s = $L,", getTargetFormat(member)); + writer.write(formatString, memberName, memberSymbol, getDefaultValue(writer, member)); + } + } + } + + + private boolean isOptionalDefault(MemberShape member) { + // If a member with a default value isn't required, it's optional. + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-default-trait + var target = model.expectShape(member.getTarget()); + return member.hasTrait(DefaultTrait.class) && !member.hasTrait(RequiredTrait.class) + && (target.isDocumentShape() || target.isListShape() || target.isMapShape()); + } + + private void writeClassDocs(boolean isError) { + if (hasDocs()) { + writer.writeDocs(() -> { + shape.getTrait(DocumentationTrait.class).ifPresent(trait -> { + writer.write(writer.formatDocs(trait.getValue())); + }); + + if (isError) { + writer.write(":param message: A message associated with the specific error."); + } + + if (!shape.members().isEmpty()) { + writer.write(""); + requiredMembers.forEach(this::writeMemberDocs); + optionalMembers.forEach(this::writeMemberDocs); + } + }); + } + } + + private List filterMessageMembers(List members) { + // Only apply this to structures that are errors. + if (!shape.hasTrait(ErrorTrait.class)) { + return members; + } + // We replace modeled message members with a static `message` member. Serialization + // and deserialization will handle assigning them properly. + return members.stream() + .filter(member -> !isErrorMessage(model, member)) + .collect(Collectors.toList()); + } + + private String getTargetFormat(MemberShape member) { + Shape target = model.expectShape(member.getTarget()); + // Recursive shapes may be referenced before they're defined even with + // a topological sort. So forward references need to be used when + // referencing them. + if (recursiveShapes.contains(target)) { + return "'$T'"; + } + return "$T"; + } + + protected String getDefaultValue(PythonWriter writer, MemberShape member) { + // The default value is defined in the model is a exposed as generic + // json, so we need to convert it to the proper type based on the target. + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-default-trait + var defaultNode = member.expectTrait(DefaultTrait.class).toNode(); + var target = model.expectShape(member.getTarget()); + if (target.isTimestampShape()) { + ZonedDateTime value = CodegenUtils.parseTimestampNode(model, member, defaultNode); + return CodegenUtils.getDatetimeConstructor(writer, value); + } else if (target.isBlobShape()) { + return String.format("b'%s'", defaultNode.expectStringNode().getValue()); + } + return switch (defaultNode.getType()) { + case NULL -> "None"; + case BOOLEAN -> defaultNode.expectBooleanNode().getValue() ? "True" : "False"; + default -> Node.printJson(defaultNode); + }; + } + + private boolean hasDocs() { + if (shape.hasTrait(DocumentationTrait.class)) { + return true; + } + for (MemberShape member : shape.members()) { + if (member.getMemberTrait(model, DocumentationTrait.class).isPresent()) { + return true; + } + } + return false; + } + + private void writeMemberDocs(MemberShape member) { + member.getMemberTrait(model, DocumentationTrait.class).ifPresent(trait -> { + String memberName = symbolProvider.toMemberName(member); + String docs = writer.formatDocs(String.format(":param %s: %s", memberName, trait.getValue())); + writer.write(docs); + }); + } + + protected void writeAsDict(boolean isError) { + writer.openBlock("def as_dict(self) -> Dict[str, Any]:", "", () -> { + writer.writeDocs(() -> { + writer.write("Converts the $L to a dictionary.\n", symbolProvider.toSymbol(shape).getName()); + writer.write(writer.formatDocs(""" + The dictionary uses the modeled shape names rather than the parameter names \ + as keys to be mostly compatible with boto3.""")); + }); + + // If there aren't any optional members, it's best to return immediately. + String dictPrefix = optionalMembers.isEmpty() ? "return" : "d: Dict[str, Any] ="; + if (requiredMembers.isEmpty() && !isError) { + writer.write("$L {}", dictPrefix); + } else { + writer.openBlock("$L {", "}", dictPrefix, () -> { + if (isError) { + writer.write("'message': self.message,"); + writer.write("'code': self.code,"); + } + for (MemberShape member : requiredMembers) { + var memberName = symbolProvider.toMemberName(member); + var target = model.expectShape(member.getTarget()); + var targetSymbol = symbolProvider.toSymbol(target); + if (target.isStructureShape() || target.isUnionShape()) { + writer.write("$S: self.$L.as_dict(),", member.getMemberName(), memberName); + } else if (targetSymbol.getProperty("asDict").isPresent()) { + var targetAsDictSymbol = targetSymbol.expectProperty("asDict", Symbol.class); + writer.write("$S: $T(self.$L),", member.getMemberName(), targetAsDictSymbol, memberName); + } else { + writer.write("$S: self.$L,", member.getMemberName(), memberName); + } + } + }); + } + + if (!optionalMembers.isEmpty()) { + writer.write(""); + for (MemberShape member : optionalMembers) { + var memberName = symbolProvider.toMemberName(member); + var target = model.expectShape(member.getTarget()); + var targetSymbol = symbolProvider.toSymbol(target); + writer.openBlock("if self.$1L is not None:", "", memberName, () -> { + if (target.isStructureShape() || target.isUnionShape()) { + writer.write("d[$S] = self.$L.as_dict()", member.getMemberName(), memberName); + } else if (targetSymbol.getProperty("asDict").isPresent()) { + var targetAsDictSymbol = targetSymbol.expectProperty("asDict", Symbol.class); + writer.write("d[$S] = $T(self.$L),", member.getMemberName(), targetAsDictSymbol, + memberName); + } else { + writer.write("d[$S] = self.$L", member.getMemberName(), memberName); + } + }); + } + writer.write("return d"); + } + }); + writer.write(""); + } + + protected void writeFromDict(boolean isError) { + writer.write("@staticmethod"); + var shapeName = symbolProvider.toSymbol(shape).getName(); + writer.openBlock("def from_dict(d: Dict[str, Any]) -> $S:", "", shapeName, () -> { + writer.writeDocs(() -> { + writer.write("Creates a $L from a dictionary.\n", shapeName); + writer.write(writer.formatDocs(""" + The dictionary is expected to use the modeled shape names rather \ + than the parameter names as keys to be mostly compatible with boto3.""")); + }); + + if (shape.members().isEmpty() && !isError) { + writer.write("return $L()", shapeName); + return; + } + + if (requiredMembers.isEmpty() && !isError) { + writer.write("kwargs: Dict[str, Any] = {}"); + } else { + writer.openBlock("kwargs: Dict[str, Any] = {", "}", () -> { + if (isError) { + writer.write("'message': d['message'],"); + } + for (MemberShape member : requiredMembers) { + var memberName = symbolProvider.toMemberName(member); + var target = model.expectShape(member.getTarget()); + Symbol targetSymbol = symbolProvider.toSymbol(target); + if (target.isStructureShape()) { + writer.write("$S: $T.from_dict(d[$S]),", memberName, targetSymbol, member.getMemberName()); + } else if (targetSymbol.getProperty("fromDict").isPresent()) { + var targetFromDictSymbol = targetSymbol.expectProperty("fromDict", Symbol.class); + writer.write("$S: $T(d[$S]),", memberName, targetFromDictSymbol, member.getMemberName()); + } else { + writer.write("$S: d[$S],", memberName, member.getMemberName()); + } + } + }); + } + writer.write(""); + + for (MemberShape member : optionalMembers) { + var memberName = symbolProvider.toMemberName(member); + var target = model.expectShape(member.getTarget()); + writer.openBlock("if $S in d:", "", member.getMemberName(), () -> { + var targetSymbol = symbolProvider.toSymbol(target); + if (target.isStructureShape()) { + writer.write("kwargs[$S] = $T.from_dict(d[$S])", memberName, targetSymbol, + member.getMemberName()); + } else if (targetSymbol.getProperty("fromDict").isPresent()) { + var targetFromDictSymbol = targetSymbol.expectProperty("fromDict", Symbol.class); + writer.write("kwargs[$S] = $T(d[$S]),", memberName, targetFromDictSymbol, + member.getMemberName()); + } else { + writer.write("kwargs[$S] = d[$S]", memberName, member.getMemberName()); + } + }); + } + + writer.write("return $L(**kwargs)", shapeName); + }); + writer.write(""); + } + + protected void writeRepr(boolean isError) { + var symbol = symbolProvider.toSymbol(shape); + writer.write(""" + def __repr__(self) -> str: + result = "$L(" + ${C|} + return result + ")" + """, symbol.getName(), (Runnable) () -> writeReprMembers(isError)); + } + + protected void writeReprMembers(boolean isError) { + if (isError) { + writer.write("result += f'message={self.message},'"); + } + var iter = shape.members().iterator(); + while (iter.hasNext()) { + var member = iter.next(); + var memberName = symbolProvider.toMemberName(member); + var trailingComma = iter.hasNext() ? ", " : ""; + if (member.hasTrait(SensitiveTrait.class)) { + // Sensitive members must not be printed + // see: https://smithy.io/2.0/spec/documentation-traits.html#smithy-api-sensitive-trait + writer.write(""" + if self.$1L is not None: + result += f"$1L=...$2L" + """, memberName, trailingComma); + } else { + writer.write(""" + if self.$1L is not None: + result += f"$1L={repr(self.$1L)}$2L" + """, memberName, trailingComma); + } + } + } + + protected void writeEq(boolean isError) { + var symbol = symbolProvider.toSymbol(shape); + writer.addStdlibImport("typing", "Any"); + var attributeList = new StringBuilder("["); + if (isError) { + attributeList.append("'message',"); + } + for (MemberShape member : shape.members()) { + attributeList.append(String.format("'%s',", symbolProvider.toMemberName(member))); + } + attributeList.append("]"); + + if (!isError && shape.members().isEmpty()) { + writer.write(""" + def __eq__(self, other: Any) -> bool: + return isinstance(other, $L) + """, symbol.getName()); + return; + } + + // Use a generator expression inside "all" here to save some space while still + // lazily evaluating each equality check. + writer.write(""" + def __eq__(self, other: Any) -> bool: + if not isinstance(other, $L): + return False + attributes: list[str] = $L + return all( + getattr(self, a) == getattr(other, a) + for a in attributes + ) + """, symbol.getName(), attributeList.toString()); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SymbolVisitor.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SymbolVisitor.java new file mode 100644 index 0000000000..615781ea85 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SymbolVisitor.java @@ -0,0 +1,444 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import static java.lang.String.format; + +import java.util.Locale; +import java.util.logging.Logger; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.codegen.core.ReservedWordSymbolProvider; +import software.amazon.smithy.codegen.core.ReservedWords; +import software.amazon.smithy.codegen.core.ReservedWordsBuilder; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.BigDecimalShape; +import software.amazon.smithy.model.shapes.BigIntegerShape; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.BooleanShape; +import software.amazon.smithy.model.shapes.ByteShape; +import software.amazon.smithy.model.shapes.CollectionShape; +import software.amazon.smithy.model.shapes.DocumentShape; +import software.amazon.smithy.model.shapes.DoubleShape; +import software.amazon.smithy.model.shapes.EnumShape; +import software.amazon.smithy.model.shapes.FloatShape; +import software.amazon.smithy.model.shapes.IntEnumShape; +import software.amazon.smithy.model.shapes.IntegerShape; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.LongShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ResourceShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.ShortShape; +import software.amazon.smithy.model.shapes.SimpleShape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.TimestampShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.model.traits.MediaTypeTrait; +import software.amazon.smithy.model.traits.SparseTrait; +import software.amazon.smithy.model.traits.StreamingTrait; +import software.amazon.smithy.utils.CaseUtils; +import software.amazon.smithy.utils.MediaType; +import software.amazon.smithy.utils.StringUtils; + +/** + * Responsible for type mapping and file/identifier formatting. + * + *

Reserved words for Python are automatically escaped so that they are + * suffixed with "_". See "reserved-words.txt" for the list of words. + */ +public class SymbolVisitor implements SymbolProvider, ShapeVisitor { + + private static final Logger LOGGER = Logger.getLogger(SymbolVisitor.class.getName()); + + protected final Model model; + protected final ReservedWordSymbolProvider.Escaper escaper; + protected final ReservedWordSymbolProvider.Escaper errorMemberEscaper; + protected final PythonSettings settings; + protected final ServiceShape service; + + public SymbolVisitor(Model model, PythonSettings settings) { + this.model = model; + this.settings = settings; + this.service = model.expectShape(settings.getService(), ServiceShape.class); + + // Load reserved words from new-line delimited files. + var reservedClassNames = new ReservedWordsBuilder() + .loadWords(SymbolVisitor.class.getResource("reserved-class-names.txt"), this::escapeWord) + .build(); + var reservedMemberNamesBuilder = new ReservedWordsBuilder() + .loadWords(SymbolVisitor.class.getResource("reserved-member-names.txt"), this::escapeWord); + + escaper = ReservedWordSymbolProvider.builder() + .nameReservedWords(reservedClassNames) + .memberReservedWords(reservedMemberNamesBuilder.build()) + // Only escape words when the symbol has a definition file to + // prevent escaping intentional references to built-in types. + .escapePredicate((shape, symbol) -> !StringUtils.isEmpty(symbol.getDefinitionFile())) + .buildEscaper(); + + // Reserved words that only apply to error members. + ReservedWords reservedErrorMembers = reservedMemberNamesBuilder + .put("code", "code_") + .build(); + + errorMemberEscaper = ReservedWordSymbolProvider.builder() + .memberReservedWords(reservedErrorMembers) + .escapePredicate((shape, symbol) -> !StringUtils.isEmpty(symbol.getDefinitionFile())) + .buildEscaper(); + } + + private String escapeWord(String word) { + return word + "_"; + } + + @Override + public Symbol toSymbol(Shape shape) { + Symbol symbol = shape.accept(this); + LOGGER.fine(() -> format("Creating symbol from %s: %s", shape, symbol)); + return escaper.escapeSymbol(shape, symbol); + } + + @Override + public String toMemberName(MemberShape shape) { + if (CodegenUtils.isErrorMessage(model, shape)) { + return "message"; + } + + var memberName = CaseUtils.toSnakeCase(escaper.escapeMemberName(shape.getMemberName())); + + // Escape words that are only reserved for error members. + if (shape.hasTrait(ErrorTrait.class)) { + memberName = errorMemberEscaper.escapeMemberName(memberName); + } + + var container = model.expectShape(shape.getContainer()); + if (container.isEnumShape() || container.isIntEnumShape()) { + memberName = memberName.toUpperCase(Locale.ENGLISH); + } + return memberName; + } + + protected String getDefaultShapeName(Shape shape) { + // Use the service-aliased name + return StringUtils.capitalize(shape.getId().getName(service)); + } + + @Override + public Symbol blobShape(BlobShape shape) { + // see: https://smithy.io/2.0/spec/streaming.html#smithy-api-streaming-trait + if (shape.hasTrait(StreamingTrait.class)) { + return createSymbolBuilder(shape, "StreamingBlob", "smithy_python.interfaces.blobs") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build(); + } + + // see: https://smithy.io/2.0/spec/protocol-traits.html#smithy-api-mediatype-trait + if (shape.hasTrait(MediaTypeTrait.class)) { + var mediaType = shape.expectTrait(MediaTypeTrait.class).getValue(); + if (MediaType.isJson(mediaType)) { + return createSymbolBuilder(shape, "bytes | bytearray | JsonBlob") + .addReference(Symbol.builder() + .name("JsonBlob") + .namespace("smithy_python.mediatypes", ".") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build()) + .build(); + } + } + return createSymbolBuilder(shape, "bytes | bytearray").build(); + } + + @Override + public Symbol booleanShape(BooleanShape shape) { + return createSymbolBuilder(shape, "bool").build(); + } + + @Override + public Symbol listShape(ListShape shape) { + Symbol reference = toSymbol(shape.getMember()); + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-sparse-trait + String type = String.format(shape.hasTrait(SparseTrait.class) ? "%s | None" : "%s", reference.getName()); + var builder = createSymbolBuilder(shape, "list[" + type + "]") + .addReference(reference); + + if (needsDictHelpers(shape)) { + builder.putProperty("asDict", createAsDictFunctionSymbol(shape)) + .putProperty("fromDict", createFromDictFunctionSymbol(shape)); + } + return builder.build(); + } + + @Override + public Symbol mapShape(MapShape shape) { + Symbol reference = toSymbol(shape.getValue()); + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-sparse-trait + String type = String.format(shape.hasTrait(SparseTrait.class) ? "%s | None" : "%s", reference.getName()); + var builder = createSymbolBuilder(shape, "dict[str, " + type + "]") + .addReference(reference); + + if (needsDictHelpers(shape)) { + builder.putProperty("asDict", createAsDictFunctionSymbol(shape)) + .putProperty("fromDict", createFromDictFunctionSymbol(shape)); + } + return builder.build(); + } + + private boolean needsDictHelpers(MapShape shape) { + Shape target = model.expectShape(shape.getValue().getTarget()); + return targetRequiresDictHelpers(target); + } + + private boolean needsDictHelpers(CollectionShape shape) { + Shape target = model.expectShape(shape.getMember().getTarget()); + return targetRequiresDictHelpers(target); + } + + /** + * Maps and collections are already dict compatible, so if a given map or + * collection only ever transitively reference dict compatible shapes, + * they don't need these dict helpers. + */ + protected boolean targetRequiresDictHelpers(Shape target) { + if (target instanceof SimpleShape) { + return false; + } + if (target instanceof CollectionShape) { + return needsDictHelpers((CollectionShape) target); + } + if (target.isMapShape()) { + return needsDictHelpers((MapShape) target); + } + return true; + } + + protected Symbol createAsDictFunctionSymbol(Shape shape) { + return Symbol.builder() + .name(String.format("_%s_as_dict", CaseUtils.toSnakeCase(shape.getId().getName()))) + .namespace(format("%s.models", settings.getModuleName()), ".") + .definitionFile(format("./%s/models.py", settings.getModuleName())) + .build(); + } + + protected Symbol createFromDictFunctionSymbol(Shape shape) { + return Symbol.builder() + .name(String.format("_%s_from_dict", CaseUtils.toSnakeCase(shape.getId().getName()))) + .namespace(format("%s.models", settings.getModuleName()), ".") + .definitionFile(format("./%s/models.py", settings.getModuleName())) + .build(); + } + + @Override + public Symbol byteShape(ByteShape shape) { + return createSymbolBuilder(shape, "int").build(); + } + + @Override + public Symbol shortShape(ShortShape shape) { + return createSymbolBuilder(shape, "int").build(); + } + + @Override + public Symbol integerShape(IntegerShape shape) { + return createSymbolBuilder(shape, "int").build(); + } + + @Override + public Symbol longShape(LongShape shape) { + return createSymbolBuilder(shape, "int").build(); + } + + @Override + public Symbol floatShape(FloatShape shape) { + return createSymbolBuilder(shape, "float").build(); + } + + @Override + public Symbol documentShape(DocumentShape shape) { + return createSymbolBuilder(shape, "Document") + .namespace("smithy_python.types", ".") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build(); + } + + @Override + public Symbol doubleShape(DoubleShape shape) { + return createSymbolBuilder(shape, "float").build(); + } + + @Override + public Symbol bigIntegerShape(BigIntegerShape shape) { + return createSymbolBuilder(shape, "int").build(); + } + + @Override + public Symbol bigDecimalShape(BigDecimalShape shape) { + return createStdlibSymbol(shape, "Decimal", "decimal"); + } + + @Override + public Symbol operationShape(OperationShape shape) { + // Operation names are escaped like members because ultimately they're + // properties on an object too. + var name = escaper.escapeMemberName(CaseUtils.toSnakeCase(shape.getId().getName(service))); + return createSymbolBuilder(shape, name, format("%s.client", settings.getModuleName())) + .definitionFile(format("./%s/client.py", settings.getModuleName())) + .build(); + } + + @Override + public Symbol resourceShape(ResourceShape shape) { + // TODO: implement resources + return createStdlibSymbol(shape, "Any", "typing"); + } + + @Override + public Symbol serviceShape(ServiceShape shape) { + var name = getDefaultShapeName(shape); + return createSymbolBuilder(shape, name, format("%s.client", settings.getModuleName())) + .definitionFile(format("./%s/client.py", settings.getModuleName())) + .build(); + } + + @Override + public Symbol stringShape(StringShape shape) { + var builder = createSymbolBuilder(shape, "str"); + // see: https://smithy.io/2.0/spec/protocol-traits.html#smithy-api-mediatype-trait + if (shape.hasTrait(MediaTypeTrait.class)) { + var mediaType = shape.expectTrait(MediaTypeTrait.class).getValue(); + if (MediaType.isJson(mediaType)) { + return createSymbolBuilder(shape, "str | JsonString") + .addReference(Symbol.builder() + .name("JsonString") + .namespace("smithy_python.mediatypes", ".") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build()) + .build(); + } + } + return builder.build(); + } + + @Override + public Symbol enumShape(EnumShape shape) { + var builder = createSymbolBuilder(shape, "str"); + String name = getDefaultShapeName(shape); + Symbol enumSymbol = createSymbolBuilder(shape, name, format("%s.models", settings.getModuleName())) + .definitionFile(format("./%s/models.py", settings.getModuleName())) + .build(); + + // We add this enum symbol as a property on a generic string symbol + // rather than returning the enum symbol directly because we only + // generate the enum constants for convenience. We actually want + // to pass around plain strings rather than what is effectively + // a namespace class. + builder.putProperty("enumSymbol", escaper.escapeSymbol(shape, enumSymbol)); + return builder.build(); + } + + @Override + public Symbol intEnumShape(IntEnumShape shape) { + var builder = createSymbolBuilder(shape, "int"); + String name = getDefaultShapeName(shape); + Symbol enumSymbol = createSymbolBuilder(shape, name, format("%s.models", settings.getModuleName())) + .definitionFile(format("./%s/models.py", settings.getModuleName())) + .build(); + + // Like string enums, int enums are plain ints when used as members. + builder.putProperty("enumSymbol", escaper.escapeSymbol(shape, enumSymbol)); + return builder.build(); + } + + @Override + public Symbol structureShape(StructureShape shape) { + String name = getDefaultShapeName(shape); + if (shape.hasTrait(ErrorTrait.class)) { + return createSymbolBuilder(shape, name, format("%s.errors", settings.getModuleName())) + .definitionFile(format("./%s/errors.py", settings.getModuleName())) + .build(); + } + return createSymbolBuilder(shape, name, format("%s.models", settings.getModuleName())) + .definitionFile(format("./%s/models.py", settings.getModuleName())) + .build(); + } + + @Override + public Symbol unionShape(UnionShape shape) { + String name = getDefaultShapeName(shape); + + var unknownName = name + "Unknown"; + var unknownSymbol = createSymbolBuilder(shape, unknownName, format("%s.models", settings.getModuleName())) + .definitionFile(format("./%s/models.py", settings.getModuleName())) + .build(); + + return createSymbolBuilder(shape, name, format("%s.models", settings.getModuleName())) + .definitionFile(format("./%s/models.py", settings.getModuleName())) + .putProperty("fromDict", createFromDictFunctionSymbol(shape)) + .putProperty("unknown", unknownSymbol) + .build(); + } + + @Override + public Symbol memberShape(MemberShape shape) { + var container = model.expectShape(shape.getContainer()); + if (container.isUnionShape()) { + // Union members, unlike other shape members, have types generated for them. + var containerSymbol = container.accept(this); + var name = containerSymbol.getName() + StringUtils.capitalize(shape.getMemberName()); + return createSymbolBuilder(shape, name, format("%s.models", settings.getModuleName())) + .definitionFile(format("./%s/models.py", settings.getModuleName())) + .build(); + } + Shape targetShape = model.getShape(shape.getTarget()) + .orElseThrow(() -> new CodegenException("Shape not found: " + shape.getTarget())); + return toSymbol(targetShape); + } + + @Override + public Symbol timestampShape(TimestampShape shape) { + return createStdlibSymbol(shape, "datetime", "datetime"); + } + + protected Symbol.Builder createSymbolBuilder(Shape shape, String typeName) { + return Symbol.builder().putProperty("shape", shape).name(typeName); + } + + protected Symbol.Builder createSymbolBuilder(Shape shape, String typeName, String namespace) { + return createSymbolBuilder(shape, typeName).namespace(namespace, "."); + } + + protected Symbol createStdlibSymbol(Shape shape, String typeName, String namespace) { + return createSymbolBuilder(shape, typeName, namespace) + .putProperty("stdlib", true) + .build(); + } + + private SymbolReference createStdlibReference(String typeName, String namespace) { + return SymbolReference.builder() + .symbol(createStdlibSymbol(null, typeName, namespace)) + .putProperty("stdlib", true) + .options(SymbolReference.ContextOption.USE) + .build(); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/UnionGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/UnionGenerator.java new file mode 100644 index 0000000000..df3ec62151 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/UnionGenerator.java @@ -0,0 +1,173 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen; + +import java.util.ArrayList; +import java.util.Set; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.utils.StringUtils; + +/** + * Renders unions. + */ +public final class UnionGenerator implements Runnable { + + private final Model model; + private final SymbolProvider symbolProvider; + private final PythonWriter writer; + private final UnionShape shape; + private final Set recursiveShapes; + + public UnionGenerator( + Model model, + SymbolProvider symbolProvider, + PythonWriter writer, + UnionShape shape, + Set recursiveShapes + ) { + this.model = model; + this.symbolProvider = symbolProvider; + this.writer = writer; + this.shape = shape; + this.recursiveShapes = recursiveShapes; + } + + @Override + public void run() { + var parentName = symbolProvider.toSymbol(shape).getName(); + writer.addStdlibImport("typing", "Dict"); + writer.addStdlibImport("typing", "Any"); + + var memberNames = new ArrayList(); + for (MemberShape member : shape.members()) { + var memberSymbol = symbolProvider.toSymbol(member); + memberNames.add(memberSymbol.getName()); + + var target = model.expectShape(member.getTarget()); + var targetSymbol = symbolProvider.toSymbol(target); + + writer.openBlock("class $L():", "", memberSymbol.getName(), () -> { + member.getMemberTrait(model, DocumentationTrait.class).ifPresent(trait -> { + writer.writeDocs(trait.getValue()); + }); + writer.openBlock("def __init__(self, value: '$T'):", "", targetSymbol, () -> { + writer.write("self.value = value"); + }); + + writer.openBlock("def as_dict(self) -> Dict[str, Any]:", "", () -> { + if (target.isStructureShape() || target.isUnionShape()) { + writer.write("return {$S: self.value.as_dict()}", member.getMemberName()); + } else if (targetSymbol.getProperty("asDict").isPresent()) { + var targetAsDictSymbol = targetSymbol.expectProperty("asDict", Symbol.class); + writer.write("return {$S: $T(self.value)}", member.getMemberName(), targetAsDictSymbol); + } else { + writer.write("return {$S: self.value}", member.getMemberName()); + } + }); + + writer.write("@staticmethod"); + writer.openBlock("def from_dict(d: Dict[str, Any]) -> $S:", "", memberSymbol.getName(), () -> { + writer.write(""" + if (len(d) != 1): + raise TypeError(f"Unions may have exactly 1 value, but found {len(d)}") + """); + if (target.isStructureShape()) { + writer.write("return $T($T.from_dict(d[$S]))", memberSymbol, targetSymbol, + member.getMemberName()); + } else if (targetSymbol.getProperty("fromDict").isPresent()) { + var targetFromDictSymbol = targetSymbol.expectProperty("fromDict", Symbol.class); + writer.write("return $T($T(d[$S]))", + memberSymbol, targetFromDictSymbol, member.getMemberName()); + } else { + writer.write("return $T(d[$S])", memberSymbol, member.getMemberName()); + } + }); + + writer.write(""" + def __repr__(self) -> str: + return f"$L(value=repr(self.value))" + """, memberSymbol.getName()); + + writer.write(""" + def __eq__(self, other: Any) -> bool: + if not isinstance(other, $1L): + return False + return self.value == other.value + """, memberSymbol.getName()); + }); + writer.write(""); + } + + // Note that the unknown variant doesn't implement __eq__. This is because + // the default implementation does exactly what we want: an instance check. + // Since the underlying value is unknown and un-comparable, that is the only + // realistic implementation. + var unknownSymbol = symbolProvider.toSymbol(shape).expectProperty("unknown", Symbol.class); + writer.write(""" + class $1L(): + \"""Represents an unknown variant. + + If you receive this value, you will need to update your library to receive the + parsed value. + + This value may not be deliberately sent. + \""" + + def __init__(self, tag: str): + self.tag = tag + + def as_dict(self) -> Dict[str, Any]: + return {"SDK_UNKNOWN_MEMBER": {"name": self.tag}} + + @staticmethod + def from_dict(d: Dict[str, Any]) -> "$1L": + if (len(d) != 1): + raise TypeError(f"Unions may have exactly 1 value, but found {len(d)}") + return $1L(d["SDK_UNKNOWN_MEMBER"]["name"]) + + def __repr__(self) -> str: + return f"$1L(tag={self.tag})" + """, unknownSymbol.getName()); + memberNames.add(unknownSymbol.getName()); + + shape.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeComment(trait.getValue())); + writer.addStdlibImport("typing", "Union"); + writer.write("$L = Union[$L]", parentName, String.join(", ", memberNames)); + + writeGlobalFromDict(); + } + + private void writeGlobalFromDict() { + var parentSymbol = symbolProvider.toSymbol(shape); + var fromDictSymbol = parentSymbol.expectProperty("fromDict", Symbol.class); + writer.openBlock("def $L(d: Dict[str, Any]) -> $T:", "", fromDictSymbol.getName(), parentSymbol, () -> { + for (MemberShape member : shape.members()) { + var memberName = parentSymbol.getName() + StringUtils.capitalize(member.getMemberName()); + writer.write(""" + if $S in d: + return $L.from_dict(d) + """, member.getMemberName(), memberName); + } + writer.write("raise TypeError(f'Unions may have exactly 1 value, but found {len(d)}')"); + }); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/AuthScheme.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/AuthScheme.java new file mode 100644 index 0000000000..a09875de93 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/AuthScheme.java @@ -0,0 +1,63 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.Collections; +import java.util.List; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.ApplicationProtocol; +import software.amazon.smithy.python.codegen.DerivedProperty; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Provides details for generating auth for an auth trait. + */ +@SmithyUnstableApi +public interface AuthScheme { + + /** + * @return Returns the auth trait this scheme implements. + */ + ShapeId getAuthTrait(); + + /** + * Gets the application protocol for the auth scheme. + * + *

The auth scheme will only be generated if its application protocol matches + * that of the service protocol. + * + * @return Returns the created application protocol. + */ + ApplicationProtocol getApplicationProtocol(); + + /** + * Gets a list of properties needed from config or input to authenticate requests. + * + * @return Returns a list of properties to gather for auth. + */ + default List getAuthProperties() { + return Collections.emptyList(); + } + + /** + * Gets a function that returns a potential auth option for a request. + * + *

This function will be given an object containing all the properties + * defined by {@code getAuthProperties}. + * + * @param context The code generation context. + * @return Returns a symbol referencing the auth option function. + */ + Symbol getAuthOptionGenerator(GenerationContext context); + + /** + * @param context The code generation context. + * @return Returns the symbol for the auth scheme implementation. + */ + Symbol getAuthSchemeSymbol(GenerationContext context); +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/DocumentMemberDeserVisitor.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/DocumentMemberDeserVisitor.java new file mode 100644 index 0000000000..eb36f980a6 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/DocumentMemberDeserVisitor.java @@ -0,0 +1,320 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.Optional; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.knowledge.HttpBinding.Location; +import software.amazon.smithy.model.knowledge.HttpBindingIndex; +import software.amazon.smithy.model.shapes.BigDecimalShape; +import software.amazon.smithy.model.shapes.BigIntegerShape; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.BooleanShape; +import software.amazon.smithy.model.shapes.ByteShape; +import software.amazon.smithy.model.shapes.DocumentShape; +import software.amazon.smithy.model.shapes.DoubleShape; +import software.amazon.smithy.model.shapes.FloatShape; +import software.amazon.smithy.model.shapes.IntegerShape; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.LongShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ResourceShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.ShortShape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.TimestampShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Visitor to generate member values for aggregate types deserialized from documents. + * + *

The standard implementations are as follows; these implementations may be + * overridden unless otherwise specified. + * + *

    + *
  • Blob: base64 decoded.
    • + *
  • BigDecimal: converted to decimal.Decimal.
    • + *
  • Timestamp: converted to datetime.datetime in UTC.
    • + *
  • Service, Operation, Resource, Member: not deserializable from documents. Not overridable.
    • + *
  • Document, List, Map, Set, Structure, Union: delegated to a deserialization function. + * Not overridable.
    • + *
  • All other types: unmodified.
    • + *
+ */ +@SmithyUnstableApi +public class DocumentMemberDeserVisitor implements ShapeVisitor { + private final GenerationContext context; + private final PythonWriter writer; + private final MemberShape member; + private final String dataSource; + private final Format defaultTimestampFormat; + + /** + * Constructor. + * + * @param context The generation context. + * @param writer The writer being written to, used for adding imports. + * @param dataSource The in-code location of the data to provide an output of + * ({@code output.foo}, {@code entry}, etc.) + * @param defaultTimestampFormat The default timestamp format used in absence + * of a TimestampFormat trait. + */ + public DocumentMemberDeserVisitor( + GenerationContext context, + PythonWriter writer, + String dataSource, + Format defaultTimestampFormat + ) { + this(context, writer, null, dataSource, defaultTimestampFormat); + } + + /** + * Constructor. + * + * @param context The generation context. + * @param writer The writer being written to, used for adding imports. + * @param member The member shape being deserialized. Used for any extra traits + * it might bear, such as the timestamp format. + * @param dataSource The in-code location of the data to provide an output of + * ({@code output.foo}, {@code entry}, etc.) + * @param defaultTimestampFormat The default timestamp format used in absence + * of a TimestampFormat trait. + */ + public DocumentMemberDeserVisitor( + GenerationContext context, + PythonWriter writer, + MemberShape member, + String dataSource, + Format defaultTimestampFormat + ) { + this.context = context; + this.writer = writer; + this.member = member; + this.dataSource = dataSource; + this.defaultTimestampFormat = defaultTimestampFormat; + } + + /** + * @return the member this visitor is being run against. Used to discover member-applied + * traits, such as @timestampFormat. + */ + protected Optional memberShape() { + return Optional.ofNullable(member); + } + + /** + * Gets the generation context. + * + * @return The generation context. + */ + protected final GenerationContext context() { + return context; + } + + /** + * Gets the PythonWriter being written to. + * + *

This should only be used to add imports. + * + * @return The writer to add imports to. + */ + protected final PythonWriter writer() { + return writer; + } + + /** + * Gets the in-code location of the data to provide an output of + * ({@code output.foo}, {@code entry}, etc.). + * + * @return The data source. + */ + protected final String dataSource() { + return dataSource; + } + + /** + * Gets the default timestamp format used in absence of a TimestampFormat trait. + * + * @return The default timestamp format. + */ + protected final Format getDefaultTimestampFormat() { + return defaultTimestampFormat; + } + + @Override + public String blobShape(BlobShape shape) { + writer.addStdlibImport("base64", "b64decode"); + writer.addImport("smithy_python.utils", "expect_type"); + return "b64decode(expect_type(str, " + dataSource + "))"; + } + + @Override + public String booleanShape(BooleanShape shape) { + writer.addImport("smithy_python.utils", "expect_type"); + return "expect_type(bool, " + dataSource + ")"; + } + + @Override + public String byteShape(ByteShape shape) { + // TODO: add bounds checks + return intShape(); + } + + @Override + public String shortShape(ShortShape shape) { + // TODO: add bounds checks + return intShape(); + } + + @Override + public String integerShape(IntegerShape shape) { + // TODO: add bounds checks + return intShape(); + } + + @Override + public String longShape(LongShape shape) { + // TODO: add bounds checks + return intShape(); + } + + private String intShape() { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "expect_type"); + return "expect_type(int, " + dataSource + ")"; + } + + @Override + public String floatShape(FloatShape shape) { + // TODO: perform a bounds check + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "limited_parse_float"); + return "limited_parse_float(" + dataSource + ")"; + } + + @Override + public String doubleShape(DoubleShape shape) { + // TODO: perform a bounds check + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "limited_parse_float"); + return "limited_parse_float(" + dataSource + ")"; + } + + @Override + public String stringShape(StringShape shape) { + // TODO: handle strings with media types + writer.addImport("smithy_python.utils", "expect_type"); + return "expect_type(str, " + dataSource + ")"; + } + + @Override + public String bigIntegerShape(BigIntegerShape shape) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "expect_type"); + return "expect_type(int, " + dataSource + ")"; + } + + @Override + public String bigDecimalShape(BigDecimalShape shape) { + writer.addStdlibImport("decimal", "Decimal", "_Decimal"); + writer.addImport("smithy_python.utils", "expect_type"); + return "_Decimal(expect_type(str" + dataSource + "))"; + } + + @Override + public final String operationShape(OperationShape shape) { + throw new CodegenException("Operation shapes cannot be bound to documents."); + } + + @Override + public final String resourceShape(ResourceShape shape) { + throw new CodegenException("Resource shapes cannot be bound to documents."); + } + + @Override + public final String serviceShape(ServiceShape shape) { + throw new CodegenException("Service shapes cannot be bound to documents."); + } + + @Override + public final String memberShape(MemberShape shape) { + throw new CodegenException("Member shapes cannot be bound to documents."); + } + + @Override + public String timestampShape(TimestampShape shape) { + var httpIndex = HttpBindingIndex.of(context.model()); + Format format; + if (memberShape().isEmpty()) { + format = httpIndex.determineTimestampFormat(shape, Location.DOCUMENT, defaultTimestampFormat); + } else { + var member = memberShape().get(); + if (!shape.getId().equals(member.getTarget())) { + throw new CodegenException( + String.format("Encountered timestamp shape %s that was not the target of member shape %s", + shape.getId(), member.getId())); + } + format = httpIndex.determineTimestampFormat(member, Location.DOCUMENT, defaultTimestampFormat); + } + + return HttpProtocolGeneratorUtils.getTimestampOutputParam(writer, dataSource, shape, format); + } + + @Override + public final String documentShape(DocumentShape shape) { + return dataSource(); + } + + @Override + public final String listShape(ListShape shape) { + return getDelegateDeserializer(shape); + } + + @Override + public final String mapShape(MapShape shape) { + return getDelegateDeserializer(shape); + } + + @Override + public final String structureShape(StructureShape shape) { + return getDelegateDeserializer(shape); + } + + @Override + public final String unionShape(UnionShape shape) { + return getDelegateDeserializer(shape); + } + + private String getDelegateDeserializer(Shape shape) { + return getDelegateDeserializer(shape, dataSource); + } + + private String getDelegateDeserializer(Shape shape, String customDataSource) { + var deserSymbol = context.protocolGenerator().getDeserializationFunction(context, shape.getId()); + writer.addImport(deserSymbol, deserSymbol.getName()); + return deserSymbol.getName() + "(" + customDataSource + ", config)"; + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/DocumentMemberSerVisitor.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/DocumentMemberSerVisitor.java new file mode 100644 index 0000000000..071d2edda1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/DocumentMemberSerVisitor.java @@ -0,0 +1,270 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.Optional; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.knowledge.HttpBinding.Location; +import software.amazon.smithy.model.knowledge.HttpBindingIndex; +import software.amazon.smithy.model.shapes.BigDecimalShape; +import software.amazon.smithy.model.shapes.BigIntegerShape; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.BooleanShape; +import software.amazon.smithy.model.shapes.ByteShape; +import software.amazon.smithy.model.shapes.DocumentShape; +import software.amazon.smithy.model.shapes.DoubleShape; +import software.amazon.smithy.model.shapes.FloatShape; +import software.amazon.smithy.model.shapes.IntegerShape; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.LongShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ResourceShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.ShortShape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.TimestampShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Visitor to serialize member values for aggregate types into document bodies. + * + *

The standard implementations are as follows; these implementations may be + * overridden unless otherwise specified. + * + *

    + *
  • Blob: base64 encoded and encoded to a utf-8 string.
    • + *
  • Timestamp: serialized to a string.
    • + *
  • Service, Operation, Resource, Member: not deserializable from documents. Not overridable.
    • + *
  • List, Map, Set, Structure, Union: delegated to a serialization function. + * Not overridable.
    • + *
  • All other types: unmodified.
    • + *
+ */ +@SmithyUnstableApi +public class DocumentMemberSerVisitor implements ShapeVisitor { + private final GenerationContext context; + private final PythonWriter writer; + private final MemberShape member; + private final String dataSource; + private final Format defaultTimestampFormat; + + /** + * @param context The generation context. + * @param writer The writer to write to. + * @param member The member shape being deserialized. Used for any extra traits + * it might bear, such as the timestamp format. + * @param dataSource The in-code location of the data to provide an output of + * ({@code output.foo}, {@code entry}, etc.) + * @param defaultTimestampFormat The default timestamp format used in absence + * of a TimestampFormat trait. + */ + public DocumentMemberSerVisitor( + GenerationContext context, + PythonWriter writer, + MemberShape member, + String dataSource, + Format defaultTimestampFormat + ) { + this.context = context; + this.writer = writer; + this.member = member; + this.dataSource = dataSource; + this.defaultTimestampFormat = defaultTimestampFormat; + } + + /** + * @return the member this visitor is being run against. Used to discover member-applied + * traits, such as @timestampFormat. + */ + protected Optional memberShape() { + return Optional.ofNullable(member); + } + + /** + * Gets the generation context. + * + * @return The generation context. + */ + protected final GenerationContext context() { + return context; + } + + /** + * Gets the PythonWriter being written to. + * + *

This should only be used to add imports. + * + * @return The writer to add imports to. + */ + protected final PythonWriter writer() { + return writer; + } + + /** + * Gets the in-code location of the data to provide an output of + * ({@code output.foo}, {@code entry}, etc.). + * + * @return The data source. + */ + protected final String dataSource() { + return dataSource; + } + + /** + * Gets the default timestamp format used in absence of a TimestampFormat trait. + * + * @return The default timestamp format. + */ + protected final Format getDefaultTimestampFormat() { + return defaultTimestampFormat; + } + + @Override + public String blobShape(BlobShape blobShape) { + writer.addStdlibImport("base64", "b64encode"); + return String.format("b64encode(%s).decode('utf-8')", dataSource()); + } + + @Override + public String booleanShape(BooleanShape booleanShape) { + return dataSource(); + } + + @Override + public String byteShape(ByteShape byteShape) { + return dataSource(); + } + + @Override + public String shortShape(ShortShape shortShape) { + return dataSource(); + } + + @Override + public String integerShape(IntegerShape integerShape) { + return dataSource(); + } + + @Override + public String longShape(LongShape longShape) { + return dataSource(); + } + + @Override + public String floatShape(FloatShape floatShape) { + return floatShapes(); + } + + @Override + public String documentShape(DocumentShape documentShape) { + return dataSource(); + } + + @Override + public String doubleShape(DoubleShape doubleShape) { + return floatShapes(); + } + + private String floatShapes() { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "limited_serialize_float"); + return String.format("limited_serialize_float(%s)", dataSource); + } + + @Override + public String bigIntegerShape(BigIntegerShape bigIntegerShape) { + return String.format("str(%s)", dataSource()); + } + + @Override + public String bigDecimalShape(BigDecimalShape bigDecimalShape) { + return String.format("str(%s.normalize())", dataSource()); + } + + @Override + public final String operationShape(OperationShape shape) { + throw new CodegenException("Operation shapes cannot be bound to documents."); + } + + @Override + public final String resourceShape(ResourceShape shape) { + throw new CodegenException("Resource shapes cannot be bound to documents."); + } + + @Override + public final String serviceShape(ServiceShape shape) { + throw new CodegenException("Service shapes cannot be bound to documents."); + } + + @Override + public final String memberShape(MemberShape shape) { + throw new CodegenException("Member shapes cannot be bound to documents."); + } + + @Override + public String stringShape(StringShape stringShape) { + return dataSource(); + } + + @Override + public String timestampShape(TimestampShape timestampShape) { + var sourceShape = memberShape().isPresent() ? member : timestampShape; + var index = HttpBindingIndex.of(context.model()); + var format = index.determineTimestampFormat(sourceShape, Location.DOCUMENT, getDefaultTimestampFormat()); + return HttpProtocolGeneratorUtils.getTimestampInputParam( + context(), writer(), dataSource(), sourceShape, format); + } + + @Override + public String listShape(ListShape listShape) { + return getDelegateSerializer(listShape); + } + + @Override + public String mapShape(MapShape mapShape) { + return getDelegateSerializer(mapShape); + } + + @Override + public String structureShape(StructureShape structureShape) { + return getDelegateSerializer(structureShape); + } + + @Override + public String unionShape(UnionShape unionShape) { + return getDelegateSerializer(unionShape); + } + + private String getDelegateSerializer(Shape shape) { + return getDelegateSerializer(shape, dataSource); + } + + private String getDelegateSerializer(Shape shape, String customDataSource) { + var serSymbol = context.protocolGenerator().getSerializationFunction(context, shape.getId()); + writer.addImport(serSymbol, serSymbol.getName()); + return serSymbol.getName() + "(" + customDataSource + ", config)"; + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpApiKeyAuth.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpApiKeyAuth.java new file mode 100644 index 0000000000..572f136d3b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpApiKeyAuth.java @@ -0,0 +1,137 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.List; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.traits.HttpApiKeyAuthTrait; +import software.amazon.smithy.python.codegen.ApplicationProtocol; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.ConfigProperty; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; + +/** + * Adds support for the http api key auth. + * {@see https://smithy.io/2.0/spec/authentication-traits.html#smithy-api-httpapikeyauth-trait} + */ +public final class HttpApiKeyAuth implements PythonIntegration { + private static final String OPTION_GENERATOR_NAME = "_generate_api_key_option"; + + @Override + public List getClientPlugins() { + return List.of( + RuntimeClientPlugin.builder() + .servicePredicate((model, service) -> service.hasTrait(HttpApiKeyAuthTrait.class)) + .addConfigProperty(ConfigProperty.builder() + .name("api_key_identity_resolver") + .documentation("Resolves the API key. Required for operations that use API key auth.") + .type(Symbol.builder() + .name("IdentityResolver[ApiKeyIdentity, IdentityProperties]") + .addReference(Symbol.builder() + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .name("IdentityResolver") + .namespace("smithy_python.interfaces.identity", ".") + .build()) + .addReference(Symbol.builder() + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .name("ApiKeyIdentity") + .namespace("smithy_python._private.api_key_auth", ".") + .build()) + .addReference(Symbol.builder() + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .name("IdentityProperties") + .namespace("smithy_python.interfaces.identity", ".") + .build()) + .build()) + .nullable(true) + .build()) + .authScheme(new ApiKeyAuthScheme()) + .build() + ); + } + + @Override + public void customize(GenerationContext context) { + if (!hasApiKeyAuth(context)) { + return; + } + var trait = context.settings().getService(context.model()).expectTrait(HttpApiKeyAuthTrait.class); + var params = CodegenUtils.getHttpAuthParamsSymbol(context.settings()); + var resolver = CodegenUtils.getHttpAuthSchemeResolverSymbol(context.settings()); + + // Add a function that generates the http auth option for api key auth. + // This needs to be generated because there's modeled parameters that + // must be accounted for. + context.writerDelegator().useFileWriter(resolver.getDefinitionFile(), resolver.getNamespace(), writer -> { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.interfaces.auth", "HTTPAuthOption"); + writer.addImport("smithy_python._private.api_key_auth", "ApiKeyLocation"); + writer.pushState(); + + // Push the scheme into the context to allow for conditionally adding + // it to the properties dict. + writer.putContext("scheme", trait.getScheme().orElse(null)); + writer.write(""" + def $1L(auth_params: $2T) -> HTTPAuthOption: + return HTTPAuthOption( + scheme_id=$3S, + identity_properties={}, + signer_properties={ + "name": $4S, + "location": ApiKeyLocation($5S), + ${?scheme} + "scheme": ${scheme:S}, + ${/scheme} + } + ) + """, OPTION_GENERATOR_NAME, params, HttpApiKeyAuthTrait.ID.toString(), + trait.getName(), trait.getIn().toString()); + writer.popState(); + }); + } + + private boolean hasApiKeyAuth(GenerationContext context) { + var service = context.settings().getService(context.model()); + return service.hasTrait(HttpApiKeyAuthTrait.class); + } + + /** + * The AuthScheme representing api key auth. + */ + private static final class ApiKeyAuthScheme implements AuthScheme { + + @Override + public ShapeId getAuthTrait() { + return HttpApiKeyAuthTrait.ID; + } + + @Override + public ApplicationProtocol getApplicationProtocol() { + return ApplicationProtocol.createDefaultHttpApplicationProtocol(); + } + + @Override + public Symbol getAuthOptionGenerator(GenerationContext context) { + var resolver = CodegenUtils.getHttpAuthSchemeResolverSymbol(context.settings()); + return Symbol.builder() + .name(OPTION_GENERATOR_NAME) + .namespace(resolver.getNamespace(), ".") + .definitionFile(resolver.getDefinitionFile()) + .build(); + } + + @Override + public Symbol getAuthSchemeSymbol(GenerationContext context) { + return Symbol.builder() + .name("ApiKeyAuthScheme") + .namespace("smithy_python._private.api_key_auth", ".") + .addDependency(SmithyPythonDependency.SMITHY_PYTHON) + .build(); + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java new file mode 100644 index 0000000000..ec68c9a572 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java @@ -0,0 +1,1493 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + + +import static java.lang.String.format; +import static software.amazon.smithy.model.knowledge.HttpBinding.Location.DOCUMENT; +import static software.amazon.smithy.model.knowledge.HttpBinding.Location.HEADER; +import static software.amazon.smithy.model.knowledge.HttpBinding.Location.LABEL; +import static software.amazon.smithy.model.knowledge.HttpBinding.Location.PAYLOAD; +import static software.amazon.smithy.model.knowledge.HttpBinding.Location.PREFIX_HEADERS; +import static software.amazon.smithy.model.knowledge.HttpBinding.Location.QUERY; +import static software.amazon.smithy.model.knowledge.HttpBinding.Location.QUERY_PARAMS; +import static software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import static software.amazon.smithy.python.codegen.integration.HttpProtocolGeneratorUtils.generateErrorDispatcher; +import static software.amazon.smithy.python.codegen.integration.HttpProtocolGeneratorUtils.getOutputShape; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.knowledge.HttpBinding; +import software.amazon.smithy.model.knowledge.HttpBinding.Location; +import software.amazon.smithy.model.knowledge.HttpBindingIndex; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.pattern.SmithyPattern; +import software.amazon.smithy.model.pattern.SmithyPattern.Segment; +import software.amazon.smithy.model.pattern.UriPattern; +import software.amazon.smithy.model.shapes.BigDecimalShape; +import software.amazon.smithy.model.shapes.BigIntegerShape; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.BooleanShape; +import software.amazon.smithy.model.shapes.ByteShape; +import software.amazon.smithy.model.shapes.DoubleShape; +import software.amazon.smithy.model.shapes.FloatShape; +import software.amazon.smithy.model.shapes.IntegerShape; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.LongShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.ShortShape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.TimestampShape; +import software.amazon.smithy.model.traits.HttpTrait; +import software.amazon.smithy.model.traits.MediaTypeTrait; +import software.amazon.smithy.model.traits.RequiresLengthTrait; +import software.amazon.smithy.model.traits.StreamingTrait; +import software.amazon.smithy.python.codegen.ApplicationProtocol; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyUnstableApi; +import software.amazon.smithy.utils.StringUtils; + +/** + * Abstract implementation useful for all protocols that use HTTP bindings. + * + *

This will implement any handling of components outside the request + * body and error handling. + */ +@SmithyUnstableApi +public abstract class HttpBindingProtocolGenerator implements ProtocolGenerator { + + // When generating operations, we add input and output shapes to these + // sets and then use them as the list of shapes we need to generate + // document serializers and deserializers for. + private final Set serializingDocumentShapes = new TreeSet<>(); + private final Set deserializingDocumentShapes = new TreeSet<>(); + + @Override + public ApplicationProtocol getApplicationProtocol() { + return ApplicationProtocol.createDefaultHttpApplicationProtocol(); + } + + /** + * Gets the default serde format for timestamps. + * + * @return Returns the default format. + */ + protected abstract Format getDocumentTimestampFormat(); + + /** + * Given a context and operation, should a default input body be written. + * + *

By default, no body will be written if there are no members bound to the input. + * + * @param context The generation context. + * @param operation The operation whose input is being serialized. + * @return True if a default body should be generated. + */ + protected boolean shouldWriteDefaultBody(GenerationContext context, OperationShape operation) { + return !HttpBindingIndex.of(context.model()).getRequestBindings(operation, DOCUMENT).isEmpty(); + } + + @Override + public void generateRequestSerializers(GenerationContext context) { + var topDownIndex = TopDownIndex.of(context.model()); + var delegator = context.writerDelegator(); + var configSymbol = CodegenUtils.getConfigSymbol(context.settings()); + var transportRequest = context.applicationProtocol().requestType(); + + for (OperationShape operation : topDownIndex.getContainedOperations(context.settings().getService())) { + var serFunction = getSerializationFunction(context, operation); + var input = context.model().expectShape(operation.getInputShape()); + var inputSymbol = context.symbolProvider().toSymbol(input); + + delegator.useFileWriter(serFunction.getDefinitionFile(), serFunction.getNamespace(), writer -> { + writer.pushState(new RequestSerializerSection(operation)); + writer.write(""" + async def $L(input: $T, config: $T) -> $T: + ${C|} + """, serFunction.getName(), inputSymbol, configSymbol, transportRequest, + writer.consumer(w -> generateRequestSerializer(context, operation, w))); + writer.popState(); + }); + } + + generateDocumentBodyShapeSerializers(context, serializingDocumentShapes); + } + + /** + * Generates the content of the operation request serializer. + * + *

Serialization of the http-level components will be inline + * since there isn't any use for them elsewhere. Serialization + * of document body components should be delegated, however, + * as they will need to be re-used in all likelihood. + * + *

This function has the following in scope: + *

    + *
  • input - the operation's input
    • + *
  • config - the client config
    • + *
+ */ + private void generateRequestSerializer( + GenerationContext context, + OperationShape operation, + PythonWriter writer + ) { + var httpTrait = operation.expectTrait(HttpTrait.class); + var bindingIndex = HttpBindingIndex.of(context.model()); + serializePath(context, writer, operation, bindingIndex); + serializeQuery(context, writer, operation, bindingIndex); + serializeBody(context, writer, operation, bindingIndex); + serializeHeaders(context, writer, operation); + + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python._private.http", "HTTPRequest", "_HTTPRequest"); + writer.addImport("smithy_python._private", "URI", "_URI"); + + writer.write(""" + return _HTTPRequest( + destination=_URI( + host="", + path=path, + scheme="https", + query=query, + ), + method=$S, + fields=headers, + body=body, + ) + """, httpTrait.getMethod()); + } + + /** + * A section that controls writing out the entire serialization function. + * + * @param operation The operation whose serializer is being generated. + */ + public record RequestSerializerSection(OperationShape operation) implements CodeSection {} + + /** + * Serializes headers, including standard headers like content-type + * and protocol-specific standard headers. + */ + private void serializeHeaders( + GenerationContext context, + PythonWriter writer, + OperationShape operation + ) { + writer.pushState(new SerializeFieldsSection(operation)); + writer.addImports("smithy_python._private", Set.of("Field", "Fields")); + writer.write(""" + headers = Fields( + [ + ${C|} + ${C|} + ${C|} + ] + ) + + """, + writer.consumer(w -> writeContentType(context, w, operation)), + writer.consumer(w -> writeContentLength(context, w, operation)), + writer.consumer(w -> writeDefaultHeaders(context, w, operation))); + serializeIndividualHeaders(context, writer, operation); + serializePrefixHeaders(context, writer, operation); + writer.popState(); + } + + /** + * Gets the default content-type when a document is synthesized in the body. + * + * @return Returns the default content-type. + */ + protected abstract String getDocumentContentType(); + + private void writeContentType(GenerationContext context, PythonWriter writer, OperationShape operation) { + // Content type can be determined by a few factors, such as the mediaType + // trait or the protocol default. The HttpBindingIndex knows about these + // factors, so it can do most of the work of finding that for us. + var httpIndex = HttpBindingIndex.of(context.model()); + var optionalContentType = httpIndex.determineRequestContentType(operation, getDocumentContentType()); + if (optionalContentType.isEmpty() && shouldWriteDefaultBody(context, operation)) { + optionalContentType = Optional.of(getDocumentContentType()); + } + optionalContentType.ifPresent(contentType -> writer.write( + "Field(name=\"Content-Type\", values=[$S]),", contentType)); + } + + private void writeContentLength(GenerationContext context, PythonWriter writer, OperationShape operation) { + // If a payload is streaming, it generally won't have a content length + // because members with that trait can be arbitrarily large and the act + // of calculating the length can be difficult, particularly if the source + // is non-seekable. + // see: https://smithy.io/2.0/spec/streaming.html#smithy-api-streaming-trait + if (isStreamingPayloadInput(context, operation)) { + // The requiresLength trait, however, can force a length calculation. + // see: https://smithy.io/2.0/spec/streaming.html#requireslength-trait + if (requiresLength(context, operation)) { + writer.write("Field(name=\"Content-Length\", values=[str(content_length)]),"); + } + return; + } + + // If there is nothing bound to the body, there's no need to add a length. + var hasBodyBindings = HttpBindingIndex.of(context.model()) + .getRequestBindings(operation).values().stream() + .anyMatch(binding -> binding.getLocation() == PAYLOAD || binding.getLocation() == DOCUMENT); + + if (hasBodyBindings) { + writer.write("Field(name=\"Content-Length\", values=[str(content_length)]),"); + } + } + + private boolean isStreamingPayloadInput(GenerationContext context, OperationShape operation) { + var payloadBinding = HttpBindingIndex.of(context.model()).getRequestBindings(operation, PAYLOAD); + if (payloadBinding.isEmpty()) { + return false; + } + // see: https://smithy.io/2.0/spec/streaming.html#smithy-api-streaming-trait + return payloadBinding.get(0).getMember().getMemberTrait(context.model(), StreamingTrait.class).isPresent(); + } + + private boolean requiresLength(GenerationContext context, OperationShape operation) { + var payloadBinding = HttpBindingIndex.of(context.model()).getRequestBindings(operation, PAYLOAD); + if (payloadBinding.isEmpty()) { + return false; + } + // see: https://smithy.io/2.0/spec/streaming.html#requireslength-trait + return payloadBinding.get(0).getMember().getMemberTrait(context.model(), RequiresLengthTrait.class).isPresent(); + } + + /** + * Writes any additional HTTP input headers required by the protocol implementation. + * + * @param context The generation context. + * @param writer The writer to write to. + * @param operation The operation whose input is being generated. + */ + protected void writeDefaultHeaders(GenerationContext context, PythonWriter writer, OperationShape operation) { + } + + /** + * Serialize headers that are bound with the httpHeader trait, where a single + * member maps to a single http header key. + * + *

{@see https://smithy.io/2.0/spec/http-bindings.html#httpheader-trait} + */ + private void serializeIndividualHeaders(GenerationContext context, PythonWriter writer, OperationShape operation) { + var index = HttpBindingIndex.of(context.model()); + var headerBindings = index.getRequestBindings(operation, HEADER); + for (HttpBinding binding : headerBindings) { + var target = context.model().expectShape(binding.getMember().getTarget()); + + // Empty strings and lists have no meaning as header values, so we don't + // want to try to serialize them. The falsey values for booleans, + // numbers, and timestamps on the other hand are serializeable. + boolean accessFalsey = !(target.isStringShape() || target.isListShape()); + + CodegenUtils.accessStructureMember(context, writer, "input", binding.getMember(), accessFalsey, () -> { + var pythonName = context.symbolProvider().toMemberName(binding.getMember()); + + if (target.isListShape()) { + var listMember = target.asListShape().get().getMember(); + var listTarget = context.model().expectShape(listMember.getTarget()); + var inputValue = listTarget.accept(new HttpMemberSerVisitor( + context, writer, binding.getLocation(), "e", listMember, + getDocumentTimestampFormat())); + + var trailer = listTarget.isStringShape() ? " if e" : ""; + writer.addImport("smithy_python._private", "tuples_to_fields"); + writer.write(""" + headers.extend(tuples_to_fields(($S, $L) for e in input.$L$L)) + """, binding.getLocationName(), inputValue, pythonName, trailer); + } else { + var dataSource = "input." + pythonName; + var inputValue = target.accept(new HttpMemberSerVisitor( + context, writer, binding.getLocation(), dataSource, binding.getMember(), + getDocumentTimestampFormat())); + writer.write( + "headers.extend(Fields([Field(name=$S, values=[$L])]))", + binding.getLocationName(), inputValue); + } + }); + } + } + + /** + * Serialize headers that are bound with the httpPrefixHeaders trait. This + * is where a dict is given as input and each key in the dict represents a + * separate header key. + * + *

{@see https://smithy.io/2.0/spec/http-bindings.html#httpprefixheaders-trait} + */ + private void serializePrefixHeaders(GenerationContext context, PythonWriter writer, OperationShape operation) { + var index = HttpBindingIndex.of(context.model()); + var prefixHeaderBindings = index.getRequestBindings(operation, PREFIX_HEADERS); + for (HttpBinding binding : prefixHeaderBindings) { + CodegenUtils.accessStructureMember(context, writer, "input", binding.getMember(), () -> { + var pythonName = context.symbolProvider().toMemberName(binding.getMember()); + var target = context.model().expectShape(binding.getMember().getTarget(), MapShape.class); + var valueTarget = context.model().expectShape(target.getValue().getTarget()); + var inputValue = valueTarget.accept(new HttpMemberSerVisitor( + context, writer, binding.getLocation(), "v", target.getValue(), + getDocumentTimestampFormat())); + writer.addImport("smithy_python._private", "tuples_to_fields"); + writer.write(""" + headers.extend( + tuples_to_fields((f'$L{k}', $L) for k, v in input.$L.items() if v) + ) + """, binding.getLocationName( + ), inputValue, pythonName); + }); + } + + } + + /** + * A section that controls serializing HTTP fields, namely headers. + * + *

By default, it handles setting protocol default values and values based on + * the smithy.api#httpHeader and smithy.api#httpPrefixHeaders traits. + * + * @param operation The operation whose fields section is being generated. + */ + public record SerializeFieldsSection(OperationShape operation) implements CodeSection {} + + /** + * Serializes the path, including resolving any path bindings. + */ + private void serializePath( + GenerationContext context, + PythonWriter writer, + OperationShape operation, + HttpBindingIndex bindingIndex + ) { + writer.pushState(new SerializePathSection(operation)); + + // Get a map of member name to label bindings. The URI pattern we fetch uses the member name + // for the content of label segments, so this lets us look up the extra info we need for those. + // see: https://smithy.io/2.0/spec/http-bindings.html#httplabel-trait + var labelBindings = bindingIndex.getRequestBindings(operation, LABEL).stream() + .collect(Collectors.toMap(HttpBinding::getMemberName, httpBinding -> httpBinding)); + + // Build up a format string that will produce the path. We could have used an f-string, but they end up + // taking a ton of space and aren't easily formatted. Using .format results in something that is much + // easier to grok. + var formatString = new StringBuilder(); + var uri = operation.expectTrait(HttpTrait.class).getUri(); + for (SmithyPattern.Segment segment : uri.getSegments()) { + formatString.append("/"); + if (segment.isLabel()) { + var httpBinding = labelBindings.get(segment.getContent()); + var memberName = context.symbolProvider().toMemberName(httpBinding.getMember()); + + // Pattern members must be non-empty and non-none, so we assert that here. + // Note that we've not actually started writing the format string out yet, which + // is why we can just write out these guard clauses here. + writer.write(""" + if not input.$1L: + raise $2T("$1L must not be empty.") + """, memberName, CodegenUtils.getServiceError(context.settings())); + + // We're creating an f-string, so here we just put the contents inside some brackets to allow + // for string interpolation. + formatString.append("{"); + formatString.append(memberName); + formatString.append("}"); + } else { + // Static segments just get inserted literally. + formatString.append(segment.getContent()); + } + } + + if (uri.getLabels().isEmpty()) { + writer.write("path = $S", formatString.toString()); + writer.popState(); + return; + } + + // Write out the f-string + writer.openBlock("path = $S.format(", ")", formatString.toString(), () -> { + writer.addStdlibImport("urllib.parse", "quote", "urlquote"); + for (Segment labelSegment : uri.getLabels()) { + var httpBinding = labelBindings.get(labelSegment.getContent()); + var memberName = context.symbolProvider().toMemberName(httpBinding.getMember()); + + // urllib.parse.quote will, by default, allow forward slashes. This is fine for + // greedy labels, which are expected to contain multiple segments. But normal + // labels aren't allowed to contain multiple segments, so we need to encode them + // there too. We do this here by adding a conditional argument specifying no safe + // characters. + var urlSafe = labelSegment.isGreedyLabel() ? "" : ", safe=''"; + + var dataSource = "input." + memberName; + var target = context.model().expectShape(httpBinding.getMember().getTarget()); + var inputValue = target.accept(new HttpMemberSerVisitor( + context, writer, httpBinding.getLocation(), dataSource, httpBinding.getMember(), + getDocumentTimestampFormat())); + writer.write("$1L=urlquote($3L$2L),", memberName, urlSafe, inputValue); + } + }); + + writer.popState(); + } + + /** + * A section that controls path serialization. + * + *

By default, it handles setting static values and labels based on + * the smithy.api#http and smithy.api#httpLabel traits. + * + * @param operation The operation whose path section is being generated. + */ + public record SerializePathSection(OperationShape operation) implements CodeSection {} + + /** + * Serializes the query in the form of a list of tuples. + */ + private void serializeQuery( + GenerationContext context, + PythonWriter writer, + OperationShape operation, + HttpBindingIndex bindingIndex + ) { + writer.pushState(new SerializeQuerySection(operation)); + var httpTrait = operation.expectTrait(HttpTrait.class); + writeStaticQuerySegment(writer, httpTrait.getUri()); + + if (hasQueryBindings(context, operation)) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.httputils", "join_query_params"); + writer.write(""" + query_params: list[tuple[str, str | None]] = [] + ${C|} + ${C|} + query = join_query_params(params=query_params, prefix=query) + + """, + writer.consumer(w -> serializeIndividualQueryParams(context, w, operation, bindingIndex)), + writer.consumer(w -> serializeQueryParamsMap(context, w, operation, bindingIndex))); + } + + writer.popState(); + } + + private boolean hasQueryBindings(GenerationContext context, OperationShape operation) { + return HttpBindingIndex.of(context.model()) + .getRequestBindings(operation).values().stream() + .anyMatch(binding -> binding.getLocation() == QUERY || binding.getLocation() == QUERY_PARAMS); + } + + // The http trait can add static query literals as part of its 'uri' property. + // see: https://smithy.io/2.0/spec/http-bindings.html#query-string-literals + private void writeStaticQuerySegment(PythonWriter writer, UriPattern uri) { + writer.writeInline("query: str = f'"); + + var queryLiterals = new ArrayList<>(uri.getQueryLiterals().entrySet()); + if (queryLiterals.size() != 0) { + writer.addStdlibImport("urllib.parse", "quote", "urlquote"); + } + + for (int i = 0; i < queryLiterals.size(); i++) { + if (i != 0) { + writer.writeInline("&"); + } + var entry = queryLiterals.get(i); + if (StringUtils.isBlank(entry.getValue())) { + writer.writeInline("{urlquote($S, safe=\"\")}", entry.getKey()); + } else { + writer.writeInline("{urlquote($S, safe=\"\")}={urlquote($S, safe=\"\")}", + entry.getKey(), entry.getValue()); + } + } + + writer.write("'\n"); + } + + /** + * Serializes individual query params bound with the queryParam trait, + * where a single member maps to a single query key. + * + *

{@see https://smithy.io/2.0/spec/http-bindings.html#httpquery-trait} + */ + private void serializeIndividualQueryParams( + GenerationContext context, + PythonWriter writer, + OperationShape operation, + HttpBindingIndex bindingIndex + ) { + var queryBindings = bindingIndex.getRequestBindings(operation, QUERY); + for (HttpBinding binding : queryBindings) { + var memberName = context.symbolProvider().toMemberName(binding.getMember()); + var locationName = binding.getLocationName(); + var target = context.model().expectShape(binding.getMember().getTarget()); + + CodegenUtils.accessStructureMember(context, writer, "input", binding.getMember(), () -> { + if (target.isListShape()) { + var listMember = target.asListShape().get().getMember(); + var listTarget = context.model().expectShape(listMember.getTarget()); + var memberSerializer = listTarget.accept(new HttpMemberSerVisitor( + context, writer, QUERY, "e", listMember, + getDocumentTimestampFormat())); + writer.write("query_params.extend(($S, $L) for e in input.$L)", + locationName, memberSerializer, memberName); + } else { + var memberSerializer = target.accept(new HttpMemberSerVisitor( + context, writer, QUERY, "input." + memberName, binding.getMember(), + getDocumentTimestampFormat())); + writer.write("query_params.append(($S, $L))", locationName, memberSerializer); + } + }); + } + } + + /** + * Serialize query params that are bound with the httpQueryParams trait. This + * is where a dict is given as input and each key in the dict represents a + * separate query key. + * + *

{@see https://smithy.io/2.0/spec/http-bindings.html#httpqueryparams-trait} + */ + private void serializeQueryParamsMap( + GenerationContext context, + PythonWriter writer, + OperationShape operation, + HttpBindingIndex bindingIndex + ) { + var queryMapBindings = bindingIndex.getRequestBindings(operation, QUERY_PARAMS); + for (HttpBinding binding : queryMapBindings) { + var memberName = context.symbolProvider().toMemberName(binding.getMember()); + var mapShape = context.model().expectShape(binding.getMember().getTarget(), MapShape.class); + var mapTarget = context.model().expectShape(mapShape.getValue().getTarget()); + + CodegenUtils.accessStructureMember(context, writer, "input", binding.getMember(), () -> { + if (mapTarget.isListShape()) { + var listMember = mapTarget.asListShape().get().getMember(); + var listMemberTarget = context.model().expectShape(listMember.getTarget()); + var memberSerializer = listMemberTarget.accept(new HttpMemberSerVisitor( + context, writer, QUERY, "v", listMember, + getDocumentTimestampFormat())); + writer.write("query_params.extend((k, $1L) for k in input.$2L for v in input.$2L[k])", + memberSerializer, memberName); + } else { + var memberSerializer = mapTarget.accept(new HttpMemberSerVisitor( + context, writer, QUERY, "v", mapShape.getValue(), + getDocumentTimestampFormat())); + writer.write("query_params.extend((k, $L) for k, v in input.$L.items())", + memberSerializer, memberName); + } + }); + } + } + + /** + * A section that controls query serialization. + * + *

By default, it handles setting static values and key-value pairs based on + * smithy.api#httpQuery and smithy.api#httpQueryParams. + * + * @param operation The operation whose query section is being generated. + */ + public record SerializeQuerySection(OperationShape operation) implements CodeSection {} + + /** + * Orchestrates body serialization. + * + *

The format of the body is going to be dependent on the specific + * protocol, so this delegates out to implementors. + */ + private void serializeBody( + GenerationContext context, + PythonWriter writer, + OperationShape operation, + HttpBindingIndex bindingIndex + ) { + writer.pushState(new SerializeBodySection(operation)); + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.interfaces.blobs", "AsyncBytesReader"); + writer.addStdlibImport("typing", "AsyncIterable"); + writer.write("body: AsyncIterable[bytes] = AsyncBytesReader(b'')"); + + // Any member that isn't explicitly bound to some part of the http + // request is bound to the document, where "document" refers to a + // structured http body value. + var documentBindings = bindingIndex.getRequestBindings(operation, DOCUMENT); + + // A payload binding is when a particular member is bound to the + // http body specifically. + // see: https://smithy.io/2.0/spec/http-bindings.html#httppayload-trait + var payloadBindings = bindingIndex.getRequestBindings(operation, PAYLOAD); + + if (!payloadBindings.isEmpty()) { + var binding = payloadBindings.get(0); + serializePayloadBody(context, writer, operation, binding); + var target = context.model().expectShape(binding.getMember().getTarget()); + serializingDocumentShapes.add(target); + } else if (!documentBindings.isEmpty() || shouldWriteDefaultBody(context, operation)) { + serializeDocumentBody(context, writer, operation, documentBindings); + for (HttpBinding binding : documentBindings) { + var target = context.model().expectShape(binding.getMember().getTarget()); + serializingDocumentShapes.add(target); + } + } + writer.popState(); + } + + /** + * A section that controls serializing the request body. + * + *

By default, it handles calling out to body serialization functions for every + * input member that is bound to the document, or which uses the smithy.api#httpPayload + * trait. + * + * @param operation The operation whose body section is being generated. + */ + public record SerializeBodySection(OperationShape operation) implements CodeSection {} + + /** + * Writes the code needed to serialize a protocol input document. + * + *

Implementations of this method are expected to set a value to the + * {@code body} variable that will be serialized as the request body. + * This variable will already be defined in scope. + * + *

Implementations MUST also set the {@code content_length} variable + * to an integer representing the length of the serialized body. + * + *

Implementations MUST properly fill the body parameter even if no + * document bindings are present. + * + *

For example: + * + * 

{@code
+     * body_params: dict[str, Any] = {}
+     *
+     * if input.spam:
+     *   body_params['spam'] = input.spam
+     *
+     * body = AsyncBytesReader(json.dumps(body_params).encode('utf-8'))
+     * }
+ * @param context The generation context. + * @param writer The writer to write to. + * @param operation The operation whose input is being generated. + * @param documentBindings The bindings to place in the document. + */ + protected abstract void serializeDocumentBody( + GenerationContext context, + PythonWriter writer, + OperationShape operation, + List documentBindings + ); + + /** + * Generates serialization functions for shapes in the given set. + * + *

These are the functions that serializeDocumentBody will call out to. + * + * @param context The generation context. + * @param shapes The shapes to generate deserialization for. + */ + protected abstract void generateDocumentBodyShapeSerializers( + GenerationContext context, + Set shapes + ); + + /** + * Writes the code needed to serialize the input payload of a request. + * + *

Implementations of this method are expected to set a value to the + * {@code body} variable that will be serialized as the request body. + * This variable will already be defined in scope. + * + *

Implementations MUST also set the {@code content_length} variable + * to an integer representing the length of the serialized body unless + * the body has the streaming trait without the requiresLength trait. + * + *

For example: + * + * 

{@code
+     * body = AsyncBytesReader(b64encode(input.body))
+     * }
+ * + * {@see https://smithy.io/2.0/spec/http-bindings.html#httppayload-trait} + * + * @param context The generation context. + * @param writer The writer to write to. + * @param operation The operation whose input is being generated. + * @param payloadBinding The payload binding to serialize. + */ + protected abstract void serializePayloadBody( + GenerationContext context, + PythonWriter writer, + OperationShape operation, + HttpBinding payloadBinding + ); + + @Override + public void generateResponseDeserializers(GenerationContext context) { + var topDownIndex = TopDownIndex.of(context.model()); + var service = context.settings().getService(context.model()); + var deserializingErrorShapes = new TreeSet(); + for (OperationShape operation : topDownIndex.getContainedOperations(context.settings().getService())) { + generateOperationResponseDeserializer(context, operation); + deserializingErrorShapes.addAll(operation.getErrors(service)); + } + for (ShapeId errorId : deserializingErrorShapes) { + var error = context.model().expectShape(errorId, StructureShape.class); + generateErrorResponseDeserializer(context, error); + } + generateDocumentBodyShapeDeserializers(context, deserializingDocumentShapes); + } + + /** + * Generates the content of the operation response deserializer. + * + *

Deserialization of the http-level components will be inline + * since there isn't any use for them elsewhere. Deserialization + * of document body components should be delegated, however, + * as they will need to be re-used in all likelihood. + * + *

This function has the following in scope: + *

    + *
  • http_response - the http-level response
    • + *
  • config - the client config
    • + *
+ */ + private void generateOperationResponseDeserializer( + GenerationContext context, + OperationShape operation + ) { + var delegator = context.writerDelegator(); + var deserFunction = getDeserializationFunction(context, operation); + var output = context.model().expectShape(operation.getOutputShape()); + var outputSymbol = context.symbolProvider().toSymbol(output); + var transportResponse = context.applicationProtocol().responseType(); + var configSymbol = CodegenUtils.getConfigSymbol(context.settings()); + var httpTrait = operation.expectTrait(HttpTrait.class); + var errorFunction = context.protocolGenerator().getErrorDeserializationFunction(context, operation); + + delegator.useFileWriter(deserFunction.getDefinitionFile(), deserFunction.getNamespace(), writer -> { + writer.pushState(new ResponseDeserializerSection(operation)); + writer.addStdlibImport("typing", "Any"); + + // The http trait can define a specific expected response code, and if we receive + // it then we shouldn't throw an exception. Otherwise, we can assume an error + // occurred if the status is 300+. + // see: https://smithy.io/2.0/spec/http-bindings.html#http-trait + writer.write(""" + async def $L(http_response: $T, config: $T) -> $T: + if http_response.status != $L and http_response.status >= 300: + raise await $T(http_response, config) + + kwargs: dict[str, Any] = {} + + ${C|} + + """, deserFunction.getName(), transportResponse, configSymbol, + outputSymbol, httpTrait.getCode(), errorFunction, + writer.consumer(w -> generateHttpResponseDeserializer(context, writer, operation))); + writer.popState(); + }); + + generateErrorDispatcher(context, operation, this::getErrorCode, this::resolveErrorCodeAndMessage); + } + + /** + * A section that controls writing out the entire deserialization function for an operation. + * + * @param operation The operation whose deserializer is being generated. + */ + public record ResponseDeserializerSection(OperationShape operation) implements CodeSection {} + + private void generateErrorResponseDeserializer(GenerationContext context, StructureShape error) { + var deserFunction = getErrorDeserializationFunction(context, error); + var errorSymbol = context.symbolProvider().toSymbol(error); + var delegator = context.writerDelegator(); + var transportResponse = context.applicationProtocol().responseType(); + var configSymbol = CodegenUtils.getConfigSymbol(context.settings()); + + delegator.useFileWriter(deserFunction.getDefinitionFile(), deserFunction.getNamespace(), writer -> { + writer.pushState(new ErrorDeserializerSection(error)); + writer.addStdlibImport("typing", "Any"); + writer.write(""" + async def $L( + http_response: $T, + config: $T, + parsed_body: dict[str, Document] | None, + default_message: str, + ) -> $T: + kwargs: dict[str, Any] = {"message": default_message} + + ${C|} + + """, deserFunction.getName(), transportResponse, configSymbol, errorSymbol, + writer.consumer(w -> generateHttpResponseDeserializer(context, writer, error))); + writer.popState(); + }); + } + + /** + * A section that controls writing out the entire deserialization function for an error. + * + * @param error The error whose deserializer is being generated. + */ + public record ErrorDeserializerSection(StructureShape error) implements CodeSection {} + + private void generateHttpResponseDeserializer( + GenerationContext context, + PythonWriter writer, + Shape operationOrError + ) { + var bindingIndex = HttpBindingIndex.of(context.model()); + var outputShape = getOutputShape(context, operationOrError); + var outputSymbol = context.symbolProvider().toSymbol(outputShape); + + writer.write(""" + ${C|} + + ${C|} + + ${C|} + + return $T(**kwargs) + """, + writer.consumer(w -> deserializeBody(context, w, operationOrError, bindingIndex)), + writer.consumer(w -> deserializeHeaders(context, w, operationOrError, bindingIndex)), + writer.consumer(w -> deserializeStatusCode(context, w, operationOrError, bindingIndex)), + outputSymbol); + } + + /** + * Maps error shapes to their error codes. + * + *

By default, this returns the error shape's name. + * + * @param error The error shape. + * @return The wire code matching the error shape. + */ + protected String getErrorCode(StructureShape error) { + return error.getId().getName(); + } + + /** + * Resolves the error code and message into the {@literal code} and {@literal message} + * variables, respectively. + * + * @param context The generation context. + * @param writer The writer to write to. + * @param canReadResponseBody If the http response body can be parsed by the delegator. + */ + protected abstract void resolveErrorCodeAndMessage( + GenerationContext context, + PythonWriter writer, + Boolean canReadResponseBody + ); + + private void deserializeHeaders( + GenerationContext context, + PythonWriter writer, + Shape operationOrError, + HttpBindingIndex bindingIndex + ) { + writer.pushState(new DeserializeFieldsSection(operationOrError)); + var individualBindings = bindingIndex.getResponseBindings(operationOrError, HEADER); + var prefixBindings = bindingIndex.getResponseBindings(operationOrError, PREFIX_HEADERS); + + if (!individualBindings.isEmpty() || !prefixBindings.isEmpty()) { + writer.write(""" + for fld in http_response.fields: + for key, value in fld.as_tuples(): + _key_lowercase = key.lower() + ${C|} + ${C|} + """, + writer.consumer(w -> deserializeIndividualHeaders(context, w, individualBindings)), + writer.consumer(w -> deserializePrefixHeaders(context, w, prefixBindings)) + ); + } + + writer.popState(); + } + + /** + * This implements deserialization for the {@literal httpHeader} trait. + * + *

{@see https://smithy.io/2.0/spec/http-bindings.html#httpheader-trait} + */ + private void deserializeIndividualHeaders( + GenerationContext context, + PythonWriter writer, + List bindings + ) { + if (bindings.isEmpty()) { + return; + } + writer.openBlock("match _key_lowercase:", "", () -> { + for (HttpBinding binding : bindings) { + // The httpHeader trait can be bound to a list, but not other + // collection types. + var target = context.model().expectShape(binding.getMember().getTarget()); + var memberName = context.symbolProvider().toMemberName(binding.getMember()); + var locationName = binding.getLocationName().toLowerCase(Locale.US); + var deserVisitor = new HttpMemberDeserVisitor( + context, writer, binding.getLocation(), "value", binding.getMember(), + getDocumentTimestampFormat() + ); + var targetHandler = target.accept(deserVisitor); + if (target.isListShape()) { + // A header list can be a comma-delimited single entry, a set of entries with + // the same header key, or a combination of the two. + writer.write(""" + case $1S: + _$2L = $3L + if $2S not in kwargs: + kwargs[$2S] = _$2L + else: + kwargs[$2S].extend(_$2L) + + """, locationName, memberName, targetHandler); + } else { + writer.write(""" + case $1S: + kwargs[$2S] = $3L + + """, locationName, memberName, targetHandler); + } + } + }); + + } + + /** + * This implements deserialization for the {@literal httpPrefixHeaders} trait. + * + *

{@see https://smithy.io/2.0/spec/http-bindings.html#httpprefixheaders-trait} + */ + private void deserializePrefixHeaders( + GenerationContext context, + PythonWriter writer, + List bindings + ) { + for (HttpBinding binding : bindings) { + var bindingTarget = context.model().expectShape(binding.getMember().getTarget()).asMapShape().get(); + var mapTarget = context.model().expectShape(bindingTarget.getValue().getTarget()); + var memberName = context.symbolProvider().toMemberName(binding.getMember()); + var locationName = binding.getLocationName().toLowerCase(Locale.US); + var deserVisitor = new HttpMemberDeserVisitor( + context, writer, binding.getLocation(), "value", bindingTarget.getValue(), + getDocumentTimestampFormat() + ); + // Prefix headers can only be maps of string to string, and they can't be sparse. + writer.write(""" + if _key_lowercase.startswith($1S): + if $2S not in kwargs: + kwargs[$2S] = {} + kwargs[$2S][key[$3L:]] = $4L + + """, locationName, memberName, locationName.length(), mapTarget.accept(deserVisitor)); + } + } + + /** + * A section that controls deserializing HTTP fields, namely headers. + * + *

By default, it handles values based on smithy.api#httpHeader and + * smithy.api#httpPrefixHeaders traits. + * + * @param operationOrError The operation or error whose fields section is being generated. + */ + public record DeserializeFieldsSection(Shape operationOrError) implements CodeSection {} + + /** + * Deserailizes the http status code to an output member. + * + *

{@see https://smithy.io/2.0/spec/http-bindings.html#httpresponsecode-trait} + */ + private void deserializeStatusCode( + GenerationContext context, + PythonWriter writer, + Shape operationOrError, + HttpBindingIndex bindingIndex + ) { + writer.pushState(new DeserializeStatusCodeSection(operationOrError)); + var statusBinding = bindingIndex.getResponseBindings(operationOrError, Location.RESPONSE_CODE); + if (!statusBinding.isEmpty()) { + var statusMember = context.symbolProvider().toMemberName(statusBinding.get(0).getMember()); + writer.write("kwargs[$S] = http_response.status", statusMember); + } + writer.popState(); + } + + /** + * A section that controls deserializing the HTTP status code. + * + * @param operationOrError The operation or error whose status code section is being generated. + */ + public record DeserializeStatusCodeSection(Shape operationOrError) implements CodeSection {} + + private void deserializeBody( + GenerationContext context, + PythonWriter writer, + Shape operationOrError, + HttpBindingIndex bindingIndex + ) { + // Any member that isn't explicitly bound to some part of the http + // request is bound to the document, where "document" refers to a + // structured http body value. + writer.pushState(new DeserializeBodySection(operationOrError)); + var documentBindings = bindingIndex.getResponseBindings(operationOrError, DOCUMENT); + if (!documentBindings.isEmpty()) { + deserializeDocumentBody(context, writer, operationOrError, documentBindings); + for (HttpBinding binding : documentBindings) { + var target = context.model().expectShape(binding.getMember().getTarget()); + deserializingDocumentShapes.add(target); + } + } + + // A payload binding is when a particular member is bound to the + // http body specifically. + // see: https://smithy.io/2.0/spec/http-bindings.html#httppayload-trait + var payloadBindings = bindingIndex.getResponseBindings(operationOrError, PAYLOAD); + if (!payloadBindings.isEmpty()) { + var binding = payloadBindings.get(0); + deserializePayloadBody(context, writer, operationOrError, binding); + var target = context.model().expectShape(binding.getMember().getTarget()); + deserializingDocumentShapes.add(target); + } + writer.popState(); + } + + /** + * A section that controls deserializing the response body. + * + *

By default, it handles calling out to body deserialization functions for every + * output member that is bound to the document, or which uses the smithy.api#httpPayload + * trait. + * + * @param operationOrError The operation or error whose body section is being generated. + */ + public record DeserializeBodySection(Shape operationOrError) implements CodeSection {} + + /** + * Writes the code needed to deserialize a protocol output document. + * + *

The contents of the response body will be available in the + * {@code http_response} variable. + * + *

For example: + * + * 

{@code
+     * data = json.loads(http_response.consume_body().decode('utf-8'))
+     * if 'spam' in data:
+     *     kwargs['spam'] = data['spam']
+     * }
+ * @param context The generation context. + * @param writer The writer to write to. + * @param operationOrError The operation or error whose output document is being deserialized. + * @param documentBindings The bindings to read from the document. + */ + protected abstract void deserializeDocumentBody( + GenerationContext context, + PythonWriter writer, + Shape operationOrError, + List documentBindings + ); + + + /** + * Generates deserialization functions for shapes in the given set. + * + *

These are the functions that deserializeDocumentBody will call out to. + * + * @param context The generation context. + * @param shapes The shapes to generate deserialization for. + */ + protected abstract void generateDocumentBodyShapeDeserializers( + GenerationContext context, + Set shapes + ); + + /** + * Writes the code needed to deserialize the output payload of a response. + * + *

{@see https://smithy.io/2.0/spec/http-bindings.html#httppayload-trait} + * + * @param context The generation context. + * @param writer The writer to write to. + * @param operationOrError The operation or error whose output payload is being deserialized. + * @param payloadBinding The payload binding to deserialize. + */ + protected abstract void deserializePayloadBody( + GenerationContext context, + PythonWriter writer, + Shape operationOrError, + HttpBinding payloadBinding + ); + + /** + * Given context and a source of data, generate an input value provider for the + * shape. This may use native types or invoke complex type serializers to + * manipulate the dataSource into the proper input content. + */ + private static class HttpMemberSerVisitor extends ShapeVisitor.Default { + private final GenerationContext context; + private final PythonWriter writer; + private final String dataSource; + private final Location bindingType; + private final MemberShape member; + private final Format defaultTimestampFormat; + + /** + * @param context The generation context. + * @param writer The writer to add dependencies to. + * @param bindingType How this value is bound to the operation input. + * @param dataSource The in-code location of the data to provide an output of + * ({@code input.foo}, {@code entry}, etc.) + * @param member The member that points to the value being provided. + * @param defaultTimestampFormat The default timestamp format to use. + */ + HttpMemberSerVisitor( + GenerationContext context, + PythonWriter writer, + Location bindingType, + String dataSource, + MemberShape member, + Format defaultTimestampFormat + ) { + this.context = context; + this.writer = writer; + this.dataSource = dataSource; + this.bindingType = bindingType; + this.member = member; + this.defaultTimestampFormat = defaultTimestampFormat; + } + + @Override + protected String getDefault(Shape shape) { + var protocolName = context.protocolGenerator().getName(); + throw new CodegenException(String.format( + "Unsupported %s binding of %s to %s in %s using the %s protocol", + bindingType, member.getMemberName(), shape.getType(), member.getContainer(), protocolName)); + } + + @Override + public String blobShape(BlobShape shape) { + if (member.getMemberTrait(context.model(), StreamingTrait.class).isPresent()) { + return dataSource; + } + writer.addStdlibImport("base64", "b64encode"); + return format("b64encode(%s).decode('utf-8')", dataSource); + } + + @Override + public String booleanShape(BooleanShape shape) { + return String.format("('true' if %s else 'false')", dataSource); + } + + @Override + public String stringShape(StringShape shape) { + if (bindingType == Location.HEADER) { + if (shape.hasTrait(MediaTypeTrait.class)) { + writer.addStdlibImport("base64", "b64encode"); + return format("b64encode(%s.encode('utf-8')).decode('utf-8')", dataSource); + } + } + return dataSource; + } + + @Override + public String byteShape(ByteShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String shortShape(ShortShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String integerShape(IntegerShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String longShape(LongShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String bigIntegerShape(BigIntegerShape shape) { + return integerShape(); + } + + private String integerShape() { + return String.format("str(%s)", dataSource); + } + + @Override + public String floatShape(FloatShape shape) { + // TODO: use strict parsing + return floatShapes(); + } + + @Override + public String doubleShape(DoubleShape shape) { + // TODO: use strict parsing + return floatShapes(); + } + + @Override + public String bigDecimalShape(BigDecimalShape shape) { + return floatShapes(); + } + + private String floatShapes() { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "serialize_float"); + return String.format("serialize_float(%s)", dataSource); + } + + @Override + public String timestampShape(TimestampShape shape) { + var httpIndex = HttpBindingIndex.of(context.model()); + var format = switch (bindingType) { + case HEADER -> httpIndex.determineTimestampFormat(member, bindingType, Format.HTTP_DATE); + case LABEL -> httpIndex.determineTimestampFormat(member, bindingType, defaultTimestampFormat); + case QUERY -> httpIndex.determineTimestampFormat(member, bindingType, Format.DATE_TIME); + default -> + throw new CodegenException("Unexpected named member shape binding location `" + bindingType + "`"); + }; + + var result = HttpProtocolGeneratorUtils.getTimestampInputParam( + context, writer, dataSource, member, format); + if (format == Format.EPOCH_SECONDS) { + result = format("str(%s)", result); + } + return result; + } + } + + /** + * Given context and a source of data, generate an output value provider for the + * shape. This may use native types (like generating a datetime for timestamps) + * converters (like a b64decode) or invoke complex type deserializers to + * manipulate the dataSource into the proper output content. + */ + private static class HttpMemberDeserVisitor extends ShapeVisitor.Default { + + private final GenerationContext context; + private final PythonWriter writer; + private final String dataSource; + private final Location bindingType; + private final MemberShape member; + private final Format defaultTimestampFormat; + + /** + * @param context The generation context. + * @param writer The writer to add dependencies to. + * @param bindingType How this value is bound to the operation output. + * @param dataSource The in-code location of the data to provide an output of + * ({@code output.foo}, {@code entry}, etc.) + * @param member The member that points to the value being provided. + * @param defaultTimestampFormat The default timestamp format to use. + */ + HttpMemberDeserVisitor( + GenerationContext context, + PythonWriter writer, + Location bindingType, + String dataSource, + MemberShape member, + Format defaultTimestampFormat + ) { + this.context = context; + this.writer = writer; + this.dataSource = dataSource; + this.bindingType = bindingType; + this.member = member; + this.defaultTimestampFormat = defaultTimestampFormat; + } + + @Override + protected String getDefault(Shape shape) { + var protocolName = context.protocolGenerator().getName(); + throw new CodegenException(String.format( + "Unsupported %s binding of %s to %s in %s using the %s protocol", + bindingType, member.getMemberName(), shape.getType(), member.getContainer(), protocolName)); + } + + @Override + public String blobShape(BlobShape shape) { + if (bindingType == PAYLOAD) { + return dataSource; + } + throw new CodegenException("Unexpected blob binding location `" + bindingType + "`"); + } + + @Override + public String booleanShape(BooleanShape shape) { + switch (bindingType) { + case QUERY, LABEL, HEADER -> { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "strict_parse_bool"); + return "strict_parse_bool(" + dataSource + ")"; + } + default -> throw new CodegenException("Unexpected boolean binding location `" + bindingType + "`"); + } + } + + @Override + public String byteShape(ByteShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String shortShape(ShortShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String integerShape(IntegerShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String longShape(LongShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String bigIntegerShape(BigIntegerShape shape) { + return integerShape(); + } + + private String integerShape() { + return switch (bindingType) { + case QUERY, LABEL, HEADER, RESPONSE_CODE -> "int(" + dataSource + ")"; + default -> throw new CodegenException("Unexpected integer binding location `" + bindingType + "`"); + }; + } + + @Override + public String floatShape(FloatShape shape) { + // TODO: use strict parsing + return floatShapes(); + } + + @Override + public String doubleShape(DoubleShape shape) { + // TODO: use strict parsing + return floatShapes(); + } + + private String floatShapes() { + switch (bindingType) { + case QUERY, LABEL, HEADER -> { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "strict_parse_float"); + return "strict_parse_float(" + dataSource + ")"; + } + default -> throw new CodegenException("Unexpected float binding location `" + bindingType + "`"); + } + } + + @Override + public String bigDecimalShape(BigDecimalShape shape) { + switch (bindingType) { + case QUERY, LABEL, HEADER -> { + writer.addStdlibImport("decimal", "Decimal", "_Decimal"); + return "_Decimal(" + dataSource + ")"; + } + default -> throw new CodegenException("Unexpected bigDecimal binding location `" + bindingType + "`"); + } + } + + @Override + public String stringShape(StringShape shape) { + if ((bindingType == HEADER || bindingType == PREFIX_HEADERS) && shape.hasTrait(MediaTypeTrait.ID)) { + writer.addStdlibImport("base64", "b64decode"); + return "b64decode(" + dataSource + ").decode('utf-8')"; + } + + return dataSource; + } + + @Override + public String timestampShape(TimestampShape shape) { + HttpBindingIndex httpIndex = HttpBindingIndex.of(context.model()); + Format format = httpIndex.determineTimestampFormat(member, bindingType, defaultTimestampFormat); + var source = dataSource; + if (format == Format.EPOCH_SECONDS) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "strict_parse_float"); + source = "strict_parse_float(" + dataSource + ")"; + } + return HttpProtocolGeneratorUtils.getTimestampOutputParam(writer, source, member, format); + } + + @Override + public String listShape(ListShape shape) { + if (bindingType != HEADER) { + throw new CodegenException("Unexpected list binding location `" + bindingType + "`"); + } + var collectionTarget = context.model().expectShape(shape.getMember().getTarget()); + writer.addImport("smithy_python.httputils", "split_header"); + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + String split = String.format("split_header(%s or '')", dataSource);; + + // Headers that have HTTP_DATE formatted timestamps may not be quoted, so we need + // to enable special handling for them. + if (isHttpDate(shape.getMember(), collectionTarget)) { + split = String.format("split_header(%s or '', True)", dataSource); + } + + var targetDeserVisitor = new HttpMemberDeserVisitor( + context, writer, bindingType, "e.strip()", shape.getMember(), defaultTimestampFormat); + return String.format("[%s for e in %s]", collectionTarget.accept(targetDeserVisitor), split); + } + + private boolean isHttpDate(MemberShape member, Shape target) { + if (target.isTimestampShape()) { + HttpBindingIndex httpIndex = HttpBindingIndex.of(context.model()); + Format format = httpIndex.determineTimestampFormat(member, bindingType, Format.HTTP_DATE); + return format == Format.HTTP_DATE; + } + return false; + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java new file mode 100644 index 0000000000..27c437a611 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java @@ -0,0 +1,236 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import static java.lang.String.format; + +import java.util.Locale; +import java.util.function.Function; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.selector.Selector; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyUnstableApi; +import software.amazon.smithy.utils.TriConsumer; + +/** + * Utility methods for generating HTTP-based protocols. + */ +@SmithyUnstableApi +public final class HttpProtocolGeneratorUtils { + // Shape is an error on an operation shape that has the httpPayload trait applied to it + // See: https://smithy.io/2.0/spec/selectors.html for more information on selectors + private static final Selector PAYLOAD_ERROR_SELECTOR = Selector.parse( + "operation -[error]-> structure :test(> member :test([trait|httpPayload]))" + ); + + private HttpProtocolGeneratorUtils() { + } + + /** + * Given a format and a source of data, generate an input value provider for the + * timestamp. + * + * @param context The generation context. + * @param writer The writer this param is being written to. Used only to add + * dependencies if necessary. + * @param dataSource The in-code location of the data to provide an input of + * ({@code input.foo}, {@code entry}, etc.) + * @param shape The shape that represents the value being provided. + * @param format The timestamp format to provide. + * @return Returns a value or expression of the input timestamp. + */ + public static String getTimestampInputParam( + GenerationContext context, + PythonWriter writer, + String dataSource, + Shape shape, + Format format + ) { + writer.addImport("smithy_python.utils", "ensure_utc"); + var result = "ensure_utc(" + dataSource + ")"; + // see: https://smithy.io/2.0/spec/protocol-traits.html#smithy-api-timestampformat-trait + switch (format) { + case DATE_TIME: + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "serialize_rfc3339"); + return String.format("serialize_rfc3339(%s)", result); + case EPOCH_SECONDS: + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "serialize_epoch_seconds"); + return String.format("serialize_epoch_seconds(%s)", result); + case HTTP_DATE: + writer.addStdlibImport("email.utils", "format_datetime"); + return "format_datetime(" + result + ", usegmt=True)"; + default: + throw new CodegenException("Unexpected timestamp format `" + format + "` on " + shape); + } + } + + /** + * Given a format and a source of data, generate an output value provider for the + * timestamp. + * + * @param writer The current writer (so that imports may be added) + * @param dataSource The in-code location of the data to provide an output of + * ({@code output.foo}, {@code entry}, etc.) + * @param shape The shape that represents the value being received. + * @param format The timestamp format to provide. + * @return Returns a value or expression of the output timestamp. + */ + public static String getTimestampOutputParam( + PythonWriter writer, + String dataSource, + Shape shape, + Format format + ) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "expect_type"); + // see: https://smithy.io/2.0/spec/protocol-traits.html#smithy-api-timestampformat-trait + switch (format) { + case DATE_TIME -> { + writer.addImport("smithy_python.utils", "ensure_utc"); + writer.addStdlibImport("datetime", "datetime"); + return format("ensure_utc(datetime.fromisoformat(expect_type(str, %s)))", dataSource); + } + case EPOCH_SECONDS -> { + writer.addImport("smithy_python.utils", "epoch_seconds_to_datetime"); + return format("epoch_seconds_to_datetime(expect_type(int | float, %s))", dataSource); + } + case HTTP_DATE -> { + writer.addImport("smithy_python.utils", "ensure_utc"); + writer.addStdlibImport("email.utils", "parsedate_to_datetime"); + return format("ensure_utc(parsedate_to_datetime(expect_type(str, %s)))", dataSource); + } + default -> throw new CodegenException("Unexpected timestamp format `" + format + "` on " + shape); + } + } + + /** + * Generates a function that inspects error codes and dispatches to the proper deserialization function. + * + * @param context The generation context. + * @param operation The operation to generate the dispatcher for. + * @param errorShapeToCode A function that maps an error structure to it's code on the wire. + * @param errorMessageCodeGenerator A consumer that generates code to extract the error message and code. + * It must set the code to the variable {@literal code}, the message to + * {@literal message}, and if it parses the body it must set the parsed + * body to {@literal parsed_body}. + */ + public static void generateErrorDispatcher( + GenerationContext context, + OperationShape operation, + Function errorShapeToCode, + TriConsumer errorMessageCodeGenerator + ) { + var configSymbol = CodegenUtils.getConfigSymbol(context.settings()); + var transportResponse = context.applicationProtocol().responseType(); + var delegator = context.writerDelegator(); + var errorDispatcher = context.protocolGenerator().getErrorDeserializationFunction(context, operation); + var apiError = CodegenUtils.getApiError(context.settings()); + var unknownApiError = CodegenUtils.getUnknownApiError(context.settings()); + var canReadResponseBody = canReadResponseBody(operation, context.model()); + delegator.useFileWriter(errorDispatcher.getDefinitionFile(), errorDispatcher.getNamespace(), writer -> { + writer.pushState(new ErrorDispatcherSection(operation, errorShapeToCode, errorMessageCodeGenerator)); + writer.addStdlibImport("typing", "Any"); + writer.write(""" + async def $1L(http_response: $2T, config: $3T) -> $4T[Any]: + ${6C|} + + match code.lower(): + ${7C|} + + case _: + return $5T(message) + """, errorDispatcher.getName(), transportResponse, configSymbol, apiError, unknownApiError, + writer.consumer(w -> errorMessageCodeGenerator.accept(context, w, canReadResponseBody)), + writer.consumer(w -> errorCases(context, w, operation, errorShapeToCode))); + writer.popState(); + }); + } + + /** + * Checks if the http_response.body can be read for a given operation shape. + *

+ * If any of the errors for an operation contain an HttpPayload then it is not safe to read + * the body of the http_response. + * + * @param operationShape operation shape to check + * @param model smithy model + */ + private static boolean canReadResponseBody(OperationShape operationShape, Model model) { + return PAYLOAD_ERROR_SELECTOR.shapes(model) + .map(Shape::getId) + .noneMatch(shapeId -> operationShape.getErrors().contains(shapeId)); + } + + private static void errorCases( + GenerationContext context, + PythonWriter writer, + OperationShape operation, + Function errorShapeToCode + ) { + var errorIds = operation.getErrors(context.settings().getService(context.model())); + for (ShapeId errorId : errorIds) { + var error = context.model().expectShape(errorId, StructureShape.class); + var code = errorShapeToCode.apply(error).toLowerCase(Locale.US); + var deserFunction = context.protocolGenerator().getErrorDeserializationFunction(context, errorId); + writer.write(""" + case $S: + return await $T(http_response, config, parsed_body, message) + """, code, deserFunction); + } + } + + /** + * Gets the output shape for an error or operation. + *

+ * If the shape is an error, the error is returned, otherwise the operation output is returned. + * + * @param context Code generation context + * @param operationOrError operation or error shape to find output shape for + * @return output shape + */ + public static Shape getOutputShape(GenerationContext context, Shape operationOrError) { + var outputShape = operationOrError; + if (operationOrError.isOperationShape()) { + outputShape = context.model().expectShape(operationOrError.asOperationShape().get().getOutputShape()); + } + return outputShape; + } + + /** + * A section that controls writing out the error dispatcher function. + * + * @param operation The operation whose deserializer is being generated. + * @param errorShapeToCode A function that maps an error structure to it's code on the wire. + * @param errorMessageCodeGenerator A consumer that generates code to extract the error message and code. + */ + public record ErrorDispatcherSection( + OperationShape operation, + Function errorShapeToCode, + TriConsumer errorMessageCodeGenerator + ) implements CodeSection {} +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonMemberSerVisitor.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonMemberSerVisitor.java new file mode 100644 index 0000000000..8f8a9540fd --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonMemberSerVisitor.java @@ -0,0 +1,89 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.Set; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.ShapeType; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.utils.SetUtils; +import software.amazon.smithy.utils.SmithyUnstableApi; + + +/** + * Visitor to serialize member values for aggregate types into JSON document bodies. + * + *

This does not delegate to serializers for lists or maps that would return them + * unchanged. A list of booleans, for example, will never need any serialization changes. + */ +@SmithyUnstableApi +public class JsonMemberSerVisitor extends DocumentMemberSerVisitor { + + private static final Set NOOP_TARGETS = SetUtils.of( + ShapeType.STRING, ShapeType.ENUM, ShapeType.BOOLEAN, ShapeType.DOCUMENT, ShapeType.BYTE, ShapeType.SHORT, + ShapeType.INTEGER, ShapeType.INT_ENUM, ShapeType.LONG, ShapeType.FLOAT, ShapeType.DOUBLE + ); + + /** + * @param context The generation context. + * @param writer The writer to write to. + * @param member The member shape being deserialized. Used for any extra traits + * it might bear, such as the timestamp format. + * @param dataSource The in-code location of the data to provide an output of + * ({@code output.foo}, {@code entry}, etc.) + * @param defaultTimestampFormat The default timestamp format used in absence + * of a TimestampFormat trait. + */ + public JsonMemberSerVisitor( + GenerationContext context, + PythonWriter writer, + MemberShape member, + String dataSource, + Format defaultTimestampFormat + ) { + super(context, writer, member, dataSource, defaultTimestampFormat); + } + + @Override + public String listShape(ListShape listShape) { + if (isNoOpMember(listShape.getMember())) { + return dataSource(); + } + return super.listShape(listShape); + } + + @Override + public String mapShape(MapShape mapShape) { + if (isNoOpMember(mapShape.getValue())) { + return dataSource(); + } + return super.mapShape(mapShape); + } + + private boolean isNoOpMember(MemberShape member) { + var target = context().model().expectShape(member.getTarget()); + if (target.isListShape()) { + return isNoOpMember(target.asListShape().get().getMember()); + } else if (target.isMapShape()) { + return isNoOpMember(target.asMapShape().get().getValue()); + } + return NOOP_TARGETS.contains(target.getType()); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonPayloadDeserVisitor.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonPayloadDeserVisitor.java new file mode 100644 index 0000000000..896ff5880c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonPayloadDeserVisitor.java @@ -0,0 +1,129 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.knowledge.HttpBinding; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.DocumentShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.StreamingTrait; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.utils.SmithyUnstableApi; + + +/** + * Visitor to generate deserialization functions for shapes marked as HttpPayloads. + */ +@SmithyUnstableApi +public final class JsonPayloadDeserVisitor extends ShapeVisitor.Default { + private static final Format DEFAULT_EPOCH_FORMAT = Format.EPOCH_SECONDS; + + private final GenerationContext context; + private final PythonWriter writer; + private final MemberShape member; + + public JsonPayloadDeserVisitor(GenerationContext context, PythonWriter writer, HttpBinding binding) { + this.context = context; + this.writer = writer; + this.member = binding.getMember(); + } + + private DocumentMemberDeserVisitor getMemberVisitor(MemberShape member, String dataSource) { + return new DocumentMemberDeserVisitor(context, writer, member, dataSource, DEFAULT_EPOCH_FORMAT); + } + + protected Void getDefault(Shape shape) { + throw new CodegenException( + "Shape " + shape + " of type " + shape.getType() + " is not supported as an httpPayload." + ); + } + + @Override + public Void blobShape(BlobShape shape) { + // see: https://smithy.io/2.0/spec/streaming.html#smithy-api-streaming-trait + if (member.getMemberTrait(context.model(), StreamingTrait.class).isPresent()) { + writer.addImport("smithy_python.interfaces.blobs", "AsyncBytesReader"); + writer.write("kwargs[$S] = AsyncBytesReader(http_response.body)", + context.symbolProvider().toMemberName(member)); + } else { + writer.write(""" + if (body := await http_response.consume_body()): + kwargs[$1S] = body + + """, + context.symbolProvider().toMemberName(member) + ); + } + + return null; + } + + @Override + public Void stringShape(StringShape shape) { + writer.write(""" + if (body := await http_response.consume_body()): + kwargs[$S] = body.decode('utf-8') + + """, + context.symbolProvider().toMemberName(member) + ); + + return null; + } + + @Override + public Void structureShape(StructureShape shape) { + generateJsonDeserializerDelegation(); + return null; + } + + @Override + public Void unionShape(UnionShape shape) { + generateJsonDeserializerDelegation(); + return null; + } + + @Override + public Void documentShape(DocumentShape shape) { + generateJsonDeserializerDelegation(); + return null; + } + + private void generateJsonDeserializerDelegation() { + writer.addStdlibImport("json"); + + var target = context.model().expectShape(member.getTarget()); + var memberVisitor = getMemberVisitor(member, "json.loads(body)"); + var memberDeserializer = target.accept(memberVisitor); + + writer.write(""" + if (body := await http_response.consume_body()): + kwargs[$S] = $L + + """, + context.symbolProvider().toMemberName(member), + memberDeserializer + ); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonShapeDeserVisitor.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonShapeDeserVisitor.java new file mode 100644 index 0000000000..8d0b105564 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonShapeDeserVisitor.java @@ -0,0 +1,271 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.Collection; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.knowledge.NullableIndex; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ResourceShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.JsonNameTrait; +import software.amazon.smithy.model.traits.SparseTrait; +import software.amazon.smithy.model.traits.StringTrait; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Visitor to generate deserialization functions for shapes in JSON document bodies. + */ +@SmithyUnstableApi +public class JsonShapeDeserVisitor extends ShapeVisitor.Default { + private final GenerationContext context; + private final PythonWriter writer; + + /** + * Constructor. + * + * @param context The code generation context. + * @param writer The writer that will be written to. Used here only to add dependencies. + */ + public JsonShapeDeserVisitor(GenerationContext context, PythonWriter writer) { + this.context = context; + this.writer = writer; + } + + private DocumentMemberDeserVisitor getMemberVisitor(MemberShape member, String dataSource) { + return new DocumentMemberDeserVisitor(context(), writer, member, dataSource, Format.EPOCH_SECONDS); + } + + /** + * Gets the generation context. + * + * @return The generation context. + */ + protected final GenerationContext context() { + return context; + } + + @Override + protected Void getDefault(Shape shape) { + return null; + } + + @Override + public final Void operationShape(OperationShape shape) { + throw new CodegenException("Operation shapes cannot be bound to documents."); + } + + @Override + public final Void resourceShape(ResourceShape shape) { + throw new CodegenException("Resource shapes cannot be bound to documents."); + } + + @Override + public final Void serviceShape(ServiceShape shape) { + throw new CodegenException("Service shapes cannot be bound to documents."); + } + + @Override + public Void listShape(ListShape shape) { + var functionName = context.protocolGenerator().getDeserializationFunctionName(context, shape.getId()); + var config = CodegenUtils.getConfigSymbol(context.settings()); + var symbol = context.symbolProvider().toSymbol(shape); + var errorSymbol = CodegenUtils.getServiceError(context.settings()); + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.types", "Document", "Document"); + + var target = context.model().expectShape(shape.getMember().getTarget()); + var memberVisitor = getMemberVisitor(shape.getMember(), "e"); + var memberDeserializer = target.accept(memberVisitor); + + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-sparse-trait + var sparseTrailer = ""; + if (shape.hasTrait(SparseTrait.class)) { + sparseTrailer = " if e is not None else None"; + } + + writer.write(""" + def $1L(output: Document, config: $2T) -> $3T: + if not isinstance(output, list): + raise $5T(f"Expected list, found {type(output)}") + return [$4L$6L for e in output] + """, functionName, config, symbol, memberDeserializer, errorSymbol, sparseTrailer); + return null; + } + + @Override + public Void mapShape(MapShape shape) { + var functionName = context.protocolGenerator().getDeserializationFunctionName(context, shape.getId()); + var config = CodegenUtils.getConfigSymbol(context.settings()); + var symbol = context.symbolProvider().toSymbol(shape); + var errorSymbol = CodegenUtils.getServiceError(context.settings()); + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.types", "Document", "Document"); + + var valueShape = context.model().expectShape(shape.getValue().getTarget()); + var valueVisitor = getMemberVisitor(shape.getValue(), "v"); + var valueDeserializer = valueShape.accept(valueVisitor); + + writer.write(""" + def $1L(output: Document, config: $2T) -> $3T: + if not isinstance(output, dict): + raise $4T(f"Expected dict, found { type(output) }") + """, functionName, config, symbol, errorSymbol); + + writer.indent(); + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-sparse-trait + if (shape.hasTrait(SparseTrait.class)) { + writer.write("return {k: $L if v is not None else None for k, v in output.items()}", valueDeserializer); + } else { + writer.write("return {k: $L for k, v in output.items() if v is not None}", valueDeserializer); + } + writer.dedent(); + return null; + } + + @Override + public Void structureShape(StructureShape shape) { + var functionName = context.protocolGenerator().getDeserializationFunctionName(context, shape.getId()); + var config = CodegenUtils.getConfigSymbol(context.settings()); + var symbol = context.symbolProvider().toSymbol(shape); + var errorSymbol = CodegenUtils.getServiceError(context.settings()); + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.types", "Document", "Document"); + writer.write(""" + def $1L(output: Document, config: $2T) -> $3T: + if not isinstance(output, dict): + raise $4T(f"Expected dict, found {type(output)}") + + kwargs: dict[str, Any] = {} + + ${5C|} + + return $3T(**kwargs) + """, functionName, config, symbol, errorSymbol, (Runnable) () -> structureMembers(shape.members())); + return null; + } + + /** + * Generate deserializers for structure members. + * + *

The structure to deserialize must exist in the variable {@literal output}, and + * the results will be stored in a dict called {@literal kwargs}, which must also + * exist. + * + * @param members The members to generate serializers for. + */ + public void structureMembers(Collection members) { + var index = NullableIndex.of(context.model()); + var errorSymbol = CodegenUtils.getServiceError(context.settings()); + for (MemberShape member : members) { + var pythonName = context.symbolProvider().toMemberName(member); + var jsonName = locationName(member); + + var target = context.model().expectShape(member.getTarget()); + + if (index.isMemberNullable(member)) { + var memberVisitor = getMemberVisitor(member, "_" + pythonName); + var memberDeserializer = target.accept(memberVisitor); + writer.write(""" + if (_$1L := output.get($2S)) is not None: + kwargs[$1S] = $3L + + """, pythonName, jsonName, memberDeserializer); + } else { + var memberVisitor = getMemberVisitor(member, "output['" + jsonName + "']"); + var memberDeserializer = target.accept(memberVisitor); + writer.write(""" + if $2S not in output: + raise $4T('Expected to find $2S in the operation output, but it was not present.') + kwargs[$1S] = $3L + + """, pythonName, jsonName, memberDeserializer, errorSymbol); + } + } + } + + /** + * Gets the JSON key that will be used for a given member. + * + * @param member The member to inspect. + * @return The string key that will be used for the member. + */ + protected String locationName(MemberShape member) { + // see: https://smithy.io/2.0/spec/protocol-traits.html#smithy-api-jsonname-trait + return member.getMemberTrait(context.model(), JsonNameTrait.class) + .map(StringTrait::getValue) + .orElse(member.getMemberName()); + } + + @Override + public final Void unionShape(UnionShape shape) { + var functionName = context.protocolGenerator().getDeserializationFunctionName(context, shape.getId()); + var config = CodegenUtils.getConfigSymbol(context.settings()); + var symbol = context.symbolProvider().toSymbol(shape); + var errorSymbol = CodegenUtils.getServiceError(context.settings()); + var unknownSymbol = symbol.expectProperty("unknown", Symbol.class); + + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.types", "Document", "Document"); + writer.write(""" + def $1L(output: Document, config: $2T) -> $3T: + if not isinstance(output, dict): + raise $5T(f"Expected dict, found {type(output)}") + + if (count := len(output)) != 1: + raise $5T(f"Unions must have exactly one member set, found {count}.") + + tag, value = list(output.items())[0] + + match tag: + ${6C|} + + case _: + return $4T(tag) + """, functionName, config, symbol, unknownSymbol, errorSymbol, (Runnable) () -> unionMembers(shape)); + return null; + } + + private void unionMembers(UnionShape shape) { + for (MemberShape member : shape.members()) { + var jsonName = locationName(member); + var memberSymbol = context.symbolProvider().toSymbol(member); + + var target = context.model().expectShape(member.getTarget()); + var memberVisitor = getMemberVisitor(member, "value"); + var memberDeserializer = target.accept(memberVisitor); + + writer.write(""" + case $1S: + return $2T($3L) + + """, jsonName, memberSymbol, memberDeserializer); + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonShapeSerVisitor.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonShapeSerVisitor.java new file mode 100644 index 0000000000..50af63103a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/JsonShapeSerVisitor.java @@ -0,0 +1,238 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.Collection; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ResourceShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.JsonNameTrait; +import software.amazon.smithy.model.traits.SparseTrait; +import software.amazon.smithy.model.traits.StringTrait; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Visitor to generate serialization functions for shapes in JSON document bodies. + */ +@SmithyUnstableApi +public class JsonShapeSerVisitor extends ShapeVisitor.Default { + private final GenerationContext context; + private final PythonWriter writer; + + /** + * Constructor. + * + * @param context The code generation context. + * @param writer The writer that will be written to. Used here only to add dependencies. + */ + public JsonShapeSerVisitor(GenerationContext context, PythonWriter writer) { + this.context = context; + this.writer = writer; + } + + /** + * Gets the serialization visitor for a member. + * + * @param member The member to be serialized. + * @param dataSource The python variable / source containing the data to serialize. + * @return The serialization visitor for the member. + */ + protected DocumentMemberSerVisitor getMemberVisitor(MemberShape member, String dataSource) { + return new JsonMemberSerVisitor(context, writer, member, dataSource, Format.EPOCH_SECONDS); + } + + @Override + protected Void getDefault(Shape shape) { + return null; + } + + @Override + public final Void operationShape(OperationShape shape) { + throw new CodegenException("Operation shapes cannot be bound to documents."); + } + + @Override + public final Void resourceShape(ResourceShape shape) { + throw new CodegenException("Resource shapes cannot be bound to documents."); + } + + @Override + public final Void serviceShape(ServiceShape shape) { + throw new CodegenException("Service shapes cannot be bound to documents."); + } + + @Override + public Void listShape(ListShape shape) { + var functionName = context.protocolGenerator().getSerializationFunctionName(context, shape); + var config = CodegenUtils.getConfigSymbol(context.settings()); + var listSymbol = context.symbolProvider().toSymbol(shape); + + var target = context.model().expectShape(shape.getMember().getTarget()); + var memberVisitor = getMemberVisitor(shape.getMember(), "e"); + var memberSerializer = target.accept(memberVisitor); + + // If we're not doing a transform, there's no need to have a function for it. + if (memberSerializer.equals("e")) { + return null; + } + + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-sparse-trait + var sparseTrailer = ""; + if (shape.hasTrait(SparseTrait.class)) { + sparseTrailer = " if e is not None else None"; + } + + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.types", "Document"); + writer.write(""" + def $1L(input: $2T, config: $3T) -> list[Document]: + return [$4L$5L for e in input] + """, functionName, listSymbol, config, memberSerializer, sparseTrailer); + return null; + } + + @Override + public Void mapShape(MapShape shape) { + var functionName = context.protocolGenerator().getSerializationFunctionName(context, shape); + var config = CodegenUtils.getConfigSymbol(context.settings()); + var mapSymbol = context.symbolProvider().toSymbol(shape); + + var target = context.model().expectShape(shape.getValue().getTarget()); + var valueVisitor = getMemberVisitor(shape.getValue(), "v"); + var valueSerializer = target.accept(valueVisitor); + + // If we're not doing a transform, there's no need to have a function for it. + if (valueSerializer.equals("v")) { + return null; + } + + // see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-sparse-trait + var sparseTrailer = ""; + if (shape.hasTrait(SparseTrait.class)) { + sparseTrailer = " if v is not None else None"; + } + + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.types", "Document"); + writer.write(""" + def $1L(input: $2T, config: $3T) -> dict[str, Document]: + return {k: $4L$5L for k, v in input.items()} + """, functionName, mapSymbol, config, valueSerializer, sparseTrailer); + return null; + } + + @Override + public Void structureShape(StructureShape shape) { + var functionName = context.protocolGenerator().getSerializationFunctionName(context, shape); + var config = CodegenUtils.getConfigSymbol(context.settings()); + var structureSymbol = context.symbolProvider().toSymbol(shape); + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.types", "Document"); + + writer.write(""" + def $1L(input: $2T, config: $3T) -> dict[str, Document]: + result: dict[str, Document] = {} + + ${4C|} + return result + """, functionName, structureSymbol, config, (Runnable) () -> structureMembers(shape.members())); + return null; + } + + /** + * Generate serializers for structure members. + * + *

The structure to serialize must exist in the variable {@literal input}, and + * the output will be stored in a dict called {@literal result}, which must also + * exist. + * + * @param members The members to generate serializers for. + */ + public void structureMembers(Collection members) { + for (MemberShape member : members) { + var pythonName = context.symbolProvider().toMemberName(member); + var jsonName = locationName(member); + var target = context.model().expectShape(member.getTarget()); + + var memberVisitor = getMemberVisitor(member, "input." + pythonName); + var memberSerializer = target.accept(memberVisitor); + + CodegenUtils.accessStructureMember(context, writer, "input", member, () -> { + writer.write("result[$S] = $L\n", jsonName, memberSerializer); + }); + } + } + + /** + * Gets the JSON key that will be used for a given member. + * + * @param member The member to inspect. + * @return The string key that will be used for the member. + */ + protected String locationName(MemberShape member) { + // see: https://smithy.io/2.0/spec/protocol-traits.html#smithy-api-jsonname-trait + return member.getMemberTrait(context.model(), JsonNameTrait.class) + .map(StringTrait::getValue) + .orElse(member.getMemberName()); + } + + @Override + public Void unionShape(UnionShape shape) { + var functionName = context.protocolGenerator().getSerializationFunctionName(context, shape); + var config = CodegenUtils.getConfigSymbol(context.settings()); + var unionSymbol = context.symbolProvider().toSymbol(shape); + var errorSymbol = CodegenUtils.getServiceError(context.settings()); + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.types", "Document"); + + writer.write(""" + def $1L(input: $2T, config: $3T) -> dict[str, Document]: + match input: + ${5C|} + case _: + raise $4T(f"Unexpected union variant: {type(input)}") + """, functionName, unionSymbol, config, errorSymbol, (Runnable) () -> unionMembers(shape.members())); + return null; + } + + private void unionMembers(Collection members) { + for (MemberShape member : members) { + var jsonName = locationName(member); + var memberSymbol = context.symbolProvider().toSymbol(member); + var target = context.model().expectShape(member.getTarget()); + var memberVisitor = getMemberVisitor(member, "input.value"); + var memberSerializer = target.accept(memberVisitor); + + writer.write(""" + case $T(): + return {$S: $L} + """, memberSymbol, jsonName, memberSerializer); + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/ProtocolGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/ProtocolGenerator.java new file mode 100644 index 0000000000..a7c4069007 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/ProtocolGenerator.java @@ -0,0 +1,170 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import static java.lang.String.format; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ToShapeId; +import software.amazon.smithy.python.codegen.ApplicationProtocol; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.utils.CaseUtils; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Generates code to implement a protocol for both servers and clients. + */ +@SmithyUnstableApi +public interface ProtocolGenerator { + /** + * Gets the supported protocol {@link ShapeId}. + * + * @return Returns the protocol supported + */ + ShapeId getProtocol(); + + /** + * Gets the name of the protocol. + * + *

The default implementation is the ShapeId name of the protocol trait in + * Smithy models (e.g., "aws.protocols#restJson1" would return "restJson1"). + * + * @return Returns the protocol name. + */ + default String getName() { + return getProtocol().getName(); + } + + /** + * Creates an application protocol for the generator. + * + * @return Returns the created application protocol. + */ + ApplicationProtocol getApplicationProtocol(); + + + /** + * Generates the name of a serializer function for shapes of a service that is not protocol-specific. + * + * @param context The code generation context. + * @param shapeId The shape the serializer function is being generated for. + * @return Returns the generated function name. + */ + default String getSerializationFunctionName(GenerationContext context, ToShapeId shapeId) { + var name = context.settings().getService(context.model()).getContextualName(shapeId); + return "_serialize_" + CaseUtils.toSnakeCase(name); + } + + /** + * Generates the symbol for a serializer function for shapes of a service that is not protocol-specific. + * + * @param context The code generation context. + * @param shapeId The shape the serializer function is being generated for. + * @return Returns the generated symbol. + */ + default Symbol getSerializationFunction(GenerationContext context, ToShapeId shapeId) { + return Symbol.builder() + .name(getSerializationFunctionName(context, shapeId)) + .namespace(format("%s.serialize", context.settings().getModuleName()), "") + .definitionFile(format("./%s/serialize.py", context.settings().getModuleName())) + .build(); + } + + /** + * Generates the name of a deserializer function for shapes of a service that is not protocol-specific. + * + * @param context The code generation context. + * @param shapeId The shape the deserializer function is being generated for. + * @return Returns the generated function name. + */ + default String getDeserializationFunctionName(GenerationContext context, ToShapeId shapeId) { + var name = context.settings().getService(context.model()).getContextualName(shapeId); + return "_deserialize_" + CaseUtils.toSnakeCase(name); + } + + /** + * Generates the symbol for a deserializer function for shapes of a service that is not protocol-specific. + * + * @param context The code generation context. + * @param shapeId The shape the deserializer function is being generated for. + * @return Returns the generated symbol. + */ + default Symbol getDeserializationFunction(GenerationContext context, ToShapeId shapeId) { + return Symbol.builder() + .name(getDeserializationFunctionName(context, shapeId)) + .namespace(format("%s.deserialize", context.settings().getModuleName()), "") + .definitionFile(format("./%s/deserialize.py", context.settings().getModuleName())) + .build(); + } + + /** + * Generates the symbol for the error deserializer function for an shape of a service that is not + * protocol-specific. + * + * @param context The code generation context. + * @param shapeId The shape id of the shape the error deserializer function is being generated for. + * @return Returns the generated symbol. + */ + default Symbol getErrorDeserializationFunction(GenerationContext context, ToShapeId shapeId) { + var name = context.settings().getService(context.model()).getContextualName(shapeId); + return Symbol.builder() + .name("_deserialize_error_" + CaseUtils.toSnakeCase(name)) + .namespace(format("%s.deserialize", context.settings().getModuleName()), "") + .definitionFile(format("./%s/deserialize.py", context.settings().getModuleName())) + .build(); + } + + /** + * Generates any standard code for service request/response serde. + * + * @param context Serde context. + */ + default void generateSharedSerializerComponents(GenerationContext context) { + } + + /** + * Generates the code used to serialize the shapes of a service + * for requests. + * + * @param context Serialization context. + */ + void generateRequestSerializers(GenerationContext context); + + /** + * Generates any standard code for service response deserialization. + * + * @param context Serde context. + */ + default void generateSharedDeserializerComponents(GenerationContext context) { + } + + /** + * Generates the code used to deserialize the shapes of a service + * for responses. + * + * @param context Deserialization context. + */ + void generateResponseDeserializers(GenerationContext context); + + /** + * Generates the code for validating the generated protocol's serializers and deserializers. + * + * @param context Generation context + */ + default void generateProtocolTests(GenerationContext context) { + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/PythonIntegration.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/PythonIntegration.java new file mode 100644 index 0000000000..c9ddeb1868 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/PythonIntegration.java @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.Collections; +import java.util.List; +import software.amazon.smithy.codegen.core.SmithyIntegration; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Java SPI for customizing Python code generation, registering + * new protocol code generators, renaming shapes, modifying the model, + * adding custom code, etc. + */ +@SmithyUnstableApi +public interface PythonIntegration extends SmithyIntegration { + + /** + * Gets a list of protocol generators to register. + * + * @return Returns the list of protocol generators to register. + */ + default List getProtocolGenerators() { + return Collections.emptyList(); + } + + /** + * Gets a list of plugins to apply to the generated client. + * + * @return Returns the list of RuntimePlugins to apply to the client. + */ + default List getClientPlugins() { + return Collections.emptyList(); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/RestJsonIntegration.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/RestJsonIntegration.java new file mode 100644 index 0000000000..762156a726 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/RestJsonIntegration.java @@ -0,0 +1,30 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.List; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Integration that registers {@link RestJsonProtocolGenerator}. + */ +@SmithyInternalApi +public final class RestJsonIntegration implements PythonIntegration { + @Override + public List getProtocolGenerators() { + return List.of(new RestJsonProtocolGenerator()); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/RestJsonProtocolGenerator.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/RestJsonProtocolGenerator.java new file mode 100644 index 0000000000..e85b5ba937 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/RestJsonProtocolGenerator.java @@ -0,0 +1,320 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import software.amazon.smithy.aws.traits.protocols.RestJson1Trait; +import software.amazon.smithy.model.knowledge.HttpBinding; +import software.amazon.smithy.model.knowledge.NeighborProviderIndex; +import software.amazon.smithy.model.neighbor.Walker; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.traits.RequiresLengthTrait; +import software.amazon.smithy.model.traits.StreamingTrait; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; +import software.amazon.smithy.protocoltests.traits.HttpMessageTestCase; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.HttpProtocolTestGenerator; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Abstract implementation of JSON-based protocols that use REST bindings. + * + *

This class will be capable of generating a functional protocol based on + * the semantics of Amazon's RestJson1 protocol. Extension hooks will be + * provided where necessary in the few cases where that protocol uses + * Amazon-specific terminology or functionality. + */ +@SmithyUnstableApi +public class RestJsonProtocolGenerator extends HttpBindingProtocolGenerator { + + private static final Set TESTS_TO_SKIP = Set.of( + // These two tests essentially try to assert nan == nan, + // which is never true. We should update the generator to + // make specific assertions for these. + "RestJsonSupportsNaNFloatHeaderOutputs", + "RestJsonSupportsNaNFloatInputs", + + // This requires support of idempotency autofill + "RestJsonQueryIdempotencyTokenAutoFill", + + // This requires support of the httpChecksumRequired trait + "RestJsonHttpChecksumRequired", + + // These require support of the endpoint trait + "RestJsonEndpointTraitWithHostLabel", + "RestJsonEndpointTrait", + + // TODO: support the request compression trait + // https://smithy.io/2.0/spec/behavior-traits.html#smithy-api-requestcompression-trait + "SDKAppliedContentEncoding_restJson1", + "SDKAppendedGzipAfterProvidedEncoding_restJson1" + ); + + @Override + public ShapeId getProtocol() { + return RestJson1Trait.ID; + } + + @Override + protected Format getDocumentTimestampFormat() { + return Format.EPOCH_SECONDS; + } + + @Override + protected String getDocumentContentType() { + return "application/json"; + } + + // This is here rather than in HttpBindingProtocolGenerator because eventually + // it will need to generate some protocol-specific comparators. + @Override + public void generateProtocolTests(GenerationContext context) { + context.writerDelegator().useFileWriter("./tests/test_protocol.py", "tests.test_protocol", writer -> { + new HttpProtocolTestGenerator( + context, getProtocol(), writer, (shape, testCase) -> filterTests(testCase) + ).run(); + }); + } + + private boolean filterTests(HttpMessageTestCase testCase) { + return TESTS_TO_SKIP.contains(testCase.getId()); + } + + @Override + protected void serializeDocumentBody( + GenerationContext context, + PythonWriter writer, + OperationShape operation, + List documentBindings + ) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.types", "Document"); + writer.write("result: dict[str, Document] = {}\n"); + + var bodyMembers = documentBindings.stream() + .map(HttpBinding::getMember) + .collect(Collectors.toSet()); + + var serVisitor = new JsonShapeSerVisitor(context, writer); + serVisitor.structureMembers(bodyMembers); + + writer.addImport("smithy_python.interfaces.blobs", "AsyncBytesReader"); + writer.addStdlibImport("json"); + + var defaultTrailer = shouldWriteDefaultBody(context, operation) ? "" : " if result else b''"; + writer.write(""" + content = json.dumps(result).encode('utf-8')$L + content_length = len(content) + body = AsyncBytesReader(content) + """, defaultTrailer); + } + + @Override + protected void serializePayloadBody( + GenerationContext context, + PythonWriter writer, + OperationShape operation, + HttpBinding payloadBinding + ) { + var target = context.model().expectShape(payloadBinding.getMember().getTarget()); + var dataSource = "input." + context.symbolProvider().toMemberName(payloadBinding.getMember()); + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + + // The streaming trait can either be bound to a union, meaning it's an event stream, + // or a blob, meaning it's some potentially big collection of bytes. + // See also: https://smithy.io/2.0/spec/streaming.html#smithy-api-streaming-trait + if (payloadBinding.getMember().getMemberTrait(context.model(), StreamingTrait.class).isPresent()) { + // TODO: support event streams + if (target.isUnionShape()) { + return; + } + + // If the stream requires a length, we need to calculate that. We can't initialize the + // variable in the property accessor because then it would be out of scope, so we do + // it here instead. + // See also https://smithy.io/2.0/spec/streaming.html#smithy-api-requireslength-trait + if (requiresLength(context, payloadBinding.getMember())) { + writer.write("content_length: int = 0"); + } + + CodegenUtils.accessStructureMember(context, writer, "input", payloadBinding.getMember(), () -> { + if (requiresLength(context, payloadBinding.getMember())) { + // Since we need to calculate the length, we need to use a seekable stream because + // we can't assume that the source is seekable or safe to read more than once. + writer.addImport("smithy_python.interfaces.blobs", "SeekableAsyncBytesReader"); + writer.write(""" + body = SeekableAsyncBytesReader($L) + await body.seek(0, 2) + content_length = body.tell() + await body.seek(0, 0) + """, dataSource); + } else { + writer.addStdlibImport("typing", "AsyncIterator"); + writer.addImport("smithy_python.interfaces.blobs", "AsyncBytesReader"); + writer.write(""" + if isinstance($1L, AsyncIterator): + body = $1L + else: + body = AsyncBytesReader($1L) + """, dataSource); + } + }); + return; + } + + var memberVisitor = new JsonMemberSerVisitor( + context, writer, payloadBinding.getMember(), dataSource, Format.EPOCH_SECONDS); + var memberSerializer = target.accept(memberVisitor); + writer.addImport("smithy_python.interfaces.blobs", "AsyncBytesReader"); + writer.write("content_length: int = 0"); + + CodegenUtils.accessStructureMember(context, writer, "input", payloadBinding.getMember(), () -> { + if (target.isBlobShape()) { + writer.write("content_length = len($L)", dataSource); + writer.write("body = AsyncBytesReader($L)", dataSource); + return; + } + + if (target.isStringShape()) { + writer.write("content = $L.encode('utf-8')", memberSerializer); + } else { + writer.write("content = json.dumps($L).encode('utf-8')", memberSerializer); + } + writer.write("content_length = len(content)"); + writer.write("body = AsyncBytesReader(content)"); + }); + if (target.isStructureShape()) { + writer.write(""" + else: + content_length = 2 + body = AsyncBytesReader(b'{}') + """); + } + } + + private boolean requiresLength(GenerationContext context, MemberShape member) { + // see: https://smithy.io/2.0/spec/streaming.html#smithy-api-requireslength-trait + return member.getMemberTrait(context.model(), RequiresLengthTrait.class).isPresent(); + } + + @Override + protected void generateDocumentBodyShapeSerializers(GenerationContext context, Set shapes) { + for (Shape shape : getConnectedShapes(context, shapes)) { + var serFunction = context.protocolGenerator().getSerializationFunction(context, shape); + context.writerDelegator().useFileWriter(serFunction.getDefinitionFile(), + serFunction.getNamespace(), writer -> { + shape.accept(new JsonShapeSerVisitor(context, writer)); + }); + } + } + + @Override + protected void deserializeDocumentBody( + GenerationContext context, + PythonWriter writer, + Shape operationOrError, + List documentBindings + ) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.types", "Document"); + writer.addStdlibImport("json"); + + if (operationOrError.isOperationShape()) { + writer.write(""" + output: dict[str, Document] = {} + if (body := await http_response.consume_body()): + output = json.loads(body) + + """); + } else { + // The method that parses the error code will also pre-parse the body if there are no errors + // on that operation with an http payload. If the operation has at least 1 error with an + // http payload then the body cannot be safely pre-parsed and must be parsed here + // within the deserializer + writer.addStdlibImport("typing", "cast"); + writer.write(""" + if (parsed_body is None) and (body := await http_response.consume_body()): + parsed_body = json.loads(body) + + output: dict[str, Document] = parsed_body if parsed_body is not None else {} + """); + } + + var bodyMembers = documentBindings.stream() + .map(HttpBinding::getMember) + .collect(Collectors.toSet()); + + var deserVisitor = new JsonShapeDeserVisitor(context, writer); + deserVisitor.structureMembers(bodyMembers); + } + + @Override + protected void generateDocumentBodyShapeDeserializers( + GenerationContext context, + Set shapes + ) { + for (Shape shape : getConnectedShapes(context, shapes)) { + var deserFunction = context.protocolGenerator().getDeserializationFunction(context, shape); + + context.writerDelegator().useFileWriter(deserFunction.getDefinitionFile(), + deserFunction.getNamespace(), writer -> { + shape.accept(new JsonShapeDeserVisitor(context, writer)); + }); + } + } + + @Override + protected void deserializePayloadBody(GenerationContext context, + PythonWriter writer, + Shape operationOrError, + HttpBinding payloadBinding + ) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + var visitor = new JsonPayloadDeserVisitor(context, writer, payloadBinding); + var target = context.model().expectShape(payloadBinding.getMember().getTarget()); + target.accept(visitor); + } + + private Set getConnectedShapes(GenerationContext context, Set initialShapes) { + var shapeWalker = new Walker(NeighborProviderIndex.of(context.model()).getProvider()); + var connectedShapes = new TreeSet<>(initialShapes); + initialShapes.forEach(shape -> connectedShapes.addAll(shapeWalker.walkShapes(shape))); + return connectedShapes; + } + + @Override + protected void resolveErrorCodeAndMessage(GenerationContext context, + PythonWriter writer, + Boolean canReadResponseBody + ) { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.protocolutils", "parse_rest_json_error_info"); + writer.writeInline("code, message, parsed_body = await parse_rest_json_error_info(http_response"); + if (!canReadResponseBody) { + writer.writeInline(", False"); + } + writer.write(")"); + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/RuntimeClientPlugin.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/RuntimeClientPlugin.java new file mode 100644 index 0000000000..7471b0f54b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/RuntimeClientPlugin.java @@ -0,0 +1,280 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.integration; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiPredicate; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.python.codegen.ConfigProperty; +import software.amazon.smithy.utils.SmithyBuilder; +import software.amazon.smithy.utils.SmithyUnstableApi; +import software.amazon.smithy.utils.ToSmithyBuilder; + +/** + * Represents a runtime plugin for a client that hooks into various aspects + * of Python code generation, including adding configuration settings + * to clients and interceptors to both clients and commands. + * + *

These runtime client plugins are registered through the + * {@link PythonIntegration} SPI and applied to the code generator at + * build-time. + */ +@SmithyUnstableApi +public final class RuntimeClientPlugin implements ToSmithyBuilder { + + private static final OperationPredicate OPERATION_ALWAYS_FALSE = (model, service, operation) -> false; + + private final BiPredicate servicePredicate; + private final OperationPredicate operationPredicate; + private final List configProperties; + private final SymbolReference pythonPlugin; + + private final AuthScheme authScheme; + + private RuntimeClientPlugin(Builder builder) { + servicePredicate = builder.servicePredicate; + operationPredicate = builder.operationPredicate; + configProperties = Collections.unmodifiableList(builder.configProperties); + this.pythonPlugin = builder.pythonPlugin; + this.authScheme = builder.authScheme; + } + + /** + * Predicate that tests whether a plugin should be applied to an individual operation. + */ + @FunctionalInterface + public interface OperationPredicate { + /** + * Tests if this should be applied to an individual operation. + * + * @param model Model the operation belongs to. + * @param service Service the operation belongs to. + * @param operation Operation to test. + * @return Returns true if interceptors / plugins should be applied to the operation. + */ + boolean test(Model model, ServiceShape service, OperationShape operation); + } + + /** + * Returns true if this plugin applies to the given service. + * + *

By default, a plugin applies to all services but not to specific + * commands. You an configure a plugin to apply only to a subset of + * services (for example, only apply to a known service or a service + * with specific traits) or to no services at all (for example, if + * the plugin is meant to by command-specific and not on every + * command executed by the service). + * + * @param model The model the service belongs to. + * @param service Service shape to test against. + * @return Returns true if the plugin is applied to the given service. + * @see #matchesOperation(Model, ServiceShape, OperationShape) + */ + public boolean matchesService(Model model, ServiceShape service) { + return servicePredicate.test(model, service); + } + + /** + * Returns true if this plugin applies to the given operation. + * + * @param model Model the operation belongs to. + * @param service Service the operation belongs to. + * @param operation Operation to test against. + * @return Returns true if the plugin is applied to the given operation. + * @see #matchesService(Model, ServiceShape) + */ + public boolean matchesOperation(Model model, ServiceShape service, OperationShape operation) { + return operationPredicate.test(model, service, operation); + } + + /** + * Gets the config properties that will be added to the client config by this plugin. + * + * @return Returns the config properties to add to the client config. + */ + public List getConfigProperties() { + return configProperties; + } + + /** + * @return Returns an optional reference to a callable that modifies client config. + */ + public Optional getPythonPlugin() { + return Optional.ofNullable(pythonPlugin); + } + + /** + * @return Returns an optional auth scheme to register. + */ + public Optional getAuthScheme() { + return Optional.ofNullable(authScheme); + } + + /** + * @return Returns a new builder for a {@link RuntimeClientPlugin}. + */ + public static Builder builder() { + return new Builder(); + } + + @Override + public SmithyBuilder toBuilder() { + var builder = builder() + .pythonPlugin(pythonPlugin) + .authScheme(authScheme) + .configProperties(configProperties); + + if (operationPredicate == OPERATION_ALWAYS_FALSE) { + builder.servicePredicate(servicePredicate); + } else { + builder.operationPredicate(operationPredicate); + } + + return builder; + } + + /** + * Builds a {@link RuntimeClientPlugin}. + */ + public static final class Builder implements SmithyBuilder { + private BiPredicate servicePredicate = (model, service) -> true; + private OperationPredicate operationPredicate = OPERATION_ALWAYS_FALSE; + private List configProperties = new ArrayList<>(); + private SymbolReference pythonPlugin = null; + private AuthScheme authScheme = null; + + Builder() { + } + + @Override + public RuntimeClientPlugin build() { + return new RuntimeClientPlugin(this); + } + + /** + * Sets a predicate that determines if the plugin applies to a + * specific operation. + * + *

When this method is called, the {@code servicePredicate} is + * automatically configured to return false for every service. + * + *

By default, a plugin applies globally to a service, which thereby + * applies to every operation when the interceptors list is copied. + * + * @param operationPredicate Operation matching predicate. + * @return Returns the builder. + * @see #servicePredicate(BiPredicate) + */ + public Builder operationPredicate(OperationPredicate operationPredicate) { + this.operationPredicate = Objects.requireNonNull(operationPredicate); + servicePredicate = (model, service) -> false; + return this; + } + + /** + * Configures a predicate that makes a plugin only apply to a set of + * operations that match one or more of the set of given shape names, + * and ensures that the plugin is not applied globally to services. + * + *

By default, a plugin applies globally to a service, which thereby + * applies to every operation when the interceptors list is copied. + * + * @param operationNames Set of operation names. + * @return Returns the builder. + */ + public Builder appliesOnlyToOperations(Set operationNames) { + operationPredicate((model, service, operation) -> operationNames.contains(operation.getId().getName())); + return servicePredicate((model, service) -> false); + } + + /** + * Configures a predicate that applies the plugin to a service if the + * predicate matches a given model and service. + * + *

When this method is called, the {@code operationPredicate} is + * automatically configured to return false for every operation, + * causing the plugin to only apply to services and not to individual + * operations. + * + *

By default, a plugin applies globally to a service, which + * thereby applies to every operation when the interceptors list is + * copied. Setting a custom service predicate is useful for plugins + * that should only be applied to specific services or only applied + * at the operation level. + * + * @param servicePredicate Service predicate. + * @return Returns the builder. + */ + public Builder servicePredicate(BiPredicate servicePredicate) { + this.servicePredicate = Objects.requireNonNull(servicePredicate); + operationPredicate = OPERATION_ALWAYS_FALSE; + return this; + } + + /** + * Sets the list of config properties to add to the client's config object. + * + * @param configProperties The list of config properties to add to the client's config object. + * @return Returns the builder. + */ + public Builder configProperties(List configProperties) { + this.configProperties = configProperties; + return this; + } + + /** + * Adds a single config property which will be added to the client's config object. + * + * @param configProperty A config property to add to the client's config object. + * @return Returns the builder. + */ + public Builder addConfigProperty(ConfigProperty configProperty) { + this.configProperties.add(configProperty); + return this; + } + + /** + * Configures a python plugin to automatically add to the service or operation. + * + * @param pythonPlugin A reference to a callable that modifies the client config. + * @return Returns the builder. + */ + public Builder pythonPlugin(SymbolReference pythonPlugin) { + this.pythonPlugin = pythonPlugin; + return this; + } + + /** + * Configures an auth scheme which will be added to the client's supported + * auth schemes. + * + * @param authScheme The auth scheme to register. + * @return Returns the builder. + */ + public Builder authScheme(AuthScheme authScheme) { + this.authScheme = authScheme; + return this; + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ConfigSection.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ConfigSection.java new file mode 100644 index 0000000000..efcae8d383 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ConfigSection.java @@ -0,0 +1,18 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.python.codegen.sections; + +import java.util.List; +import software.amazon.smithy.python.codegen.ConfigProperty; +import software.amazon.smithy.utils.CodeSection; + +/** + * Section that contains the entire generated config object. + * + * @param properties The list of properties that need to be present on the config. + */ +public record ConfigSection(List properties) implements CodeSection { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthParametersSection.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthParametersSection.java new file mode 100644 index 0000000000..18075c2478 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthParametersSection.java @@ -0,0 +1,19 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.python.codegen.sections; + +import java.util.Map; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.utils.CodeSection; + +/** + * A section that controls generating the HttpAuthParameters class. + * + * @param properties A map of property names to types for properties that must + * be present on the parameters object. + */ +public record GenerateHttpAuthParametersSection(Map properties) implements CodeSection { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthSchemeResolverSection.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthSchemeResolverSection.java new file mode 100644 index 0000000000..80e7d6497b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/GenerateHttpAuthSchemeResolverSection.java @@ -0,0 +1,18 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.python.codegen.sections; + +import java.util.List; +import software.amazon.smithy.python.codegen.integration.AuthScheme; +import software.amazon.smithy.utils.CodeSection; + +/** + * A code section that controls generating the entire auth scheme resolver. + * + * @param authSchemes A list of supported auth schemes discovered on the path. + */ +public record GenerateHttpAuthSchemeResolverSection(List authSchemes) implements CodeSection { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/InitializeHttpAuthParametersSection.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/InitializeHttpAuthParametersSection.java new file mode 100644 index 0000000000..42eb5b8d56 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/InitializeHttpAuthParametersSection.java @@ -0,0 +1,14 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.python.codegen.sections; + +import software.amazon.smithy.utils.CodeSection; + +/** + * A section that controls writing out the auth scheme parameters. + */ +public record InitializeHttpAuthParametersSection() implements CodeSection { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/PyprojectSection.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/PyprojectSection.java new file mode 100644 index 0000000000..74586e316f --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/PyprojectSection.java @@ -0,0 +1,36 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.sections; + +import java.util.Map; +import software.amazon.smithy.codegen.core.SymbolDependency; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * This section controls the entire pyproject.toml + * + *

An integration may want to append to this if, for instance, it is adding + * some new dependency that needs to be configured. Note that dependencies + * themselves should be added when they're introduced in the writers, not here. + * + * @param dependencies A mapping of {@link software.amazon.smithy.python.codegen.PythonDependency.Type} + * to a mapping of the package name to {@link SymbolDependency}. + * This contains all the dependencies for the generated client. + */ +@SmithyUnstableApi +public record PyprojectSection(Map> dependencies) implements CodeSection { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ReadmeSection.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ReadmeSection.java new file mode 100644 index 0000000000..fba45db5ff --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ReadmeSection.java @@ -0,0 +1,30 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.sections; + + +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * This section controls the entire generated README.md + * + *

An integration may want to use this if they want to programmatically + * overwrite the generated README. + */ +@SmithyUnstableApi +public record ReadmeSection() implements CodeSection { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ResolveEndpointSection.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ResolveEndpointSection.java new file mode 100644 index 0000000000..2d55abeb47 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ResolveEndpointSection.java @@ -0,0 +1,29 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.sections; + +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * This section resolves the service's endpoint and adds it to the transport + * request, if necessary. + * + *

This is implemented by default only for HTTP protocols. + */ +@SmithyUnstableApi +public record ResolveEndpointSection() implements CodeSection { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ResolveIdentitySection.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ResolveIdentitySection.java new file mode 100644 index 0000000000..1c9df50612 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/ResolveIdentitySection.java @@ -0,0 +1,14 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.python.codegen.sections; + +import software.amazon.smithy.utils.CodeSection; + +/** + * A section responsible for resolving the caller's identity. + */ +public record ResolveIdentitySection() implements CodeSection { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/SendRequestSection.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/SendRequestSection.java new file mode 100644 index 0000000000..c9da498dc2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/SendRequestSection.java @@ -0,0 +1,28 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.python.codegen.sections; + +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * This section is responsible for sending the client's request to the service. + * + *

This is implemented by default only for HTTP protocols. + */ +@SmithyUnstableApi +public record SendRequestSection() implements CodeSection { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/SignRequestSection.java b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/SignRequestSection.java new file mode 100644 index 0000000000..9fdebbf01e --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/sections/SignRequestSection.java @@ -0,0 +1,14 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.python.codegen.sections; + +import software.amazon.smithy.utils.CodeSection; + +/** + * A section responsible for signing requests before sending them. + */ +public record SignRequestSection() implements CodeSection { +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin new file mode 100644 index 0000000000..9df651510d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin @@ -0,0 +1,6 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +software.amazon.smithy.python.codegen.PythonClientCodegenPlugin diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration new file mode 100644 index 0000000000..10665fde4f --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration @@ -0,0 +1,7 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +software.amazon.smithy.python.codegen.integration.RestJsonIntegration +software.amazon.smithy.python.codegen.integration.HttpApiKeyAuth diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/software/amazon/smithy/python/codegen/reserved-class-names.txt b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/software/amazon/smithy/python/codegen/reserved-class-names.txt new file mode 100644 index 0000000000..a0a78e1853 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/software/amazon/smithy/python/codegen/reserved-class-names.txt @@ -0,0 +1,15 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Python reserved words for class names +# +# Most of python's reserved words are lower-case, so there isn't much +# here. Furthermore, import aliases can resolve any other conflicts +# with non-reserved builtins or other conflicting classes. +# +# A full list of reserved words can be found here: +# https://docs.python.org/3/reference/lexical_analysis.html#keywords +True +False +None diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt new file mode 100644 index 0000000000..97a0cfd515 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-codegen/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt @@ -0,0 +1,83 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Python reserved words for members. + +# The following are reserved words that can't be used as identifiers at all. +# For example, the following would produce a syntax error: +# +# class Foo: +# pass: int +# +# A full list of these can be found here: +# +# https://docs.python.org/3/reference/lexical_analysis.html#keywords +# +and +as +assert +async +await +break +class +continue +def +del +elif +else +except +False +finally +for +from +global +if +import +in +is +lambda +None +nonlocal +not +or +pass +raise +return +True +try +while +with +yield + +# The following aren't reserved words, but are built-in types / functions that +# would break if you ever tried to refer to the type again in scope. For +# example: +# +# class Foo: +# str: str +# +# def __init__(self, str: str): +# pass +# +# That would have an exception in the definition of __init__ since when you use +# `str` as the type after you've defined `str` in scope, it thinks you're +# referencing `Foo.str` rather than the built-in type (or a type at all). +# +# A listing of these types can be found here: +# https://docs.python.org/3/library/stdtypes.html +# +# Note though that we only need to escape the types we use. +bool +bytes +bytearray +dict +float +int +list +str + +# For the exact same reason as above, these are names of common types +# that are likely imported in the generated code (e.g. datetime) +# We only escape the types we use. +datetime diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-protocol-test/README.md b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-protocol-test/README.md new file mode 100644 index 0000000000..3ca38ab339 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-protocol-test/README.md @@ -0,0 +1,75 @@ +## Smithy Python Protocol Test + +This package generates and runs [protocol tests +](https://smithy.io/2.0/additional-specs/http-protocol-compliance-tests.html) +for all supported protocols. Protocol tests are defined in the Smithy model itself +using traits, and are generated along with the client. These test the actual +functionality of the generated client and protocol implementation using expected +inputs and outputs. The tests are generated by [`HttpProtocolTestGenerator` +](https://github.com/awslabs/smithy-python/blob/develop/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java). + +For AWS protocols, these tests are defined [here +](https://github.com/awslabs/smithy/tree/main/smithy-aws-protocol-tests). + +To run these tests, run `make test-protocols` from the repository root, ensuring that +you have a [Python virtual environment](https://docs.python.org/3/library/venv.html) +active. + +### When should I change this package? + +This package should only be changed when support for a new protocol is initially +introduced. + +### How can I add new protocol tests to an exiting protocol? + +To add new protocol tests, the source protocol test package must be updated. For +example, AWS protocol tests must be defined in the [smithy-aws-protocol-tests +](https://github.com/awslabs/smithy/tree/main/smithy-aws-protocol-tests) package. +After a new test case is added to the Smithy model, publish the change locally by +running `./gradlew build publishToMavenLocal`. Make sure that the version range of the +dependency here contains the version of the protocol test package you are building. + +### How can I add protocol tests for a new protocol? + +First, add a dependency on the package that contains the tests for the protocol. For +AWS protocols, that already exists. Next, add a new [projection +](https://smithy.io/2.0/guides/building-models/build-config.html#projections) +in `smithy-build.json` for each service that has tests for the protocol. For example: + +```json +{ + "version": "1.0", + "projections": { + "rest-json-1": { + "transforms": [{ + "name": "includeServices", + "args": { + "services": ["aws.protocoltests.restjson#RestJson"] + } + }], + "plugins": { + "python-client-codegen": { + "service": "aws.protocoltests.restjson#RestJson", + "module": "restjson", + "moduleVersion": "0.0.1" + } + } + }, + "my-new-protocol": { + "transforms": [{ + "name": "includeServices", + "args": { + "services": ["com.example#MyNewProtocolTestService"] + } + }], + "plugins": { + "python-client-codegen": { + "service": "com.example#MyNewProtocolTestService", + "module": "mynewprotocol", + "moduleVersion": "0.0.1" + } + } + } + } +} +``` diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-protocol-test/build.gradle.kts b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-protocol-test/build.gradle.kts new file mode 100644 index 0000000000..ffc6d5d57f --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-protocol-test/build.gradle.kts @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +extra["displayName"] = "Smithy :: Python :: Protocol :: Test" +extra["moduleName"] = "software.amazon.smithy.python.protocol.test" + +tasks["jar"].enabled = false + +val smithyVersion: String by project + +buildscript { + val smithyVersion: String by project + + repositories { + mavenCentral() + } + dependencies { + "classpath"("software.amazon.smithy:smithy-cli:$smithyVersion") + } +} + +plugins { + val smithyGradleVersion: String by project + id("software.amazon.smithy").version(smithyGradleVersion) +} + +repositories { + mavenLocal() + mavenCentral() +} + +dependencies { + implementation(project(":smithy-python-codegen")) + implementation("software.amazon.smithy:smithy-aws-protocol-tests:$smithyVersion") +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-protocol-test/smithy-build.json b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-protocol-test/smithy-build.json new file mode 100644 index 0000000000..68443aef8b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/codegen/smithy-python-protocol-test/smithy-build.json @@ -0,0 +1,20 @@ +{ + "version": "1.0", + "projections": { + "rest-json-1": { + "transforms": [{ + "name": "includeServices", + "args": { + "services": ["aws.protocoltests.restjson#RestJson"] + } + }], + "plugins": { + "python-client-codegen": { + "service": "aws.protocoltests.restjson#RestJson", + "module": "restjson", + "moduleVersion": "0.0.1" + } + } + } + } +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/designs/http-interfaces.md b/codegen/smithy-dafny-codegen-modules/smithy-python/designs/http-interfaces.md new file mode 100644 index 0000000000..a08ebc408d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/designs/http-interfaces.md @@ -0,0 +1,326 @@ +# Abstract + +This document will go over the proposed interfaces required for making HTTP +requests in the context of Smithy generated service clients. + +# Motivation + +The HTTP interfaces and data classes defined in this document will serve as the +basis for all SDK clients built on top of smithy-python and will therefore aim +to provide the simplest interface that is correct. These interfaces will +directly be used by consumers of the smithy-python library, both in the context +of custom Smithy service clients as well as all AWS service clients. These +interfaces will serve as guidance and should not require any specific HTTP +library, concurrency paradigm, or require a runtime dependency on the +smithy-python package to implement. + +# Specification + +## Requests and Responses + +Requests and responses are represented by minimal, async-compatible interfaces. +Since Smithy clients are expected to be capable of using non-HTTP transport protocols, +such as MQTT, any HTTP-specific properties will exist in their own sub-interfaces. + +Request bodies are defined as an `AsyncIterable[bytes]` instead of some file-like +object. This allows flexibility both within protocols and specific transfer settings +to defer message framing to a lower layer. There are mechanisms within protocols, such +as HTTP’s `Content-Encoding`, which enable application-specific content framing within +a message. + +```python +class Request(Protocol): + """Protocol-agnostic representation of a request.""" + + destination: URI + body: AsyncIterable[bytes] + + async def consume_body(self) -> bytes: + """Iterate over request body and return as bytes.""" + ... + + +class Response(Protocol): + """Protocol-agnostic representation of a response.""" + + @property + def body(self) -> AsyncIterable[bytes]: + """The response payload as iterable of chunks of bytes.""" + ... + + async def consume_body(self) -> bytes: + """Iterate over response body and return as bytes.""" + ... + + +class HTTPRequest(Request, Protocol): + """HTTP primitive for an Exchange to construct a version agnostic HTTP message. + + :param destination: The URI where the request should be sent to. + :param method: The HTTP method of the request, for example "GET". + :param fields: ``Fields`` object containing HTTP headers and trailers. + :param body: A streamable collection of bytes. + """ + + method: str + fields: Fields + + +class HTTPResponse(Response, Protocol): + """HTTP primitives returned from an Exchange, used to construct a client + response.""" + + @property + def status(self) -> int: + """The 3 digit response status code (1xx, 2xx, 3xx, 4xx, 5xx).""" + ... + + @property + def fields(self) -> Fields: + """``Fields`` object containing HTTP headers and trailers.""" + ... + + @property + def reason(self) -> str | None: + """Optional string provided by the server explaining the status.""" + ... +``` + +## URI + +URIs are represented by an explicit interface rather than an arbitrary string. This +avoids joining and splitting an endpoint multiple times in the request/response +lifecycle, like we do in botocore. + +It will be the responsibility of an HTTP client implementation to take the information +present in the `URI` object and render it into an appropriate representation of the URI +for the HTTP client being used. + +```python +class URI(Protocol): + """Universal Resource Identifier, target location for a :py:class:`Request`.""" + + scheme: str + """For example ``http`` or ``mqtts``.""" + + username: str | None + """Username part of the userinfo URI component.""" + + password: str | None + """Password part of the userinfo URI component.""" + + host: str + """The hostname, for example ``amazonaws.com``.""" + + port: int | None + """An explicit port number.""" + + path: str | None + """Path component of the URI.""" + + query: str | None + """Query component of the URI as string.""" + + fragment: str | None + """The fragment component of the URI.""" + + def build(self) -> str: + """Construct URI string representation. + + Returns a string of the form + ``{scheme}://{username}:{password}@{host}:{port}{path}?{query}#{fragment}`` + """ + ... +``` + +## Fields + +Most HTTP users will be familiar with the concept of headers. These were introduced in +HTTP/1.0 and have since evolved through HTTP 1.1/2/3 to include things like trailers +and other arbitrary metadata. Starting in RFC 7230 (HTTP/1.1), the term `Header` began +being referred to interchangeably as a `Field` or `Header Field`. Starting in RFC 9114 +(HTTP/3), these are now strictly referred to as `HTTP Fields` or `Fields`. + +This design uses the modern `Field` concept as interfaces to more closely reflect the +current RFCs. Notably absent is the concept of a direct header map or field map. This +reflects the reality that headers and other fields have always allowed multiple values +for a given key. Built-in joining methods are included to support HTTP client +implementations that only understand headers as a simple map. + +```python +class FieldPosition(Enum): + """The type of a field. + + Defines its placement in a request or response. + """ + + HEADER = 0 + """Header field. + + In HTTP this is a header as defined in RFC 9110 Section 6.3. Implementations of + other protocols may use this FieldPosition for similar types of metadata. + """ + + TRAILER = 1 + """Trailer field. + + In HTTP this is a trailer as defined in RFC 9110 Section 6.5. Implementations of + other protocols may use this FieldPosition for similar types of metadata. + """ + + +class Field(Protocol): + """A name-value pair representing a single field in a request or response. + + The kind will dictate metadata placement within an the message, for example as + header or trailer field in a HTTP request as defined in RFC 9110 Section 5. + + All field names are case insensitive and case-variance must be treated as + equivalent. Names may be normalized but should be preserved for accuracy during + transmission. + """ + + name: str + values: list[str] + kind: FieldPosition = FieldPosition.HEADER + + def add(self, value: str) -> None: + """Append a value to a field.""" + ... + + def set(self, values: list[str]) -> None: + """Overwrite existing field values.""" + ... + + def remove(self, value: str) -> None: + """Remove all matching entries from list.""" + ... + + def as_string(self) -> str: + """Serialize the ``Field``'s values into a single line string.""" + ... + + def as_tuples(self) -> list[tuple[str, str]]: + """Get list of ``name``, ``value`` tuples where each tuple represents one + value.""" + ... + + +class Fields(Protocol): + """Protocol agnostic mapping of key-value pair request metadata, such as HTTP + fields.""" + + # Entries are keyed off the name of a provided Field + entries: OrderedDict[str, Field] + encoding: str | None = "utf-8" + + def set_field(self, field: Field) -> None: + """Set entry for a Field name.""" + ... + + def get_field(self, name: str) -> Field: + """Retrieve Field entry.""" + ... + + def remove_field(self, name: str) -> None: + """Delete entry from collection.""" + ... + + def get_by_type(self, kind: FieldPosition) -> list[Field]: + """Helper function for retrieving specific types of fields. + + Used to grab all headers or all trailers. + """ + ... + + def extend(self, other: "Fields") -> None: + """Merges ``entries`` of ``other`` into the current ``entries``. + + For every `Field` in the ``entries`` of ``other``: If the normalized name + already exists in the current ``entries``, the values from ``other`` are + appended. Otherwise, the ``Field`` is added to the list of ``entries``. + """ + ... + + def __iter__(self) -> Iterator[Field]: + """Allow iteration over entries.""" + ... +``` + +## HTTP client interface + +HTTP clients are represented by a simple interface that defines a single `send` method, +which takes a request and some configuration and asynchronously return a response. +Having a minimal interface makes it much easier to implement these interfaces on top of +a variety http libraries. + +```python +@dataclass(kw_only=True) +class HTTPRequestConfiguration: + """Request-level HTTP configuration. + + :param read_timeout: How long, in seconds, the client will attempt to read the + first byte over an established, open connection before timing out. + """ + + read_timeout: float | None = None + + +class HTTPClient(Protocol): + """An asynchronous HTTP client interface.""" + + async def send( + self, *, request: HTTPRequest, request_config: HTTPRequestConfiguration | None + ) -> HTTPResponse: + """Send HTTP request over the wire and return the response. + + :param request: The request including destination URI, fields, payload. + :param request_config: Configuration specific to this request. + """ + ... +``` + +# FAQs + +## Why use protocols instead of just defining base classes, etc.? + +Protocols allow us to define an interface that can be implemented without +requiring implementations to have a runtime dependency on the interfaces. +Validation that an implemenation meets the interfaces can happen as a step +during testing, or at the point the implementation is used under a context +where one of these protocols is expected. + +## What if we need to add additional fields to these interfaces? + +Adding new fields to these interfaces is presumably being done to support some +end feature in the SDKs, which will effectively require the new field. This +means that custom implementations or older implementations we've created will +be incompatible with newer versions of the AWS SDK. This should be relatively +easy to work around in the white label or AWS SDKs by bumping the HTTP client +implementation version floor. However, for custom implementations this may be a +harder sell. Given that the addition of fields to this interface will result in +a typing error for incomplete implementations we should convey to customers +creating custom implementations that these interfaces may grow over time and +that using custom HTTP client implementations are not always guaranteed to be +forwards compatible. + +## What about exceptions and handling exceptions? + +Different implementations of these interfaces will raise different exceptions +in the same logical scenarios. This will potentially be problematic down the +road when we begin to implement logic that cares about exceptions such as +retries. That being said, there isn't really a way to model exceptions at a +typing level, and certainly not in a manner that would decouple the interface +definitions and the runtime of implementations. This may be something that we +need to revisit and will need to either modify these interfaces or work around +via other means. A couple of very loose ideas for handling this: + +* Exceptions only matter if they can be recovered from, e.g. retried. +Meaningful exceptions should actually defined as part of the interface +definition. This could mean modeling responses as `Tuple[Response, +Optional[HttpError]]` or something similar. Where an `HttpError` can be +marked as retryable, etc. +* Custom HTTP implementations will also require a custom retry handler +implementation +* Custom HTTP implementations will also require a custom set of retryable +exceptions if you want retries to work properly. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/designs/shapes.md b/codegen/smithy-dafny-codegen-modules/smithy-python/designs/shapes.md new file mode 100644 index 0000000000..bed5b2b8ad --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/designs/shapes.md @@ -0,0 +1,713 @@ +# Abstract + +This document will describe how [simple shapes](https://smithy.io/2.0/spec/simple-types.html) +and [aggregate shapes](https://smithy.io/2.0/spec/aggregate-types.html) +are generated into python types and type hints from a Smithy model. + +# Specification + +## Simple shapes + +| Shape Type | Python Type | +|------------|-------------| +| blob | Union[bytes, bytearray] | +| boolean | bool | +| string | str | +| byte | int | +| short | int | +| integer | int | +| long | int | +| float | float | +| double | float | +| bigInteger | int | +| bigDecimal | decimal.Decimal | +| timestamp | datetime.datetime | +| document | Document = dict[str, 'Document'] | list['Document'] | str | int | float | bool | None | + +## Trait-influenced shapes + +### enum + +```python +class EnumWithNames: + SPAM = "spam" + EGGS = "eggs" + SPAM_EGGS = "spam:eggs" + + values = frozenset(SPAM, EGGS, SPAM_EGGS) +``` + +Enums are classes with a `values` property that contains an immutable set of +all the possible values of the enum in addition to static properties for each +entry. + +This provides customers with a way to access the simplified names for easier +development, as well as giving them a programmatic ability to check known +values. + +Members targeting enums will continue to use plain strings to enable forwards +compatibility. Documentation for those members will reference the enum classes +for discoverability. + +#### Alternative: native enums + +Python 3.4 introduced native enums, and they're about what you'd expect: + +```python +class MyEnum(Enum): + SPAM = "spam" + EGGS = "eggs" + SPAM_EGGS = "spam:eggs" +``` + +Defining an enum in this way gives you iterators, comparators, and a more +helpful string representation (``) for free. + +Unfortuantely, native enums aren't forwards compatible. To support forwards +compatibility, we need to also support passing plain strings. If a client were +handling an enum value not represented in their version, then they would break +upon updating because `MyEnum.SPAM != "spam"`. + +While we could generate them anyway for helpers, it would be very confusing +to use as you would have to pass `MyEnum.SPAM.value` instead of `MyEnum.SPAM`. + +#### Alternative: native enums with generated accessors and unknown variant + +```python +class MyEnum(Enum): + SPAM = "spam" + EGGS = "eggs" + SPAM_EGGS = "spam:eggs" + UNKNOWN_TO_SDK_VERSION = "" + + @staticmethod + def from_string(value: str) -> 'MyEnum': + try: + return MyEnum(value) + except ValueError: + return MyEnum.UNKNOWN_TO_SDK_VERSION + + +class Struct: + def __init__(self, *, my_enum: Union[MyEnum, str]): + if isinstance(my_enum, MyEnum): + self._my_enum = my_enum + self._my_enum_as_string = my_enum.value + else: + self._my_enum = MyEnum.from_string(my_enum) + self._my_enum_as_string = my_enum + + @property + def my_enum(self) -> MyEnum: + return self._my_enum + + @my_enum.setter + def my_enum(self, value: MyEnum): + self._my_enum = value + self._my_enum_as_string = value + + @property + def my_enum_as_string(self) -> str: + return self._my_enum_as_string + + @my_enum_as_string.setter + def my_enum_as_string(self, value: str): + self._my_enum_as_string = value + self._my_enum = MyEnum.from_string(value) +``` + +This alternative uses separate properties and a canonical unknown variant to +allow customers to use the native enums without risking a breaking change. +The tradeoff is we have to add another property, which may cause confusion. +We also have to generate getters/setters to make sure they stay in sync, which +dramatically increases the amount of code generated since these need to +appear in every referencing structure. + +### intEnums + +```python +class MyIntEnum(IntEnum): + SPAM = 1 + EGGS = 2 +``` + +IntEnums will use the native `IntEnum` type. This will allow customers to +easily specify the enum value without having to reference the actual number. It +also gives them a programmatic way to list known values. + +Like string enums, members targeting IntEnums will use plain integers to enable +forwards compatibility and type checking. Documentation for those members will +reference the generated classes for discoverability. + +#### Alternative: target the generated enums + +In this alternative, members targeting IntEnums would reference the generated +classes. This wasn't chosen for both forwards and backwards compatibility reasons. +While one can mostly use IntEnums and integers interchangeably, the types *are* +different. Type checking would fail if you provided a base integer, known or +unknown. + +### streaming blobs + +A blob with the streaming trait will continue to support `bytes` as input. +Additionally, it will support passing in a `ByteStream` which is any class that +implements a `read` method that accepts a `size` param and returns bytes. +It will also accept an async variant of that same type or an `AsyncIterable`. + +Both `ByteStream` and `AsyncByteStream` will be implemented as [Protocols +](https://www.python.org/dev/peps/pep-0544/), which are a way of defining +structural subtyping in Python. + +```python +@runtime_checkable +class ByteStream(Protocol): + def read(self, size: int = -1) -> bytes: + ... + + +@runtime_checkable +class AsyncByteStream(Protocol): + async def read(self, size: int = -1) -> bytes: + ... + + +StreamingBlob = Union[ + ByteStream, + AsyncByteStream, + bytes, + bytearray, + AsyncIterable, +] +``` + +The type signature of members targeting blobs with the streaming trait will be +the union `StreamingBlob`. + +### mediaType + +Python is very generous in allowing subtyping of built in types, so a string or +blob modeled with the mediaType can accept or return helper classes. These can +be passed around and used exactly like normal strings/blobs. + +```python +class JsonString(str): + _json = "" + + def as_json(self) -> Any: + if not self._json: + self._json = json.loads(self) + return self._json + + @staticmethod + def from_json(json: Any) -> 'JsonString': + json_string = JsonString(json.dumps(json)) + json_string._json = json + return json_string + +class JsonBlob(bytes): + _json = b"" + + def as_json(self) -> Any: + if not self._json: + self._json = json.loads(self.decode(encoding="utf-8")) + return self._json + + @staticmethod + def from_json(json: Any) -> 'JsonString': + json_string = JsonBlob(json.dumps(json).encode(encoding="utf-8")) + json_string._json = json + return json_string +``` + +A member with a json media type could then accept any json-compatible type in +addition to their base types. Deserializers would always deserialize into a +`JsonString` or `JsonBlob` to ensure that the parsing is lazy and that adding +the mediaType trait is backwards-compatible. + +Example usage: + +```python +import json + +my_json = {"spam": "eggs"} + +# Without JsonBlob +client.send_json(json=b'{"spam": "eggs"}') +client.send_json(json=json.dumps(my_json.encode(encoding="utf-8"))) +returned_json = json.loads(client.get_json().decode(encoding="utf-8")) + +# With JsonBlob. All of the above are also possible. +client.send_json(json=my_json) +client.send_json(json=JsonBlob.from_json(my_json)) +returned_json = client.get_json().as_json() +``` + +By default only json helpers will be supported. More can be added later by +demand. Additionally, more can be be added with plugins to the code generators. + +## Simple aggregate shapes + +| Shape Type | Python Type | Type Hint | +|------------|-------------|-----------| +| list | list | List[str] | +| set | set | Set[str] | +| map | dict | Mapping[str, str] | + +## Structures + +Structures are simple python objects with `as_dict` and `from_dict` methods +whose constructors only allow keyword arguments. For example: + +```python +class ExampleStructure: + required_param: str + struct_param: OtherStructure + optional_param: str | None + + def __init__( + self, + *, # This prevents positional arguments + required_param: str, + struct_param: OtherStructure, + optional_param: str | None = None + ): + self.required_param = required_param + self.struct_param = struct_param + self.optional_param = optional_param + + def as_dict(self) -> Dict: + d = { + "RequiredParam": self.required_param, + "StructParam": self.struct_param.as_dict(), + } + + if self.optional_param is not None: + d["OptionalParam"] = self.optional_param + + @staticmethod + def from_dict(d: Dict) -> ExampleStructure: + return ExampleStructure( + required_param=d["RequiredParam"], + struct_param=OtherStructure.from_dict(d["StructParam"]), + optional_param=d.get("OptionalParam"), + ) +``` + +Disallowing positional arguments prevents errors from arising when future +updates add additional structure members. + +The `as_dict` and `from_dict` methods exist primarily to make migration +from `boto3` easier, as users will be able to use them to convert to/from +`boto3` style arguments freely. To facilitate that migration, keys in the +generated dicts use shape names as defined in the model rather than the +snake cased variants used in the constructor. + +### Alternative: Dataclasses + +Python 3.7 introduced dataclasses, which are a simple way of defining simple +classes which have auto-generated implementations of a bunch of common +functions. + +```python +@dataclass +class ExampleStructure: + required_param: str + struct_param: OtherStructure + optional_param: Optional[str] = None +``` + +This will auto-generate `__init__`, `__repr__`, `__eq__`, `__hash__`, and +optionally a number of other magic methods. `dataclasses` also provides a +number of other useful methods, like an `as_dict` function. + +Unfortunately, the generated constructors allow for positional arguments. +Constructor generation can be disabled, and a custom constructer can be +written instead. Still, this immediately starts eating away at the utility +of the decorator. + +Similarly, the prebuilt `as_dict` function is fairly rigid. There is currently +no way to customize the dict representation. This means that we wouldn't be +able to have compatibility with `boto3` unless we implement the function +ourselves. And there is no built in way to convert an existing dict into +a given dataclass. + +Since none of the methods we want can be properly generated by `dataclass`, +there is not much reason to use them. The free `eq`, `hash`, and `repr` +integrations are nice, but not worth adding a dependency for. + +Adding any dependency, even one on the standard library, should be approached +with caution. Updates can introduce potentially breaking features that we have +little ability to resolve. The standard library in particular is impossible for +us to keep up to date, as we have no control over the environment our customers +run code in. + +### Alternative: Plain dicts + +Rather than generating classes, plain dicts could be used: + +```python +{ + "RequiredParam": "foo", + "OptionalParam": "bar", + "StructParam": {} +} +``` + +Since no classes need to be generated, the amount of generated code would be +significantly reduced. These dicts would be directly compatible with `boto3`, +making migration extremely easy. + +The major downside is that these are extremely difficult to write useful type +hints for. `TypedDict` does exist, but the lack of support for recursive +definitions make it flaky to use. Without adequate type hints, or adequate +tooling that handles them, the development experience is greatly reduced. +Autocomplete, for example, is only sparsely supported for `TypedDict` and is +not at all supported for dicts without type hints. This can and does lead to +bugs where parameters are misspelled and not caught until runtime, if at all. + +Additionally, using dicts is just more cumbersome than using plain python +objects. Consider the following: + +```python +example_class = ExampleStruct(required_param="foo") +print(example_class.optional_param) # None + +example_dict = {"RequiredParam": "foo"} +print(example_dict["OptionalParam"]) # KeyError: 'OptionalParam' +print(example_dict.get("OptionalParam")) # None +``` + +This is a small example of a minor annoyance, but one that you must always be +aware of when using dicts. + +### Default Values + +Default values use python's built-in default values system. Shapes that have +immutable defaults, such as integers, have their values directly assigned in +the constructor default. Shapes that can have immutable defaults (i.e. lists, +maps, and documents) are assigned to `None` and have their default value set +inside the constructor body. + +```python +class StructWithDefaults: + default_int: int + default_list: list[int] + + def __init__( + self, + *, + default_int: int = 7, + default_list: Optional[list[int]] = None, + ): + self.default_int = default_int + self.default_list = default_list if default_list is not None else [] +``` + +## Errors + +Modeled errors are specialized structures that have a `code` and `message`. + +```python +# Defined in a shared library, used for all common errors in generic library +# code. +class SmithyError(Exception): + pass + + +# This is just an example of a direct implementation of a SmithyError. This +# would also be defined in generic library code. +class HttpClientError(SmithyError): + def __init__( + self, + error: Exception, + request: Optional[HttpRequest] = None, + response: Optional[HttpResponse] = None + ): + super().__init__( + f"An HTTP client raised an unhaneled exception: {error}") + self.request = request + self.response = response + self.error = error + + +# Generated per service and used for all service-specific errors. This would +# be the base error for customizations, for instance. +class ServiceError(SmithyError): + pass + + +# Generated per service and used for all modeled errors. +class ApiError(ServiceError, Generic[T]): + code: T + + def __init__(self, message: str): + super().__init__(message) + self.message = message + + +class ModeledException(ApiException[Literal["ModeledException"]]): + code: Literal["ModeledException"] = "ModeledException" + + def __init__( + self, + *, + message: str, + modeled_member: List[str] + ): + super().__init__(message) + self.modeled_member = modeled_member + + +class UnknownException(ApiException[Literal["Unknown"]]): + code: Literal["Unknown"] = "Unknown" +``` + +All errors will inherit from a base `SmithyError` class. This will allow +customers to catch nearly any exception generated by client if they so choose. +This class will be hand written inside a shared library. + +Each generated service will have two static errors generated: `ServiceError` +and `ApiError`. `ServiceError` will be inherited by ever error specific to the +service, including errors raised in service-specific customizations. `ApiError` +will be inherited by all modeled errors. + +Modeled errors will differ from normal structures in three respects: + +* They will inherit from `ApiError`. +* They will have a static `code` property generated, which will default to the + name of the error shape. +* They will have a consistent `message` property, which will replace any member + with a name case-insensitively matching `message`, `error_message`, or + `errormessage`. + +### Alternative: ServiceError wraps SmithyError + +In this alternative, the generated `ServiceError` would not inherit from +`SmithyError`. Instead it would have a subclass that wraps it. + +```python +class SmithyError(Exception): + pass + + +class ServiceError(Exception): + pass + + +class WrappedSmithyError(ServiceError): + def __init__(self, smithy_error: SmithyError): + super().__init__(str(smithy_error)) + self.error = smithy_error +``` + +A generated client would then catch and wrap any instances of `SmithyError` +thrown. This has the advantage of allowing a customer to catch *all* errors +thrown for a given service. This could potentially be useful for code bases +that make use of several clients. + +The downside is that this means a customer can't catch any particular subclass +of `SmithyError` in the normal way. Instead, they'd have to first catch the +wrapped error and then manually access the `error` property to dispatch, +remembering to re-raise if it isn't what they're looking for. + +## Unions + +Unions are separate classes groupd by a `Union` type hint. + +```python +class MyUnionMemberA(MyUnion[bytes]): + def __init__(self, value: bytes): + self.value = value + + def as_dict(self): + return {"MemberA": self.value} + + @staticmethod + def from_dict(d: Dict[str, Any]) -> MyUnionMemberA: + if len(d) != 1: + raise TypeError(f"Unions may have exactly 1 value, but found {len(d)}") + return MyUnionMemberA(d["MemberA"]) + + +class MyUnionMemberB(MyUnion[str]): + def __init__(self, value: str): + self.value = value + + def as_dict(self): + return {"MemberB": self.value} + + @staticmethod + def from_dict(d: Dict[str, Any]) -> MyUnionMemberA: + if len(d) != 1: + raise TypeError(f"Unions may have exactly 1 value, but found {len(d)}") + return MyUnionMemberB(d["MemberB"]) + + +class MyUnionUnknown: + def __init__(self, tag: str, value: bytes): + self.tag = tag + + def as_dict(self) -> Dict[str, Any]: + return {"SDK_UNKNOWN_MEMBER": {"name": self.tag}} + + @staticmethod + def from_dict(d: Dict[str, Any]) -> "AnnouncementsUnknown": + if len(d) != 1: + raise TypeError(f"Unions may have exactly 1 value, but found {len(d)}") + return AnnouncementsUnknown(d["SDK_UNKNOWN_MEMBER"]["name"]) + + +MyUnion = Union[MyUnionMemberA, MyUnionMemberB, MyUnionUnknown] + + +class SampleStruct: + def __init__(self, *, union_member: MyUnion): + self.union_member = union_member +``` + +This allows for exhaustiveness checks since all variants are known ahead of time, +and as of python 3.10+ it also allows isinstance checks as if inheritance were +being used. + +```python +def handle_with_instance(my_union: Any): + if not isinstance(my_union, MyUnion): + return + if isinstance(my_union, MyUnionMemberA): + print(value.decode(encoding="utf-8")) + elif isinstance(my_union, MyUnionMemberB): + for v in my_union.value: + print(v) + else: + raise Exception(f"Unhandled union type: {my_union}") + + +# This is only possible at all in python 3.10 and up +def handle_with_match(my_union: MyUnion): + # A type checker can see that MyUnionMemberB isn't accounted + # for. This implies that updates could cause type checking to fail if + # there's no default case. There is no way to avoid this, and we wouldn't + # want to even if we could. This would expose the error at type checking + # time rather than runtime, which is what we want. + match my_union: + case MyUnionUnknown: + raise Exception(f"Unknown union type: {my_union.tag}") + case MyUnionMemberA: + print(value.decode(encoding="utf-8")) + # A default case could suppress the type check error. + # case _: + # raise Exception(f"Unhandled union type: {my_union}") +``` + +### Alternative: inheritance + +In this alternative, unions are classes with a typed `value` property grouped +by a parent class. + +```python +V = TypeVar("V") + + +class MyUnion(ABC, Generic[V]): + value: V + + @abstractmethod + def as_dict(self): pass + + @abstractmethod + @staticmethod + def from_dict(d: Dict[str, Any]) -> "MyUnion[V]": pass + + +class MyUnionMemberA(MyUnion[bytes]): + def __init__(self, value: bytes): + self.value = value + + def as_dict(self): + return {"MemberA": self.value} + + @staticmethod + def from_dict(d: Dict[str, Any]) -> MyUnionMemberA: + if len(d) != 1: + raise TypeError(f"Unions may have exactly 1 value, but found {len(d)}") + return MyUnionMemberA(d["MemberA"]) + + +class MyUnionMemberB(MyUnion[str]): + def __init__(self, value: str): + self.value = value + + def as_dict(self): + return {"MemberB": self.value} + + @staticmethod + def from_dict(d: Dict[str, Any]) -> MyUnionMemberA: + if len(d) != 1: + raise TypeError(f"Unions may have exactly 1 value, but found {len(d)}") + return MyUnionMemberB(d["MemberB"]) + + +class MyUnionUnknown(MyUnion[None]): + def __init__(self, tag: str): + self.tag = tag + self.value = None + + def as_dict(self) -> Dict[str, Any]: + return {"SDK_UNKNOWN_MEMBER": {"name": self.tag}} + + @staticmethod + def from_dict(d: Dict[str, Any]) -> "AnnouncementsUnknown": + if len(d) != 1: + raise TypeError(f"Unions may have exactly 1 value, but found {len(d)}") + return AnnouncementsUnknown(d["SDK_UNKNOWN_MEMBER"]["name"]) + + +class SampleStruct: + def __init__(self, *, union_member: MyUnion[Any]): + self.union_member = union_member +``` + +This design has a number of disadvantages over using native unions: + +* A type checker cannot know every possible subclass of the base class, so + you won't get exhaustiveness checks or gradual type refinement. An early + version of the pattern matching PEP included a `@sealed` decorator that + would have helped here, but it was removed in the accepted version. + + Even if `@sealed` were introduced in another PEP, it wouldn't be available + until 3.11, meaning if we wanted to use it we'd have to do something like: + +```python +try: + from typing import sealed +except ImportError: + # Identity decorator to allow us to gracefully upgrade to sealed on newer + # python versions. + def sealed(wrapped: Any): + return wrapped +``` + + This would potentially mean a user upgrading from 3.10 to 3.11 would start + seeing type check failures even though they haven't updated the library + or changed any of their code. + +* A user might try to use the parent type directly in their own typing with + some generic type, e.g. `MyUnion[str]`. This could obscure the fact that + multiple variants of a union may have the same member type and lead to bugs. + +* The unknown variant has a `value` property even though it has no value. This + could lead to bugs if the user expects the value to not be none, as it allows + them to not deal with the fact that the unknown variant exits. + +The only advantage this has over using native unions is the ability to use +isinstance checks on versions prior to 3.10. + + +# FAQs + +## Why not use built-in class subtypes for unions? + +Unions can have multiple members targeting the same shape, so there would be +no way to automatically determine what a users intent was if they passed in +the base class. Since they would, therefore, have to always pass in our +concrete types, there would be no advantage to subclassing built-ins. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/black b/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/black new file mode 100644 index 0000000000..7a8cfe5b7b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/black @@ -0,0 +1,188 @@ +// This lockfile was autogenerated by Pants. To regenerate, run: +// +// ./pants generate-lockfiles --resolve=black +// +// --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- +// { +// "version": 3, +// "valid_for_interpreter_constraints": [ +// "CPython>=3.11" +// ], +// "generated_with_requirements": [ +// "black==22.10.0", +// "typing-extensions>=3.10.0.0; python_version < \"3.10\"" +// ], +// "manylinux": "manylinux2014", +// "requirement_constraints": [], +// "only_binary": [], +// "no_binary": [] +// } +// --- END PANTS LOCKFILE METADATA --- + +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "5d8f74030e67087b219b032aa33a919fae8806d49c867846bfacde57f43972ef", + "url": "https://files.pythonhosted.org/packages/a6/84/5c3f3ffc4143fa7e208d745d2239d915e74d3709fdbc64c3e98d3fd27e56/black-22.10.0-1fixedarch-cp311-cp311-macosx_11_0_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "f513588da599943e0cde4e32cc9879e825d58720d6557062d1098c5ad80080e1", + "url": "https://files.pythonhosted.org/packages/a3/89/629fca2eea0899c06befaa58dc0f49d56807d454202bb2e54bd0d98c77f3/black-22.10.0.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "b8b49776299fece66bffaafe357d929ca9451450f5466e997a7285ab0fe28e3b", + "url": "https://files.pythonhosted.org/packages/b0/9e/fa912c5ae4b8eb6d36982fc8ac2d779cf944dbd7c3c1fe7a28acf462c1ed/black-22.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458", + "url": "https://files.pythonhosted.org/packages/ce/6f/74492b8852ee4f2ad2178178f6b65bc8fc80ad539abe56c1c23eab6732e2/black-22.10.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "5b9b29da4f564ba8787c119f37d174f2b69cdfdf9015b7d8c5c16121ddc054ae", + "url": "https://files.pythonhosted.org/packages/e2/2f/a8406a9e337a213802aa90a3e9fbf90c86f3edce92f527255fd381309b77/black-22.10.0-cp311-cp311-macosx_11_0_arm64.whl" + } + ], + "project_name": "black", + "requires_dists": [ + "aiohttp>=3.7.4; extra == \"d\"", + "click>=8.0.0", + "colorama>=0.4.3; extra == \"colorama\"", + "ipython>=7.8.0; extra == \"jupyter\"", + "mypy-extensions>=0.4.3", + "pathspec>=0.9.0", + "platformdirs>=2", + "tokenize-rt>=3.2.0; extra == \"jupyter\"", + "tomli>=1.1.0; python_full_version < \"3.11.0a7\"", + "typed-ast>=1.4.2; python_version < \"3.8\" and implementation_name == \"cpython\"", + "typing-extensions>=3.10.0.0; python_version < \"3.10\"", + "uvloop>=0.15.2; extra == \"uvloop\"" + ], + "requires_python": ">=3.7", + "version": "22.10" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48", + "url": "https://files.pythonhosted.org/packages/c2/f1/df59e28c642d583f7dacffb1e0965d0e00b218e0186d7858ac5233dce840/click-8.1.3-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", + "url": "https://files.pythonhosted.org/packages/59/87/84326af34517fca8c58418d148f2403df25303e02736832403587318e9e8/click-8.1.3.tar.gz" + } + ], + "project_name": "click", + "requires_dists": [ + "colorama; platform_system == \"Windows\"", + "importlib-metadata; python_version < \"3.8\"" + ], + "requires_python": ">=3.7", + "version": "8.1.3" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "url": "https://files.pythonhosted.org/packages/5c/eb/975c7c080f3223a5cdaff09612f3a5221e4ba534f7039db34c35d95fa6a5/mypy_extensions-0.4.3-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8", + "url": "https://files.pythonhosted.org/packages/63/60/0582ce2eaced55f65a4406fc97beba256de4b7a95a0034c6576458c6519f/mypy_extensions-0.4.3.tar.gz" + } + ], + "project_name": "mypy-extensions", + "requires_dists": [ + "typing>=3.5.3; python_version < \"3.5\"" + ], + "requires_python": null, + "version": "0.4.3" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "3c95343af8b756205e2aba76e843ba9520a24dd84f68c22b9f93251507509dd6", + "url": "https://files.pythonhosted.org/packages/3c/29/c07c3a976dbe37c56e381e058c11e8738cb3a0416fc842a310461f8bb695/pathspec-0.10.3-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "56200de4077d9d0791465aa9095a01d421861e405b5096955051deefd697d6f6", + "url": "https://files.pythonhosted.org/packages/32/1a/6baf904503c3e943cae9605c9c88a43b964dea5b59785cf956091b341b08/pathspec-0.10.3.tar.gz" + } + ], + "project_name": "pathspec", + "requires_dists": [], + "requires_python": ">=3.7", + "version": "0.10.3" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490", + "url": "https://files.pythonhosted.org/packages/c1/c7/9be9d651b93efce682b45142a6267034fc4215972780748618c02e236361/platformdirs-2.6.2-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2", + "url": "https://files.pythonhosted.org/packages/cf/4d/198b7e6c6c2b152f4f9f4cdf975d3590e33e63f1920f2d89af7f0390e6db/platformdirs-2.6.2.tar.gz" + } + ], + "project_name": "platformdirs", + "requires_dists": [ + "appdirs==1.4.4; extra == \"test\"", + "covdefaults>=2.2.2; extra == \"test\"", + "furo>=2022.12.7; extra == \"docs\"", + "proselint>=0.13; extra == \"docs\"", + "pytest-cov>=4; extra == \"test\"", + "pytest-mock>=3.10; extra == \"test\"", + "pytest>=7.2; extra == \"test\"", + "sphinx-autodoc-typehints>=1.19.5; extra == \"docs\"", + "sphinx>=5.3; extra == \"docs\"", + "typing-extensions>=4.4; python_version < \"3.8\"" + ], + "requires_python": ">=3.7", + "version": "2.6.2" + } + ], + "platform_tag": null + } + ], + "path_mappings": {}, + "pex_version": "2.1.108", + "pip_version": "20.3.4-patched", + "prefer_older_binary": false, + "requirements": [ + "black==22.10.0", + "typing-extensions>=3.10.0.0; python_version < \"3.10\"" + ], + "requires_python": [ + ">=3.11" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/docformatter b/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/docformatter new file mode 100644 index 0000000000..e2efdaf615 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/docformatter @@ -0,0 +1,126 @@ +// This lockfile was autogenerated by Pants. To regenerate, run: +// +// ./pants generate-lockfiles --resolve=docformatter +// +// --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- +// { +// "version": 3, +// "valid_for_interpreter_constraints": [ +// "CPython>=3.11" +// ], +// "generated_with_requirements": [ +// "docformatter==1.5.1" +// ], +// "manylinux": "manylinux2014", +// "requirement_constraints": [], +// "only_binary": [], +// "no_binary": [] +// } +// --- END PANTS LOCKFILE METADATA --- + +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f", + "url": "https://files.pythonhosted.org/packages/db/51/a507c856293ab05cdc1db77ff4bc1268ddd39f29e7dc4919aa497f0adbec/charset_normalizer-2.1.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", + "url": "https://files.pythonhosted.org/packages/a1/34/44964211e5410b051e4b8d2869c470ae8a68ae274953b1c7de6d98bbcf94/charset-normalizer-2.1.1.tar.gz" + } + ], + "project_name": "charset-normalizer", + "requires_dists": [ + "unicodedata2; extra == \"unicode_backport\"" + ], + "requires_python": ">=3.6.0", + "version": "2.1.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "05d6e4c528278b3a54000e08695822617a38963a380f5aef19e12dd0e630f19a", + "url": "https://files.pythonhosted.org/packages/81/21/ffe70580b217d1e6c7ca5f3e67b050abf750d312d5403c6b4463074d1aeb/docformatter-1.5.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "3fa3cdb90cdbcdee82747c58410e47fc7e2e8c352b82bed80767915eb03f2e43", + "url": "https://files.pythonhosted.org/packages/1c/5b/337824ecca1b5a593cd96aa3404ac7e6d7fa65ae750d94c0e57e255b6168/docformatter-1.5.1.tar.gz" + } + ], + "project_name": "docformatter", + "requires_dists": [ + "charset_normalizer<3.0.0,>=2.0.0", + "tomli<2.0.0; python_version < \"3.7\" and extra == \"tomli\"", + "tomli<3.0.0,>=2.0.0; python_version >= \"3.7\"", + "untokenize<0.2.0,>=0.1.1" + ], + "requires_python": "<4.0,>=3.6", + "version": "1.5.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f", + "url": "https://files.pythonhosted.org/packages/c0/3f/d7af728f075fb08564c5949a9c95e44352e23dee646869fa104a3b2060a3/tomli-2.0.1.tar.gz" + } + ], + "project_name": "tomli", + "requires_dists": [], + "requires_python": ">=3.7", + "version": "2.0.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "3865dbbbb8efb4bb5eaa72f1be7f3e0be00ea8b7f125c69cbd1f5fda926f37a2", + "url": "https://files.pythonhosted.org/packages/f7/46/e7cea8159199096e1df52da20a57a6665da80c37fb8aeb848a3e47442c32/untokenize-0.1.1.tar.gz" + } + ], + "project_name": "untokenize", + "requires_dists": [], + "requires_python": null, + "version": "0.1.1" + } + ], + "platform_tag": null + } + ], + "path_mappings": {}, + "pex_version": "2.1.108", + "pip_version": "20.3.4-patched", + "prefer_older_binary": false, + "requirements": [ + "docformatter==1.5.1" + ], + "requires_python": [ + ">=3.11" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/flake8 b/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/flake8 new file mode 100644 index 0000000000..782b70a9a4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/flake8 @@ -0,0 +1,128 @@ +// This lockfile was autogenerated by Pants. To regenerate, run: +// +// ./pants generate-lockfiles --resolve=flake8 +// +// --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- +// { +// "version": 3, +// "valid_for_interpreter_constraints": [ +// "CPython>=3.11" +// ], +// "generated_with_requirements": [ +// "flake8<6.1" +// ], +// "manylinux": "manylinux2014", +// "requirement_constraints": [], +// "only_binary": [], +// "no_binary": [] +// } +// --- END PANTS LOCKFILE METADATA --- + +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7", + "url": "https://files.pythonhosted.org/packages/d9/6a/bb0122ebe280476c924470779d2595f1403878cafe3c8a343ac56a5a9c0e/flake8-6.0.0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181", + "url": "https://files.pythonhosted.org/packages/66/53/3ad4a3b74d609b3b9008a10075c40e7c8909eae60af53623c3888f7a529a/flake8-6.0.0.tar.gz" + } + ], + "project_name": "flake8", + "requires_dists": [ + "mccabe<0.8.0,>=0.7.0", + "pycodestyle<2.11.0,>=2.10.0", + "pyflakes<3.1.0,>=3.0.0" + ], + "requires_python": ">=3.8.1", + "version": "6" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", + "url": "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "url": "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz" + } + ], + "project_name": "mccabe", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "0.7" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610", + "url": "https://files.pythonhosted.org/packages/a2/54/001fdc0d69e8d0bb86c3423a6fa6dfada8cc26317c2635ab543e9ac411bd/pycodestyle-2.10.0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", + "url": "https://files.pythonhosted.org/packages/06/6b/5ca0d12ef7dcf7d20dfa35287d02297f3e0f9e515da5183654c03a9636ce/pycodestyle-2.10.0.tar.gz" + } + ], + "project_name": "pycodestyle", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "2.10" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf", + "url": "https://files.pythonhosted.org/packages/af/4c/b1c7008aa7788b3e26c06c60aa18da7d3aa1f00e344aa3f18ac92768854b/pyflakes-3.0.1-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd", + "url": "https://files.pythonhosted.org/packages/f2/51/506ddcfab10d708e8460554cc1cf37c727a6a2cccbad8dfe57766cfce33c/pyflakes-3.0.1.tar.gz" + } + ], + "project_name": "pyflakes", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "3.0.1" + } + ], + "platform_tag": null + } + ], + "path_mappings": {}, + "pex_version": "2.1.108", + "pip_version": "20.3.4-patched", + "prefer_older_binary": false, + "requirements": [ + "flake8<6.1" + ], + "requires_python": [ + ">=3.11" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/mypy b/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/mypy new file mode 100644 index 0000000000..c70da64d72 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/mypy @@ -0,0 +1,135 @@ +// This lockfile was autogenerated by Pants. To regenerate, run: +// +// ./pants generate-lockfiles --resolve=mypy +// +// --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- +// { +// "version": 3, +// "valid_for_interpreter_constraints": [ +// "CPython>=3.11" +// ], +// "generated_with_requirements": [ +// "mypy==1.0.0" +// ], +// "manylinux": "manylinux2014", +// "requirement_constraints": [], +// "only_binary": [], +// "no_binary": [] +// } +// --- END PANTS LOCKFILE METADATA --- + +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "2efa963bdddb27cb4a0d42545cd137a8d2b883bd181bbc4525b568ef6eca258f", + "url": "https://files.pythonhosted.org/packages/5f/4b/7b65392ae3a1fbc924b4ebb6b80708c6b06f86e8123739f883c7499c5bc4/mypy-1.0.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "f34495079c8d9da05b183f9f7daec2878280c2ad7cc81da686ef0b484cea2ecf", + "url": "https://files.pythonhosted.org/packages/1d/06/9a40050ef10f0e9ddfd667f29e98dd650db31612128e3e8925cda6621944/mypy-1.0.0.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "14d776869a3e6c89c17eb943100f7868f677703c8a4e00b3803918f86aafbc52", + "url": "https://files.pythonhosted.org/packages/59/f9/cd5f17593583bf08944a30311b3d92362643db19d6078847b36cfaebe014/mypy-1.0.0-cp311-cp311-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "8845125d0b7c57838a10fd8925b0f5f709d0e08568ce587cc862aacce453e3dd", + "url": "https://files.pythonhosted.org/packages/8f/84/9a37fd92f19edf66b6127fa22146a79214e130e27ed21a68ffbde16487aa/mypy-1.0.0-cp311-cp311-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "5cfca124f0ac6707747544c127880893ad72a656e136adc935c8600740b21ff5", + "url": "https://files.pythonhosted.org/packages/a9/d3/5a3ec1a0413b16ea79abb511703424ef0f4bb538b5bb713a721c7d04ecb2/mypy-1.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "bb2782a036d9eb6b5a6efcdda0986774bf798beef86a62da86cb73e2a10b423d", + "url": "https://files.pythonhosted.org/packages/f9/4d/60037c331964be7e5ef98e7b5b696ac10516d254d71d9ab6459f231f3de2/mypy-1.0.0-cp311-cp311-macosx_11_0_arm64.whl" + } + ], + "project_name": "mypy", + "requires_dists": [ + "lxml; extra == \"reports\"", + "mypy-extensions>=0.4.3", + "pip; extra == \"install-types\"", + "psutil>=4.0; extra == \"dmypy\"", + "tomli>=1.1.0; python_version < \"3.11\"", + "typed-ast<2,>=1.4.0; extra == \"python2\"", + "typed-ast<2,>=1.4.0; python_version < \"3.8\"", + "typing-extensions>=3.10" + ], + "requires_python": ">=3.7", + "version": "1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", + "url": "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", + "url": "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz" + } + ], + "project_name": "mypy-extensions", + "requires_dists": [], + "requires_python": ">=3.5", + "version": "1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4", + "url": "https://files.pythonhosted.org/packages/31/25/5abcd82372d3d4a3932e1fa8c3dbf9efac10cc7c0d16e78467460571b404/typing_extensions-4.5.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb", + "url": "https://files.pythonhosted.org/packages/d3/20/06270dac7316220643c32ae61694e451c98f8caf4c8eab3aa80a2bedf0df/typing_extensions-4.5.0.tar.gz" + } + ], + "project_name": "typing-extensions", + "requires_dists": [], + "requires_python": ">=3.7", + "version": "4.5" + } + ], + "platform_tag": null + } + ], + "path_mappings": {}, + "pex_version": "2.1.108", + "pip_version": "20.3.4-patched", + "prefer_older_binary": false, + "requirements": [ + "mypy==1.0.0" + ], + "requires_python": [ + ">=3.11" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/pytest b/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/pytest new file mode 100644 index 0000000000..fd7f628069 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/lockfiles/pytest @@ -0,0 +1,366 @@ +// This lockfile was autogenerated by Pants. To regenerate, run: +// +// ./pants generate-lockfiles --resolve=pytest +// +// --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- +// { +// "version": 3, +// "valid_for_interpreter_constraints": [ +// "CPython>=3.11" +// ], +// "generated_with_requirements": [ +// "freezegun<1.3.0", +// "pytest-asyncio<0.21.0", +// "pytest-cov<3.1", +// "pytest<7.3" +// ], +// "manylinux": "manylinux2014", +// "requirement_constraints": [], +// "only_binary": [], +// "no_binary": [] +// } +// --- END PANTS LOCKFILE METADATA --- + +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836", + "url": "https://files.pythonhosted.org/packages/fb/6e/6f83bf616d2becdf333a1640f1d463fef3150e2e926b7010cb0f81c95e88/attrs-22.2.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99", + "url": "https://files.pythonhosted.org/packages/21/31/3f468da74c7de4fcf9b25591e682856389b3400b4b62f201e65f15ea3e07/attrs-22.2.0.tar.gz" + } + ], + "project_name": "attrs", + "requires_dists": [ + "attrs[docs,tests]; extra == \"dev\"", + "attrs[tests-no-zope]; extra == \"tests\"", + "attrs[tests]; extra == \"cov\"", + "cloudpickle; platform_python_implementation == \"CPython\" and extra == \"tests-no-zope\"", + "cloudpickle; platform_python_implementation == \"CPython\" and extra == \"tests_no_zope\"", + "coverage-enable-subprocess; extra == \"cov\"", + "coverage[toml]>=5.3; extra == \"cov\"", + "furo; extra == \"docs\"", + "hypothesis; extra == \"tests-no-zope\"", + "hypothesis; extra == \"tests_no_zope\"", + "mypy<0.990,>=0.971; platform_python_implementation == \"CPython\" and extra == \"tests-no-zope\"", + "mypy<0.990,>=0.971; platform_python_implementation == \"CPython\" and extra == \"tests_no_zope\"", + "myst-parser; extra == \"docs\"", + "pympler; extra == \"tests-no-zope\"", + "pympler; extra == \"tests_no_zope\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version < \"3.11\") and extra == \"tests-no-zope\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version < \"3.11\") and extra == \"tests_no_zope\"", + "pytest-xdist[psutil]; extra == \"tests-no-zope\"", + "pytest-xdist[psutil]; extra == \"tests_no_zope\"", + "pytest>=4.3.0; extra == \"tests-no-zope\"", + "pytest>=4.3.0; extra == \"tests_no_zope\"", + "sphinx-notfound-page; extra == \"docs\"", + "sphinx; extra == \"docs\"", + "sphinxcontrib-towncrier; extra == \"docs\"", + "towncrier; extra == \"docs\"", + "zope.interface; extra == \"docs\"", + "zope.interface; extra == \"tests\"" + ], + "requires_python": ">=3.6", + "version": "22.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "09643fb0df8e29f7417adc3f40aaf379d071ee8f0350ab290517c7004f05360b", + "url": "https://files.pythonhosted.org/packages/bf/4e/bb6008789e813f6930179757acdd409f0056e48ef687416bed819464a79c/coverage-7.2.1-cp311-cp311-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "80559eaf6c15ce3da10edb7977a1548b393db36cbc6cf417633eca05d84dd1ed", + "url": "https://files.pythonhosted.org/packages/24/8d/d9d880cb7319cc06eab02757a0fb3f623c6e7613d16d297cfdf249d4926d/coverage-7.2.1-cp311-cp311-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "9e872b082b32065ac2834149dc0adc2a2e6d8203080501e1e3c3c77851b466f9", + "url": "https://files.pythonhosted.org/packages/2d/24/06ad2452717337ed45a928107fc5d91601a4a79692012ee86dc06782ab51/coverage-7.2.1-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "d9256d4c60c4bbfec92721b51579c50f9e5062c21c12bec56b55292464873508", + "url": "https://files.pythonhosted.org/packages/47/83/d5353ffb69cd7cfb32e146475d10b6ebba930d9eb323e508933df0d02434/coverage-7.2.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "fac6343bae03b176e9b58104a9810df3cdccd5cfed19f99adfa807ffbf43cf9b", + "url": "https://files.pythonhosted.org/packages/4e/6b/7d9c6c23aa227b91bc2f85f197406b899a4469c6e0d182d499eb2a515e91/coverage-7.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "570c21a29493b350f591a4b04c158ce1601e8d18bdcd21db136fbb135d75efa6", + "url": "https://files.pythonhosted.org/packages/51/61/2bab4add265c0fcf0a4372ab9e647405f157a9c5cdcbab1e0b7b117f92fa/coverage-7.2.1-cp311-cp311-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "c77f2a9093ccf329dd523a9b2b3c854c20d2a3d968b6def3b820272ca6732242", + "url": "https://files.pythonhosted.org/packages/8d/4a/3518606d4b110df4f3e77bd52c241ae8a84c6dc74fac7c2a8e809449e541/coverage-7.2.1.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "abacd0a738e71b20e224861bc87e819ef46fedba2fb01bc1af83dfd122e9c319", + "url": "https://files.pythonhosted.org/packages/af/ff/bf04eeb95213c25a5ef718e1e70b6e476f4e6f48b00d62860f3a8facd3ef/coverage-7.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "0bd7e628f6c3ec4e7d2d24ec0e50aae4e5ae95ea644e849d92ae4805650b4c4e", + "url": "https://files.pythonhosted.org/packages/d9/c5/8a6ad089d9d9a15f94f40957d804e7712767f8ac458eca55b02a73d249d6/coverage-7.2.1-cp311-cp311-musllinux_1_1_i686.whl" + } + ], + "project_name": "coverage", + "requires_dists": [ + "tomli; python_full_version <= \"3.11.0a6\" and extra == \"toml\"" + ], + "requires_python": ">=3.7", + "version": "7.2.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "ea1b963b993cb9ea195adbd893a48d573fda951b0da64f60883d7e988b606c9f", + "url": "https://files.pythonhosted.org/packages/50/cd/ba1c8319c002727ccfa03049127218d1767232a77219924d03ba170e0601/freezegun-1.2.2-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "cd22d1ba06941384410cd967d8a99d5ae2442f57dfafeff2fda5de8dc5c05446", + "url": "https://files.pythonhosted.org/packages/1d/97/002ac49ec52858538b4aa6f6831f83c2af562c17340bdf6043be695f39ac/freezegun-1.2.2.tar.gz" + } + ], + "project_name": "freezegun", + "requires_dists": [ + "python-dateutil>=2.7" + ], + "requires_python": ">=3.6", + "version": "1.2.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", + "url": "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", + "url": "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz" + } + ], + "project_name": "iniconfig", + "requires_dists": [], + "requires_python": ">=3.7", + "version": "2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2", + "url": "https://files.pythonhosted.org/packages/ed/35/a31aed2993e398f6b09a790a181a7927eb14610ee8bbf02dc14d31677f1c/packaging-23.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97", + "url": "https://files.pythonhosted.org/packages/47/d5/aca8ff6f49aa5565df1c826e7bf5e85a6df852ee063600c1efa5b932968c/packaging-23.0.tar.gz" + } + ], + "project_name": "packaging", + "requires_dists": [], + "requires_python": ">=3.7", + "version": "23" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3", + "url": "https://files.pythonhosted.org/packages/9e/01/f38e2ff29715251cf25532b9082a1589ab7e4f571ced434f98d0139336dc/pluggy-1.0.0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", + "url": "https://files.pythonhosted.org/packages/a1/16/db2d7de3474b6e37cbb9c008965ee63835bba517e22cdb8c35b5116b5ce1/pluggy-1.0.0.tar.gz" + } + ], + "project_name": "pluggy", + "requires_dists": [ + "importlib-metadata>=0.12; python_version < \"3.8\"", + "pre-commit; extra == \"dev\"", + "pytest-benchmark; extra == \"testing\"", + "pytest; extra == \"testing\"", + "tox; extra == \"dev\"" + ], + "requires_python": ">=3.6", + "version": "1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5", + "url": "https://files.pythonhosted.org/packages/cc/02/8f59bf194c9a1ceac6330850715e9ec11e21e2408a30a596c65d54cf4d2a/pytest-7.2.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42", + "url": "https://files.pythonhosted.org/packages/e5/6c/f3a15217ac72912c28c5d7a7a8e87ff6d6475c9530595ae9f0f8dedd8dd8/pytest-7.2.1.tar.gz" + } + ], + "project_name": "pytest", + "requires_dists": [ + "argcomplete; extra == \"testing\"", + "attrs>=19.2.0", + "colorama; sys_platform == \"win32\"", + "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", + "hypothesis>=3.56; extra == \"testing\"", + "importlib-metadata>=0.12; python_version < \"3.8\"", + "iniconfig", + "mock; extra == \"testing\"", + "nose; extra == \"testing\"", + "packaging", + "pluggy<2.0,>=0.12", + "pygments>=2.7.2; extra == \"testing\"", + "requests; extra == \"testing\"", + "tomli>=1.0.0; python_version < \"3.11\"", + "xmlschema; extra == \"testing\"" + ], + "requires_python": ">=3.7", + "version": "7.2.1" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "f129998b209d04fcc65c96fc85c11e5316738358909a8399e93be553d7656442", + "url": "https://files.pythonhosted.org/packages/45/74/9421cfde8def10c265b4f9ae19c95b8f4dc227f639cb8b89287d4946ac97/pytest_asyncio-0.20.3-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "83cbf01169ce3e8eb71c6c278ccb0574d1a7a3bb8eaaf5e50e0ad342afb33b36", + "url": "https://files.pythonhosted.org/packages/6e/06/38b0ca5d53582bb49697626975b5540435ea064762d852b5c66646c729e9/pytest-asyncio-0.20.3.tar.gz" + } + ], + "project_name": "pytest-asyncio", + "requires_dists": [ + "coverage>=6.2; extra == \"testing\"", + "flaky>=3.5.0; extra == \"testing\"", + "hypothesis>=5.7.1; extra == \"testing\"", + "mypy>=0.931; extra == \"testing\"", + "pytest-trio>=0.7.0; extra == \"testing\"", + "pytest>=6.1.0", + "sphinx-rtd-theme>=1.0; extra == \"docs\"", + "sphinx>=5.3; extra == \"docs\"", + "typing-extensions>=3.7.2; python_version < \"3.8\"" + ], + "requires_python": ">=3.7", + "version": "0.20.3" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6", + "url": "https://files.pythonhosted.org/packages/20/49/b3e0edec68d81846f519c602ac38af9db86e1e71275528b3e814ae236063/pytest_cov-3.0.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470", + "url": "https://files.pythonhosted.org/packages/61/41/e046526849972555928a6d31c2068410e47a31fb5ab0a77f868596811329/pytest-cov-3.0.0.tar.gz" + } + ], + "project_name": "pytest-cov", + "requires_dists": [ + "coverage[toml]>=5.2.1", + "fields; extra == \"testing\"", + "hunter; extra == \"testing\"", + "process-tests; extra == \"testing\"", + "pytest-xdist; extra == \"testing\"", + "pytest>=4.6", + "six; extra == \"testing\"", + "virtualenv; extra == \"testing\"" + ], + "requires_python": ">=3.6", + "version": "3" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9", + "url": "https://files.pythonhosted.org/packages/36/7a/87837f39d0296e723bb9b62bbb257d0355c7f6128853c78955f57342a56d/python_dateutil-2.8.2-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "url": "https://files.pythonhosted.org/packages/4c/c4/13b4776ea2d76c115c1d1b84579f3764ee6d57204f6be27119f13a61d0a9/python-dateutil-2.8.2.tar.gz" + } + ], + "project_name": "python-dateutil", + "requires_dists": [ + "six>=1.5" + ], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7", + "version": "2.8.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254", + "url": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "url": "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz" + } + ], + "project_name": "six", + "requires_dists": [], + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7", + "version": "1.16" + } + ], + "platform_tag": null + } + ], + "path_mappings": {}, + "pex_version": "2.1.108", + "pip_version": "20.3.4-patched", + "prefer_older_binary": false, + "requirements": [ + "freezegun<1.3.0", + "pytest-asyncio<0.21.0", + "pytest-cov<3.1", + "pytest<7.3" + ], + "requires_python": [ + ">=3.11" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/pants b/codegen/smithy-dafny-codegen-modules/smithy-python/pants new file mode 100755 index 0000000000..badfe7b209 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/pants @@ -0,0 +1,389 @@ +#!/usr/bin/env bash +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# =============================== NOTE =============================== +# This ./pants bootstrap script comes from the pantsbuild/setup +# project. It is intended to be checked into your code repository so +# that other developers have the same setup. +# +# Learn more here: https://www.pantsbuild.org/docs/installation +# ==================================================================== + +set -eou pipefail + +# NOTE: To use an unreleased version of Pants from the pantsbuild/pants main branch, +# locate the main branch SHA, set PANTS_SHA= in the environment, and run this script as usual. +# +# E.g., PANTS_SHA=725fdaf504237190f6787dda3d72c39010a4c574 ./pants --version + +PYTHON_BIN_NAME="${PYTHON:-unspecified}" + +# Set this to specify a non-standard location for this script to read the Pants version from. +# NB: This will *not* cause Pants itself to use this location as a config file. +# You can use PANTS_CONFIG_FILES or --pants-config-files to do so. +PANTS_TOML=${PANTS_TOML:-pants.toml} + +PANTS_BIN_NAME="${PANTS_BIN_NAME:-$0}" + +PANTS_SETUP_CACHE="${PANTS_SETUP_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/pants/setup}" +# If given a relative path, we fix it to be absolute. +if [[ "$PANTS_SETUP_CACHE" != /* ]]; then + PANTS_SETUP_CACHE="${PWD}/${PANTS_SETUP_CACHE}" +fi + +PANTS_BOOTSTRAP="${PANTS_SETUP_CACHE}/bootstrap-$(uname -s)-$(uname -m)" + +_PEX_VERSION=2.1.62 +_PEX_URL="https://github.com/pantsbuild/pex/releases/download/v${_PEX_VERSION}/pex" +_PEX_EXPECTED_SHA256="56668b1ca147bd63141e586ffee97c7cc51ce8e6eac6c9b7a4bf1215b94396e5" + +VIRTUALENV_VERSION=20.4.7 +VIRTUALENV_REQUIREMENTS=$( +cat << EOF +virtualenv==${VIRTUALENV_VERSION} --hash sha256:2b0126166ea7c9c3661f5b8e06773d28f83322de7a3ff7d06f0aed18c9de6a76 +filelock==3.0.12 --hash sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836 +six==1.16.0 --hash sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 +distlib==0.3.2 --hash sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c +appdirs==1.4.4 --hash sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128 +importlib-resources==5.1.4; python_version < "3.7" --hash sha256:e962bff7440364183203d179d7ae9ad90cb1f2b74dcb84300e88ecc42dca3351 +importlib-metadata==4.5.0; python_version < "3.8" --hash sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00 +zipp==3.4.1; python_version < "3.10" --hash sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098 +typing-extensions==3.10.0.0; python_version < "3.8" --hash sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84 +EOF +) + +COLOR_RED="\x1b[31m" +COLOR_GREEN="\x1b[32m" +COLOR_YELLOW="\x1b[33m" +COLOR_RESET="\x1b[0m" + +function log() { + echo -e "$@" 1>&2 +} + +function die() { + (($# > 0)) && log "${COLOR_RED}$*${COLOR_RESET}" + exit 1 +} + +function green() { + (($# > 0)) && log "${COLOR_GREEN}$*${COLOR_RESET}" +} + +function warn() { + (($# > 0)) && log "${COLOR_YELLOW}$*${COLOR_RESET}" +} + +function tempdir { + mkdir -p "$1" + mktemp -d "$1"/pants.XXXXXX +} + +function get_exe_path_or_die { + local exe="$1" + if ! command -v "${exe}"; then + die "Could not find ${exe}. Please ensure ${exe} is on your PATH." + fi +} + +function get_pants_config_string_value { + local config_key="$1" + local optional_space="[[:space:]]*" + local prefix="^${config_key}${optional_space}=${optional_space}" + local raw_value + raw_value="$(sed -ne "/${prefix}/ s|${prefix}||p" "${PANTS_TOML}")" + local optional_suffix="${optional_space}(#.*)?$" + echo "${raw_value}" \ + | sed -E \ + -e "s|^'([^']*)'${optional_suffix}|\1|" \ + -e 's|^"([^"]*)"'"${optional_suffix}"'$|\1|' \ + && return 0 + return 0 +} + +function get_python_major_minor_version { + local python_exe="$1" + "$python_exe" </dev/null 2>&1; then + continue + fi + if [[ -n "$(check_python_exe_compatible_version "${interpreter_path}")" ]]; then + echo "${interpreter_path}" && return 0 + fi + done +} + +function determine_python_exe { + local pants_version="$1" + set_supported_python_versions "${pants_version}" + local requirement_str="For \`pants_version = \"${pants_version}\"\`, Pants requires Python ${supported_message} to run." + + local python_exe + if [[ "${PYTHON_BIN_NAME}" != 'unspecified' ]]; then + python_exe="$(get_exe_path_or_die "${PYTHON_BIN_NAME}")" || exit 1 + if [[ -z "$(check_python_exe_compatible_version "${python_exe}")" ]]; then + die "Invalid Python interpreter version for ${python_exe}. ${requirement_str}" + fi + else + python_exe="$(determine_default_python_exe)" + if [[ -z "${python_exe}" ]]; then + die "No valid Python interpreter found. ${requirement_str} Please check that a valid interpreter is installed and on your \$PATH." + fi + fi + echo "${python_exe}" +} + +function compute_sha256 { + local python="$1" + local path="$2" + + "$python" <&2 || exit 1 + fi + echo "${bootstrapped}" +} + +function scrub_PEX_env_vars { + # Ensure the virtualenv PEX runs as shrink-wrapped. + # See: https://github.com/pantsbuild/setup/issues/105 + if [[ -n "${!PEX_@}" ]]; then + warn "Scrubbing ${!PEX_@}" + unset "${!PEX_@}" + fi +} + +function bootstrap_virtualenv { + local python="$1" + local bootstrapped="${PANTS_BOOTSTRAP}/virtualenv-${VIRTUALENV_VERSION}/virtualenv.pex" + if [[ ! -f "${bootstrapped}" ]]; then + ( + green "Creating the virtualenv PEX." + pex_path="$(bootstrap_pex "${python}")" || exit 1 + mkdir -p "${PANTS_BOOTSTRAP}" + local staging_dir + staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") + cd "${staging_dir}" + echo "${VIRTUALENV_REQUIREMENTS}" > requirements.txt + ( + scrub_PEX_env_vars + "${python}" "${pex_path}" -r requirements.txt -c virtualenv -o virtualenv.pex + ) + mkdir -p "$(dirname "${bootstrapped}")" + mv -f "${staging_dir}/virtualenv.pex" "${bootstrapped}" + rm -rf "${staging_dir}" + ) 1>&2 || exit 1 + fi + echo "${bootstrapped}" +} + +function find_links_url { + local pants_version="$1" + local pants_sha="$2" + echo -n "https://binaries.pantsbuild.org/wheels/pantsbuild.pants/${pants_sha}/${pants_version/+/%2B}/index.html" +} + +function get_version_for_sha { + local sha="$1" + + # Retrieve the Pants version associated with this commit. + local pants_version + pants_version="$(curl --proto "=https" \ + --tlsv1.2 \ + --fail \ + --silent \ + --location \ + "https://raw.githubusercontent.com/pantsbuild/pants/${sha}/src/python/pants/VERSION")" + + # Construct the version as the release version from src/python/pants/VERSION, plus the string `+gitXXXXXXXX`, + # where the XXXXXXXX is the first 8 characters of the SHA. + echo "${pants_version}+git${sha:0:8}" +} + +function bootstrap_pants { + local pants_version="$1" + local python="$2" + local pants_sha="${3:-}" + + local pants_requirement="pantsbuild.pants==${pants_version}" + local maybe_find_links + if [[ -z "${pants_sha}" ]]; then + maybe_find_links="" + else + maybe_find_links="--find-links=$(find_links_url "${pants_version}" "${pants_sha}")" + fi + local python_major_minor_version + python_major_minor_version="$(get_python_major_minor_version "${python}")" + local target_folder_name="${pants_version}_py${python_major_minor_version}" + local bootstrapped="${PANTS_BOOTSTRAP}/${target_folder_name}" + + if [[ ! -d "${bootstrapped}" ]]; then + ( + green "Bootstrapping Pants using ${python}" + local staging_dir + staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") + local virtualenv_path + virtualenv_path="$(bootstrap_virtualenv "${python}")" || exit 1 + green "Installing ${pants_requirement} into a virtual environment at ${bootstrapped}" + ( + scrub_PEX_env_vars + # shellcheck disable=SC2086 + "${python}" "${virtualenv_path}" --quiet --no-download "${staging_dir}/install" && \ + # Grab the latest pip, but don't advance setuptools past 58 which drops support for the + # `setup` kwarg `use_2to3` which Pants 1.x sdist dependencies (pystache) use. + "${staging_dir}/install/bin/pip" install --quiet -U pip "setuptools<58" && \ + "${staging_dir}/install/bin/pip" install ${maybe_find_links} --quiet --progress-bar off "${pants_requirement}" + ) && \ + ln -s "${staging_dir}/install" "${staging_dir}/${target_folder_name}" && \ + mv "${staging_dir}/${target_folder_name}" "${bootstrapped}" && \ + green "New virtual environment successfully created at ${bootstrapped}." + ) 1>&2 || exit 1 + fi + echo "${bootstrapped}" +} + +# Ensure we operate from the context of the ./pants buildroot. +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +pants_version="$(determine_pants_version)" +python="$(determine_python_exe "${pants_version}")" +pants_dir="$(bootstrap_pants "${pants_version}" "${python}" "${PANTS_SHA:-}")" || exit 1 + +pants_python="${pants_dir}/bin/python" +pants_binary="${pants_dir}/bin/pants" +pants_extra_args="" +if [[ -n "${PANTS_SHA:-}" ]]; then + pants_extra_args="${pants_extra_args} --python-repos-repos=$(find_links_url "$pants_version" "$PANTS_SHA")" +fi + +# shellcheck disable=SC2086 +exec "${pants_python}" "${pants_binary}" ${pants_extra_args} \ + --pants-bin-name="${PANTS_BIN_NAME}" --pants-version=${pants_version} "$@" diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/pants.toml b/codegen/smithy-dafny-codegen-modules/smithy-python/pants.toml new file mode 100644 index 0000000000..309df29331 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/pants.toml @@ -0,0 +1,71 @@ +[GLOBAL] +pants_version = "2.14.0" + +backend_packages = [ + "pants.backend.python", + "pants.backend.python.lint.black", + "pants.backend.python.lint.isort", + "pants.backend.python.lint.flake8", + "pants.backend.python.typecheck.mypy", + "pants.backend.experimental.python.lint.pyupgrade", + "pants.backend.python.lint.docformatter", +] + +[source] +root_patterns = [ + "python-packages/*" +] + +[python] +interpreter_constraints = [">=3.11"] + +[black] +version = "black==22.10.0" +args = ["-t py311"] +lockfile = "lockfiles/black" +interpreter_constraints = [">=3.11"] + +[pytest] +version = "pytest<7.3" +extra_requirements = [ + "pytest-asyncio<0.21.0", + "pytest-cov<3.1", + "freezegun<1.3.0" +] +lockfile = "lockfiles/pytest" + +[coverage-py] +interpreter_constraints = [">=3.11"] +report = [ + "xml", + "html" +] + +[mypy] +version = "mypy==1.0.0" +lockfile = "lockfiles/mypy" +interpreter_constraints = [">=3.11"] +extra_type_stubs = [ + "types-freezegun<1.2.0", +] + +[flake8] +version = "flake8<6.1" +lockfile = "lockfiles/flake8" +args = "--extend-ignore=W503" + +[pyupgrade] +args = ["--py311-plus"] +interpreter_constraints = [">=3.11"] + +[docformatter] +version = "docformatter==1.5.1" +args = ["--wrap-summaries 88", "--wrap-descriptions 88"] +lockfile = "lockfiles/docformatter" +interpreter_constraints = [">=3.11"] + +[poetry] +interpreter_constraints = [">=3.11"] + +[anonymous-telemetry] +enabled = false diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/pyproject.toml b/codegen/smithy-dafny-codegen-modules/smithy-python/pyproject.toml new file mode 100644 index 0000000000..edf18cffa4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/pyproject.toml @@ -0,0 +1,7 @@ +[tool.mypy] +strict = true +warn_unused_configs = true + +[[tool.mypy.overrides]] +module = ["awscrt", "pytest"] +ignore_missing_imports = true diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/BUILD b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/BUILD new file mode 100644 index 0000000000..e7e3a24ef9 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/BUILD @@ -0,0 +1,28 @@ +resource(name="pyproject", source="pyproject.toml") +resource(name="readme", source="README.md") +resource(name="notice", source="NOTICE") + +python_distribution( + name="dist", + dependencies=[ + ":pyproject", + ":readme", + ":notice", + ":requirements", + "python-packages/aws-smithy-python/aws_smithy_python:source", + "python-packages/smithy-python:dist", + ], + provides=python_artifact( + name="aws_smithy_python", + version="0.0.1", + ) +) + +python_requirements( + name="dev-requirements", + source="requirements-dev.txt", +) + +python_requirements( + name="requirements", +) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/MANIFEST.in b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/MANIFEST.in new file mode 100644 index 0000000000..24770aae6c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/MANIFEST.in @@ -0,0 +1 @@ +include aws_smithy_python/py.typed diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/NOTICE b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/NOTICE new file mode 100644 index 0000000000..616fc58894 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/NOTICE @@ -0,0 +1 @@ +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/README.md b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/README.md new file mode 100644 index 0000000000..686a09c3d6 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/README.md @@ -0,0 +1,4 @@ +# aws-smithy-python + +This is the core package that provides AWS specific interfaces for both client and server +tooling generated by Smithy. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/BUILD b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/BUILD new file mode 100644 index 0000000000..e8723ee522 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/BUILD @@ -0,0 +1,7 @@ +resource(name="pytyped", source="py.typed") + +python_sources( + name="source", + dependencies=[":pytyped"], + sources=["**/*.py"], +) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/__init__.py new file mode 100644 index 0000000000..9a8a050633 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/__init__.py @@ -0,0 +1,15 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + + +__version__ = "0.0.1" diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/identity.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/identity.py new file mode 100644 index 0000000000..a6d3bcb341 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/identity.py @@ -0,0 +1,54 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from datetime import datetime + +from smithy_python._private.identity import Identity + + +class AWSCredentialIdentity(Identity): + """Container for AWS authentication credentials.""" + + def __init__( + self, + *, + access_key_id: str, + secret_access_key: str, + session_token: str | None = None, + expiration: datetime | None = None, + ) -> None: + """Initialize the AWSCredentialIdentity. + + :param access_key_id: A unique identifier for an AWS user or role. + :param secret_access_key: A secret key used in conjunction with the access key + ID to authenticate programmatic access to AWS services. + :param session_token: A temporary token used to specify the current session for + the supplied credentials. + :param expiration: The expiration time of the identity. If time zone is provided, + it is updated to UTC. The value must always be in UTC. + """ + super().__init__(expiration=expiration) + self._access_key_id: str = access_key_id + self._secret_access_key: str = secret_access_key + self._session_token: str | None = session_token + + @property + def access_key_id(self) -> str: + return self._access_key_id + + @property + def secret_access_key(self) -> str: + return self._secret_access_key + + @property + def session_token(self) -> str | None: + return self._session_token diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/interfaces/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/interfaces/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/py.typed b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/py.typed new file mode 100644 index 0000000000..f5642f79f2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/aws_smithy_python/py.typed @@ -0,0 +1 @@ +Marker diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/pyproject.toml b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/pyproject.toml new file mode 100644 index 0000000000..3f6fce9e58 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +requires = ["setuptools", "setuptools-scm", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "aws_smithy_python" +version = "0.0.1" +description = "Core libraries for Smithy defined AWS services in Python." +readme = "README.md" +authors = [{name = "Amazon Web Services"}] +keywords = ["aws", "python", "sdk", "amazon", "smithy", "codegen"] +requires-python = ">=3.11" +license = {text = "Apache License 2.0"} +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Libraries" +] +dependencies = [ + "smithy-python==0.0.1" +] + +[project.urls] +source = "https://github.com/awslabs/smithy-python/tree/develop/python-packages/aws-smithy-python" +changelog = "https://github.com/awslabs/smithy-python/blob/develop/CHANGES.md" + +[tool.setuptools] +license-files = ["NOTICE"] +include-package-data = true + +[tool.setuptools.packages.find] +exclude=["tests*", "codegen", "designs"] + +[tool.isort] +profile = "black" +honor_noqa = true +src_paths = ["aws_smithy_python", "tests"] diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/requirements-dev.txt b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/requirements-dev.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/requirements.txt b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/functional/BUILD b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/functional/BUILD new file mode 100644 index 0000000000..57341b1358 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/functional/BUILD @@ -0,0 +1,3 @@ +python_tests( + name="tests", +) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/functional/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/functional/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/integration/BUILD b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/integration/BUILD new file mode 100644 index 0000000000..57341b1358 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/integration/BUILD @@ -0,0 +1,3 @@ +python_tests( + name="tests", +) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/BUILD b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/BUILD new file mode 100644 index 0000000000..57341b1358 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/BUILD @@ -0,0 +1,3 @@ +python_tests( + name="tests", +) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/LICENSE b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/NOTICE b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/NOTICE new file mode 100644 index 0000000000..0ceb2983da --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/NOTICE @@ -0,0 +1,2 @@ +AWS Signature Version 4 Test Suite +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.authz new file mode 100644 index 0000000000..ade3ec7537 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=c9d5ea9f3f72853aea855b47ea873832890dbdd183b4468f858259531a5138ea \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.creq new file mode 100644 index 0000000000..fa8f49a1cf --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.creq @@ -0,0 +1,9 @@ +GET +/ + +host:example.amazonaws.com +my-header1:value2,value2,value1 +x-amz-date:20150830T123600Z + +host;my-header1;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.req new file mode 100644 index 0000000000..08a0364c82 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.req @@ -0,0 +1,6 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value2 +My-Header1:value2 +My-Header1:value1 +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sreq new file mode 100644 index 0000000000..f0166e18c2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sreq @@ -0,0 +1,7 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value2 +My-Header1:value2 +My-Header1:value1 +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=c9d5ea9f3f72853aea855b47ea873832890dbdd183b4468f858259531a5138ea \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sts new file mode 100644 index 0000000000..48a135eced --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +dc7f04a3abfde8d472b0ab1a418b741b7c67174dad1551b4117b15527fbe966c \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.authz new file mode 100644 index 0000000000..9f455693b0 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cfd34249e4b1c8d6b91ef74165d41a32e5fab3306300901bb65a51a73575eefd \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.creq new file mode 100644 index 0000000000..8cb54769dd --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.creq @@ -0,0 +1,9 @@ +GET +/ + +host:example.amazonaws.com +my-header1:value1 value2 value3 +x-amz-date:20150830T123600Z + +host;my-header1;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.req new file mode 100644 index 0000000000..7caa6acc23 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.req @@ -0,0 +1,6 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value1 + value2 + value3 +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sreq new file mode 100644 index 0000000000..49513e3bd5 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sreq @@ -0,0 +1,7 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value1 + value2 + value3 +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cfd34249e4b1c8d6b91ef74165d41a32e5fab3306300901bb65a51a73575eefd \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sts new file mode 100644 index 0000000000..97c7430991 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +e99419459a677bc11de234014be3c4e72c1ea5b454ceb58b613061f5d7a162e8 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.authz new file mode 100644 index 0000000000..c0409ab2a3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.creq new file mode 100644 index 0000000000..e336bc94b9 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.creq @@ -0,0 +1,9 @@ +GET +/ + +host:example.amazonaws.com +my-header1:value4,value1,value3,value2 +x-amz-date:20150830T123600Z + +host;my-header1;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.req new file mode 100644 index 0000000000..f7bd9e6685 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.req @@ -0,0 +1,7 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value4 +My-Header1:value1 +My-Header1:value3 +My-Header1:value2 +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.sreq new file mode 100644 index 0000000000..79e16a9537 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.sreq @@ -0,0 +1,8 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value4 +My-Header1:value1 +My-Header1:value3 +My-Header1:value2 +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.sts new file mode 100644 index 0000000000..711a8d4d69 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-order/get-header-value-order.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +31ce73cd3f3d9f66977ad3dd957dc47af14df92fcd8509f59b349e9137c58b86 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.authz new file mode 100644 index 0000000000..4874ac0b1f --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;my-header2;x-amz-date, Signature=acc3ed3afb60bb290fc8d2dd0098b9911fcaa05412b367055dee359757a9c736 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.creq new file mode 100644 index 0000000000..a59087c9a4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.creq @@ -0,0 +1,10 @@ +GET +/ + +host:example.amazonaws.com +my-header1:value1 +my-header2:"a b c" +x-amz-date:20150830T123600Z + +host;my-header1;my-header2;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.req new file mode 100644 index 0000000000..901f36c359 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.req @@ -0,0 +1,5 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1: value1 +My-Header2: "a b c" +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.sreq new file mode 100644 index 0000000000..98224c9bde --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.sreq @@ -0,0 +1,6 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +My-Header1: value1 +My-Header2: "a b c" +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;my-header2;x-amz-date, Signature=acc3ed3afb60bb290fc8d2dd0098b9911fcaa05412b367055dee359757a9c736 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.sts new file mode 100644 index 0000000000..a0b15cc704 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-header-value-trim/get-header-value-trim.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +a726db9b0df21c14f559d0a978e563112acb1b9e05476f0a6a1c7d68f28605c7 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.authz new file mode 100644 index 0000000000..2943ec89d2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=07ef7494c76fa4850883e2b006601f940f8a34d404d0cfa977f52a65bbf5f24f \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.creq new file mode 100644 index 0000000000..8af54df27e --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.creq @@ -0,0 +1,8 @@ +GET +/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.req new file mode 100644 index 0000000000..da760cdb32 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.req @@ -0,0 +1,3 @@ +GET /-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.sreq new file mode 100644 index 0000000000..8001b3d6b5 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.sreq @@ -0,0 +1,4 @@ +GET /-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=07ef7494c76fa4850883e2b006601f940f8a34d404d0cfa977f52a65bbf5f24f \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.sts new file mode 100644 index 0000000000..e9dc541460 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-unreserved/get-unreserved.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +6a968768eefaa713e2a6b16b589a8ea192661f098f37349f4e2c0082757446f9 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.authz new file mode 100644 index 0000000000..738b3fbd86 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=8318018e0b0f223aa2bbf98705b62bb787dc9c0e678f255a891fd03141be5d85 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.creq new file mode 100644 index 0000000000..5d4b9f619d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.creq @@ -0,0 +1,8 @@ +GET +/%E1%88%B4 + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.req new file mode 100644 index 0000000000..da4808d0bc --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.req @@ -0,0 +1,3 @@ +GET /ሴ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.sreq new file mode 100644 index 0000000000..94eadb6d2b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.sreq @@ -0,0 +1,4 @@ +GET /ሴ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=8318018e0b0f223aa2bbf98705b62bb787dc9c0e678f255a891fd03141be5d85 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.sts new file mode 100644 index 0000000000..5edc8f456b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-utf8/get-utf8.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +2a0a97d02205e45ce2e994789806b19270cfbbb0921b278ccf58f5249ac42102 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.authz new file mode 100644 index 0000000000..65b5c7ce4e --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=a67d582fa61cc504c4bae71f336f98b97f1ea3c7a6bfe1b6e45aec72011b9aeb \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.creq new file mode 100644 index 0000000000..c6cdceda17 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.creq @@ -0,0 +1,8 @@ +GET +/ +Param1=value1 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.req new file mode 100644 index 0000000000..970d0a050e --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.req @@ -0,0 +1,3 @@ +GET /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sreq new file mode 100644 index 0000000000..f0815913fb --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sreq @@ -0,0 +1,4 @@ +GET /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=a67d582fa61cc504c4bae71f336f98b97f1ea3c7a6bfe1b6e45aec72011b9aeb \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sts new file mode 100644 index 0000000000..c4ed216c13 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +1e24db194ed7d0eec2de28d7369675a243488e08526e8c1c73571282f7c517ab \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.authz new file mode 100644 index 0000000000..99e9725717 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=371d3713e185cc334048618a97f809c9ffe339c62934c032af5a0e595648fcac \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.creq new file mode 100644 index 0000000000..0c8ba21f3d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.creq @@ -0,0 +1,8 @@ +GET +/ +%E1%88%B4=Value1&Param=Value2&Param-3=Value3 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.req new file mode 100644 index 0000000000..c539437dde --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.req @@ -0,0 +1,3 @@ +GET /?Param-3=Value3&Param=Value2&%E1%88%B4=Value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.sreq new file mode 100644 index 0000000000..7d43616449 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.sreq @@ -0,0 +1,4 @@ +GET /?Param-3=Value3&Param=Value2&%E1%88%B4=Value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=371d3713e185cc334048618a97f809c9ffe339c62934c032af5a0e595648fcac \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.sts new file mode 100644 index 0000000000..bf674ad638 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-encoded/get-vanilla-query-order-encoded.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +868294f5c38bd141c4972a373a76654f1418a8e4fc18b2e7903ae45e8ae0ec71 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.authz new file mode 100644 index 0000000000..c781fe665e --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.creq new file mode 100644 index 0000000000..8ae02cd600 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.creq @@ -0,0 +1,8 @@ +GET +/ +Param1=value1&Param2=value2 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.req new file mode 100644 index 0000000000..8a56f15f74 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.req @@ -0,0 +1,3 @@ +GET /?Param2=value2&Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sreq new file mode 100644 index 0000000000..aa3162d8e9 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sreq @@ -0,0 +1,4 @@ +GET /?Param2=value2&Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sts new file mode 100644 index 0000000000..f773de5947 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +816cd5b414d056048ba4f7c5386d6e0533120fb1fcfa93762cf0fc39e2cf19e0 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.authz new file mode 100644 index 0000000000..812cd3fdf1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=eedbc4e291e521cf13422ffca22be7d2eb8146eecf653089df300a15b2382bd1 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.creq new file mode 100644 index 0000000000..36c3cdfaef --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.creq @@ -0,0 +1,8 @@ +GET +/ +Param1=Value1&Param1=value2 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.req new file mode 100644 index 0000000000..375a496558 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.req @@ -0,0 +1,3 @@ +GET /?Param1=value2&Param1=Value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.sreq new file mode 100644 index 0000000000..bc8e652013 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.sreq @@ -0,0 +1,4 @@ +GET /?Param1=value2&Param1=Value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=eedbc4e291e521cf13422ffca22be7d2eb8146eecf653089df300a15b2382bd1 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.sts new file mode 100644 index 0000000000..fd43a414ce --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-key/get-vanilla-query-order-key.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +704b4cef673542d84cdff252633f065e8daeba5f168b77116f8b1bcaf3d38f89 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.authz new file mode 100644 index 0000000000..b8ad91f661 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.creq new file mode 100644 index 0000000000..26898ebebf --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.creq @@ -0,0 +1,8 @@ +GET +/ +Param1=value1&Param1=value2 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.req new file mode 100644 index 0000000000..9255bee055 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.req @@ -0,0 +1,3 @@ +GET /?Param1=value2&Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.sreq new file mode 100644 index 0000000000..4793e218c3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.sreq @@ -0,0 +1,4 @@ +GET /?Param1=value2&Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.sts new file mode 100644 index 0000000000..90e66b8da5 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-order-value/get-vanilla-query-order-value.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +c968629d70850097a2d8781c9bf7edcb988b04cac14cca9be4acc3595f884606 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.authz new file mode 100644 index 0000000000..a44ca5be80 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=9c3e54bfcdf0b19771a7f523ee5669cdf59bc7cc0884027167c21bb143a40197 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.creq new file mode 100644 index 0000000000..5249be3bf8 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.creq @@ -0,0 +1,8 @@ +GET +/ +-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.req new file mode 100644 index 0000000000..d2833b32f9 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.req @@ -0,0 +1,3 @@ +GET /?-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sreq new file mode 100644 index 0000000000..ba1ef40235 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sreq @@ -0,0 +1,4 @@ +GET /?-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=9c3e54bfcdf0b19771a7f523ee5669cdf59bc7cc0884027167c21bb143a40197 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sts new file mode 100644 index 0000000000..24a97d209b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +c30d4703d9f799439be92736156d47ccfb2d879ddf56f5befa6d1d6aab979177 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.authz new file mode 100644 index 0000000000..551c0271d4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.creq new file mode 100644 index 0000000000..ed91561f4a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.creq @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.req new file mode 100644 index 0000000000..0f7a9bfae3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.req @@ -0,0 +1,3 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.sreq new file mode 100644 index 0000000000..d739b01fd1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.sreq @@ -0,0 +1,4 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.sts new file mode 100644 index 0000000000..b187649cb3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-query/get-vanilla-query.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.authz new file mode 100644 index 0000000000..e016c3da09 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.creq new file mode 100644 index 0000000000..a835c9e491 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.creq @@ -0,0 +1,8 @@ +GET +/ +%E1%88%B4=bar +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.req new file mode 100644 index 0000000000..cc2757e167 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.req @@ -0,0 +1,3 @@ +GET /?ሴ=bar HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sreq new file mode 100644 index 0000000000..7baf4c82f3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sreq @@ -0,0 +1,4 @@ +GET /?ሴ=bar HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sts new file mode 100644 index 0000000000..51ee71b749 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +eb30c5bed55734080471a834cc727ae56beb50e5f39d1bff6d0d38cb192a7073 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.authz new file mode 100644 index 0000000000..cb7ae61e1a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=07ec1639c89043aa0e3e2de82b96708f198cceab042d4a97044c66dd9f74e7f8 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.creq new file mode 100644 index 0000000000..ccacdeb490 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.creq @@ -0,0 +1,9 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z +x-amz-security-token:6e86291e8372ff2a2260956d9b8aae1d763fbf315fa00fa31553b73ebf194267 + +host;x-amz-date;x-amz-security-token +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.req new file mode 100644 index 0000000000..0f7a9bfae3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.req @@ -0,0 +1,3 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.sreq new file mode 100644 index 0000000000..406ac5690c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.sreq @@ -0,0 +1,5 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +X-Amz-Security-Token:6e86291e8372ff2a2260956d9b8aae1d763fbf315fa00fa31553b73ebf194267 +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.sts new file mode 100644 index 0000000000..742b880cb0 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla-with-session-token/get-vanilla-with-session-token.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +067b36aa60031588cea4a4cde1f21215227a047690c72247f1d70b32fbbfad2b \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.authz new file mode 100644 index 0000000000..551c0271d4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.creq new file mode 100644 index 0000000000..ed91561f4a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.creq @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.req new file mode 100644 index 0000000000..0f7a9bfae3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.req @@ -0,0 +1,3 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.sreq new file mode 100644 index 0000000000..d739b01fd1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.sreq @@ -0,0 +1,4 @@ +GET / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.sts new file mode 100644 index 0000000000..b187649cb3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/get-vanilla/get-vanilla.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.authz new file mode 100644 index 0000000000..551c0271d4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.creq new file mode 100644 index 0000000000..ed91561f4a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.creq @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.req new file mode 100644 index 0000000000..cfd4e8b74c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.req @@ -0,0 +1,3 @@ +GET /example1/example2/../.. HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sreq new file mode 100644 index 0000000000..cbdebe2cca --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sreq @@ -0,0 +1,4 @@ +GET /example1/example2/../.. HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sts new file mode 100644 index 0000000000..b187649cb3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.authz new file mode 100644 index 0000000000..551c0271d4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.creq new file mode 100644 index 0000000000..ed91561f4a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.creq @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.req new file mode 100644 index 0000000000..9d6d7ca20a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.req @@ -0,0 +1,3 @@ +GET /example/.. HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.sreq new file mode 100644 index 0000000000..4f59e7d20c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.sreq @@ -0,0 +1,4 @@ +GET /example/.. HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.sts new file mode 100644 index 0000000000..b187649cb3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-relative/get-relative.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.authz new file mode 100644 index 0000000000..551c0271d4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.creq new file mode 100644 index 0000000000..ed91561f4a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.creq @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.req new file mode 100644 index 0000000000..f3537b7095 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.req @@ -0,0 +1,3 @@ +GET /./ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sreq new file mode 100644 index 0000000000..23a2b41ced --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sreq @@ -0,0 +1,4 @@ +GET /./ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sts new file mode 100644 index 0000000000..b187649cb3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.authz new file mode 100644 index 0000000000..b76ca1e2d4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=ef75d96142cf21edca26f06005da7988e4f8dc83a165a80865db7089db637ec5 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.creq new file mode 100644 index 0000000000..915c57f214 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.creq @@ -0,0 +1,8 @@ +GET +/example + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.req new file mode 100644 index 0000000000..3c9107171a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.req @@ -0,0 +1,3 @@ +GET /./example HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sreq new file mode 100644 index 0000000000..8096609653 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sreq @@ -0,0 +1,4 @@ +GET /./example HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=ef75d96142cf21edca26f06005da7988e4f8dc83a165a80865db7089db637ec5 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sts new file mode 100644 index 0000000000..7429923e6b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +214d50c111a8edc4819da6a636336472c916b5240f51e9a51b5c3305180cf702 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.authz new file mode 100644 index 0000000000..551c0271d4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.creq new file mode 100644 index 0000000000..ed91561f4a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.creq @@ -0,0 +1,8 @@ +GET +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.req new file mode 100644 index 0000000000..ede8e3c8ea --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.req @@ -0,0 +1,3 @@ +GET // HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.sreq new file mode 100644 index 0000000000..cde31b4381 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.sreq @@ -0,0 +1,4 @@ +GET // HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.sts new file mode 100644 index 0000000000..b187649cb3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slash/get-slash.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.authz new file mode 100644 index 0000000000..307c1051d5 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=9a624bd73a37c9a373b5312afbebe7a714a789de108f0bdfe846570885f57e84 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.creq new file mode 100644 index 0000000000..2bdaf7479b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.creq @@ -0,0 +1,8 @@ +GET +/example/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.req new file mode 100644 index 0000000000..a4307ce425 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.req @@ -0,0 +1,3 @@ +GET //example// HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.sreq new file mode 100644 index 0000000000..c84a80d56a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.sreq @@ -0,0 +1,4 @@ +GET //example// HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=9a624bd73a37c9a373b5312afbebe7a714a789de108f0bdfe846570885f57e84 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.sts new file mode 100644 index 0000000000..95d1fc2584 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-slashes/get-slashes.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +cb96b4ac96d501f7c5c15bc6d67b3035061cfced4af6585ad927f7e6c985c015 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.authz new file mode 100644 index 0000000000..832d8a50d2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.creq new file mode 100644 index 0000000000..124a7096a1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.creq @@ -0,0 +1,8 @@ +GET +/example%20space/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.req new file mode 100644 index 0000000000..b7d5e8bb95 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.req @@ -0,0 +1,3 @@ +GET /example space/ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.sreq new file mode 100644 index 0000000000..eefa20c48c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.sreq @@ -0,0 +1,4 @@ +GET /example space/ HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.sts new file mode 100644 index 0000000000..a633f0c052 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-space/get-space.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +63ee75631ed7234ae61b5f736dfc7754cdccfedbff4b5128a915706ee9390d86 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.authz new file mode 100644 index 0000000000..858b601bb3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=a853c9b21b528b19643d00910d35b83a10c366a10833ceefb45edd6c80e40f27 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.creq new file mode 100644 index 0000000000..236b8f27d5 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.creq @@ -0,0 +1,8 @@ +GET +/example/%24delete + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.req new file mode 100644 index 0000000000..e657d8858a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.req @@ -0,0 +1,3 @@ +GET /example/$delete HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.sreq new file mode 100644 index 0000000000..d3a607188a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.sreq @@ -0,0 +1,4 @@ +GET /example/$delete HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=a853c9b21b528b19643d00910d35b83a10c366a10833ceefb45edd6c80e40f27 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.sts new file mode 100644 index 0000000000..df29bc44df --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/get-special-character/get-special-character.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +4053e45b5cef7cec5e17f736b1c12b3faf0388fd4c0bd24326386f132039ce5c \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/normalize-path.txt b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/normalize-path.txt new file mode 100644 index 0000000000..caaf34fb5e --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/normalize-path/normalize-path.txt @@ -0,0 +1,3 @@ +A note about signing requests to Amazon S3: + +In exception to this, you do not normalize URI paths for requests to Amazon S3. For example, if you have a bucket with an object named my-object//example//photo.user, use that path. Normalizing the path to my-object/example/photo.user will cause the request to fail. For more information, see Task 1: Create a Canonical Request in the Amazon Simple Storage Service API Reference: http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html#canonical-request \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.authz new file mode 100644 index 0000000000..89e572e609 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.creq new file mode 100644 index 0000000000..5c3a9434ec --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.creq @@ -0,0 +1,8 @@ +POST +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.req new file mode 100644 index 0000000000..3dc4179013 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.req @@ -0,0 +1,3 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.sreq new file mode 100644 index 0000000000..a5ada0d940 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.sreq @@ -0,0 +1,4 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.sts new file mode 100644 index 0000000000..a636703949 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-case/post-header-key-case.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +553f88c9e4d10fc9e109e2aeb65f030801b70c2f6468faca261d401ae622fc87 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.authz new file mode 100644 index 0000000000..a62589ff7e --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=c5410059b04c1ee005303aed430f6e6645f61f4dc9e1461ec8f8916fdf18852c \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.creq new file mode 100644 index 0000000000..ebe943e895 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.creq @@ -0,0 +1,9 @@ +POST +/ + +host:example.amazonaws.com +my-header1:value1 +x-amz-date:20150830T123600Z + +host;my-header1;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.req new file mode 100644 index 0000000000..0253f19456 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.req @@ -0,0 +1,4 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value1 +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.sreq new file mode 100644 index 0000000000..b4b78a1668 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.sreq @@ -0,0 +1,5 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:value1 +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=c5410059b04c1ee005303aed430f6e6645f61f4dc9e1461ec8f8916fdf18852c \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.sts new file mode 100644 index 0000000000..eb66362697 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-key-sort/post-header-key-sort.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +9368318c2967cf6de74404b30c65a91e8f6253e0a8659d6d5319f1a812f87d65 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.authz new file mode 100644 index 0000000000..d9e52a379a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.creq new file mode 100644 index 0000000000..af824c8899 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.creq @@ -0,0 +1,9 @@ +POST +/ + +host:example.amazonaws.com +my-header1:VALUE1 +x-amz-date:20150830T123600Z + +host;my-header1;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.req new file mode 100644 index 0000000000..3f9987af7f --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.req @@ -0,0 +1,4 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:VALUE1 +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.sreq new file mode 100644 index 0000000000..99c3210c99 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.sreq @@ -0,0 +1,5 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +My-Header1:VALUE1 +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.sts new file mode 100644 index 0000000000..40062c79f8 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-header-value-case/post-header-value-case.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +d51ced243e649e3de6ef63afbbdcbca03131a21a7103a1583706a64618606a93 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.authz new file mode 100644 index 0000000000..89e572e609 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.creq new file mode 100644 index 0000000000..5c3a9434ec --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.creq @@ -0,0 +1,8 @@ +POST +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.req new file mode 100644 index 0000000000..3dc4179013 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.req @@ -0,0 +1,3 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sreq new file mode 100644 index 0000000000..291ed0756b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sreq @@ -0,0 +1,5 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +X-Amz-Security-Token:AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sts new file mode 100644 index 0000000000..a636703949 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +553f88c9e4d10fc9e109e2aeb65f030801b70c2f6468faca261d401ae622fc87 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.authz new file mode 100644 index 0000000000..64aa046dbb --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=85d96828115b5dc0cfc3bd16ad9e210dd772bbebba041836c64533a82be05ead \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.creq new file mode 100644 index 0000000000..1d5a462ee2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.creq @@ -0,0 +1,9 @@ +POST +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z +x-amz-security-token:AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== + +host;x-amz-date;x-amz-security-token +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.req new file mode 100644 index 0000000000..9d917755f4 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.req @@ -0,0 +1,4 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +X-Amz-Security-Token:AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sreq new file mode 100644 index 0000000000..37b2f04190 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sreq @@ -0,0 +1,5 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +X-Amz-Security-Token:AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=85d96828115b5dc0cfc3bd16ad9e210dd772bbebba041836c64533a82be05ead \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sts new file mode 100644 index 0000000000..bc39ccfc5b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +c237e1b440d4c63c32ca95b5b99481081cb7b13c7e40434868e71567c1a882f6 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/readme.txt b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/readme.txt new file mode 100644 index 0000000000..3731a30128 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-sts-token/readme.txt @@ -0,0 +1,15 @@ +A note about using temporary security credentials: + +You can use temporary security credentials provided by the AWS Security Token Service (AWS STS) to sign a request. The process is the same as using long-term credentials but requires an additional HTTP header or query string parameter for the security token. The name of the header or query string parameter is X-Amz-Security-Token, and the value is the session token (the string that you received from AWS STS when you obtained temporary security credentials). + +When you add X-Amz-Security-Token, some services require that you include this parameter in the canonical (signed) request. For other services, you add this parameter at the end, after you calculate the signature. For details see the API reference documentation for that service. + +The test suite has 2 examples: + +post-sts-header-before - The X-Amz-Security-Token header is part of the canonical request. + +post-sts-header-after - The X-Amz-Security-Token header is added to the request after you calculate the signature. + +The test suite uses this example value for X-Amz-Security-Token: + +AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.authz new file mode 100644 index 0000000000..44280cd7bb --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.creq new file mode 100644 index 0000000000..f5058d430b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.creq @@ -0,0 +1,8 @@ +POST +/ +Param1=value1 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.req new file mode 100644 index 0000000000..9157bc74de --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.req @@ -0,0 +1,3 @@ +POST /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sreq new file mode 100644 index 0000000000..82af1505e2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sreq @@ -0,0 +1,4 @@ +POST /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sts new file mode 100644 index 0000000000..ca7cc661d1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +9d659678c1756bb3113e2ce898845a0a79dbbc57b740555917687f1b3340fbbd \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.authz new file mode 100644 index 0000000000..44280cd7bb --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.creq new file mode 100644 index 0000000000..f5058d430b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.creq @@ -0,0 +1,8 @@ +POST +/ +Param1=value1 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.req new file mode 100644 index 0000000000..9157bc74de --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.req @@ -0,0 +1,3 @@ +POST /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.sreq new file mode 100644 index 0000000000..82af1505e2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.sreq @@ -0,0 +1,4 @@ +POST /?Param1=value1 HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.sts new file mode 100644 index 0000000000..ca7cc661d1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla-query/post-vanilla-query.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +9d659678c1756bb3113e2ce898845a0a79dbbc57b740555917687f1b3340fbbd \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.authz new file mode 100644 index 0000000000..89e572e609 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.creq new file mode 100644 index 0000000000..5c3a9434ec --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.creq @@ -0,0 +1,8 @@ +POST +/ + +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +host;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.req new file mode 100644 index 0000000000..3dc4179013 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.req @@ -0,0 +1,3 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.sreq new file mode 100644 index 0000000000..a5ada0d940 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.sreq @@ -0,0 +1,4 @@ +POST / HTTP/1.1 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.sts new file mode 100644 index 0000000000..a636703949 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-vanilla/post-vanilla.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +553f88c9e4d10fc9e109e2aeb65f030801b70c2f6468faca261d401ae622fc87 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.authz new file mode 100644 index 0000000000..531b89b45b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=1a72ec8f64bd914b0e42e42607c7fbce7fb2c7465f63e3092b3b0d39fa77a6fe \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.creq new file mode 100644 index 0000000000..8ec0d6cf0a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.creq @@ -0,0 +1,9 @@ +POST +/ + +content-type:application/x-www-form-urlencoded; charset=utf8 +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +content-type;host;x-amz-date +9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.req new file mode 100644 index 0000000000..5ce537e674 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.req @@ -0,0 +1,6 @@ +POST / HTTP/1.1 +Content-Type:application/x-www-form-urlencoded; charset=utf8 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z + +Param1=value1 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sreq new file mode 100644 index 0000000000..88beb82a38 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sreq @@ -0,0 +1,7 @@ +POST / HTTP/1.1 +Content-Type:application/x-www-form-urlencoded; charset=utf8 +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=1a72ec8f64bd914b0e42e42607c7fbce7fb2c7465f63e3092b3b0d39fa77a6fe + +Param1=value1 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sts new file mode 100644 index 0000000000..3e83c524b0 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +2e1cf7ed91881a30569e46552437e4156c823447bf1781b921b5d486c568dd1c \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.authz b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.authz new file mode 100644 index 0000000000..d7baf53540 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.authz @@ -0,0 +1 @@ +AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=ff11897932ad3f4e8b18135d722051e5ac45fc38421b1da7b9d196a0fe09473a \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.creq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.creq new file mode 100644 index 0000000000..d7197f17ec --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.creq @@ -0,0 +1,9 @@ +POST +/ + +content-type:application/x-www-form-urlencoded +host:example.amazonaws.com +x-amz-date:20150830T123600Z + +content-type;host;x-amz-date +9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.req b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.req new file mode 100644 index 0000000000..ada7f87760 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.req @@ -0,0 +1,6 @@ +POST / HTTP/1.1 +Content-Type:application/x-www-form-urlencoded +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z + +Param1=value1 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sreq b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sreq new file mode 100644 index 0000000000..9bac9311ba --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sreq @@ -0,0 +1,7 @@ +POST / HTTP/1.1 +Content-Type:application/x-www-form-urlencoded +Host:example.amazonaws.com +X-Amz-Date:20150830T123600Z +Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=ff11897932ad3f4e8b18135d722051e5ac45fc38421b1da7b9d196a0fe09473a + +Param1=value1 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sts b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sts new file mode 100644 index 0000000000..65ab663719 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/aws-smithy-python/tests/unit/auth/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sts @@ -0,0 +1,4 @@ +AWS4-HMAC-SHA256 +20150830T123600Z +20150830/us-east-1/service/aws4_request +42a5e5bb34198acb3e84da4f085bb7927f2bc277ca766e6d19c73c2154021281 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/MANIFEST.in b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/MANIFEST.in new file mode 100644 index 0000000000..50093a0b71 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/MANIFEST.in @@ -0,0 +1 @@ +include smithy_python/py.typed diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/NOTICE b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/NOTICE new file mode 100644 index 0000000000..616fc58894 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/NOTICE @@ -0,0 +1 @@ +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/README.md b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/README.md new file mode 100644 index 0000000000..80737e8984 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/README.md @@ -0,0 +1,4 @@ +# smithy-python + +This is the core package that provides interfaces for both client and server +tooling generated by Smithy. diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/pyproject.toml b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/pyproject.toml new file mode 100644 index 0000000000..6f61403ecf --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/pyproject.toml @@ -0,0 +1,50 @@ +[build-system] +requires = ["setuptools", "setuptools-scm", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "smithy_python" +version = "0.0.1" +description = "Core libraries for Smithy defined services in Python" +readme = "README.md" +authors = [{name = "Amazon Web Services"}] +keywords = ["python", "sdk", "amazon", "smithy", "codegen"] +requires-python = ">=3.11" +license = {text = "Apache License 2.0"} +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Libraries" +] +dependencies = [ + "awscrt>=0.15,<1.0", + "aiohttp>=3.9.0" +] + +[project.urls] +source = "https://github.com/awslabs/smithy-python/tree/develop/python-packages/smithy-python" +changelog = "https://github.com/awslabs/smithy-python/blob/develop/CHANGES.md" + +[tool.setuptools] +license-files = ["NOTICE"] +include-package-data = true + +[tool.setuptools.packages.find] +exclude=["tests*", "codegen", "designs"] + +[tool.isort] +profile = "black" +honor_noqa = true +src_paths = ["smithy_python", "tests"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/requirements-dev.txt b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/requirements-dev.txt new file mode 100644 index 0000000000..ff6665bceb --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/requirements-dev.txt @@ -0,0 +1,6 @@ +black==21.11b1 +flake8<6.1 +mypy==1.0.0 +pytest<7.3 +pytest-asyncio<0.21.0 +pytest-cov<3.1 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/requirements.txt b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/requirements.txt new file mode 100644 index 0000000000..13a3e19b9d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/requirements.txt @@ -0,0 +1,2 @@ +awscrt>=0.15,<1.0 +aiohttp>=3.9.0 \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/BUILD b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/BUILD new file mode 100644 index 0000000000..e8723ee522 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/BUILD @@ -0,0 +1,7 @@ +resource(name="pytyped", source="py.typed") + +python_sources( + name="source", + dependencies=[":pytyped"], + sources=["**/*.py"], +) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/__init__.py new file mode 100644 index 0000000000..9a8a050633 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/__init__.py @@ -0,0 +1,15 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + + +__version__ = "0.0.1" diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/__init__.py new file mode 100644 index 0000000000..124e7f751c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/__init__.py @@ -0,0 +1,341 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from collections import Counter, OrderedDict +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from enum import Enum +from functools import cached_property +from urllib.parse import urlunparse + +from .. import interfaces +from ..exceptions import SmithyHTTPException +from ..interfaces import FieldPosition as FieldPosition # re-export +from . import abnf + + +class HostType(Enum): + """Enumeration of possible host types.""" + + IPv6 = "IPv6" + """Host is an IPv6 address.""" + + IPv4 = "IPv4" + """Host is an IPv4 address.""" + + DOMAIN = "DOMAIN" + """Host type is a domain name.""" + + UNKNOWN = "UNKNOWN" + """Host type is unknown.""" + + +@dataclass(kw_only=True, frozen=True) +class URI(interfaces.URI): + """Universal Resource Identifier, target location for a :py:class:`HTTPRequest`.""" + + scheme: str = "https" + """For example ``http`` or ``https``.""" + + username: str | None = None + """Username part of the userinfo URI component.""" + + password: str | None = None + """Password part of the userinfo URI component.""" + + host: str + """The hostname, for example ``amazonaws.com``.""" + + port: int | None = None + """An explicit port number.""" + + path: str | None = None + """Path component of the URI.""" + + query: str | None = None + """Query component of the URI as string.""" + + fragment: str | None = None + """Part of the URI specification, but may not be transmitted by a client.""" + + def __post_init__(self) -> None: + """Validate host component.""" + if not abnf.HOST_MATCHER.match(self.host) and not abnf.IPv6_MATCHER.match( + f"[{self.host}]" + ): + raise SmithyHTTPException(f"Invalid host: {self.host}") + + @cached_property + def netloc(self) -> str: + """Construct netloc string in format ``{username}:{password}@{host}:{port}`` + + ``username``, ``password``, and ``port`` are only included if set. ``password`` + is ignored, unless ``username`` is also set. Add square brackets around the host + if it is a valid IPv6 endpoint URI per :rfc:`3986#section-3.2.2`. + """ + if self.username is not None: + password = "" if self.password is None else f":{self.password}" + userinfo = f"{self.username}{password}@" + else: + userinfo = "" + + if self.port is not None: + port = f":{self.port}" + else: + port = "" + + if self.host_type == HostType.IPv6: + host = f"[{self.host}]" + else: + host = self.host + + return f"{userinfo}{host}{port}" + + @cached_property + def host_type(self) -> HostType: + """Return the type of host.""" + if abnf.IPv6_MATCHER.match(f"[{self.host}]"): + return HostType.IPv6 + if abnf.IPv4_MATCHER.match(self.host): + return HostType.IPv4 + if abnf.HOST_MATCHER.match(self.host): + return HostType.DOMAIN + return HostType.UNKNOWN + + def build(self) -> str: + """Construct URI string representation. + + Validate host. Returns a string of the form + ``{scheme}://{username}:{password}@{host}:{port}{path}?{query}#{fragment}`` + """ + components = ( + self.scheme, + self.netloc, + self.path or "", + "", # params + self.query, + self.fragment, + ) + return urlunparse(components) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, URI): + return False + return ( + self.scheme == other.scheme + and self.host == other.host + and self.port == other.port + and self.path == other.path + and self.query == other.query + and self.username == other.username + and self.password == other.password + and self.fragment == other.fragment + ) + + +class Field(interfaces.Field): + """A name-value pair representing a single field in an HTTP Request or Response. + + The kind will dictate metadata placement within an HTTP message. + + All field names are case insensitive and case-variance must be treated as + equivalent. Names may be normalized but should be preserved for accuracy during + transmission. + """ + + def __init__( + self, + *, + name: str, + values: Iterable[str] | None = None, + kind: FieldPosition = FieldPosition.HEADER, + ): + self.name = name + self.values: list[str] = [val for val in values] if values is not None else [] + self.kind = kind + + def add(self, value: str) -> None: + """Append a value to a field.""" + self.values.append(value) + + def set(self, values: list[str]) -> None: + """Overwrite existing field values.""" + self.values = values + + def remove(self, value: str) -> None: + """Remove all matching entries from list.""" + try: + while True: + self.values.remove(value) + except ValueError: + return + + def as_string(self) -> str: + """Get comma-delimited string of all values. + + If the ``Field`` has zero values, the empty string is returned. If the ``Field`` + has exactly one value, the value is returned unmodified. + + For ``Field``s with more than one value, the values are joined by a comma and a + space. For such multi-valued ``Field``s, any values that already contain + commas or double quotes will be surrounded by double quotes. Within any values + that get quoted, pre-existing double quotes and backslashes are escaped with a + backslash. + """ + value_count = len(self.values) + if value_count == 0: + return "" + if value_count == 1: + return self.values[0] + return ", ".join(quote_and_escape_field_value(val) for val in self.values) + + def as_tuples(self) -> list[tuple[str, str]]: + """Get list of ``name``, ``value`` tuples where each tuple represents one + value.""" + return [(self.name, val) for val in self.values] + + def __eq__(self, other: object) -> bool: + """Name, values, and kind must match. + + Values order must match. + """ + if not isinstance(other, Field): + return False + return ( + self.name == other.name + and self.kind is other.kind + and self.values == other.values + ) + + def __repr__(self) -> str: + return f"Field(name={self.name!r}, value={self.values!r}, kind={self.kind!r})" + + +class Fields(interfaces.Fields): + def __init__( + self, + initial: Iterable[interfaces.Field] | None = None, + *, + encoding: str = "utf-8", + ): + """Collection of header and trailer entries mapped by name. + + :param initial: Initial list of ``Field`` objects. ``Field``s can alse be added + with :func:`set_field` and later removed with :func:`remove_field`. + :param encoding: The string encoding to be used when converting the ``Field`` + name and value from ``str`` to ``bytes`` for transmission. + """ + init_fields = [fld for fld in initial] if initial is not None else [] + init_field_names = [self._normalize_field_name(fld.name) for fld in init_fields] + fname_counter = Counter(init_field_names) + repeated_names_exist = ( + len(init_fields) > 0 and fname_counter.most_common(1)[0][1] > 1 + ) + if repeated_names_exist: + non_unique_names = [name for name, num in fname_counter.items() if num > 1] + raise ValueError( + "Field names of the initial list of fields must be unique. The " + "following normalized field names appear more than once: " + f"{', '.join(non_unique_names)}." + ) + init_tuples = zip(init_field_names, init_fields) + self.entries: OrderedDict[str, interfaces.Field] = OrderedDict(init_tuples) + self.encoding: str = encoding + + def set_field(self, field: interfaces.Field) -> None: + """Set entry for a Field name.""" + normalized_name = self._normalize_field_name(field.name) + self.entries[normalized_name] = field + + def get_field(self, name: str) -> interfaces.Field: + """Retrieve Field entry.""" + normalized_name = self._normalize_field_name(name) + return self.entries[normalized_name] + + def remove_field(self, name: str) -> None: + """Delete entry from collection.""" + normalized_name = self._normalize_field_name(name) + del self.entries[normalized_name] + + def get_by_type(self, kind: FieldPosition) -> list[interfaces.Field]: + """Helper function for retrieving specific types of fields. + + Used to grab all headers or all trailers. + """ + return [entry for entry in self.entries.values() if entry.kind is kind] + + def extend(self, other: interfaces.Fields) -> None: + """Merges ``entries`` of ``other`` into the current ``entries``. + + For every `Field` in the ``entries`` of ``other``: If the normalized name + already exists in the current ``entries``, the values from ``other`` are + appended. Otherwise, the ``Field`` is added to the list of ``entries``. + """ + for other_field in other: + try: + cur_field = self.get_field(name=other_field.name) + for other_value in other_field.values: + cur_field.add(other_value) + except KeyError: + self.set_field(other_field) + + def _normalize_field_name(self, name: str) -> str: + """Normalize field names. + + For use as key in ``entries``. + """ + return name.lower() + + def __eq__(self, other: object) -> bool: + """Encoding must match. + + Entries must match in values and order. + """ + if not isinstance(other, Fields): + return False + return self.encoding == other.encoding and self.entries == other.entries + + def __iter__(self) -> Iterator[interfaces.Field]: + yield from self.entries.values() + + +def quote_and_escape_field_value(value: str) -> str: + """Escapes and quotes a single :class:`Field` value if necessary. + + See :func:`Field.as_string` for quoting and escaping logic. + """ + chars_to_quote = (",", '"') + if any(char in chars_to_quote for char in value): + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + else: + return value + + +def tuples_to_fields( + tuples: Iterable[tuple[str, str]], *, kind: FieldPosition | None = None +) -> Fields: + """Convert ``name``, ``value`` tuples to ``Fields`` object. Each tuple represents + one Field value. + + :param kind: The Field kind to define for all tuples. + """ + fields = Fields() + for name, value in tuples: + try: + fields.get_field(name).add(value) + except KeyError: + fields.set_field( + Field(name=name, values=[value], kind=kind or FieldPosition.HEADER) + ) + + return fields diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/abnf.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/abnf.py new file mode 100644 index 0000000000..d2188e13da --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/abnf.py @@ -0,0 +1,120 @@ +# Copyright 2014 Ian Cordasco, Rackspace + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module vended from rfc3986 ``abnf_rexexp.py`` and ``misc.py``. + +https://github.com/python-hyper/rfc3986/blob/main/src/rfc3986/abnf_regexp.py +https://github.com/python-hyper/rfc3986/blob/main/src/rfc3986/misc.py +""" +import re + +# ######################### +# Start abnf_regexp.py +# ######################### + +# Escape the '*' for use in regular expressions +SUB_DELIMITERS_RE = r"!$&'()\*+,;=" +# We need to escape the '-' in this case: +UNRESERVED_RE = r"A-Za-z0-9._~\-" + +# Percent encoded character values +PERCENT_ENCODED = PCT_ENCODED = "%[A-Fa-f0-9]{2}" +PCHAR = "([" + UNRESERVED_RE + SUB_DELIMITERS_RE + ":@]|%s)" % PCT_ENCODED + +# ######################### +# Authority Matcher Section +# ######################### + +# Host patterns, see: http://tools.ietf.org/html/rfc3986#section-3.2.2 +# The pattern for a regular name, e.g., www.google.com, api.github.com +REGULAR_NAME_RE = REG_NAME = "((?:{}|[{}])*)".format( + "%[0-9A-Fa-f]{2}", SUB_DELIMITERS_RE + UNRESERVED_RE +) +# The pattern for an IPv4 address, e.g., 192.168.255.255, 127.0.0.1, +IPv4_RE = r"([0-9]{1,3}\.){3}[0-9]{1,3}" +# Hexadecimal characters used in each piece of an IPv6 address +HEXDIG_RE = "[0-9A-Fa-f]{1,4}" +# Least-significant 32 bits of an IPv6 address +LS32_RE = "({hex}:{hex}|{ipv4})".format(hex=HEXDIG_RE, ipv4=IPv4_RE) +# Substitutions into the following patterns for IPv6 patterns defined +# http://tools.ietf.org/html/rfc3986#page-20 +_subs = {"hex": HEXDIG_RE, "ls32": LS32_RE} + +# Below: h16 = hexdig, see: https://tools.ietf.org/html/rfc5234 for details +# about ABNF (Augmented Backus-Naur Form) use in the comments +variations = [ + # 6( h16 ":" ) ls32 + "(%(hex)s:){6}%(ls32)s" % _subs, + # "::" 5( h16 ":" ) ls32 + "::(%(hex)s:){5}%(ls32)s" % _subs, + # [ h16 ] "::" 4( h16 ":" ) ls32 + "(%(hex)s)?::(%(hex)s:){4}%(ls32)s" % _subs, + # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 + "((%(hex)s:)?%(hex)s)?::(%(hex)s:){3}%(ls32)s" % _subs, + # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 + "((%(hex)s:){0,2}%(hex)s)?::(%(hex)s:){2}%(ls32)s" % _subs, + # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 + "((%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s" % _subs, + # [ *4( h16 ":" ) h16 ] "::" ls32 + "((%(hex)s:){0,4}%(hex)s)?::%(ls32)s" % _subs, + # [ *5( h16 ":" ) h16 ] "::" h16 + "((%(hex)s:){0,5}%(hex)s)?::%(hex)s" % _subs, + # [ *6( h16 ":" ) h16 ] "::" + "((%(hex)s:){0,6}%(hex)s)?::" % _subs, +] + +IPv6_RE = "(({})|({})|({})|({})|({})|({})|({})|({})|({}))".format(*variations) + +IPv_FUTURE_RE = r"v[0-9A-Fa-f]+\.[%s]+" % (UNRESERVED_RE + SUB_DELIMITERS_RE + ":") + +# RFC 6874 Zone ID ABNF +ZONE_ID = "(?:[" + UNRESERVED_RE + "]|" + PCT_ENCODED + ")+" + +IPv6_ADDRZ_RFC4007_RE = IPv6_RE + "(?:(?:%25|%)" + ZONE_ID + ")?" +IPv6_ADDRZ_RE = IPv6_RE + "(?:%25" + ZONE_ID + ")?" + +IP_LITERAL_RE = r"\[({}|{})\]".format( + IPv6_ADDRZ_RFC4007_RE, + IPv_FUTURE_RE, +) + +# Pattern for matching the host piece of the authority +HOST_RE = HOST_PATTERN = "({}|{}|{})".format( + REG_NAME, + IPv4_RE, + IP_LITERAL_RE, +) + +# ######################### +# End abnf_regexp.py +# ######################### + + +# ######################### +# Start misc.py +# ######################### + +# These are enumerated for the named tuple used as a superclass of +# URIReference + +HOST_MATCHER = re.compile("^" + HOST_RE + "$") +IPv4_MATCHER = re.compile("^" + IPv4_RE + "$") +IPv6_MATCHER = re.compile(r"^\[" + IPv6_ADDRZ_RFC4007_RE + r"\]$") + +# Used by host validator +IPv6_NO_RFC4007_MATCHER = re.compile(r"^\[%s\]$" % (IPv6_ADDRZ_RE)) + +# ######################### +# End misc.py +# ######################### diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/api_key_auth.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/api_key_auth.py new file mode 100644 index 0000000000..cb95b1701d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/api_key_auth.py @@ -0,0 +1,145 @@ +from dataclasses import dataclass +from enum import Enum +from typing import NotRequired, Protocol, TypedDict + +from ..exceptions import SmithyIdentityException +from ..interfaces.auth import HTTPAuthScheme, HTTPSigner +from ..interfaces.http import HTTPRequest +from ..interfaces.identity import IdentityProperties, IdentityResolver +from . import URI, Field +from .identity import Identity + + +class ApiKeyIdentity(Identity): + """The identity for auth that uses an api key.""" + + def __init__(self, *, api_key: str) -> None: + super().__init__(expiration=None) + self.api_key = api_key + + +class ApiKeyIdentityResolver(IdentityResolver[ApiKeyIdentity, IdentityProperties]): + """Loads the api key identity from the configuration.""" + + def __init__(self, *, api_key: str | ApiKeyIdentity) -> None: + """ + :param api_key: The API key to authenticate with. + """ + match api_key: + case str(): + self._identity = ApiKeyIdentity(api_key=api_key) + case ApiKeyIdentity(): + self._identity = api_key + + async def get_identity( + self, *, identity_properties: IdentityProperties + ) -> ApiKeyIdentity: + """Load the user's api key identity from this resolver. + + :param identity_properties: Properties used to help determine the + identity to return. + :returns: The api key identity. + """ + return self._identity + + +class ApiKeyLocation(Enum): + """The locations that the api key could be placed in the signed request.""" + + HEADER = "header" + QUERY = "query" + + +class ApiKeySigningProperties(TypedDict): + """The properties needed to sign a request with api key auth. + + seealso:: The `Smithy API Key auth trait docs `_ + , which have more details on these properties, including examples. + """ + + name: str + """The name of the HTTP header or query string parameter containing the key.""" + + scheme: NotRequired[str] + """The :rfc:`9110#section-11.4` scheme to prefix a header value with.""" + + location: ApiKeyLocation + """Where the key is serialized.""" + + +class ApiKeyConfig(Protocol): + api_key_identity_resolver: IdentityResolver[ + ApiKeyIdentity, IdentityProperties + ] | None + + +@dataclass(init=False) +class ApiKeyAuthScheme( + HTTPAuthScheme[ + ApiKeyIdentity, ApiKeyConfig, IdentityProperties, ApiKeySigningProperties + ] +): + """An auth scheme containing necessary data and tools for api key auth.""" + + scheme_id: str + signer: HTTPSigner[ApiKeyIdentity, ApiKeySigningProperties] + + def __init__( + self, + *, + signer: HTTPSigner[ApiKeyIdentity, ApiKeySigningProperties] | None = None, + ) -> None: + """Constructor. + + :param identity_resolver: The identity resolver to extract the api key identity. + :param signer: The signer used to sign the request. + """ + self.scheme_id = "smithy.api#httpApiKeyAuth" + self.signer = signer or ApiKeySigner() + + def identity_resolver( + self, *, config: ApiKeyConfig + ) -> IdentityResolver[ApiKeyIdentity, IdentityProperties]: + if not config.api_key_identity_resolver: + raise SmithyIdentityException( + "Attempted to use API key auth, but api_key_identity_resolver was not" + "set on the config." + ) + return config.api_key_identity_resolver + + +class ApiKeySigner(HTTPSigner[ApiKeyIdentity, ApiKeySigningProperties]): + """A signer that signs http requests with an api key.""" + + async def sign( + self, + *, + http_request: HTTPRequest, + identity: ApiKeyIdentity, + signing_properties: ApiKeySigningProperties, + ) -> HTTPRequest: + match signing_properties["location"]: + case ApiKeyLocation.QUERY: + query = http_request.destination.query or "" + if query: + query += "&" + query += f"{signing_properties['name']}={identity.api_key}" + http_request.destination = URI( + scheme=http_request.destination.scheme, + username=http_request.destination.username, + password=http_request.destination.password, + host=http_request.destination.host, + port=http_request.destination.port, + path=http_request.destination.password, + query=query, + fragment=http_request.destination.fragment, + ) + case ApiKeyLocation.HEADER: + value = identity.api_key + if "scheme" in signing_properties and signing_properties["scheme"]: + value = f"{signing_properties['scheme']} {value}" + http_request.fields.set_field( + Field(name=signing_properties["name"], values=[value]) + ) + + return http_request diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/auth.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/auth.py new file mode 100644 index 0000000000..e8bf986c34 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/auth.py @@ -0,0 +1,42 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from ..interfaces.auth import HTTPSigner as HTTPSignerInterface +from ..interfaces.auth import SigningPropertiesType_contra +from ..interfaces.http import HTTPRequest as HTTPRequestInterface +from ..interfaces.identity import IdentityType_contra +from .http import HTTPRequest + + +class HTTPSigner( + HTTPSignerInterface[IdentityType_contra, SigningPropertiesType_contra] +): + """An interface for generating a signed HTTP request.""" + + async def sign( + self, + *, + http_request: HTTPRequestInterface, + identity: IdentityType_contra, + signing_properties: SigningPropertiesType_contra, + ) -> HTTPRequest: + """Generate a new signed HTTPRequest based on the one provided. + + :param http_request: The HTTP request to sign. + + :param identity: The signing identity. + + :param signing_properties: Additional properties loaded to modify the + signing process. + """ + raise NotImplementedError() diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/http/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/http/__init__.py new file mode 100644 index 0000000000..c25471cd03 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/http/__init__.py @@ -0,0 +1,116 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +# TODO: move all of this out of _private + +from collections.abc import AsyncIterable +from dataclasses import dataclass, field +from urllib.parse import urlparse + +from ... import interfaces +from ...interfaces import http as http_interface +from .. import URI, Fields + + +@dataclass(kw_only=True) +class HTTPRequest(http_interface.HTTPRequest): + """HTTP primitives for an Exchange to construct a version agnostic HTTP message.""" + + destination: interfaces.URI + body: AsyncIterable[bytes] + method: str + fields: interfaces.Fields + + async def consume_body(self) -> bytes: + """Iterate over request body and return as bytes.""" + body = b"" + async for chunk in self.body: + body += chunk + return body + + +# HTTPResponse implements interfaces.http.HTTPResponse but cannot be explicitly +# annotated to reflect this because doing so causes Python to raise an AttributeError. +# See https://github.com/python/typing/discussions/903#discussioncomment-4866851 for +# details. +@dataclass(kw_only=True) +class HTTPResponse: + """Basic implementation of :py:class:`...interfaces.http.HTTPResponse`. + + Implementations of :py:class:`...interfaces.http.HTTPClient` may return instances of + this class or of custom response implementations. + """ + + body: AsyncIterable[bytes] + """The response payload as iterable of chunks of bytes.""" + + status: int + """The 3 digit response status code (1xx, 2xx, 3xx, 4xx, 5xx).""" + + fields: interfaces.Fields + """HTTP header and trailer fields.""" + + reason: str | None = None + """Optional string provided by the server explaining the status.""" + + async def consume_body(self) -> bytes: + """Iterate over response body and return as bytes.""" + body = b"" + async for chunk in self.body: + body += chunk + return body + + +@dataclass +class Endpoint(http_interface.Endpoint): + uri: interfaces.URI + headers: interfaces.Fields = field(default_factory=Fields) + + +@dataclass +class StaticEndpointParams: + """Static endpoint params. + + :param uri: A static URI to route requests to. + """ + + uri: str | interfaces.URI + + +class StaticEndpointResolver(http_interface.EndpointResolver[StaticEndpointParams]): + """A basic endpoint resolver that forwards a static URI.""" + + async def resolve_endpoint(self, params: StaticEndpointParams) -> Endpoint: + # If it's not a string, it's already a parsed URI so just pass it along. + if not isinstance(params.uri, str): + return Endpoint(uri=params.uri) + + # Does crt have implementations of these parsing methods? Using the standard + # library is probably fine. + parsed = urlparse(params.uri) + + # This will end up getting wrapped in the client. + if parsed.hostname is None: + raise ValueError( + f"Unable to parse hostname from provided URI: {params.uri}" + ) + + return Endpoint( + uri=URI( + host=parsed.hostname, + path=parsed.path, + scheme=parsed.scheme, + query=parsed.query, + port=parsed.port, + ) + ) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/http/aiohttp_client.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/http/aiohttp_client.py new file mode 100644 index 0000000000..a24d32da7c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/http/aiohttp_client.py @@ -0,0 +1,113 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from copy import copy, deepcopy +from itertools import chain +from typing import Any +from urllib.parse import parse_qs, urlunparse + +import aiohttp + +from ... import interfaces +from ...async_utils import async_list +from .. import Field, FieldPosition, Fields +from . import HTTPResponse + + +class AIOHTTPClientConfig(interfaces.http.HTTPClientConfiguration): + pass + + +class AIOHTTPClient(interfaces.http.HTTPClient): + """Implementation of :py:class:`...interfaces.http.HTTPClient` using aiohttp.""" + + def __init__( + self, + *, + client_config: AIOHTTPClientConfig | None = None, + _session: aiohttp.ClientSession | None = None, + ) -> None: + """ + :param client_config: Configuration that applies to all requests made with this + client. + """ + self._config = client_config or AIOHTTPClientConfig() + self._session = _session or aiohttp.ClientSession() + + async def send( + self, + *, + request: interfaces.http.HTTPRequest, + request_config: interfaces.http.HTTPRequestConfiguration | None = None, + ) -> HTTPResponse: + """Send HTTP request using aiohttp client. + + :param request: The request including destination URI, fields, payload. + :param request_config: Configuration specific to this request. + """ + request_config = ( + interfaces.http.HTTPRequestConfiguration() # todo: should be an implementation + if request_config is None + else request_config + ) + + headers_list = list( + chain.from_iterable( + fld.as_tuples() + for fld in request.fields.get_by_type(FieldPosition.HEADER) + ) + ) + + async with self._session.request( + method=request.method, + url=self._serialize_uri_without_query(request.destination), + params=parse_qs(request.destination.query), + headers=headers_list, + data=request.body, + ) as resp: + return await self._marshal_response(resp) + + def _serialize_uri_without_query(self, uri: interfaces.URI) -> str: + """Serialize all parts of the URI up to and including the path.""" + components = (uri.scheme, uri.host, uri.path or "", "", "", "") + return urlunparse(components) + + async def _marshal_response( + self, aiohttp_resp: aiohttp.ClientResponse + ) -> HTTPResponse: + """Convert a ``aiohttp.ClientResponse`` to a + ``smithy_python.http.HTTPResponse``""" + headers = Fields() + for header_name, header_val in aiohttp_resp.headers.items(): + try: + headers.get_field(header_name).add(header_val) + except KeyError: + headers.set_field( + Field( + name=header_name, + values=[header_val], + kind=FieldPosition.HEADER, + ) + ) + + return HTTPResponse( + status=aiohttp_resp.status, + fields=headers, + body=async_list([await aiohttp_resp.read()]), + reason=aiohttp_resp.reason, + ) + + def __deepcopy__(self, memo: Any) -> "AIOHTTPClient": + return AIOHTTPClient( + client_config=deepcopy(self._config), + _session=copy(self._session), + ) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/http/crt.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/http/crt.py new file mode 100644 index 0000000000..ccc58027d8 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/http/crt.py @@ -0,0 +1,296 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + + +import asyncio +from collections.abc import AsyncIterable +from concurrent.futures import Future +from io import BytesIO +from threading import Lock +from typing import Any, AsyncGenerator, Awaitable + +from awscrt import http as crt_http +from awscrt import io as crt_io + +from ... import interfaces +from ...exceptions import SmithyHTTPException +from .. import Field, FieldPosition, Fields + + +class _AWSCRTEventLoop: + def __init__(self) -> None: + self.bootstrap = self._initialize_default_loop() + + def _initialize_default_loop(self) -> crt_io.ClientBootstrap: + event_loop_group = crt_io.EventLoopGroup(1) + host_resolver = crt_io.DefaultHostResolver(event_loop_group) + return crt_io.ClientBootstrap(event_loop_group, host_resolver) + + +class AWSCRTHTTPResponse(interfaces.http.HTTPResponse): + def __init__(self) -> None: + self._stream: crt_http.HttpClientStream | None = None + self._status_code_future: Future[int] = Future() + self._headers_future: Future[Fields] = Future() + self._chunk_futures: list[Future[bytes]] = [] + self._received_chunks: list[bytes] = [] + self._chunk_lock: Lock = Lock() + + def _set_stream(self, stream: crt_http.HttpClientStream) -> None: + if self._stream is not None: + raise SmithyHTTPException("Stream already set on AWSCRTHTTPResponse object") + self._stream = stream + self._stream.completion_future.add_done_callback(self._on_complete) + self._stream.activate() + + def _on_headers( + self, status_code: int, headers: list[tuple[str, str]], **kwargs: Any + ) -> None: # pragma: crt-callback + fields = Fields() + for header_name, header_val in headers: + try: + fields.get_field(header_name).add(header_val) + except KeyError: + fields.set_field( + Field( + name=header_name, + values=[header_val], + kind=FieldPosition.HEADER, + ) + ) + self._status_code_future.set_result(status_code) + self._headers_future.set_result(fields) + + def _on_body(self, chunk: bytes, **kwargs: Any) -> None: # pragma: crt-callback + with self._chunk_lock: + # TODO: update back pressure window once CRT supports it + if self._chunk_futures: + future = self._chunk_futures.pop(0) + future.set_result(chunk) + else: + self._received_chunks.append(chunk) + + def _get_chunk_future(self) -> Future[bytes]: + if self._stream is None: + raise SmithyHTTPException("Stream not set") + with self._chunk_lock: + future: Future[bytes] = Future() + # TODO: update backpressure window once CRT supports it + if self._received_chunks: + chunk = self._received_chunks.pop(0) + future.set_result(chunk) + elif self._stream.completion_future.done(): + future.set_result(b"") + else: + self._chunk_futures.append(future) + return future + + def _on_complete( + self, completion_future: Future[int] + ) -> None: # pragma: crt-callback + with self._chunk_lock: + if self._chunk_futures: + future = self._chunk_futures.pop(0) + future.set_result(b"") + + @property + def body(self) -> AsyncIterable[bytes]: + return self.chunks() + + @property + def status(self) -> int: + """The 3 digit response status code (1xx, 2xx, 3xx, 4xx, 5xx).""" + return self._status_code_future.result() + + @property + def fields(self) -> Fields: + """List of HTTP header fields.""" + if self._stream is None: + raise SmithyHTTPException("Stream not set") + if not self._headers_future.done(): + raise SmithyHTTPException("Headers not received yet") + return self._headers_future.result() + + @property + def reason(self) -> str | None: + """Optional string provided by the server explaining the status.""" + # TODO: See how CRT exposes reason. + return None + + def get_chunk(self) -> Awaitable[bytes]: + future = self._get_chunk_future() + return asyncio.wrap_future(future) + + async def chunks(self) -> AsyncGenerator[bytes, None]: + while True: + chunk = await self.get_chunk() + if chunk: + yield chunk + else: + break + + async def consume_body(self) -> bytes: + """Iterate over request body and return as bytes.""" + body = b"" + async for chunk in self.body: + body += chunk + return body + + +ConnectionPoolKey = tuple[str, str, int | None] +ConnectionPoolDict = dict[ConnectionPoolKey, crt_http.HttpClientConnection] + + +class AWSCRTHTTPClientConfig(interfaces.http.HTTPClientConfiguration): + pass + + +class AWSCRTHTTPClient(interfaces.http.HTTPClient): + _HTTP_PORT = 80 + _HTTPS_PORT = 443 + + def __init__( + self, + eventloop: _AWSCRTEventLoop | None = None, + client_config: AWSCRTHTTPClientConfig | None = None, + ) -> None: + """ + :param client_config: Configuration that applies to all requests made with this + client. + """ + self._config = ( + AWSCRTHTTPClientConfig() if client_config is None else client_config + ) + if eventloop is None: + eventloop = _AWSCRTEventLoop() + self._eventloop = eventloop + self._client_bootstrap = self._eventloop.bootstrap + self._tls_ctx = crt_io.ClientTlsContext(crt_io.TlsContextOptions()) + self._socket_options = crt_io.SocketOptions() + self._connections: ConnectionPoolDict = {} + + async def send( + self, + request: interfaces.http.HTTPRequest, + request_config: interfaces.http.HTTPRequestConfiguration | None = None, + ) -> AWSCRTHTTPResponse: + """Send HTTP request using awscrt client. + + :param request: The request including destination URI, fields, payload. + :param request_config: Configuration specific to this request. + """ + crt_request = await self._marshal_request(request) + connection = await self._get_connection(request.destination) + crt_response = AWSCRTHTTPResponse() + crt_stream = connection.request( + crt_request, + crt_response._on_headers, + crt_response._on_body, + ) + crt_response._set_stream(crt_stream) + return crt_response + + async def _create_connection( + self, url: interfaces.URI + ) -> crt_http.HttpClientConnection: + """Builds and validates connection to ``url``, returns it as + ``asyncio.Future``""" + connect_future = self._build_new_connection(url) + connection = await asyncio.wrap_future(connect_future) + self._validate_connection(connection) + return connection + + async def _get_connection( + self, url: interfaces.URI + ) -> crt_http.HttpClientConnection: + # TODO: Use CRT connection pooling instead of this basic kind + connection_key = (url.scheme, url.host, url.port) + if connection_key in self._connections: + return self._connections[connection_key] + else: + connection = await self._create_connection(url) + self._connections[connection_key] = connection + return connection + + def _build_new_connection( + self, url: interfaces.URI + ) -> Future[crt_http.HttpClientConnection]: + if url.scheme == "http": + port = self._HTTP_PORT + tls_connection_options = None + elif url.scheme == "https": + port = self._HTTPS_PORT + tls_connection_options = self._tls_ctx.new_connection_options() + tls_connection_options.set_server_name(url.host) + # TODO: Support TLS configuration, including alpn + tls_connection_options.set_alpn_list(["h2", "http/1.1"]) + else: + raise SmithyHTTPException( + f"AWSCRTHTTPClient does not support URL scheme {url.scheme}" + ) + if url.port is not None: + port = url.port + + connect_future: Future[ + crt_http.HttpClientConnection + ] = crt_http.HttpClientConnection.new( + bootstrap=self._client_bootstrap, + host_name=url.host, + port=port, + socket_options=self._socket_options, + tls_connection_options=tls_connection_options, + ) + return connect_future + + def _validate_connection(self, connection: crt_http.HttpClientConnection) -> None: + """Validates an existing connection against the client config. + + Checks performed: + * If ``force_http_2`` is enabled: Is the connection HTTP/2? + """ + force_http_2 = self._config.force_http_2 + if force_http_2 and connection.version is not crt_http.HttpVersion.Http2: + connection.close() + negotiated = crt_http.HttpVersion(connection.version).name + raise SmithyHTTPException(f"HTTP/2 could not be negotiated: {negotiated}") + + def _render_path(self, url: interfaces.URI) -> str: + path = url.path if url.path is not None else "/" + query = f"?{url.query}" if url.query is not None else "" + return f"{path}{query}" + + async def _marshal_request( + self, request: interfaces.http.HTTPRequest + ) -> crt_http.HttpRequest: + """ + Create :py:class:`awscrt.http.HttpRequest` from + :py:class:`smithy_python.interfaces.http.HTTPRequest` + """ + headers_list = [] + for fld in request.fields.entries.values(): + if fld.kind != FieldPosition.HEADER: + continue + for val in fld.values: + headers_list.append((fld.name, val)) + + path = self._render_path(request.destination) + headers = crt_http.HttpHeaders(headers_list) + body = BytesIO(await request.consume_body()) + + crt_request = crt_http.HttpRequest( + method=request.method, + path=path, + headers=headers, + body_stream=body, + ) + return crt_request diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/identity.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/identity.py new file mode 100644 index 0000000000..125aa8ee6c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/identity.py @@ -0,0 +1,41 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from datetime import datetime, timezone + +from ..interfaces import identity as identity_interface +from ..utils import ensure_utc + + +class Identity(identity_interface.Identity): + """An entity available to the client representing who the user is.""" + + def __init__( + self, + *, + expiration: datetime | None = None, + ) -> None: + """Initialize an identity. + + :param expiration: The expiration time of the identity. If time zone is + provided, it is updated to UTC. The value must always be in UTC. + """ + if expiration is not None: + expiration = ensure_utc(expiration) + self.expiration: datetime | None = expiration + + @property + def is_expired(self) -> bool: + """Whether the identity is expired.""" + if self.expiration is None: + return False + return datetime.now(tz=timezone.utc) >= self.expiration diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/retries/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/retries/__init__.py new file mode 100644 index 0000000000..d4a917f4ae --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/_private/retries/__init__.py @@ -0,0 +1,256 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import random +from dataclasses import dataclass +from enum import Enum +from typing import Callable + +from ...exceptions import SmithyRetryException +from ...interfaces import retries as retries_interface + + +class ExponentialBackoffJitterType(Enum): + """Jitter mode for exponential backoff. + + For use with :py:class:`ExponentialRetryBackoffStrategy`. + """ + + DEFAULT = 1 + """Truncated binary exponential backoff delay with equal jitter: + + .. code-block:: python + + capped = min(max_backoff, backoff_scale_value * 2 ** (retry_attempt - 1)) + (capped / 2) + random_between(0, capped / 2) + + Also known as "Equal Jitter". Similar to :py:var:`FULL` but always keep some of the + backoff and jitters by a smaller amount. + """ + + NONE = 2 + """Truncated binary exponential backoff delay without jitter: + + .. code-block:: python + + min(max_backoff, backoff_scale_value * 2 ** (retry_attempt - 1)) + """ + + FULL = 3 + """Truncated binary exponential backoff delay with full jitter: + + .. code-block:: python + + random_between( + max_backoff, + min(max_backoff, backoff_scale_value * 2 ** (retry_attempt - 1)) + ) + """ + + DECORRELATED = 4 + """Truncated binary exponential backoff delay with decorrelated jitter: + + .. code-block:: python + + min(max_backoff, random_between(backoff_scale_value, t_(i-1) * 3)) + + Similar to :py:var:`FULL`, but also increases the maximum jitter at each retry. + """ + + +class ExponentialRetryBackoffStrategy(retries_interface.RetryBackoffStrategy): + def __init__( + self, + *, + backoff_scale_value: float = 0.025, + max_backoff: float = 20, + jitter_type: ExponentialBackoffJitterType = ExponentialBackoffJitterType.DEFAULT, + random: Callable[[], float] = random.random, + ): + """Exponential backoff with optional jitter. + + .. seealso:: https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + + :param backoff_scale_value: Factor that linearly adjusts returned backoff delay + values. See the methods ``_next_delay_*`` for the formula used to calculate the + delay for each jitter type. If set to ``None`` (the default), :py:attr:`random` + will be called to generate a value. + + :param max_backoff: Upper limit for backoff delay values returned, in seconds. + + :param jitter_type: Determines the formula used to apply jitter to the backoff + delay. + + :param random: A callable that returns random numbers between ``0`` and ``1``. + Use the default ``random.random`` unless you require an alternate source of + randomness or a non-uniform distribution. + """ + self._backoff_scale_value = backoff_scale_value + self._max_backoff = max_backoff + self._jitter_type = jitter_type + self._random = random + self._previous_delay_seconds = self._backoff_scale_value + + def compute_next_backoff_delay(self, retry_attempt: int) -> float: + """Calculate timespan in seconds to delay before next retry. + + See the methods ``_next_delay_*`` for the formula used to calculate the delay + for each jitter type for values of ``retry_attempt > 0``. + + :param retry_attempt: The index of the retry attempt that is about to be made + after the delay. The initial attempt, before any retries, is index ``0``, and + will return a delay of ``0``. The first retry attempt after a failed initial + attempt is index ``1``, and so on. + """ + if retry_attempt == 0: + return 0 + + match self._jitter_type: + case ExponentialBackoffJitterType.NONE: + seconds = self._next_delay_no_jitter(retry_attempt=retry_attempt) + case ExponentialBackoffJitterType.DEFAULT: + seconds = self._next_delay_equal_jitter(retry_attempt=retry_attempt) + case ExponentialBackoffJitterType.FULL: + seconds = self._next_delay_full_jitter(retry_attempt=retry_attempt) + case ExponentialBackoffJitterType.DECORRELATED: + seconds = self._next_delay_decorrelated_jitter( + previous_delay=self._previous_delay_seconds + ) + + self._previous_delay_seconds = seconds + return seconds + + def _jitter_free_uncapped_delay(self, retry_attempt: int) -> float: + """The basic exponential delay without jitter or upper bound: + + .. code-block:: python + + backoff_scale_value * 2 ** (retry_attempt - 1) + """ + return self._backoff_scale_value * (2.0 ** (retry_attempt - 1)) + + def _next_delay_no_jitter(self, retry_attempt: int) -> float: + """Calculates truncated binary exponential backoff delay without jitter. + + Used when :py:var:`jitter_type` is :py:attr:`ExponentialBackoffJitterType.NONE`. + """ + no_jitter_delay = self._jitter_free_uncapped_delay(retry_attempt) + return min(no_jitter_delay, self._max_backoff) + + def _next_delay_full_jitter(self, retry_attempt: int) -> float: + """Calculates truncated binary exponential backoff delay with full jitter. + + Used when :py:var:`jitter_type` is :py:attr:`ExponentialBackoffJitterType.FULL`. + """ + + no_jitter_delay = self._jitter_free_uncapped_delay(retry_attempt) + return self._random() * min(no_jitter_delay, self._max_backoff) + + def _next_delay_equal_jitter(self, retry_attempt: int) -> float: + """Calculates truncated binary exponential backoff delay with equal jitter: + + Used when :py:var:`jitter_type` is + :py:attr:`ExponentialBackoffJitterType.DEFAULT`. + """ + no_jitter_delay = self._jitter_free_uncapped_delay(retry_attempt) + return (self._random() * 0.5 + 0.5) * min(no_jitter_delay, self._max_backoff) + + def _next_delay_decorrelated_jitter(self, previous_delay: float) -> float: + """Calculates truncated binary exp. backoff delay with decorrelated jitter: + + Used when :py:var:`jitter_type` is + :py:attr:`ExponentialBackoffJitterType.DECORRELATED`. + """ + return min( + self._backoff_scale_value + self._random() * previous_delay * 3, + self._max_backoff, + ) + + +@dataclass(kw_only=True) +class SimpleRetryToken: + """Basic retry token that stores only the attempt count and backoff strategy. + + Retry tokens should always be obtained from an implementation of + :py:class:`retries_interface.RetryStrategy`. + """ + + retry_count: int + """Retry count is the total number of attempts minus the initial attempt.""" + + retry_delay: float + """Delay in seconds to wait before the retry attempt.""" + + @property + def attempt_count(self) -> int: + """The total number of attempts including the initial attempt and retries.""" + return self.retry_count + 1 + + +class SimpleRetryStrategy(retries_interface.RetryStrategy): + def __init__( + self, + *, + backoff_strategy: retries_interface.RetryBackoffStrategy | None = None, + max_attempts: int = 5, + ): + """Basic retry strategy that simply invokes the given backoff strategy. + + :param backoff_strategy: The backoff strategy used by returned tokens to compute + the retry delay. Defaults to :py:class:`ExponentialRetryBackoffStrategy`. + + :param max_attempts: Upper limit on total number of attempts made, including + initial attempt and retries. + """ + self.backoff_strategy = backoff_strategy or ExponentialRetryBackoffStrategy() + self.max_attempts = max_attempts + + def acquire_initial_retry_token( + self, *, token_scope: str | None = None + ) -> SimpleRetryToken: + """Called before any retries (for the first attempt at the operation). + + :param token_scope: This argument is ignored by this retry strategy. + """ + retry_delay = self.backoff_strategy.compute_next_backoff_delay(0) + return SimpleRetryToken(retry_count=0, retry_delay=retry_delay) + + def refresh_retry_token_for_retry( + self, + *, + token_to_renew: retries_interface.RetryToken, + error_info: retries_interface.RetryErrorInfo, + ) -> SimpleRetryToken: + """Replace an existing retry token from a failed attempt with a new token. + + This retry strategy always returns a token until the attempt count stored in + the new token exceeds the ``max_attempts`` value. + + :param token_to_renew: The token used for the previous failed attempt. + + :param error_info: If no further retry is allowed, this information is used to + construct the exception. + + :raises SmithyRetryException: If no further retry attempts are allowed. + """ + retry_count = token_to_renew.retry_count + 1 + if retry_count >= self.max_attempts: + raise SmithyRetryException( + f"Reached maximum number of allowed attempts: {self.max_attempts}" + ) + retry_delay = self.backoff_strategy.compute_next_backoff_delay(retry_count) + return SimpleRetryToken(retry_count=retry_count, retry_delay=retry_delay) + + def record_success(self, *, token: retries_interface.RetryToken) -> None: + """Not used by this retry strategy.""" + pass diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/async_utils.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/async_utils.py new file mode 100644 index 0000000000..5810990975 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/async_utils.py @@ -0,0 +1,25 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from asyncio import sleep +from collections.abc import AsyncIterable, Iterable +from typing import TypeVar + +_ListEl = TypeVar("_ListEl") + + +async def async_list(lst: Iterable[_ListEl]) -> AsyncIterable[_ListEl]: + """Turn an Iterable into an AsyncIterable.""" + for x in lst: + await sleep(0) + yield x diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/exceptions.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/exceptions.py new file mode 100644 index 0000000000..77eed206ff --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/exceptions.py @@ -0,0 +1,32 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + + +class SmithyException(Exception): + """Base exception type for all exceptions raised by smithy-python.""" + + +class SmithyRetryException(SmithyException): + """Base exception type for all exceptions raised in retry strategies.""" + + +class ExpectationNotMetException(SmithyException): + """Exception type for exceptions thrown by unmet assertions.""" + + +class SmithyIdentityException(SmithyException): + """Base exception type for all exceptions raised in identity resolution.""" + + +class SmithyHTTPException(SmithyException): + """Base exception type for all exceptions raised in HTTP clients.""" diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/httputils.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/httputils.py new file mode 100644 index 0000000000..3182e0e4a7 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/httputils.py @@ -0,0 +1,133 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from urllib.parse import quote as urlquote + +from .exceptions import SmithyException + + +def split_header(given: str, handle_unquoted_http_date: bool = False) -> list[str]: + """Splits a header value into a list of strings. + + The format is based on RFC9110's list production found in secion 5.6.1 with + the quoted string production found in section 5.6.4. In short: + + A list is 1 or more elements surrounded by optional whitespace and separated by + commas. Elements may be quoted with double quotes (``"``) to contain leading or + trailing whitespace, commas, or double quotes. Inside the the double quotes, a + value may be escaped with a backslash (``\\``). Elements that contain no contents + are ignored. + + If the list is known to contain unquoted IMF-fixdate formatted timestamps, the + ``handle_unquoted_http_date`` parameter can be set to ensure the list isn't + split on the commas inside the timestamps. + + :param given: The header value to split. + :param handle_unquoted_http_date: Support splitting IMF-fixdate lists without + quotes. Defaults to False. + :returns: The header value split on commas. + """ + result: list[str] = [] + + i = 0 + while i < len(given): + if given[i].isspace(): + # Skip any leading space. + i += 1 + elif given[i] == '"': + # Grab the contents of the quoted value and append it. + entry, i = _consume_until(given, i + 1, '"', escape_char="\\") + result.append(entry) + + if i > len(given) or given[i - 1] != '"': + raise SmithyException( + f"Invalid header list syntax: expected end quote but reached end " + f"of value: `{given}`" + ) + + # Skip until the next comma. + excess, i = _consume_until(given, i, ",") + if excess.strip(): + raise SmithyException( + f"Invalid header list syntax: Found quote contents after " + f"end-quote: `{excess}` in `{given}`" + ) + else: + entry, i = _consume_until( + given, i, ",", skip_first=handle_unquoted_http_date + ) + if stripped := entry.strip(): + result.append(stripped) + + return result + + +def _consume_until( + given: str, + start_index: int, + end_char: str, + escape_char: str | None = None, + skip_first: bool = False, +) -> tuple[str, int]: + """Creates a slice of the given string from the start index to the end character. + + This also handles resolving escaped characters using the given escape_char if + provided. + + If `skip_first` is true, the first instance of the end character will be skipped. + This is to enable support for unquoted IMF fixdate timestamps. + + :param given: The whole header string. + :param start_index: The index at which to start slicing. + :param end_char: The character to split on. This is not included in the output. + :param escape_char: The character to escape with, e.g. ``\\``. + :param skip_first: Whether to skip the first instance of the end character. + :returns: A substring from the start index to the first instance of the end char. + """ + should_skip = skip_first + end_index = start_index + result = "" + escaped = False + while end_index < len(given): + if escaped: + result += given[end_index] + escaped = False + elif given[end_index] == escape_char: + escaped = True + elif given[end_index] == end_char: + if should_skip: + result += given[end_index] + should_skip = False + else: + break + else: + result += given[end_index] + end_index += 1 + return result, end_index + 1 + + +def join_query_params(params: list[tuple[str, str | None]], prefix: str = "") -> str: + """Join a list of query parameter key-value tuples. + + :param params: The list of key-value query parameter tuples. + :param prefix: An optional query prefix. + """ + query: str = prefix + for param in params: + if query: + query += "&" + if param[1] is None: + query += urlquote(param[0], safe="") + else: + query += f"{urlquote(param[0], safe='')}={urlquote(param[1], safe='')}" + return query diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/__init__.py new file mode 100644 index 0000000000..322bd62833 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/__init__.py @@ -0,0 +1,181 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from collections import OrderedDict +from collections.abc import AsyncIterable, Iterator +from enum import Enum +from typing import Protocol + + +class URI(Protocol): + """Universal Resource Identifier, target location for a :py:class:`Request`.""" + + scheme: str + """For example ``http`` or ``mqtts``.""" + + username: str | None + """Username part of the userinfo URI component.""" + + password: str | None + """Password part of the userinfo URI component.""" + + host: str + """The hostname, for example ``amazonaws.com``.""" + + port: int | None + """An explicit port number.""" + + path: str | None + """Path component of the URI.""" + + query: str | None + """Query component of the URI as string.""" + + fragment: str | None + """Part of the URI specification, but may not be transmitted by a client.""" + + def build(self) -> str: + """Construct URI string representation. + + Returns a string of the form + ``{scheme}://{username}:{password}@{host}:{port}{path}?{query}#{fragment}`` + """ + ... + + @property + def netloc(self) -> str: + """Construct netloc string in format ``{username}:{password}@{host}:{port}``""" + ... + + +class Request(Protocol): + """Protocol-agnostic representation of a request.""" + + destination: URI + body: AsyncIterable[bytes] + + async def consume_body(self) -> bytes: + """Iterate over request body and return as bytes.""" + ... + + +class Response(Protocol): + """Protocol-agnostic representation of a response.""" + + @property + def body(self) -> AsyncIterable[bytes]: + """The response payload as iterable of chunks of bytes.""" + ... + + async def consume_body(self) -> bytes: + """Iterate over response body and return as bytes.""" + ... + + +class FieldPosition(Enum): + """The type of a field. + + Defines its placement in a request or response. + """ + + HEADER = 0 + """Header field. + + In HTTP this is a header as defined in RFC 9110 Section 6.3. Implementations of + other protocols may use this FieldPosition for similar types of metadata. + """ + + TRAILER = 1 + """Trailer field. + + In HTTP this is a trailer as defined in RFC 9110 Section 6.5. Implementations of + other protocols may use this FieldPosition for similar types of metadata. + """ + + +class Field(Protocol): + """A name-value pair representing a single field in a request or response. + + The kind will dictate metadata placement within an the message, for example as + header or trailer field in a HTTP request as defined in RFC 9110 Section 5. + + All field names are case insensitive and case-variance must be treated as + equivalent. Names may be normalized but should be preserved for accuracy during + transmission. + """ + + name: str + values: list[str] + kind: FieldPosition = FieldPosition.HEADER + + def add(self, value: str) -> None: + """Append a value to a field.""" + ... + + def set(self, values: list[str]) -> None: + """Overwrite existing field values.""" + ... + + def remove(self, value: str) -> None: + """Remove all matching entries from list.""" + ... + + def as_string(self) -> str: + """Serialize the ``Field``'s values into a single line string.""" + ... + + def as_tuples(self) -> list[tuple[str, str]]: + """Get list of ``name``, ``value`` tuples where each tuple represents one + value.""" + ... + + +class Fields(Protocol): + """Protocol agnostic mapping of key-value pair request metadata, such as HTTP + fields.""" + + # Entries are keyed off the name of a provided Field + entries: OrderedDict[str, Field] + encoding: str | None = "utf-8" + + def set_field(self, field: Field) -> None: + """Set entry for a Field name.""" + ... + + def get_field(self, name: str) -> Field: + """Retrieve Field entry.""" + ... + + def remove_field(self, name: str) -> None: + """Delete entry from collection.""" + ... + + def get_by_type(self, kind: FieldPosition) -> list[Field]: + """Helper function for retrieving specific types of fields. + + Used to grab all headers or all trailers. + """ + ... + + def extend(self, other: "Fields") -> None: + """Merges ``entries`` of ``other`` into the current ``entries``. + + For every `Field` in the ``entries`` of ``other``: If the normalized name + already exists in the current ``entries``, the values from ``other`` are + appended. Otherwise, the ``Field`` is added to the list of ``entries``. + """ + ... + + def __iter__(self) -> Iterator[Field]: + """Allow iteration over entries.""" + ... diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/auth.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/auth.py new file mode 100644 index 0000000000..b815a8d791 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/auth.py @@ -0,0 +1,122 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from dataclasses import dataclass +from typing import Any, Protocol, TypedDict, TypeVar + +from .http import HTTPRequest +from .identity import ( + IdentityConfig_contra, + IdentityPropertiesType_contra, + IdentityResolver, + IdentityType, + IdentityType_contra, +) + + +class SigningProperties(TypedDict): + """Additional properties loaded to modify the signing process.""" + + ... + + +SigningPropertiesType = TypeVar("SigningPropertiesType", bound=SigningProperties) +SigningPropertiesType_contra = TypeVar( + "SigningPropertiesType_contra", bound=SigningProperties, contravariant=True +) + + +class HTTPSigner(Protocol[IdentityType_contra, SigningPropertiesType_contra]): + """An interface for generating a signed HTTP request.""" + + async def sign( + self, + *, + http_request: HTTPRequest, + identity: IdentityType_contra, + signing_properties: SigningPropertiesType_contra, + ) -> HTTPRequest: + """Generate a new signed HTTPRequest based on the one provided. + + :param http_request: The HTTP request to sign. + + :param identity: The signing identity. + + :param signing_properties: Additional properties loaded to modify the + signing process. + """ + ... + + +class HTTPAuthScheme( + Protocol[ + IdentityType, + IdentityConfig_contra, + IdentityPropertiesType_contra, + SigningPropertiesType, + ] +): + """Represents a way a service will authenticate the user's identity.""" + + # A unique identifier for the authentication scheme. + scheme_id: str + + # An API that can be used to sign HTTP requests. + signer: HTTPSigner[IdentityType, SigningPropertiesType] + + def identity_resolver( + self, *, config: IdentityConfig_contra + ) -> IdentityResolver[IdentityType, IdentityPropertiesType_contra]: + """An API that can be queried to resolve identity.""" + ... + + +@dataclass(kw_only=True) +class HTTPAuthOption: + """Auth scheme used for signing and identity resolution.""" + + # The ID of the scheme to use. This string matches the one returned by + # HttpAuthScheme.scheme_id + scheme_id: str + + # Parameters to pass to IdentityResolver.get_identity. + identity_properties: dict[str, Any] + + # Parameters to pass to HttpSigner.sign. + signer_properties: dict[str, Any] + + +@dataclass(kw_only=True) +class AuthSchemeParameters: + """The input to the auth scheme resolver. + + A code-generated interface for passing in the data required for determining the + authentication scheme. By default, this only includes the operation name. + """ + + # The service operation being invoked by the client. + operation: str + + +class AuthSchemeResolver(Protocol): + """Determines which authentication scheme to use for a given service.""" + + def resolve_auth_scheme( + self, *, auth_parameters: AuthSchemeParameters + ) -> list[HTTPAuthOption]: + """Resolve an ordered list of applicable auth schemes. + + :param auth_parameters: The parameters required for determining which + authentication schemes to potentially use. + """ + ... diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/blobs.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/blobs.py new file mode 100644 index 0000000000..6d8d88e610 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/blobs.py @@ -0,0 +1,305 @@ +from asyncio import iscoroutinefunction +from io import BytesIO +from typing import ( + AsyncIterable, + AsyncIterator, + Awaitable, + Callable, + Protocol, + Self, + Union, + cast, + runtime_checkable, +) + +# The default chunk size for iterating streams. +_DEFAULT_CHUNK_SIZE = 1024 + + +@runtime_checkable +class ByteStream(Protocol): + """A file-like object with a read method that returns bytes.""" + + def read(self, size: int = -1) -> bytes: + pass + + +@runtime_checkable +class AsyncByteStream(Protocol): + """A file-like object with an async read method.""" + + async def read(self, size: int = -1) -> bytes: + pass + + +# A union of all acceptable streaming blob types. Deserialized payloads will +# always return a ByteStream, or AsyncByteStream if async is enabled. +StreamingBlob = Union[ + ByteStream, + AsyncByteStream, + bytes, + bytearray, + AsyncIterable[bytes], +] + + +# asyncio has a StreamReader class which you might think would be appropriate here, +# but it is unfortunately tied to the asyncio http interfaces. +class AsyncBytesReader: + """A file-like object with an async read method.""" + + # BytesIO *is* a ByteStream, but mypy will nevertheless complain if it isn't here. + _data: ByteStream | AsyncByteStream | AsyncIterable[bytes] | BytesIO | None + _closed = False + + def __init__(self, data: StreamingBlob): + """Initializes self. + + Data is read from the source on an as-needed basis and is not buffered. + + :param data: The source data to read from. + """ + self._remainder = b"" + # pylint: disable-next=isinstance-second-argument-not-valid-type + if isinstance(data, bytes | bytearray): + self._data = BytesIO(data) + else: + self._data = data + + async def read(self, size: int = -1) -> bytes: + """Read a number of bytes from the stream. + + :param size: The maximum number of bytes to read. If less than 0, all bytes + will be read. + """ + if self._closed or not self._data: + raise ValueError("I/O operation on closed file.") + + if isinstance(self._data, ByteStream) and not iscoroutinefunction( + self._data.read + ): + # Python's runtime_checkable can't actually tell the difference between + # sync and async, so we have to check ourselves. + return self._data.read(size) + + if isinstance(self._data, AsyncByteStream): + return await self._data.read(size) + + return await self._read_from_iterable( + cast(AsyncIterable[bytes], self._data), size + ) + + async def _read_from_iterable( + self, iterator: AsyncIterable[bytes], size: int + ) -> bytes: + # This takes the iterator as an arg here just to avoid mypy complaints, since + # we know it's an iterator where this is called. + result = self._remainder + if size < 0: + async for element in iterator: + result += element + self._remainder = b"" + return result + + async for element in iterator: + result += element + if len(result) >= size: + break + + self._remainder = result[size:] + return result[:size] + + def __aiter__(self) -> AsyncIterator[bytes]: + return self.iter_chunks() + + def iter_chunks( + self, chunk_size: int = _DEFAULT_CHUNK_SIZE + ) -> AsyncIterator[bytes]: + """Iterate over the reader in chunks of a given size. + + :param chunk_size: The maximum size of each chunk. If less than 0, the entire + reader will be read into one chunk. + """ + return _AsyncByteStreamIterator(self.read, chunk_size) + + def readable(self) -> bool: + """Returns whether the stream is readable.""" + return True + + def writeable(self) -> bool: + """Returns whether the stream is writeable.""" + return False + + def seekable(self) -> bool: + """Returns whether the stream is seekable.""" + return False + + @property + def closed(self) -> bool: + """Returns whether the stream is closed.""" + return self._closed + + def close(self) -> None: + """Closes the stream, as well as the underlying stream where possible.""" + if (close := getattr(self._data, "close", None)) is not None: + close() + self._data = None + self._closed = True + + +class SeekableAsyncBytesReader: + """A file-like object with async read and seek methods.""" + + def __init__(self, data: StreamingBlob): + """Initializes self. + + Data is read from the source on an as-needed basis and buffered internally so + that it can be rewound safely. + + :param data: The source data to read from. + """ + # pylint: disable-next=isinstance-second-argument-not-valid-type + if isinstance(data, bytes | bytearray): + self._buffer = BytesIO(data) + self._data_source = None + elif isinstance(data, AsyncByteStream) and iscoroutinefunction(data.read): + # Note that we need that iscoroutine check because python won't actually check + # whether or not the read function is async. + self._buffer = BytesIO() + self._data_source = data + else: + self._buffer = BytesIO() + self._data_source = AsyncBytesReader(data) + + async def read(self, size: int = -1) -> bytes: + """Read a number of bytes from the stream. + + :param size: The maximum number of bytes to read. If less than 0, all bytes + will be read. + """ + if self._data_source is None or size == 0: + return self._buffer.read(size) + + start = self._buffer.tell() + current_buffer_size = self._buffer.seek(0, 2) + + if size < 0: + await self._read_into_buffer(size) + elif (target := start + size) > current_buffer_size: + amount_to_read = target - current_buffer_size + await self._read_into_buffer(amount_to_read) + + self._buffer.seek(start, 0) + return self._buffer.read(size) + + async def seek(self, offset: int, whence: int = 0) -> int: + """Moves the cursor to a position relatve to the position indicated by whence. + + Whence can have one of three values: + + * 0 => The offset is relative to the start of the stream. + + * 1 => The offset is relative to the current location of the cursor. + + * 2 => The offset is relative to the end of the stream. + + :param offset: The amount of movement to be done relative to whence. + :param whence: The location the offset is relative to. + :returns: Returns the new position of the cursor. + """ + if self._data_source is None: + return self._buffer.seek(offset, whence) + + if whence >= 2: + # If the seek is relative to the end of the stream, we need to read the + # whole thing in from the source. + self._buffer.seek(0, 2) + self._buffer.write(await self._data_source.read()) + return self._buffer.seek(offset, whence) + + start = self.tell() + target = offset + if whence == 1: + target += start + + current_buffer_size = self._buffer.seek(0, 2) + if current_buffer_size < target: + await self._read_into_buffer(target - current_buffer_size) + + return self._buffer.seek(target, 0) + + async def _read_into_buffer(self, size: int) -> None: + if self._data_source is None: + return + + read_bytes = await self._data_source.read(size) + if len(read_bytes) < size or size < 0: + self._data_source = None + + self._buffer.seek(0, 2) + self._buffer.write(read_bytes) + + def tell(self) -> int: + """Returns the position of the cursor.""" + return self._buffer.tell() + + def __aiter__(self) -> AsyncIterator[bytes]: + return self.iter_chunks() + + def iter_chunks( + self, chunk_size: int = _DEFAULT_CHUNK_SIZE + ) -> AsyncIterator[bytes]: + """Iterate over the reader in chunks of a given size. + + :param chunk_size: The maximum size of each chunk. If less than 0, the entire + reader will be read into one chunk. + """ + return _AsyncByteStreamIterator(self.read, chunk_size) + + def readable(self) -> bool: + """Returns whether the stream is readable.""" + return True + + def writeable(self) -> bool: + """Returns whether the stream is writeable.""" + return False + + def seekable(self) -> bool: + """Returns whether the stream is seekable.""" + return True + + @property + def closed(self) -> bool: + """Returns whether the stream is closed.""" + return self._buffer.closed + + def close(self) -> None: + """Closes the stream, as well as the underlying stream where possible.""" + if callable(close_fn := getattr(self._data_source, "close", None)): + close_fn() # pylint: disable=not-callable + self._data_source = None + self._buffer.close() + + +class _AsyncByteStreamIterator: + """An async bytes iterator that operates over an async read method.""" + + def __init__(self, read: Callable[[int], Awaitable[bytes]], chunk_size: int): + """Initializes self. + + :param read: An async callable that reads a given number of bytes from some + source. + + :param chunk_size: The number of bytes to read in each iteration. + """ + self._read = read + self._chunk_size = chunk_size + + def __aiter__(self) -> Self: + return self + + async def __anext__(self) -> bytes: + data = await self._read(self._chunk_size) + if data: + return data + raise StopAsyncIteration diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/http.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/http.py new file mode 100644 index 0000000000..370525ccdf --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/http.py @@ -0,0 +1,112 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from dataclasses import dataclass +from typing import Protocol, TypeVar + +from . import URI, Fields, Request, Response + +QueryParamsList = list[tuple[str, str]] + + +class HTTPRequest(Request, Protocol): + """HTTP primitive for an Exchange to construct a version agnostic HTTP message. + + :param destination: The URI where the request should be sent to. + :param method: The HTTP method of the request, for example "GET". + :param fields: ``Fields`` object containing HTTP headers and trailers. + :param body: A streamable collection of bytes. + """ + + method: str + fields: Fields + + +class HTTPResponse(Response, Protocol): + """HTTP primitives returned from an Exchange, used to construct a client + response.""" + + @property + def status(self) -> int: + """The 3 digit response status code (1xx, 2xx, 3xx, 4xx, 5xx).""" + ... + + @property + def fields(self) -> Fields: + """``Fields`` object containing HTTP headers and trailers.""" + ... + + @property + def reason(self) -> str | None: + """Optional string provided by the server explaining the status.""" + ... + + +class Endpoint(Protocol): + uri: URI + headers: Fields + + +# EndpointParams are defined in the generated client, so we use a TypeVar here. +# More specific EndpointParams implementations are subtypes of less specific ones. But +# consumers of less specific EndpointParams implementations are subtypes of consumers +# of more specific ones. +EndpointParams = TypeVar("EndpointParams", contravariant=True) + + +class EndpointResolver(Protocol[EndpointParams]): + """Resolves an operation's endpoint based given parameters.""" + + async def resolve_endpoint(self, params: EndpointParams) -> Endpoint: + raise NotImplementedError() + + +@dataclass(kw_only=True) +class HTTPClientConfiguration: + """Client-level HTTP configuration. + + :param force_http_2: Whether to require HTTP/2. + """ + + force_http_2: bool = False + + +@dataclass(kw_only=True) +class HTTPRequestConfiguration: + """Request-level HTTP configuration. + + :param read_timeout: How long, in seconds, the client will attempt to read the + first byte over an established, open connection before timing out. + """ + + read_timeout: float | None = None + + +class HTTPClient(Protocol): + """An asynchronous HTTP client interface.""" + + def __init__(self, *, client_config: HTTPClientConfiguration | None) -> None: + """ + :param client_config: Configuration that applies to all requests made with this + client. + """ + ... + + async def send( + self, *, request: HTTPRequest, request_config: HTTPRequestConfiguration | None + ) -> HTTPResponse: + """Send HTTP request over the wire and return the response. + + :param request: The request including destination URI, fields, payload. + :param request_config: Configuration specific to this request. + """ + ... diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/identity.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/identity.py new file mode 100644 index 0000000000..3688993b5a --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/identity.py @@ -0,0 +1,64 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from datetime import datetime +from typing import Protocol, TypedDict, TypeVar + + +class Identity(Protocol): + """An entity available to the client representing who the user is.""" + + # The expiration time of the identity. If time zone is provided, + # it is updated to UTC. The value must always be in UTC. + expiration: datetime | None = None + + @property + def is_expired(self) -> bool: + """Whether the identity is expired.""" + ... + + +IdentityType = TypeVar("IdentityType", bound=Identity) +IdentityType_contra = TypeVar("IdentityType_contra", bound=Identity, contravariant=True) +IdentityType_cov = TypeVar("IdentityType_cov", bound=Identity, covariant=True) + + +class IdentityProperties(TypedDict): + """Properties used to help determine the identity to return.""" + + ... + + +IdentityPropertiesType = TypeVar("IdentityPropertiesType", bound=IdentityProperties) +IdentityPropertiesType_contra = TypeVar( + "IdentityPropertiesType_contra", bound=IdentityProperties, contravariant=True +) + +IdentityConfig_contra = TypeVar("IdentityConfig_contra", contravariant=True) + + +class IdentityResolver(Protocol[IdentityType_cov, IdentityPropertiesType_contra]): + """Used to load a user's `Identity` from a given source. + + Each `Identity` may have one or more resolver implementations. + """ + + async def get_identity( + self, *, identity_properties: IdentityPropertiesType_contra + ) -> IdentityType_cov: + """Load the user's identity from this resolver. + + :param identity_properties: Properties used to help determine the + identity to return. + """ + ... diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/interceptor.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/interceptor.py new file mode 100644 index 0000000000..bb6f4e483e --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/interceptor.py @@ -0,0 +1,606 @@ +from copy import copy, deepcopy +from typing import Any, Generic, TypeVar + +Request = TypeVar("Request") +Response = TypeVar("Response") +TransportRequest = TypeVar("TransportRequest") +TransportResponse = TypeVar("TransportResponse") + + +class InterceptorContext( + Generic[Request, Response, TransportRequest, TransportResponse] +): + def __init__( + self, + *, + request: Request, + response: Response | Exception, + transport_request: TransportRequest, + transport_response: TransportResponse, + ): + """A container for the current data available to an interceptor. + + :param request: The modeled request for the operation being invoked. + :param response: The modeled response for the operation being invoked. This will + only be available once the transport_response has been deserialized or the + attempt/execution has failed. + :param transport_request: The transmittable request for the operation being + invoked. This will only be available once request serialization has + completed. + :param transport_response: The transmitted response for the operation being + invoked. This will only be available once transmission has completed. + """ + self._request = request + self._response = response + self._transport_request = transport_request + self._transport_response = transport_response + self._properties: dict[str, Any] = {} + + @property + def request(self) -> Request: + """Retrieve the modeled request for the operation being invoked.""" + return self._request + + @property + def response(self) -> Response | Exception: + """Retrieve the modeled response for the operation being invoked. + + This will only be available once the transport_response has been deserialized or + the attempt/execution has failed. + """ + return self._response + + # Note that TransportRequest (and TransportResponse below) aren't resolved types, + # but rather TypeVars. This is very important, because in the actual Interceptor + # interface class these are sometimes typed as None rather than, say, HTTPRequest. + # That lets us use the type system to tell people when something will be set and + # when it will not be set without leaking nullability into the cases where the + # property will ALWAYS be set. + @property + def transport_request(self) -> TransportRequest: + """Retrieve the transmittable request for the operation being invoked. + + This will only be available once request serialization has completed. + """ + return self._transport_request + + @property + def transport_response(self) -> TransportResponse: + """Retrieve the transmitted response for the operation being invoked. + + This will only be available once transmission has completed. + """ + return self._transport_response + + @property + def properties(self) -> dict[str, Any]: + """Retrieve the generic property bag. + + These untyped properties will be made available to all other interceptors or + hooks that are called for this execution. + """ + return self._properties + + # The static properties of this class are made 'read-only' like this to discourage + # people from trying to modify the context outside of the specific hooks where that + # is allowed. + def copy( + self, + *, + request: Request | None = None, + response: Response | Exception | None = None, + transport_request: TransportRequest | None = None, + transport_response: TransportResponse | None = None, + ) -> "InterceptorContext[Request, Response, TransportRequest, TransportResponse]": + """Copy the context object, optionally overriding certain properties.""" + if transport_request is None: + transport_request = copy(self._transport_request) + + if transport_response is None: + transport_response = copy(self._transport_response) + + context = InterceptorContext( + request=request if request is not None else self._request, + response=response if response is not None else self._response, + transport_request=transport_request, + transport_response=transport_response, + ) + context._properties = deepcopy(self._properties) + return context + + +class Interceptor(Generic[Request, Response, TransportRequest, TransportResponse]): + """Allows injecting code into the SDK's request execution pipeline. + + Terminology: + + * execution - An execution is one end-to-end invocation against a client. + * attempt - An attempt is an attempt at performing an execution. By default, + executions are retried multiple times based on the client's retry strategy. + * hook - A hook is a single method on the interceptor, allowing injection of code + into a specific part of the SDK's request execution pipeline. Hooks are either + "read" hooks, which make it possible to read in-flight request or response + messages, or "read/write" hooks, which make it possible to modify in-flight + requests or responses. + """ + + def read_before_execution( + self, context: InterceptorContext[Request, None, None, None] + ) -> None: + """A hook called at the start of an execution, before the SDK does anything + else. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per execution. The duration between invocation + of this hook and `read_after_execution` is very close to full duration of the + execution. + + The `request` of the context will always be available. Other static properties + will be None. + + Exceptions thrown by this hook will be stored until all interceptors have had + their `read_before_execution` invoked. Other hooks will then be skipped and + execution will jump to `modify_before_completion` with the thrown exception as + the `response`. If multiple `read_before_execution` methods throw exceptions, + the latest will be used and earlier ones will be logged and dropped. + """ + pass + + def modify_before_serialization( + self, context: InterceptorContext[Request, None, None, None] + ) -> Request: + """A hook called before the request is serialized into a transport request. + + This method has the ability to modify and return a new request of the same + type. + + This will ALWAYS be called once per execution, except when a failure occurs + earlier in the request pipeline. + + The `request` of the context will always be available. This `request` may have + been modified by earlier `modify_before_serialization` hooks, and may be + modified further by later hooks. Other static properites will be None. + + If exceptions are thrown by this hook, execution will jump to + `modify_before_completion` with the thrown exception as the `response`. + + The request returned by this hook MUST be the same type of request + message passed into this hook. If not, an exception will immediately occur. + """ + return context.request + + def read_before_serialization( + self, context: InterceptorContext[Request, None, None, None] + ) -> None: + """A hook called before the input message is serialized into a transport + request. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per execution, except when a failure occurs + earlier in the request pipeline. The duration between invocation of this hook + and `read_after_serialization` is very close to the amount of time spent + marshalling the request. + + The `request` of the context will always be available. Other static properties + will be None. + + If exceptions are thrown by this hook, execution will jump to + `modify_before_completion` with the thrown exception as the `response`. + """ + pass + + def read_after_serialization( + self, context: InterceptorContext[Request, None, TransportRequest, None] + ) -> None: + """A hook called after the input message is serialized into a transport request. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per execution, except when a failure occurs + earlier in the request pipeline. The duration between + `read_before_serialization` and the invocation of this hook is very close to + the amount of time spent serializing the request. + + The `request` and `transport_request` of the context will always be available. + Other static properties will be None. + + If exceptions are thrown by this hook, execution will jump to + `modify_before_completion` with the thrown exception as the `response`. + """ + pass + + def modify_before_retry_loop( + self, context: InterceptorContext[Request, None, TransportRequest, None] + ) -> TransportRequest: + """A hook called before the retry loop is entered. + + This method has the ability to modify and return a new transport request of the + same type. + + This will always be called once per execution, except when a failure occurs + earlier in the request pipeline. + + If exceptions are thrown by this hook, execution will jump to + `modify_before_completion` with the thrown exception as the `response`. + + The transport request returned by this hook MUST be the same type of request + passed into this hook. If not, an exception will immediately occur. + """ + return context.transport_request + + def read_before_attempt( + self, context: InterceptorContext[Request, None, TransportRequest, None] + ) -> None: + """A hook called before each attempt at sending the transport request to the + service. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per attempt, except when a failure occurs + earlier in the request pipeline. This method will be called multiple times in + the event of retries. + + The `request` and `transport_request` of the context will always be available. + Other static properties will be None. In the event of retries, the context will + not include changes made in previous attempts (e.g. by request signers or other + interceptors). + + Exceptions thrown by this hook will be stored until all interceptors have had + their `read_before_attempt` invoked. Other hooks will then be skipped and + execution will jump to `modify_before_attempt_completion` with the thrown + exception as the `response` If multiple `read_before_attempt` methods throw + exceptions, the latest will be used and earlier ones will be logged and dropped. + """ + pass + + def modify_before_signing( + self, context: InterceptorContext[Request, None, TransportRequest, None] + ) -> TransportRequest: + """A hook called before the transport request is signed. + + This method has the ability to modify and return a new transport request of the + same type. + + This will always be called once per attempt, except when a failure occurs + earlier in the request pipeline. This method will be called multiple times in + the event of retries. + + The `request` and `transport_request` of the context will always be available. + Other static properties will be None. The `transport_request` may have been + modified by earlier `modify_before_signing` hooks, and may be modified further + by later hooks. In the event of retries, the context will not include changes + made in previous attempts (e.g. by request signers or other interceptors). + + If exceptions are thrown by this hook, execution will jump to + `modify_before_attempt_completion` with the thrown exception as the `response`. + + The transport request returned by this hook MUST be the same type of request + passed into this hook. If not, an exception will immediately occur. + """ + return context.transport_request + + def read_before_signing( + self, context: InterceptorContext[Request, None, TransportRequest, None] + ) -> None: + """A hook called before the transport request is signed. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per attempt, except when a failure occurs + earlier in the request pipeline. This method may be called multiple times in + the event of retries. The duration between invocation of this hook and + `read_after_signing` is very close to the amount of time spent signing the + request. + + The `request` and `transport_request` of the context will always be available. + Other static properties will be None. In the event of retries, the context will + not include changes made in previous attempts (e.g. by request signers or other + interceptors). + + If exceptions are thrown by this hook, execution will jump to + `modify_before_attempt_completion` with the thrown exception as the `response`. + """ + pass + + def read_after_signing( + self, context: InterceptorContext[Request, None, TransportRequest, None] + ) -> None: + """A hook called after the transport request is signed. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per attempt, except when a failure occurs + earlier in the request pipeline. This method may be called multiple times in + the event of retries. The duration between `read_before_signing` and the + invocation of this hook is very close to the amount of time spent signing the + request. + + The `request` and `transport_request` of the context will always be available. + Other static properties will be None. In the event of retries, the context will + not include changes made in previous attempts (e.g. by request signers or other + interceptors). + + If exceptions are thrown by this hook, execution will jump to + `modify_before_attempt_completion` with the thrown exception as the `response`. + """ + pass + + def modify_before_transmit( + self, context: InterceptorContext[Request, None, TransportRequest, None] + ) -> TransportRequest: + """A hook called before the transport request is sent to the service. + + This method has the ability to modify and return a new transport request of the + same type. + + This will always be called once per attempt, except when a failure occurs + earlier in the request pipeline. This method may be called multiple times in + the event of retries. + + The `request` and `transport_request` of the context will always be available. + Other static properties will be None. The `transport_request` may have been + modified by earlier `modify_before_signing` hooks, and may be modified further + by later hooks. In the event of retries, the context will not include changes + made in previous attempts (e.g. by request signers or other interceptors). + + If exceptions are thrown by this hook, execution will jump to + `modify_before_attempt_completion` with the thrown exception as the `response`. + + The transport request returned by this hook MUST be the same type of request + passed into this hook. If not, an exception will immediately occur. + """ + return context.transport_request + + def read_before_transmit( + self, context: InterceptorContext[Request, None, TransportRequest, None] + ) -> None: + """A hook called before the transport request is sent to the service. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per attempt, except when a failure occurs + earlier in the request pipeline. This method may be called multiple times in + the event of retries. The duration between invocation of this hook and + `read_after_transmit` is very close to the amount of time spent communicating + with the service. Depending on the protocol, the duration may not include the + time spent reading the response data. + + The `request` and `transport_request` of the context will always be available. + Other static properties will be None. In the event of retries, the context will + not include changes made in previous attempts (e.g. by request signers or other + interceptors). + + If exceptions are thrown by this hook, execution will jump to + `modify_before_attempt_completion` with the thrown exception as the `response`. + """ + pass + + def read_after_transmit( + self, + context: InterceptorContext[Request, None, TransportRequest, TransportResponse], + ) -> None: + """A hook called after the transport request is sent to the service and a + transport response is received. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per attempt, except when a failure occurs + earlier in the request pipeline. This method may be called multiple times in + the event of retries. The duration between `read_before_transmit` and the + invocation of this hook is very close to the amount of time spent communicating + with the service. Depending on the protocol, the duration may not include the + time spent reading the response data. + + The `request`, `transport_request`, and `transport_response` of the context + will always be available. Other static properties will be None. In the event of + retries, the context will not include changes made in previous attempts (e.g. + by request signers or other interceptors). + + If exceptions are thrown by this hook, execution will jump to + `modify_before_attempt_completion` with the thrown exception as the `response`. + """ + pass + + def modify_before_deserialization( + self, + context: InterceptorContext[Request, None, TransportRequest, TransportResponse], + ) -> TransportResponse: + """A hook called before the transport response is deserialized. + + This method has the ability to modify and return a new transport response of the + same type. + + This will always be called once per attempt, except when a failure occurs + earlier in the request pipeline. This method may be called multiple times in + the event of retries. + + The `request`, `transport_request`, and `transport_response` of the context + will always be available. Other static properties will be None. In the event of + retries, the context will not include changes made in previous attempts (e.g. + by request signers or other interceptors). The `transport_response` may have + been modified by earlier `modify_before_deserialization` hooks, and may be + modified further by later hooks. In the event of retries, the context will not + include changes made in previous attempts (e.g. by request signers or other + interceptors). + + If exceptions are thrown by this hook, execution will jump to + `modify_before_attempt_completion` with the thrown exception as the `response`. + + The transport response returned by this hook MUST be the same type of + response passed into this hook. If not, an exception will immediately occur. + """ + return context.transport_response + + def read_before_deserialization( + self, + context: InterceptorContext[Request, None, TransportRequest, TransportResponse], + ) -> None: + """A hook called before the transport response is deserialized. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per attempt, except when a failure occurs + earlier in the request pipeline. This method may be called multiple times in + the event of retries. The duration between invocation of this hook and + `read_after_deserialization` is very close to the amount of time spent + deserializing the service response. Depending on the protocol and operation, + the duration may include the time spent downloading the response data. + + The `request`, `transport_request`, and `transport_response` of the context + will always be available. Other static properties will be None. In the event of + retries, the context will not include changes made in previous attempts (e.g. + by request signers or other interceptors). + + If exceptions are thrown by this hook, execution will jump to + `modify_before_attempt_completion` with the thrown exception as the `response`. + """ + pass + + def read_after_deserialization( + self, + context: InterceptorContext[ + Request, Response, TransportRequest, TransportResponse + ], + ) -> None: + """A hook called after the transport response is deserialized. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per attempt, except when a failure occurs + earlier in the request pipeline. This method may be called multiple times in + the event of retries. The duration between `read_before_deserialization` + and the invocation of this hook is very close to the amount of time spent + deserializing the service response. Depending on the protocol and operation, + the duration may include the time spent downloading the response data. + + The `request`, `response`, `transport_request`, and `transport_response` of the + context will always be available. In the event of retries, the context will not + include changes made in previous attempts (e.g. by request signers or other + interceptors). + + If exceptions are thrown by this hook, execution will jump to + `modify_before_attempt_completion` with the thrown exception as the `response`. + """ + pass + + def modify_before_attempt_completion( + self, + context: InterceptorContext[ + Request, Response, TransportRequest, TransportResponse | None + ], + ) -> Response | Exception: + """A hook called when an attempt is completed. + + This method has the ability to modify and return a new output message or + exception matching the currently-executing operation. + + This will ALWAYS be called once per attempt, except when a failure occurs + before `read_before_attempt`. This method may be called multiple times in the + event of retries. + + The `request`, `response`, and `transport_request` of the context will always + be available. The `transport_response` will be available if a response was + received by the service for this attempt. In the event of retries, the context + will not include changes made in previous attempts (e.g. by request signers or + other interceptors). + + If exceptions are thrown by this hook, execution will jump to + `read_after_attempt` with the thrown exception as the `response`. + + Any output returned by this hook MUST match the operation being invoked. Any + exception type can be returned, replacing the `response` currently in the + context. + """ + return context.response + + def read_after_attempt( + self, + context: InterceptorContext[ + Request, Response, TransportRequest, TransportResponse | None + ], + ) -> None: + """A hook called when an attempt is completed. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will ALWAYS be called once per attempt, as long as `read_before_attempt` + has been executed. + + The `request`, `response`, and `transport_request` of the context will always + be available. The `transport_response` will be available if a response was + received by the service for this attempt. In the event of retries, the context + will not include changes made in previous attempts (e.g. by request signers or + other interceptors). + + Exceptions thrown by this hook will be stored until all interceptors have had + their `read_after_attempt` invoked. If multiple `read_after_attempt` methods + throw exceptions, the latest will be used and earlier ones will be logged and + dropped. If the retry strategy determines that the execution is retryable, + execution will then jump to `read_before_attempt`. Otherwise, execution will + jump to `modify_before_completion` with the thrown exception as the `response`. + """ + pass + + def modify_before_completion( + self, + context: InterceptorContext[ + Request, Response, TransportRequest | None, TransportResponse | None + ], + ) -> Response | Exception: + """A hook called when an execution is completed. + + This method has the ability to modify and return a new output message or + exception matching the currently-executing operation. + + This will always be called once per execution. + + The `request` and `response` of the context will always be available. The + `transport_request` and `transport_response` will be available if the execution + proceeded far enough for them to be generated. + + If exceptions are thrown by this hook, execution will jump to + `read_after_execution` with the thrown exception as the `response`. + + Any output returned by this hook MUST match the operation being invoked. Any + exception type can be returned, replacing the `response` currently in the context. + """ + return context.response + + def read_after_execution( + self, + context: InterceptorContext[ + Request, Response, TransportRequest | None, TransportResponse | None + ], + ) -> None: + """A hook called when an execution is completed. + + Implementations MUST NOT modify the `request`, `response`, `transport_request`, + or `transport_response` in this hook. + + This will always be called once per execution. The duration between + `read_before_execution` and the invocation of this hook is very close to the + full duration of the execution. + + The `request` and `response` of the context will always be available. The + `transport_request` and `transport_response` will be available if the execution + proceeded far enough for them to be generated. + + Exceptions thrown by this hook will be stored until all interceptors have had + their `read_after_execution` invoked. The exception will then be treated as the + final response. If multiple `read_after_execution` methods throw exceptions, + the latest will be used and earlier ones will be logged and dropped. + """ + pass diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/retries.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/retries.py new file mode 100644 index 0000000000..1609549055 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/interfaces/retries.py @@ -0,0 +1,132 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from dataclasses import dataclass +from enum import Enum +from typing import Protocol + + +class RetryErrorType(Enum): + """Classification of errors based on desired retry behavior.""" + + TRANSIENT = 1 + """A connection level error such as a socket timeout, socket connect error, TLS + negotiation timeout.""" + + THROTTLING = 2 + """The server explicitly told the client to back off, for example with HTTP status + 429 or 503.""" + + SERVER_ERROR = 3 + """A server error that should be retried and does not match the definition of + ``THROTTLING``.""" + + CLIENT_ERROR = 4 + """Doesn't count against any budgets. + + This could be something like a 401 challenge in HTTP. + """ + + +@dataclass(kw_only=True) +class RetryErrorInfo: + """Container for information about a retryable error.""" + + error_type: RetryErrorType + """Classification of error based on desired retry behavior.""" + + retry_after_hint: float | None = None + """Protocol hint for computing the timespan to delay before the next retry. + + This could come from HTTP's 'retry-after' header or similar mechanisms in other + protocols. + """ + + +class RetryBackoffStrategy(Protocol): + """Stateless strategy for computing retry delays based on retry attempt account.""" + + def compute_next_backoff_delay(self, retry_attempt: int) -> float: + """Calculate timespan in seconds to delay before next retry. + + :param retry_attempt: The index of the retry attempt that is about to be made + after the delay. The initial attempt, before any retries, is index ``0``, the + first retry attempt after the initial attempt failed is index ``1``, and so on. + """ + ... + + +@dataclass(kw_only=True) +class RetryToken(Protocol): + """Token issued by a :py:class:`RetryStrategy` for the next attempt.""" + + retry_count: int + """Retry count is the total number of attempts minus the initial attempt.""" + + retry_delay: float + """Delay in seconds to wait before the retry attempt.""" + + +class RetryStrategy(Protocol): + """Issuer of :py:class:`RetryToken`s.""" + + backoff_strategy: RetryBackoffStrategy + """The strategy used by returned tokens to compute delay duration values.""" + + max_attempts: int + """Upper limit on total attempt count (initial attempt plus retries).""" + + def acquire_initial_retry_token( + self, *, token_scope: str | None = None + ) -> RetryToken: + """Called before any retries (for the first attempt at the operation). + + :param token_scope: An arbitrary string accepted by the retry strategy to + separate tokens into scopes. + + :returns: A retry token, to be used for determining the retry delay, refreshing + the token after a failure, and recording success after success. + + :raises SmithyRetryException: If the retry strategy has no available tokens. + """ + ... + + def refresh_retry_token_for_retry( + self, *, token_to_renew: RetryToken, error_info: RetryErrorInfo + ) -> RetryToken: + """Replace an existing retry token from a failed attempt with a new token. + + After a failed operation call, this method is called to exchange a retry token + that was previously obtained by calling :py:func:`acquire_initial_retry_token` + or this method with a new retry token for the next attempt. This method can + either choose to allow another retry and send a new or updated token, or reject + the retry attempt and raise the error as exception. + + :param token_to_renew: The token used for the previous failed attempt. + + :param error_info: If no further retry is allowed, this information is used to + construct the exception. + + :raises SmithyRetryException: If no further retry attempts are allowed. + """ + ... + + def record_success(self, *, token: RetryToken) -> None: + """Return token after successful completion of an operation. + + Upon successful completion of the operation, a user calls this function + to record that the operation was successful. + + :param token: The token used for the previous successful attempt. + """ + ... diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/mediatypes.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/mediatypes.py new file mode 100644 index 0000000000..ecb4eb1bf3 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/mediatypes.py @@ -0,0 +1,36 @@ +import json +from typing import Any + + +class JsonString(str): + """A string that contains json data which can be lazily loaded.""" + + _json = None + + def as_json(self) -> Any: + if not self._json: + self._json = json.loads(self) + return self._json + + @staticmethod + def from_json(j: Any) -> "JsonString": + json_string = JsonString(json.dumps(j)) + json_string._json = j + return json_string + + +class JsonBlob(bytes): + """Bytes that contain json data which can be lazily loaded.""" + + _json = None + + def as_json(self) -> Any: + if not self._json: + self._json = json.loads(self.decode(encoding="utf-8")) + return self._json + + @staticmethod + def from_json(j: Any) -> "JsonBlob": + json_string = JsonBlob(json.dumps(j).encode(encoding="utf-8")) + json_string._json = j + return json_string diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/protocolutils.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/protocolutils.py new file mode 100644 index 0000000000..6c26f4263d --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/protocolutils.py @@ -0,0 +1,69 @@ +import json +from typing import NamedTuple + +from .interfaces.http import HTTPResponse +from .types import Document +from .utils import expect_type + +_REST_JSON_CODE_HEADER = "x-amzn-errortype" + +_REST_JSON_CODE_KEYS = {"__type", "code"} + +_REST_JSON_MESSAGE_KEYS = {"message", "errormessage", "error_message"} + + +class RestJsonErrorInfo(NamedTuple): + """Generic error information from a RestJson protocol error.""" + + code: str + """The error code.""" + + message: str + """The generic error message. + + A modeled error may have the error bound somewhere else. This is based off of + checking the most common locations and is intended for use with excpetions that + either didn't model the message or which are unknown. + """ + + json_body: dict[str, Document] | None = None + """The HTTP response body parsed as JSON.""" + + +async def parse_rest_json_error_info( + http_response: HTTPResponse, check_body: bool = True +) -> RestJsonErrorInfo: + """Parses generic RestJson error info from an HTTP response. + + :param http_response: The HTTP response to parse. + :param check_body: Whether to check the body for the code / message. + :returns: The parsed error information. + """ + code: str | None = None + message: str | None = None + json_body: dict[str, Document] | None = None + + for field in http_response.fields: + if field.name.lower() == _REST_JSON_CODE_HEADER: + code = field.values[0] + + if check_body: + if body := await http_response.consume_body(): + json_body = json.loads(body) + + if json_body: + for key, value in json_body.items(): + key_lower = key.lower() + if not code and key_lower in _REST_JSON_CODE_KEYS: + code = expect_type(str, value) + if not message and key_lower in _REST_JSON_MESSAGE_KEYS: + message = expect_type(str, value) + + # Normalize the error code. Some services may try to send a fully-qualified shape + # ID or a URI, but we don't want to include those. + if code: + if "#" in code: + code = code.split("#")[1] + code = code.split(":")[0] + + return RestJsonErrorInfo(code or "Unknown", message or "Unknown", json_body) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/py.typed b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/py.typed new file mode 100644 index 0000000000..f5642f79f2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/py.typed @@ -0,0 +1 @@ +Marker diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/types.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/types.py new file mode 100644 index 0000000000..5c317982a6 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/types.py @@ -0,0 +1,5 @@ +from typing import Mapping, Sequence, TypeAlias + +Document: TypeAlias = ( + Mapping[str, "Document"] | Sequence["Document"] | str | int | float | bool | None +) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/utils.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/utils.py new file mode 100644 index 0000000000..b79e0eab59 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/smithy_python/utils.py @@ -0,0 +1,275 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import re +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from math import isinf, isnan +from types import UnionType +from typing import Any, TypeVar, overload + +from .exceptions import ExpectationNotMetException + +RFC3339 = "%Y-%m-%dT%H:%M:%SZ" +# Same as RFC3339, but with microsecond precision. +RFC3339_MICRO = "%Y-%m-%dT%H:%M:%S.%fZ" + + +def ensure_utc(value: datetime) -> datetime: + """Ensures that the given datetime is a UTC timezone-aware datetime. + + If the datetime isn't timezone-aware, its timezone is set to UTC. If it is + aware, it's replaced with the equivalent datetime under UTC. + + :param value: A datetime object that may or may not be timezone-aware. + :returns: A UTC timezone-aware equivalent datetime. + """ + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + else: + return value.astimezone(timezone.utc) + + +# Python is way more permissive on value of non-numerical floats than Smithy is, so we +# need to compare potential string values against this set of values that Smithy +# generally permits. +_NON_NUMERICAL_FLOATS = {"NaN", "Infinity", "-Infinity"} + + +def limited_parse_float(value: Any) -> float: + """Asserts a value is a float or a limited set of non-numerical strings and returns + it as a float. + + :param value: An object that is expected to be a float. + :returns: The given value as a float. + :raises SmithyException: If the value is not a float or one of the strings ``NaN``, + ``Infinity``, or ``-Infinity``. + """ + # TODO: add limited bounds checking + if isinstance(value, str) and value in _NON_NUMERICAL_FLOATS: + return float(value) + + return expect_type(float, value) + + +def epoch_seconds_to_datetime(value: int | float) -> datetime: + """Parse numerical epoch timestamps (seconds since 1970) into a datetime in UTC. + + Falls back to using ``timedelta`` when ``fromtimestamp`` raises ``OverflowError``. + From Python's ``fromtimestamp`` documentation: "This may raise OverflowError, if the + timestamp is out of the range of values supported by the platform C localtime() + function, and OSError on localtime() failure. It's common for this to be restricted + to years from 1970 through 2038." This affects 32-bit systems. + """ + try: + return datetime.fromtimestamp(value, tz=timezone.utc) + except OverflowError: + epoch_zero = datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + return epoch_zero + timedelta(seconds=value) + + +_T = TypeVar("_T") + + +@overload +def expect_type(typ: type[_T], value: Any) -> _T: + ... + + +# For some reason, mypy and other type checkers don't treat Union like a full type +# despite it being checkable with isinstance and other methods. This essentially means +# we can't pass back the given type when we're given a union. So instead we have to +# return Any. +@overload +def expect_type(typ: UnionType, value: Any) -> Any: + ... + + +def expect_type(typ: UnionType | type, value: Any) -> Any: + """Asserts a value is of the given type and returns it unchanged. + + This performs both a runtime assertion and type narrowing during type checking + similar to ``typing.cast``. If the runtime assertion is not needed, ``typing.cast`` + should be preferred. + + :param typ: The expected type. + :param value: The value which is expected to be the given type. + :returns: The given value cast as the given type. + :raises SmithyException: If the value does not match the type. + """ + if not isinstance(value, typ): + raise ExpectationNotMetException( + f"Expected {typ}, found {type(value)}: {value}" + ) + return value + + +def split_every(given: str, split_char: str, n: int) -> list[str]: + """Splits a string every nth instance of the given character. + + :param given: The string to split. + :param split_char: The character to split on. + :param n: The number of instances of split_char to see before each split. + :returns: A list of strings. + """ + split = given.split(split_char) + return [split_char.join(split[i : i + n]) for i in range(0, len(split), n)] + + +def strict_parse_bool(given: str) -> bool: + """Strictly parses a boolean from string. + + :param given: A string that is expected to contain either "true" or "false". + :returns: The given string parsed to a boolean. + :raises ExpectationNotMetException: if the given string is neither "true" nor + "false". + """ + match given: + case "true": + return True + case "false": + return False + case _: + raise ExpectationNotMetException( + f"Expected 'true' or 'false', found: {given}" + ) + + +# A regex for Smithy floats. It matches JSON-style numbers. +_FLOAT_REGEX = re.compile( + r""" + ( # Opens the numeric float group. + -? # The integral may start with a negative sign, but not a positive one. + (?:0|[1-9]\d*) # The integral may not have leading 0s unless it is exactly 0. + (?:\.\d+)? # There may be a fraction starting with a period and containing at + # least one number. + (?: # Opens the exponent group. + [eE] # The exponent starts with a case-insensitive e + [+-]? # The exponent may have a positive or negative sign. + \d+ # The exponent must have one or more digits. + )? # Closes the exponent group and makes it optional. + ) # Closes the numeric float group. + |(-?Infinity) # If the float isn't numeric, it may be Infinity or -Infinity + |(NaN) # If the float isn't numeric, it may also be NaN + """, + re.VERBOSE, +) + + +def strict_parse_float(given: str) -> float: + """Strictly parses a float from a string. + + Unlike float(), this forbids the use of "inf" and case-sensitively matches + Infinity and NaN. + + :param given: A string that is expected to contain a float. + :returns: The given string parsed to a float. + :raises ExpectationNotMetException: If the given string isn't a float. + """ + if _FLOAT_REGEX.fullmatch(given): + return float(given) + raise ExpectationNotMetException(f"Expected float, found: {given}") + + +def serialize_float(given: float | Decimal) -> str: + """Serializes a float to a string. + + This ensures non-numeric floats are serialized correctly, and ensures that there is + a fractional part. + + :param given: A float or Decimal to be serialized. + :returns: The string representation of the given float. + """ + if isnan(given): + return "NaN" + if isinf(given): + return "-Infinity" if given < 0 else "Infinity" + + if isinstance(given, Decimal): + given = given.normalize() + + result = str(given) + if result.isnumeric(): + result += ".0" + return result + + +def limited_serialize_float(given: float) -> str | float: + """Serializes non-numeric floats to strings. + + Numeric floats are returned without alteration. + + :param given: A float to be conditionally serialized. + :returns: The given float as a float or string. + """ + if isnan(given): + return "NaN" + if isinf(given): + return "-Infinity" if given < 0 else "Infinity" + + return given + + +def serialize_rfc3339(given: datetime) -> str: + """Serializes a datetime into an RFC3339 string representation. + + If ``microseconds`` is 0, no fractional part is serialized. + + :param given: The datetime to serialize. + :returns: An RFC3339 formatted timestamp. + """ + if given.microsecond != 0: + return given.strftime(RFC3339_MICRO) + else: + return given.strftime(RFC3339) + + +def serialize_epoch_seconds(given: datetime) -> float: + """Serializes a datetime into a string containing the epoch seconds. + + If ``microseconds`` is 0, no fractional part is serialized. + + :param given: The datetime to serialize. + :returns: A string containing the seconds since the UNIX epoch. + """ + result = given.timestamp() + if given.microsecond == 0: + result = int(result) + return result + + +def remove_dot_segments(path: str, remove_consecutive_slashes: bool = False) -> str: + """Removes dot segments from a path per :rfc:`3986#section-5.2.4`. + + Optionally removes consecutive slashes. + + :param path: The path to modify. + :param remove_consecutive_slashes: Whether to remove consecutive slashes. + :returns: The path with dot segments removed. + """ + output = [] + for segment in path.split("/"): + if segment == ".": + continue + elif segment != "..": + output.append(segment) + elif output: + output.pop() + if path.startswith("/") and (not output or output[0]): + output.insert(0, "") + if output and path.endswith(("/.", "/..")): + output.append("") + result = "/".join(output) + if remove_consecutive_slashes: + result = result.replace("//", "/") + return result diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/BUILD b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/BUILD new file mode 100644 index 0000000000..5699a8f0bd --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/BUILD @@ -0,0 +1,35 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +python_test_utils( + name="test_utils", + sources=[ + "**/conftest.py", # pytest's conftest.py file + "**/*_utils.py", # all other utils files must be named to match this pattern + ] +) + +resource(name="pytyped", source="py.typed") + +python_tests( + name="tests", + dependencies=[":test_utils", ":pytyped"], + sources=[ + "unit/**/test_*.py", + "unit/**/tests.py", + "functional/**/test_*.py", + "functional/**/tests.py", + "integration/**/test_*.py", + "integration/**/tests.py", + ] +) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/functional/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/functional/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/conftest.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/conftest.py new file mode 100644 index 0000000000..e74bb6eefa --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/conftest.py @@ -0,0 +1,34 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import pytest + +from smithy_python._private import URI, Field, Fields +from smithy_python._private.http import HTTPRequest +from smithy_python.async_utils import async_list + + +@pytest.fixture +def sample_request() -> HTTPRequest: + headers = Fields( + [ + Field(name="host", values=["aws.amazon.com"]), + Field(name="user-agent", values=["smithy-python-test"]), + ] + ) + return HTTPRequest( + method="GET", + destination=URI(host="aws.amazon.com"), + fields=headers, + body=async_list([]), + ) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/test_aiohttp.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/test_aiohttp.py new file mode 100644 index 0000000000..59223123d2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/test_aiohttp.py @@ -0,0 +1,27 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from smithy_python._private.http import HTTPRequest +from smithy_python._private.http.aiohttp_client import ( + AIOHTTPClient, + AIOHTTPClientConfig, +) + + +async def test_basic_request_local(sample_request: HTTPRequest) -> None: + config = AIOHTTPClientConfig() + session = AIOHTTPClient(client_config=config) + response = await session.send(request=sample_request) + assert response.status == 200 + body = await response.consume_body() + assert b"aws" in body diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/test_http_crt.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/test_http_crt.py new file mode 100644 index 0000000000..b090613c8c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/integration/http_clients/test_http_crt.py @@ -0,0 +1,38 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +# mypy: allow-untyped-defs +# mypy: allow-incomplete-defs + +from smithy_python._private.http import HTTPRequest +from smithy_python._private.http.crt import AWSCRTHTTPClient, AWSCRTHTTPClientConfig + + +async def test_basic_request_local(sample_request: HTTPRequest) -> None: + config = AWSCRTHTTPClientConfig() + session = AWSCRTHTTPClient(client_config=config) + response = await session.send(sample_request) + assert response.status == 200 + print(f"{response=}") + body = await response.consume_body() + print(f"{body=}") + assert b"aws" in body + + +async def test_basic_request_http2(sample_request: HTTPRequest) -> None: + config = AWSCRTHTTPClientConfig(force_http_2=True) + session = AWSCRTHTTPClient(client_config=config) + response = await session.send(sample_request) + assert response.status == 200 + body = await response.consume_body() + assert b"aws" in body diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/py.typed b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/py.typed new file mode 100644 index 0000000000..f5642f79f2 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/py.typed @@ -0,0 +1 @@ +Marker diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/__init__.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_api_key_auth.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_api_key_auth.py new file mode 100644 index 0000000000..0959bae85b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_api_key_auth.py @@ -0,0 +1,157 @@ +from dataclasses import dataclass +from typing import AsyncIterable, AsyncIterator + +import pytest + +from smithy_python._private import URI, Field, Fields +from smithy_python._private.api_key_auth import ( + ApiKeyAuthScheme, + ApiKeyIdentity, + ApiKeyIdentityResolver, + ApiKeyLocation, + ApiKeySigner, + ApiKeySigningProperties, +) +from smithy_python._private.http import HTTPRequest +from smithy_python.exceptions import SmithyIdentityException +from smithy_python.interfaces.identity import IdentityProperties, IdentityResolver + + +@pytest.fixture +def signer() -> ApiKeySigner: + return ApiKeySigner() + + +class _FakeBody(AsyncIterable[bytes]): + def __aiter__(self) -> AsyncIterator[bytes]: + return self + + async def __anext__(self) -> bytes: + return b"spam" + + def __eq__(self, other: object) -> bool: + return isinstance(other, _FakeBody) + + +def request(query: str | None = None, fields: Fields | None = None) -> HTTPRequest: + return HTTPRequest( + destination=URI(host="example.com", query=query), + body=_FakeBody(), + method="POST", + fields=fields or Fields(), + ) + + +async def test_identity_resolver() -> None: + api_key = "spam" + resolver = ApiKeyIdentityResolver(api_key=api_key) + identity = await resolver.get_identity(identity_properties={}) + + assert identity.api_key == api_key + + resolver = ApiKeyIdentityResolver(api_key=ApiKeyIdentity(api_key=api_key)) + identity = await resolver.get_identity(identity_properties={}) + + assert identity.api_key == api_key + + +async def test_sign_empty_query(signer: ApiKeySigner) -> None: + api_key = "spam" + identity = ApiKeyIdentity(api_key=api_key) + properties: ApiKeySigningProperties = { + "name": "eggs", + "location": ApiKeyLocation.QUERY, + } + + given = request() + expected = request(query="eggs=spam") + + actual = await signer.sign( + http_request=given, + identity=identity, + signing_properties=properties, + ) + + assert actual == expected + + +async def test_sign_non_empty_query(signer: ApiKeySigner) -> None: + api_key = "spam" + identity = ApiKeyIdentity(api_key=api_key) + properties: ApiKeySigningProperties = { + "name": "eggs", + "location": ApiKeyLocation.QUERY, + } + + given = request(query="spam=eggs") + expected = request(query="spam=eggs&eggs=spam") + + actual = await signer.sign( + http_request=given, + identity=identity, + signing_properties=properties, + ) + + assert actual == expected + + +async def test_sign_header(signer: ApiKeySigner) -> None: + api_key = "spam" + identity = ApiKeyIdentity(api_key=api_key) + properties: ApiKeySigningProperties = { + "name": "eggs", + "location": ApiKeyLocation.HEADER, + } + + given = request() + expected = request(fields=Fields([Field(name="eggs", values=["spam"])])) + + actual = await signer.sign( + http_request=given, + identity=identity, + signing_properties=properties, + ) + + assert actual == expected + + +async def test_sign_header_with_scheme(signer: ApiKeySigner) -> None: + api_key = "spam" + identity = ApiKeyIdentity(api_key=api_key) + properties: ApiKeySigningProperties = { + "name": "eggs", + "location": ApiKeyLocation.HEADER, + "scheme": "Bearer", + } + + given = request() + expected = request(fields=Fields([Field(name="eggs", values=["Bearer spam"])])) + + actual = await signer.sign( + http_request=given, + identity=identity, + signing_properties=properties, + ) + + assert actual == expected + + +@dataclass +class ApiKeyConfig: + api_key_identity_resolver: IdentityResolver[ + ApiKeyIdentity, IdentityProperties + ] | None = None + + +async def test_auth_scheme_gets_resolver() -> None: + scheme = ApiKeyAuthScheme() + resolver = ApiKeyIdentityResolver(api_key="spam") + config = ApiKeyConfig(api_key_identity_resolver=resolver) + + assert resolver == scheme.identity_resolver(config=config) + + +async def test_auth_scheme_missing_resolver() -> None: + scheme = ApiKeyAuthScheme() + with pytest.raises(SmithyIdentityException): + scheme.identity_resolver(config=ApiKeyConfig()) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_blobs.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_blobs.py new file mode 100644 index 0000000000..50732066b1 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_blobs.py @@ -0,0 +1,270 @@ +from io import BytesIO +from typing import Self + +import pytest + +from smithy_python.interfaces.blobs import AsyncBytesReader, SeekableAsyncBytesReader + + +class _AsyncIteratorWrapper: + def __init__(self, source: BytesIO, chunk_size: int = -1): + self._source = source + self._chunk_size = chunk_size + + def __aiter__(self) -> Self: + return self + + async def __anext__(self) -> bytes: + data = self._source.read(self._chunk_size) + if data: + return data + raise StopAsyncIteration + + +async def test_read_bytes() -> None: + reader = AsyncBytesReader(b"foo") + assert await reader.read() == b"foo" + + +async def test_seekable_read_byes() -> None: + reader = SeekableAsyncBytesReader(b"foo") + assert reader.tell() == 0 + assert await reader.read() == b"foo" + assert reader.tell() == 3 + + +async def test_read_bytearray() -> None: + reader = AsyncBytesReader(bytearray(b"foo")) + assert await reader.read() == b"foo" + + +async def test_seekable_read_bytearray() -> None: + reader = SeekableAsyncBytesReader(bytearray(b"foo")) + assert reader.tell() == 0 + assert await reader.read() == b"foo" + assert reader.tell() == 3 + + +async def test_read_byte_stream() -> None: + source = BytesIO(b"foo") + reader = AsyncBytesReader(source) + assert source.tell() == 0 + assert await reader.read() == b"foo" + assert source.tell() == 3 + + +async def test_seekable_read_byte_stream() -> None: + source = BytesIO(b"foo") + reader = SeekableAsyncBytesReader(source) + assert reader.tell() == 0 + assert source.tell() == 0 + assert await reader.read() == b"foo" + assert reader.tell() == 3 + assert source.tell() == 3 + + +async def test_read_async_byte_stream() -> None: + source = BytesIO(b"foo") + reader = AsyncBytesReader(AsyncBytesReader(source)) + assert source.tell() == 0 + assert await reader.read() == b"foo" + assert source.tell() == 3 + + +async def test_seekable_read_async_byte_stream() -> None: + source = BytesIO(b"foo") + reader = SeekableAsyncBytesReader(AsyncBytesReader(source)) + assert reader.tell() == 0 + assert source.tell() == 0 + assert await reader.read() == b"foo" + assert reader.tell() == 3 + assert source.tell() == 3 + + +async def test_read_async_iterator() -> None: + source = BytesIO(b"foo") + reader = AsyncBytesReader(_AsyncIteratorWrapper(source)) + assert source.tell() == 0 + assert await reader.read() == b"foo" + assert source.tell() == 3 + + source = BytesIO(b"foo,bar,baz\n") + reader = AsyncBytesReader(_AsyncIteratorWrapper(source, chunk_size=6)) + assert source.tell() == 0 + assert await reader.read(4) == b"foo," + assert source.tell() == 6 + assert await reader.read(4) == b"bar," + assert source.tell() == 12 + assert await reader.read(4) == b"baz\n" + assert source.tell() == 12 + + +async def test_seekable_read_async_iterator() -> None: + source = BytesIO(b"foo") + reader = SeekableAsyncBytesReader(_AsyncIteratorWrapper(source)) + assert reader.tell() == 0 + assert source.tell() == 0 + assert await reader.read() == b"foo" + assert reader.tell() == 3 + assert source.tell() == 3 + + source = BytesIO(b"foo,bar,baz\n") + reader = SeekableAsyncBytesReader(_AsyncIteratorWrapper(source, chunk_size=6)) + assert source.tell() == 0 + assert reader.tell() == 0 + assert await reader.read(4) == b"foo," + assert source.tell() == 6 + assert reader.tell() == 4 + assert await reader.read(4) == b"bar," + assert source.tell() == 12 + assert reader.tell() == 8 + assert await reader.read(4) == b"baz\n" + assert source.tell() == 12 + assert reader.tell() == 12 + + +async def test_close_closeable_source() -> None: + source = BytesIO(b"foo") + reader = AsyncBytesReader(source) + + assert not reader.closed + assert not source.closed + + reader.close() + + assert reader.closed + assert source.closed + + with pytest.raises(ValueError): + await reader.read() + + +async def test_close_non_closeable_source() -> None: + source = _AsyncIteratorWrapper(BytesIO(b"foo")) + reader = AsyncBytesReader(source) + + assert not reader.closed + reader.close() + assert reader.closed + + with pytest.raises(ValueError): + await reader.read() + + +async def test_seekable_close_closeable_source() -> None: + source = BytesIO(b"foo") + reader = SeekableAsyncBytesReader(source) + + assert not reader.closed + assert not source.closed + assert reader.tell() == 0 + + reader.close() + + assert reader.closed + assert source.closed + + with pytest.raises(ValueError): + await reader.read() + + with pytest.raises(ValueError): + reader.tell() + + +async def test_seekable_close_non_closeable_source() -> None: + source = _AsyncIteratorWrapper(BytesIO(b"foo")) + reader = SeekableAsyncBytesReader(source) + + assert not reader.closed + assert reader.tell() == 0 + reader.close() + assert reader.closed + + with pytest.raises(ValueError): + await reader.read() + + with pytest.raises(ValueError): + reader.tell() + + +async def test_seek() -> None: + source = BytesIO(b"foo") + reader = SeekableAsyncBytesReader(source) + + assert source.tell() == 0 + assert reader.tell() == 0 + + assert await reader.seek(2, 0) == 2 + + assert source.tell() == 2 + assert reader.tell() == 2 + + assert await reader.seek(1, 1) == 3 + + assert source.tell() == 3 + assert reader.tell() == 3 + + assert await reader.seek(0, 0) == 0 + + assert source.tell() == 3 + assert reader.tell() == 0 + + source = BytesIO(b"foo") + reader = SeekableAsyncBytesReader(source) + + assert await reader.seek(-3, 2) == 0 + + assert source.tell() == 3 + assert reader.tell() == 0 + + +async def test_read_as_iterator() -> None: + source = BytesIO(b"foo") + reader = AsyncBytesReader(source) + + result = b"" + async for chunk in reader: + result += chunk + + assert result == b"foo" + + +async def test_seekable_read_as_iterator() -> None: + source = BytesIO(b"foo") + reader = SeekableAsyncBytesReader(source) + + assert reader.tell() == 0 + + result = b"" + async for chunk in reader: + result += chunk + + assert reader.tell() == 3 + assert result == b"foo" + + +async def test_iter_chunks() -> None: + source = BytesIO(b"foo") + reader = AsyncBytesReader(source) + + result = b"" + async for chunk in reader.iter_chunks(chunk_size=1): + assert len(chunk) == 1 + result += chunk + + assert result == b"foo" + + +async def test_seekable_iter_chunks() -> None: + source = BytesIO(b"foo") + reader = SeekableAsyncBytesReader(source) + + assert reader.tell() == 0 + + result = b"" + async for chunk in reader.iter_chunks(chunk_size=1): + assert len(chunk) == 1 + result += chunk + + assert reader.tell() == 3 + assert result == b"foo" diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_http.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_http.py new file mode 100644 index 0000000000..1a22e65080 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_http.py @@ -0,0 +1,199 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import pytest + +from smithy_python._private import URI, Field, Fields, HostType +from smithy_python._private.http import ( + HTTPRequest, + HTTPResponse, + StaticEndpointParams, + StaticEndpointResolver, +) +from smithy_python.async_utils import async_list +from smithy_python.exceptions import SmithyHTTPException + + +def test_uri_basic() -> None: + uri = URI( + host="test.aws.dev", + path="/my/path", + query="foo=bar", + ) + + assert uri.host == "test.aws.dev" + assert uri.path == "/my/path" + assert uri.query == "foo=bar" + assert uri.netloc == "test.aws.dev" + assert uri.build() == "https://test.aws.dev/my/path?foo=bar" + + +def test_uri_all_fields_present() -> None: + uri = URI( + host="test.aws.dev", + path="/my/path", + scheme="http", + query="foo=bar", + port=80, + username="abc", + password="def", + fragment="frag", + ) + + assert uri.host == "test.aws.dev" + assert uri.path == "/my/path" + assert uri.scheme == "http" + assert uri.query == "foo=bar" + assert uri.port == 80 + assert uri.username == "abc" + assert uri.password == "def" + assert uri.fragment == "frag" + assert uri.netloc == "abc:def@test.aws.dev:80" + assert uri.build() == "http://abc:def@test.aws.dev:80/my/path?foo=bar#frag" + + +def test_uri_without_scheme_field() -> None: + uri = URI( + host="test.aws.dev", + path="/my/path", + query="foo=bar", + port=80, + username="abc", + password="def", + fragment="frag", + ) + # scheme should default to https + assert uri.scheme == "https" + assert uri.build() == "https://abc:def@test.aws.dev:80/my/path?foo=bar#frag" + + +def test_uri_without_port_number() -> None: + uri = URI( + host="test.aws.dev", + path="/my/path", + scheme="http", + query="foo=bar", + username="abc", + password="def", + fragment="frag", + ) + # by default, the port is omitted from computed netloc and built URI string + assert uri.port is None + assert uri.netloc == "abc:def@test.aws.dev" + assert uri.build() == "http://abc:def@test.aws.dev/my/path?foo=bar#frag" + + +def test_uri_ipv6_host() -> None: + uri = URI(host="::1") + assert uri.host == "::1" + assert uri.netloc == "[::1]" + assert uri.build() == "https://[::1]" + assert uri.host_type == HostType.IPv6 + + +def test_uri_escaped_path() -> None: + uri = URI(host="test.aws.dev", path="/%20%2F") + assert uri.path == "/%20%2F" + assert uri.build() == "https://test.aws.dev/%20%2F" + + +def test_uri_password_but_no_username() -> None: + uri = URI(host="test.aws.dev", password="def") + assert uri.password == "def" + # the password is ignored if no username is present + assert uri.netloc == "test.aws.dev" + + +async def test_request() -> None: + uri = URI(host="test.aws.dev") + headers = Fields([Field(name="foo", values=["bar"])]) + request = HTTPRequest( + method="GET", + destination=uri, + fields=headers, + body=async_list([b"test body"]), + ) + + assert request.method == "GET" + assert request.destination == uri + assert request.fields == headers + request_body = b"".join([chunk async for chunk in request.body]) + assert request_body == b"test body" + + +async def test_response() -> None: + headers = Fields([Field(name="foo", values=["bar"])]) + response = HTTPResponse( + status=200, + fields=headers, + body=async_list([b"test body"]), + ) + + assert response.status == 200 + assert response.fields == headers + response_body = await response.consume_body() + assert response_body == b"test body" + + +async def test_endpoint_provider_with_uri_string() -> None: + params = StaticEndpointParams( + uri="https://foo.example.com:8080/spam?foo=bar&foo=baz" + ) + expected = URI( + host="foo.example.com", + path="/spam", + scheme="https", + query="foo=bar&foo=baz", + port=8080, + ) + resolver = StaticEndpointResolver() + result = await resolver.resolve_endpoint(params=params) + assert result.uri == expected + assert result.headers == Fields([]) + + +async def test_endpoint_provider_with_uri_object() -> None: + expected = URI( + host="foo.example.com", + path="/spam", + scheme="https", + query="foo=bar&foo=baz", + port=8080, + ) + params = StaticEndpointParams(uri=expected) + resolver = StaticEndpointResolver() + result = await resolver.resolve_endpoint(params=params) + assert result.uri == expected + assert result.headers == Fields([]) + + +@pytest.mark.parametrize( + "input_uri, host_type", + [ + (URI(host="example.com"), HostType.DOMAIN), + (URI(host="001:db8:3333:4444:5555:6666:7777:8888"), HostType.IPv6), + (URI(host="::"), HostType.IPv6), + (URI(host="2001:db8::"), HostType.IPv6), + (URI(host="192.168.1.1"), HostType.IPv4), + ], +) +def test_host_type(input_uri: URI, host_type: HostType) -> None: + assert input_uri.host_type == host_type + + +@pytest.mark.parametrize( + "input_host", ["example.com\t", "umlaut-äöü.aws.dev", "foo\nbar.com"] +) +def test_uri_init_with_disallowed_characters(input_host: str) -> None: + with pytest.raises(SmithyHTTPException): + URI(host=input_host) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_http_fields.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_http_fields.py new file mode 100644 index 0000000000..929c903aae --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_http_fields.py @@ -0,0 +1,182 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +# mypy: allow-untyped-defs +# mypy: allow-incomplete-defs + +import pytest + +from smithy_python._private import Field, FieldPosition, Fields + + +def test_field_single_valued_basics() -> None: + field = Field(name="fname", values=["fval"], kind=FieldPosition.HEADER) + assert field.name == "fname" + assert field.kind == FieldPosition.HEADER + assert field.values == ["fval"] + assert field.as_string() == "fval" + assert field.as_tuples() == [("fname", "fval")] + + +def test_field_multi_valued_basics() -> None: + field = Field(name="fname", values=["fval1", "fval2"], kind=FieldPosition.HEADER) + assert field.name == "fname" + assert field.kind == FieldPosition.HEADER + assert field.values == ["fval1", "fval2"] + assert field.as_string() == "fval1, fval2" + assert field.as_tuples() == [("fname", "fval1"), ("fname", "fval2")] + + +@pytest.mark.parametrize( + "values,expected", + [ + # Single-valued fields are serialized without any quoting or escaping. + (["val1"], "val1"), + (['"val1"'], '"val1"'), + (['"'], '"'), + (['val"1'], 'val"1'), + (["val\\1"], "val\\1"), + # Multi-valued fields are joined with one comma and one space as separator. + (["val1", "val2"], "val1, val2"), + (["val1", "val2", "val3", "val4"], "val1, val2, val3, val4"), + (["©väl", "val2"], "©väl, val2"), + # Values containing commas must be double-quoted. + (["val1", "val2,val3", "val4"], 'val1, "val2,val3", val4'), + (["v,a,l,1", "val2"], '"v,a,l,1", val2'), + # In strings that get quoted, pre-existing double quotes are escaped with a + # single backslash. The second backslash below is for escaping the actual + # backslash in the string for Python. + (["slc", '4,196"'], 'slc, "4,196\\""'), + (['"val1"', "val2"], '"\\"val1\\"", val2'), + (["val1", '"'], 'val1, "\\""'), + (['val1:2",val3:4"', "val5"], '"val1:2\\",val3:4\\"", val5'), + # If quoting happens, backslashes are also escaped. The following case is a + # single backslash getting serialized into two backslashes. Python escaping + # accounts for each actual backslash being written as two. + (["foo,bar\\", "val2"], '"foo,bar\\\\", val2'), + ], +) +def test_field_serialization(values: list[str], expected: str): + field = Field(name="_", values=values) + assert field.as_string() == expected + + +@pytest.mark.parametrize( + "f1,f2", + [ + ( + Field(name="fname", values=["fval1", "fval2"], kind=FieldPosition.TRAILER), + Field(name="fname", values=["fval1", "fval2"], kind=FieldPosition.TRAILER), + ), + ( + Field(name="fname", values=["fval1", "fval2"]), + Field(name="fname", values=["fval1", "fval2"]), + ), + ( + Field(name="fname"), + Field(name="fname"), + ), + ], +) +def test_field_equality(f1: Field, f2: Field) -> None: + assert f1 == f2 + + +@pytest.mark.parametrize( + "f1,f2", + [ + ( + Field(name="fname", values=["fval1", "fval2"], kind=FieldPosition.HEADER), + Field(name="fname", values=["fval1", "fval2"], kind=FieldPosition.TRAILER), + ), + ( + Field(name="fname", values=["fval1", "fval2"], kind=FieldPosition.HEADER), + Field(name="fname", values=["fval2", "fval1"], kind=FieldPosition.HEADER), + ), + ( + Field(name="fname", values=["fval1", "fval2"], kind=FieldPosition.HEADER), + Field(name="fname", values=["fval1"], kind=FieldPosition.HEADER), + ), + ( + Field(name="fname1", values=["fval1", "fval2"], kind=FieldPosition.HEADER), + Field(name="fname2", values=["fval1", "fval2"], kind=FieldPosition.HEADER), + ), + ], +) +def test_field_inqueality(f1: Field, f2: Field) -> None: + assert f1 != f2 + + +@pytest.mark.parametrize( + "fs1,fs2", + [ + ( + Fields([Field(name="fname", values=["fval1", "fval2"])]), + Fields([Field(name="fname", values=["fval1", "fval2"])]), + ), + ], +) +def test_fields_equality(fs1: Fields, fs2: Fields) -> None: + assert fs1 == fs2 + + +@pytest.mark.parametrize( + "fs1,fs2", + [ + ( + Fields(), + Fields([Field(name="fname")]), + ), + ( + Fields([Field(name="fname1")]), + Fields([Field(name="fname2")]), + ), + ( + Fields(encoding="utf-1"), + Fields(encoding="utf-2"), + ), + ( + Fields([Field(name="fname", values=["val1"])]), + Fields([Field(name="fname", values=["val2"])]), + ), + ( + Fields([Field(name="fname", values=["val2", "val1"])]), + Fields([Field(name="fname", values=["val1", "val2"])]), + ), + ( + Fields([Field(name="f1"), Field(name="f2")]), + Fields([Field(name="f2"), Field(name="f1")]), + ), + ], +) +def test_fields_inequality(fs1: Fields, fs2: Fields) -> None: + assert fs1 != fs2 + + +@pytest.mark.parametrize( + "initial_fields", + [ + [ + Field(name="fname1", values=["val1"]), + Field(name="fname1", values=["val2"]), + ], + # uniqueness is checked _after_ normaling field names + [ + Field(name="fNaMe1", values=["val1"]), + Field(name="fname1", values=["val2"]), + ], + ], +) +def test_repeated_initial_field_names(initial_fields: list[Field]) -> None: + with pytest.raises(ValueError): + Fields(initial_fields) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_http_utils.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_http_utils.py new file mode 100644 index 0000000000..5d0cad2f12 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_http_utils.py @@ -0,0 +1,98 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + + +import pytest + +from smithy_python.exceptions import SmithyException +from smithy_python.httputils import join_query_params, split_header + + +@pytest.mark.parametrize( + "given, expected", + [ + ("", []), + (",", []), + (", ,,", []), + ('"\\""', ['"']), + ('"\\\\"', ["\\"]), + ('"\\a"', ["a"]), + ("a,b,c", ["a", "b", "c"]), + ("a, b, c", ["a", "b", "c"]), + ("1, 2, 3", ["1", "2", "3"]), + ("true, false, true", ["true", "false", "true"]), + (" foo , bar ", ["foo", "bar"]), + ('" foo "," bar "', [" foo ", " bar "]), + ("foo,bar", ["foo", "bar"]), + ("foo ,bar,", ["foo", "bar"]), + ("foo , ,bar,charlie", ["foo", "bar", "charlie"]), + ('"b,c", "\\"def\\"", a', ["b,c", '"def"', "a"]), + ], +) +def test_split_header(given: str, expected: list[str]) -> None: + assert split_header(given) == expected + + +@pytest.mark.parametrize( + "given, expected", + [ + ( + "Mon, 16 Dec 2019 23:48:18 GMT, Mon, 16 Dec 2019 23:48:18 GMT", + ["Mon, 16 Dec 2019 23:48:18 GMT", "Mon, 16 Dec 2019 23:48:18 GMT"], + ), + ( + '"Mon, 16 Dec 2019 23:48:18 GMT", Mon, 16 Dec 2019 23:48:18 GMT', + ["Mon, 16 Dec 2019 23:48:18 GMT", "Mon, 16 Dec 2019 23:48:18 GMT"], + ), + ], +) +def test_split_imf_fixdate_header(given: str, expected: list[str]) -> None: + assert split_header(given, handle_unquoted_http_date=True) == expected + + +@pytest.mark.parametrize( + "given", + [ + ('"'), + ('",foo'), + ('"foo" bar'), + ], +) +def test_split_header_raises(given: str) -> None: + with pytest.raises(SmithyException): + split_header(given) + + +@pytest.mark.parametrize( + "given, expected", + [ + ([("foo", None)], "foo"), + ([("foo&bar", None)], "foo%26bar"), + ([("foo", "")], "foo="), + ([("foo&bar", "")], "foo%26bar="), + ([("foo", "bar")], "foo=bar"), + ([("foo&bar", "spam&eggs")], "foo%26bar=spam%26eggs"), + ([("foo", "bar"), ("spam", "eggs")], "foo=bar&spam=eggs"), + ( + [("foo&bar", "spam&eggs"), ("foo&bar", "spam&eggs")], + "foo%26bar=spam%26eggs&foo%26bar=spam%26eggs", + ), + ], +) +def test_join_query_params(given: list[tuple[str, str | None]], expected: str) -> None: + assert join_query_params(given) == expected + + +def test_join_query_params_adds_and_if_prefix_non_empty() -> None: + actual = join_query_params(params=[("spam", "eggs")], prefix="foo") + assert actual == "foo&spam=eggs" diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_identity.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_identity.py new file mode 100644 index 0000000000..5e430b390c --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_identity.py @@ -0,0 +1,67 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from datetime import datetime, timedelta, timezone + +import pytest +from freezegun import freeze_time + +from smithy_python._private.identity import Identity + + +@pytest.mark.parametrize( + "time_zone", + [ + None, + timezone(timedelta(hours=3)), + timezone(timedelta(hours=-3)), + timezone.utc, + timezone(timedelta(hours=0)), + timezone(timedelta(hours=0, minutes=30)), + timezone(timedelta(hours=0, minutes=-30)), + ], +) +def test_expiration_timezone(time_zone: timezone) -> None: + expiration = datetime.now(tz=time_zone) + identity = Identity(expiration=expiration) + assert identity.expiration is not None + assert identity.expiration.tzinfo == timezone.utc + + +@pytest.mark.parametrize( + "identity, expected_expired", + [ + ( + Identity( + expiration=datetime(year=2023, month=1, day=1, tzinfo=timezone.utc), + ), + True, + ), + (Identity(), False), + ( + Identity( + expiration=datetime(year=2023, month=1, day=2, tzinfo=timezone.utc), + ), + False, + ), + ( + Identity( + expiration=datetime(year=2022, month=12, day=31, tzinfo=timezone.utc), + ), + True, + ), + ], +) +@freeze_time("2023-01-01") +def test_is_expired(identity: Identity, expected_expired: bool) -> None: + assert identity.is_expired is expected_expired diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_mediatypes.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_mediatypes.py new file mode 100644 index 0000000000..ec35da188b --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_mediatypes.py @@ -0,0 +1,53 @@ +from smithy_python.mediatypes import JsonBlob, JsonString + + +def test_json_string() -> None: + json_string = JsonString("{}") + assert json_string == "{}" + assert json_string.as_json() == {} + assert isinstance(json_string, str) + + +def test_json_string_is_lazy() -> None: + json_string = JsonString("{}") + + # Since as_json hasn't been called yet, the json shouldn't have been + # parsed yet. + assert json_string._json is None + + json_string.as_json() + + # Now that as_json has been called, the parsed result should be + # cached. + assert json_string._json == {} + + +def test_string_from_json_immediately_caches() -> None: + json_string = JsonString.from_json({}) + assert json_string._json == {} + + +def test_json_blob() -> None: + json_blob = JsonBlob(b"{}") + assert json_blob == b"{}" + assert json_blob.as_json() == {} + assert isinstance(json_blob, bytes) + + +def test_json_blob_is_lazy() -> None: + json_blob = JsonBlob(b"{}") + + # Since as_json hasn't been called yet, the json shouldn't have been + # parsed yet. + assert json_blob._json is None + + json_blob.as_json() + + # Now that as_json has been called, the parsed result should be + # cached. + assert json_blob._json == {} + + +def test_blob_from_json_immediately_caches() -> None: + json_blob = JsonBlob.from_json({}) + assert json_blob._json == {} diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_protocolutils.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_protocolutils.py new file mode 100644 index 0000000000..2c9ce5b976 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_protocolutils.py @@ -0,0 +1,129 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import json +from collections.abc import AsyncIterator + +import pytest + +from smithy_python._private import tuples_to_fields +from smithy_python._private.http import HTTPResponse +from smithy_python.async_utils import async_list +from smithy_python.protocolutils import RestJsonErrorInfo, parse_rest_json_error_info +from smithy_python.types import Document + + +@pytest.mark.parametrize( + "headers, body, expected", + [ + ([], {}, RestJsonErrorInfo("Unknown", "Unknown", {})), + ([("x-amzn-errortype", "foo")], {}, RestJsonErrorInfo("foo", "Unknown", {})), + ([("X-Amzn-Errortype", "foo")], {}, RestJsonErrorInfo("foo", "Unknown", {})), + ( + [("x-amzn-errortype", "com.example#foo")], + {}, + RestJsonErrorInfo("foo", "Unknown", {}), + ), + ( + [("x-amzn-errortype", "foo:https://example.com/")], + {}, + RestJsonErrorInfo("foo", "Unknown", {}), + ), + ( + [("x-amzn-errortype", "com.example#foo:https://example.com/")], + {}, + RestJsonErrorInfo("foo", "Unknown", {}), + ), + ( + [], + {"__type": "foo"}, + RestJsonErrorInfo("foo", "Unknown", {"__type": "foo"}), + ), + ([], {"code": "foo"}, RestJsonErrorInfo("foo", "Unknown", {"code": "foo"})), + ( + [("X-Amzn-Errortype", "foo")], + {"__type": "baz"}, + RestJsonErrorInfo("foo", "Unknown", {"__type": "baz"}), + ), + ( + [], + {"message": "bar"}, + RestJsonErrorInfo("Unknown", "bar", {"message": "bar"}), + ), + ( + [], + {"error_message": "bar"}, + RestJsonErrorInfo("Unknown", "bar", {"error_message": "bar"}), + ), + ( + [], + {"errormessage": "bar"}, + RestJsonErrorInfo("Unknown", "bar", {"errormessage": "bar"}), + ), + ( + [], + {"mEsSaGe": "bar"}, + RestJsonErrorInfo("Unknown", "bar", {"mEsSaGe": "bar"}), + ), + ( + [("x-amzn-errortype", "foo")], + {"message": "bar"}, + RestJsonErrorInfo("foo", "bar", {"message": "bar"}), + ), + ], +) +async def test_parse_rest_json_error_info( + headers: list[tuple[str, str]], body: Document, expected: RestJsonErrorInfo +) -> None: + response = HTTPResponse( + status=400, + fields=tuples_to_fields(headers), + body=async_list([json.dumps(body).encode()]), + ) + actual = await parse_rest_json_error_info(response) + assert actual == expected + + +class _ExceptionThrowingBody: + def __aiter__(self) -> AsyncIterator[bytes]: + raise Exception("Body unexpectedly accessed") + + +@pytest.mark.parametrize( + "headers, expected", + [ + ([], RestJsonErrorInfo("Unknown", "Unknown", None)), + ([("x-amzn-errortype", "foo")], RestJsonErrorInfo("foo", "Unknown", None)), + ([("X-Amzn-Errortype", "foo")], RestJsonErrorInfo("foo", "Unknown", None)), + ( + [("x-amzn-errortype", "com.example#foo")], + RestJsonErrorInfo("foo", "Unknown", None), + ), + ( + [("x-amzn-errortype", "foo:https://example.com/")], + RestJsonErrorInfo("foo", "Unknown", None), + ), + ( + [("x-amzn-errortype", "com.example#foo:https://example.com/")], + RestJsonErrorInfo("foo", "Unknown", None), + ), + ], +) +async def test_parse_rest_json_error_info_without_body( + headers: list[tuple[str, str]], expected: RestJsonErrorInfo +) -> None: + response = HTTPResponse( + status=400, fields=tuples_to_fields(headers), body=_ExceptionThrowingBody() + ) + actual = await parse_rest_json_error_info(response, check_body=False) + assert actual == expected diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_retries.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_retries.py new file mode 100644 index 0000000000..213a793dee --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_retries.py @@ -0,0 +1,89 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import pytest + +from smithy_python._private.retries import ExponentialBackoffJitterType as EBJT +from smithy_python._private.retries import ( + ExponentialRetryBackoffStrategy, + SimpleRetryStrategy, +) +from smithy_python.exceptions import SmithyRetryException +from smithy_python.interfaces.retries import RetryErrorInfo, RetryErrorType + + +@pytest.mark.parametrize( + "jitter_type, scale_value, max_backoff, expected_delays", + [ + # no jitter + (EBJT.NONE, 2, 20, [0, 2.0, 4.0, 8.0, 16.0, 20.0, 20.0]), + (EBJT.NONE, 2.0, 20.0, [0, 2.0, 4.0, 8.0, 16.0, 20.0, 20.0]), + (EBJT.NONE, 1.0, 20.0, [0, 1.0, 2.0, 4.0, 8.0, 16.0, 20.0]), + (EBJT.NONE, 4.0, 2.0, [0, 2.0, 2.0, 2.0]), + (EBJT.NONE, 23.4, 76.5, [0, 23.4, 46.8, 76.5, 76.5]), + # full jitter + (EBJT.FULL, 2.0, 20.0, [0, 1.0, 2.0, 4.0, 8.0, 10.0, 10.0]), + (EBJT.FULL, 5.0, 20.0, [0, 2.5, 5.0, 10.0, 10.0]), + (EBJT.FULL, 5.0, 10.0, [0, 2.5, 5.0, 5.0, 5.0]), + (EBJT.FULL, 23.4, 76.5, [0, 11.7, 23.4, 38.25, 38.25]), + # equal jitter + (EBJT.DEFAULT, 2.0, 20.0, [0, 1.5, 3.0, 6.0, 12.0, 15.0, 15.0]), + (EBJT.DEFAULT, 23.4, 76.5, [0, 17.55, 35.1, 57.375, 57.375]), + # decorrelated jitter + (EBJT.DECORRELATED, 2.0, 20.0, [0, 5.0, 9.5, 16.25, 20.0, 20.0]), + (EBJT.DECORRELATED, 23.4, 76.5, [0, 58.5, 76.5, 76.5]), + # edge cases with zeros + (EBJT.NONE, 5.0, 0.0, [0, 0, 0, 0]), + (EBJT.NONE, 0.0, 5.0, [0, 0, 0, 0]), + (EBJT.NONE, 0.0, 0.0, [0, 0, 0, 0]), + (EBJT.FULL, 5.0, 0.0, [0, 0, 0, 0]), + (EBJT.FULL, 0.0, 5.0, [0, 0, 0, 0]), + (EBJT.FULL, 0.0, 0.0, [0, 0, 0, 0]), + ], +) +def test_exponential_backoff_strategy( + jitter_type: EBJT, + scale_value: float, + max_backoff: float, + expected_delays: list[float], +) -> None: + bos = ExponentialRetryBackoffStrategy( + backoff_scale_value=scale_value, + max_backoff=max_backoff, + jitter_type=jitter_type, + random=lambda: 0.5, # every generated "random" value equals 0.5 + ) + + for delay_index, delay_expected in enumerate(expected_delays): + delay_actual = bos.compute_next_backoff_delay(retry_attempt=delay_index) + delay_expected2 = delay_expected + print(f"{delay_index=} {delay_actual=} {delay_expected2=}") + assert delay_actual == pytest.approx(delay_expected) + + +@pytest.mark.parametrize("max_attempts", [2, 3, 10]) +def test_simple_retry_strategy(max_attempts: int) -> None: + strategy = SimpleRetryStrategy( + backoff_strategy=ExponentialRetryBackoffStrategy(backoff_scale_value=5), + max_attempts=max_attempts, + ) + error_info = RetryErrorInfo(error_type=RetryErrorType.THROTTLING) + token = strategy.acquire_initial_retry_token() + for _ in range(max_attempts - 1): + token = strategy.refresh_retry_token_for_retry( + token_to_renew=token, error_info=error_info + ) + with pytest.raises(SmithyRetryException): + strategy.refresh_retry_token_for_retry( + token_to_renew=token, error_info=error_info + ) diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_utils.py b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_utils.py new file mode 100644 index 0000000000..1ee3450795 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python/tests/unit/test_utils.py @@ -0,0 +1,327 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +# mypy: allow-untyped-defs +# mypy: allow-incomplete-defs + +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from math import isnan +from typing import Any, NamedTuple +from unittest.mock import Mock + +import pytest + +from smithy_python.exceptions import ExpectationNotMetException +from smithy_python.utils import ( + ensure_utc, + epoch_seconds_to_datetime, + expect_type, + limited_parse_float, + limited_serialize_float, + remove_dot_segments, + serialize_epoch_seconds, + serialize_float, + serialize_rfc3339, + strict_parse_bool, + strict_parse_float, +) + + +@pytest.mark.parametrize( + "given, expected", + [ + (datetime(2017, 1, 1), datetime(2017, 1, 1, tzinfo=timezone.utc)), + ( + datetime(2017, 1, 1, tzinfo=timezone.utc), + datetime(2017, 1, 1, tzinfo=timezone.utc), + ), + ( + datetime(2017, 1, 1, tzinfo=timezone(timedelta(hours=1))), + datetime(2016, 12, 31, 23, tzinfo=timezone.utc), + ), + ], +) +def test_ensure_utc(given: datetime, expected: datetime) -> None: + assert ensure_utc(given) == expected + + +@pytest.mark.parametrize( + "typ, value", + [ + (str, ""), + (int, 1), + (bool, True), + (float | int, 1), + (float | int, 1.1), + ], +) +def test_expect_type(typ: Any, value: Any) -> None: + assert expect_type(typ, value) == value + + +@pytest.mark.parametrize( + "typ, value", + [ + (str, b""), + (int, ""), + (int, 1.0), + (bool, 0), + (bool, ""), + (float | int, "1"), + ], +) +def test_expect_type_raises(typ: Any, value: Any) -> None: + with pytest.raises(ExpectationNotMetException): + expect_type(typ, value) + + +@pytest.mark.parametrize( + "given, expected", + [ + (1.0, 1.0), + ("Infinity", float("Infinity")), + ("-Infinity", float("-Infinity")), + ], +) +def test_limited_parse_float(given: float | str, expected: float) -> None: + assert limited_parse_float(given) == expected + + +@pytest.mark.parametrize( + "given", + [ + (1), + ("1.0"), + ("nan"), + ("infinity"), + ("inf"), + ("-infinity"), + ("-inf"), + ], +) +def test_limited_parse_float_raises(given: float | str) -> None: + with pytest.raises(ExpectationNotMetException): + limited_parse_float(given) + + +def test_limited_parse_float_nan() -> None: + assert isnan(limited_parse_float("NaN")) + + +def test_strict_parse_bool() -> None: + assert strict_parse_bool("true") is True + assert strict_parse_bool("false") is False + with pytest.raises(ExpectationNotMetException): + strict_parse_bool("") + + +@pytest.mark.parametrize( + "given, expected", + [ + ("1.0", 1.0), + ("-1.0", -1.0), + ("1e1", 10.0), + ("1E1", 10.0), + ("1e-1", 0.1), + ("-1e-1", -0.1), + ("0.1", 0.1), + ("Infinity", float("Infinity")), + ("-Infinity", float("-Infinity")), + ], +) +def test_strict_parse_float(given: str, expected: float) -> None: + assert strict_parse_float(given) == expected + + +def test_strict_parse_float_nan() -> None: + assert isnan(strict_parse_float("NaN")) + + +@pytest.mark.parametrize( + "given", + [ + ("01"), + ("-01"), + ("nan"), + ("infinity"), + ("inf"), + ("-infinity"), + ("-inf"), + ], +) +def test_strict_parse_float_raises(given: str) -> None: + with pytest.raises(ExpectationNotMetException): + strict_parse_float(given) + + +@pytest.mark.parametrize( + "given, expected", + [ + (1, "1.0"), + (1.0, "1.0"), + (1.1, "1.1"), + # It's not particularly important whether the result of this is "1.1e3" or + # "1100.0" since both are valid representations. This is how float behaves + # by default in python though, and there's no reason to do extra work to + # change it. + (1.1e3, "1100.0"), + (1e1, "10.0"), + (32.100, "32.1"), + (0.321000e2, "32.1"), + # It's at about this point that floats start using scientific notation. + (1e16, "1e+16"), + (float("NaN"), "NaN"), + (float("Infinity"), "Infinity"), + (float("-Infinity"), "-Infinity"), + (Decimal("1"), "1.0"), + (Decimal("1.0"), "1.0"), + (Decimal("1.1"), "1.1"), + (Decimal("1.1e3"), "1.1E+3"), + (Decimal("1e1"), "1E+1"), + (Decimal("32.100"), "32.1"), + (Decimal("0.321000e+2"), "32.1"), + (Decimal("1e16"), "1E+16"), + (Decimal("NaN"), "NaN"), + (Decimal("Infinity"), "Infinity"), + (Decimal("-Infinity"), "-Infinity"), + ], +) +def test_serialize_float(given: float | Decimal, expected: str) -> None: + assert serialize_float(given) == expected + + +class DateTimeTestcase(NamedTuple): + dt_object: datetime + rfc3339_str: str + epoch_seconds_num: int | float + epoch_seconds_str: str + + +DATETIME_TEST_CASES: list[DateTimeTestcase] = [ + DateTimeTestcase( + dt_object=datetime(2017, 1, 1, tzinfo=timezone.utc), + rfc3339_str="2017-01-01T00:00:00Z", + epoch_seconds_num=1483228800, + epoch_seconds_str="1483228800", + ), + DateTimeTestcase( + dt_object=datetime(2017, 1, 1, microsecond=1, tzinfo=timezone.utc), + rfc3339_str="2017-01-01T00:00:00.000001Z", + epoch_seconds_num=1483228800.000001, + epoch_seconds_str="1483228800.000001", + ), + DateTimeTestcase( + dt_object=datetime(1969, 12, 31, 23, 59, 59, tzinfo=timezone.utc), + rfc3339_str="1969-12-31T23:59:59Z", + epoch_seconds_num=-1, + epoch_seconds_str="-1", + ), + # The first second affected by the Year 2038 problem where fromtimestamp raises an + # OverflowError on 32-bit systems for dates beyond 2038-01-19 03:14:07 UTC. + DateTimeTestcase( + dt_object=datetime(2038, 1, 19, 3, 14, 8, tzinfo=timezone.utc), + rfc3339_str="2038-01-19T03:14:08Z", + epoch_seconds_num=2147483648, + epoch_seconds_str="2147483648", + ), +] + + +@pytest.mark.parametrize( + "given, expected", + [ + (1.0, 1.0), + (float("NaN"), "NaN"), + (float("Infinity"), "Infinity"), + (float("-Infinity"), "-Infinity"), + ], +) +def test_limited_serialize_float(given: float, expected: str | float) -> None: + assert limited_serialize_float(given) == expected + + +@pytest.mark.parametrize( + "given, expected", + [(case.dt_object, case.rfc3339_str) for case in DATETIME_TEST_CASES], +) +def test_serialize_rfc3339(given: datetime, expected: str) -> None: + assert serialize_rfc3339(given) == expected + + +@pytest.mark.parametrize( + "given, expected", + [(case.dt_object, case.epoch_seconds_num) for case in DATETIME_TEST_CASES], +) +def test_serialize_epoch_seconds(given: datetime, expected: int) -> None: + assert serialize_epoch_seconds(given) == expected + + +@pytest.mark.parametrize( + "given, expected", + [(case.epoch_seconds_num, case.dt_object) for case in DATETIME_TEST_CASES], +) +def test_epoch_seconds_to_datetime(given: int | float, expected: datetime) -> None: + assert epoch_seconds_to_datetime(given) == expected + + +def test_epoch_seconds_to_datetime_with_overflow_error(monkeypatch): + # Emulate the Year 2038 problem by always raising an OverflowError. + datetime_mock = Mock(wraps=datetime) + datetime_mock.fromtimestamp = Mock(side_effect=OverflowError()) + monkeypatch.setattr("smithy_python.utils.datetime", datetime_mock) + dt_object = datetime(2038, 1, 19, 3, 14, 8, tzinfo=timezone.utc) + epoch_seconds_to_datetime(2147483648) == dt_object + + +@pytest.mark.parametrize( + "input_path, remove_consecutive_slashes, expected_path", + [ + ("/foo/bar", False, "/foo/bar"), + ("/foo/bar/", False, "/foo/bar/"), + ("/foo/bar/.", False, "/foo/bar/"), + ("/foo/bar/..", False, "/foo/"), + ("/foo/bar/../", False, "/foo/"), + ("/foo/bar/../baz", False, "/foo/baz"), + ("/foo/bar/../baz/", False, "/foo/baz/"), + ("/foo/bar/./baz", False, "/foo/bar/baz"), + ("/foo/bar/./baz/", False, "/foo/bar/baz/"), + ("/foo/bar/././baz", False, "/foo/bar/baz"), + ("/foo/bar/././baz/", False, "/foo/bar/baz/"), + ("/foo/bar/./../baz", False, "/foo/baz"), + ("/foo/bar/./../baz/", False, "/foo/baz/"), + ("/foo/bar/.././baz", False, "/foo/baz"), + ("/foo/bar/.././baz/", False, "/foo/baz/"), + ("/foo/bar/../../baz", False, "/baz"), + ("", False, ""), + ("/", False, "/"), + ("/.", False, "/"), + ("/..", False, "/"), + ("/./", False, "/"), + ("/../", False, ""), + ("/./.", False, "/"), + ("./", False, ""), + ("../", False, ""), + ("..", False, ""), + (".", False, ""), + ("/foo/bar", True, "/foo/bar"), + ("/foo/bar/", True, "/foo/bar/"), + ("/foo//bar/.", True, "/foo/bar/"), + ("/foo/bar//..", True, "/foo/bar/"), + ("//foo//bar//..//", True, "/foo/bar/"), + ], +) +def test_remove_dot_segments( + input_path: str, remove_consecutive_slashes: bool, expected_path: str +) -> None: + assert remove_dot_segments(input_path, remove_consecutive_slashes) == expected_path diff --git a/codegen/smithy-dafny-codegen-modules/smithy-python/setup.cfg b/codegen/smithy-dafny-codegen-modules/smithy-python/setup.cfg new file mode 100644 index 0000000000..bdd5ec0560 --- /dev/null +++ b/codegen/smithy-dafny-codegen-modules/smithy-python/setup.cfg @@ -0,0 +1,9 @@ +# NOTE: changes in this file likely should be reflected in the setup.cfg that +# is copied into generated code. +[metadata] +requires_dist = + # TODO + +[flake8] +# We ignore E203, E501 for this project due to black +ignore = E203,E501 diff --git a/codegen/smithy-dafny-codegen/build.gradle.kts b/codegen/smithy-dafny-codegen/build.gradle.kts index b9908cd2a4..e8c645a90b 100644 --- a/codegen/smithy-dafny-codegen/build.gradle.kts +++ b/codegen/smithy-dafny-codegen/build.gradle.kts @@ -36,6 +36,9 @@ dependencies { implementation("software.amazon.awssdk:codegen:2.20.26") implementation("com.squareup:javapoet:1.13.0") + // Smithy-Python + implementation(project(":smithy-python-codegen")) + // Used for parsing-based tests testImplementation("org.antlr:antlr4:4.9.2") } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java index a96c84248a..9589b124e4 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java @@ -34,14 +34,33 @@ import software.amazon.polymorph.smithyjava.generator.awssdk.v2.JavaAwsSdkV2; import software.amazon.polymorph.smithyjava.generator.library.JavaLibrary; import software.amazon.polymorph.smithyjava.generator.library.TestJavaLibrary; +import software.amazon.polymorph.smithypython.awssdk.extensions.DafnyPythonAwsSdkClientCodegenPlugin; +import software.amazon.polymorph.smithypython.localservice.extensions.DafnyPythonLocalServiceClientCodegenPlugin; +import software.amazon.polymorph.smithypython.wrappedlocalservice.extensions.DafnyPythonWrappedLocalServiceClientCodegenPlugin; import software.amazon.polymorph.utils.IOUtils; import software.amazon.polymorph.utils.ModelUtils; import software.amazon.smithy.aws.traits.ServiceTrait; +import software.amazon.smithy.build.FileManifest; +import software.amazon.smithy.build.PluginContext; import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.node.ObjectNode; import software.amazon.smithy.model.shapes.ServiceShape; import software.amazon.smithy.utils.IoUtils; import software.amazon.smithy.utils.Pair; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + public class CodegenEngine { private static final Logger LOGGER = LoggerFactory.getLogger( @@ -69,6 +88,8 @@ public class CodegenEngine { // To be initialized in constructor private final Model model; private final ServiceShape serviceShape; + private final Map dependencyModuleNames; + private final Optional moduleName; /** * This should only be called by {@link Builder#build()}, @@ -90,7 +111,10 @@ private CodegenEngine( final boolean generateProjectFiles, final Path libraryRoot, final Optional patchFilesDir, - final boolean updatePatchFiles + final boolean updatePatchFiles, + final Map dependencyModuleNames, + final Optional moduleName + ) { // To be provided to constructor this.fromSmithyBuildPlugin = fromSmithyBuildPlugin; @@ -107,6 +131,8 @@ private CodegenEngine( this.libraryRoot = libraryRoot; this.patchFilesDir = patchFilesDir; this.updatePatchFiles = updatePatchFiles; + this.dependencyModuleNames = dependencyModuleNames; + this.moduleName = moduleName; this.model = this.awsSdkStyle @@ -144,6 +170,7 @@ public void run() { case JAVA -> generateJava(outputDir); case DOTNET -> generateDotnet(outputDir); case RUST -> generateRust(outputDir); + case PYTHON -> generatePython(); default -> throw new UnsupportedOperationException( "Cannot generate code for target language %s".formatted(lang.name()) ); @@ -460,10 +487,10 @@ private void handlePatching(TargetLanguage targetLanguage, Path outputDir) { if (Files.exists(patchFilesForLanguage)) { List> sortedPatchFiles = Files - .list(patchFilesForLanguage) - .map(file -> Pair.of(getDafnyVersionForPatchFile(file), file)) - .sorted(Collections.reverseOrder(Map.Entry.comparingByKey())) - .toList(); + .list(patchFilesForLanguage) + .map(file -> Pair.of(getDafnyVersionForPatchFile(file), file)) + .sorted(Collections.reverseOrder(Map.Entry.comparingByKey())) + .toList(); for (Pair patchFilePair : sortedPatchFiles) { if (dafnyVersion.compareTo(patchFilePair.getKey()) >= 0) { Path patchFile = patchFilePair.getValue(); @@ -478,6 +505,44 @@ private void handlePatching(TargetLanguage targetLanguage, Path outputDir) { } } + private void generatePython() { + + if (moduleName.isEmpty()) { + throw new IllegalArgumentException("Python codegen requires a module name"); + } + + ObjectNode.Builder pythonSettingsBuilder = ObjectNode.builder() + .withMember("service", serviceShape.getId().toString()) + .withMember("module", moduleName.get()) + + // Smithy-Python requires some string to be present here, but this is unused. + // Any references to this version are deleted as part of code generation. + .withMember("moduleVersion", "0.0.1"); + + final PluginContext pluginContext = PluginContext.builder() + .model(model) + .fileManifest(FileManifest.create(targetLangOutputDirs.get(TargetLanguage.PYTHON))) + .settings(pythonSettingsBuilder.build()) + .build(); + + final Map smithyNamespaceToPythonModuleNameMap = new HashMap<>(dependencyModuleNames); + smithyNamespaceToPythonModuleNameMap.put(serviceShape.getId().getNamespace(), moduleName.get()); + + if (this.awsSdkStyle) { + DafnyPythonAwsSdkClientCodegenPlugin dafnyPythonAwsSdkClientCodegenPlugin + = new DafnyPythonAwsSdkClientCodegenPlugin(smithyNamespaceToPythonModuleNameMap); + dafnyPythonAwsSdkClientCodegenPlugin.execute(pluginContext); + } else if (this.localServiceTest) { + DafnyPythonWrappedLocalServiceClientCodegenPlugin pythonClientCodegenPlugin + = new DafnyPythonWrappedLocalServiceClientCodegenPlugin(smithyNamespaceToPythonModuleNameMap); + pythonClientCodegenPlugin.execute(pluginContext); + } else { + DafnyPythonLocalServiceClientCodegenPlugin pythonClientCodegenPlugin + = new DafnyPythonLocalServiceClientCodegenPlugin(smithyNamespaceToPythonModuleNameMap); + pythonClientCodegenPlugin.execute(pluginContext); + } + } + private String runCommand(Path workingDir, String... args) { List argsList = List.of(args); StringBuilder output = new StringBuilder(); @@ -517,6 +582,8 @@ public static class Builder { private Path libraryRoot; private Path patchFilesDir; private boolean updatePatchFiles = false; + private Map dependencyModuleNames; + private String moduleName; public Builder() {} @@ -544,6 +611,22 @@ public Builder withNamespace(final String namespace) { return this; } + /** + * Sets the directories in which to search for dependent model file(s). + */ + public Builder withDependencyModuleNames(final Map dependencyModuleNames) { + this.dependencyModuleNames = dependencyModuleNames; + return this; + } + + /** + * Sets the Python module name for any generated Python code. + */ + public Builder withModuleName(final String moduleName) { + this.moduleName = moduleName; + return this; + } + /** * Sets the target language(s) for which to generate code, * along with the directory(-ies) into which to output each language's generated code. @@ -672,6 +755,10 @@ public CodegenEngine build() { throw new IllegalStateException("No namespace provided"); } + final Map dependencyModuleNames = this.dependencyModuleNames == null + ? new HashMap<>() + : this.dependencyModuleNames; + final Map targetLangOutputDirsRaw = Objects.requireNonNull(this.targetLangOutputDirs); targetLangOutputDirsRaw.replaceAll((_lang, path) -> @@ -708,6 +795,8 @@ public CodegenEngine build() { ); } + final Optional moduleName = Optional.ofNullable(this.moduleName); + final Path libraryRoot = this.libraryRoot.toAbsolutePath().normalize(); final Optional patchFilesDir = Optional @@ -734,7 +823,9 @@ public CodegenEngine build() { this.generateProjectFiles, libraryRoot, patchFilesDir, - updatePatchFiles + updatePatchFiles, + dependencyModuleNames, + moduleName ); } } @@ -744,5 +835,6 @@ public enum TargetLanguage { JAVA, DOTNET, RUST, + PYTHON, } } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/README.md b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/README.md new file mode 100644 index 0000000000..003b29b554 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/README.md @@ -0,0 +1,23 @@ +TODO-Python: Add more content here + +Top-level file overview: + +``` +├── awssdk - Generates a boto3 wrapper to call from Dafny-generated Python code +├── common - Common code across generation targets +├── localservice - Generates a Smithy client that wraps a Dafny-generated Python localService implementation +└── wrappedlocalservice - Generates a wrapper for the `localservice` code to call the Smithy client from Dafny-generated Python code +``` + +Each subfolder follows a similar structure: + +``` +├── customize - Classes referenced from a plugin's `PythonIntegration.customize` function. +│ Generates new files or adds new code to Smithy-Python generated files. +├── extensions - Classes that extend or replace Smithy-Python codegen components. +├── nameresolver - Utility classes to map Smithy model shapes to strings used in generated code. +└── shapevisitor - Classes that generate code to convert to/from Smithy client Python shapes + │ (or AWS SDK shapes) and Dafny implementation shapes. + └── conversionwriter - Classes that generate functions that convert to/from Smithy client Python shapes + (or AWS SDK shapes) and Dafny implementation shapes for StructureShapes and UnionShapes. +``` \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/AwsSdkCodegenConstants.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/AwsSdkCodegenConstants.java new file mode 100644 index 0000000000..15eff94a3f --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/AwsSdkCodegenConstants.java @@ -0,0 +1,5 @@ +package software.amazon.polymorph.smithypython.awssdk; + +public class AwsSdkCodegenConstants { + public static String AWS_SDK_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME = "awssdk_codegen_todelete"; +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/DafnyPythonAwsSdkIntegration.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/DafnyPythonAwsSdkIntegration.java new file mode 100644 index 0000000000..51909a5f6f --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/DafnyPythonAwsSdkIntegration.java @@ -0,0 +1,70 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.awssdk; + +import java.util.ArrayList; +import java.util.List; +import software.amazon.polymorph.smithypython.awssdk.customize.AwsSdkShimFileWriter; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.integration.ProtocolGenerator; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; + +public final class DafnyPythonAwsSdkIntegration implements PythonIntegration { + + /** + * Generate all Smithy-Dafny Python AWS SDK custom code. + * + * @param codegenContext Code generation context that can be queried when writing additional + * files. + */ + @Override + public void customize(GenerationContext codegenContext) { + // ONLY run this integration's customizations IF the codegen is using its ApplicationProtocol + if (!codegenContext + .applicationProtocol() + .equals(DafnyPythonAwsSdkProtocolGenerator.DAFNY_PYTHON_AWS_SDK_PROTOCOL)) { + return; + } + + customizeForServiceShape( + codegenContext.settings().getService(codegenContext.model()), codegenContext); + } + + /** + * Generate any code for the serviceShape. + * + * @param serviceShape + * @param codegenContext + */ + private void customizeForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) { + new AwsSdkShimFileWriter().customizeFileForServiceShape(serviceShape, codegenContext); + } + + /** + * Creates the Dafny ApplicationProtocol object. Smithy-Python requests this object as part of the + * ProtocolGenerator implementation. This uses the {@link + * software.amazon.polymorph.traits.DafnyAwsSdkProtocolTrait}. + * + * @return Returns the created application protocol. + */ + @Override + public List getProtocolGenerators() { + List protocolGenerators = new ArrayList<>(); + protocolGenerators.add( + new DafnyPythonAwsSdkProtocolGenerator() { + // Setting a Polymorph-specific protocol allows any services that + // have this protocol trait to be generated using this PythonIntegration. + // See DafnyAwsSdkProtocolTrait class. + @Override + public ShapeId getProtocol() { + return ShapeId.fromParts("aws.polymorph", "awsSdk"); + } + }); + + return protocolGenerators; + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/DafnyPythonAwsSdkProtocolGenerator.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/DafnyPythonAwsSdkProtocolGenerator.java new file mode 100644 index 0000000000..38e8e45827 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/DafnyPythonAwsSdkProtocolGenerator.java @@ -0,0 +1,38 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.awssdk; + +import software.amazon.smithy.python.codegen.ApplicationProtocol; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.integration.ProtocolGenerator; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** This will implement any handling of components outside the request body and error handling. */ +@SmithyUnstableApi +public abstract class DafnyPythonAwsSdkProtocolGenerator implements ProtocolGenerator { + + public static ApplicationProtocol DAFNY_PYTHON_AWS_SDK_PROTOCOL = + new ApplicationProtocol( + // Define the `dafny-python-aws-sdk-protocol` ApplicationProtocol. + // ApplicationProtocol is distinct from the Protocols used in ProtocolGenerators. + // We define an ApplicationProtocol that will be used by all AWS SDK shims for + // Smithy-plugin-integrated code generators. + // The ApplicationProtocol is used within our code to determine which code should be + // generated. + // The `null`s reflect that this ApplicationProtocol does not have request + // or response object types, since it does not use Smithy-generated clients, + // but is instead a wrapper for boto3. + "dafny-python-aws-sdk-application-protocol", null, null); + + @Override + public ApplicationProtocol getApplicationProtocol() { + return DAFNY_PYTHON_AWS_SDK_PROTOCOL; + } + + @Override + public void generateRequestSerializers(GenerationContext context) {} + + @Override + public void generateResponseDeserializers(GenerationContext context) {} +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/customize/AwsSdkShimFileWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/customize/AwsSdkShimFileWriter.java new file mode 100644 index 0000000000..d36e80a894 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/customize/AwsSdkShimFileWriter.java @@ -0,0 +1,234 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.awssdk.customize; + +import java.util.TreeSet; +import java.util.stream.Collectors; +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.awssdk.shapevisitor.AwsSdkToDafnyShapeVisitor; +import software.amazon.polymorph.smithypython.awssdk.shapevisitor.DafnyToAwsSdkShapeVisitor; +import software.amazon.polymorph.smithypython.common.customize.CustomFileWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.Utils; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * Writes the shim.py file for AWS SDKs.The shim wraps boto3 calls. Its inputs are Dafny-modelled + * requests; its outputs are Dafny-modelled responses. Internally, the shim will convert the + * Dafny-modelled requests to dictionaries passed to boto3 via kwargs, call boto3 with the request + * and receive a response, convert the boto3 response dictionary into a Dafny-modelled response, and + * return the Dafny-modelled response. Other Dafny-generated Python code will use the shim to call + * AWS services (e.g. KMS, DDB). + */ +public class AwsSdkShimFileWriter implements CustomFileWriter { + + @Override + public void customizeFileForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) { + String typesModulePrelude = + DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(serviceShape.getId()); + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + codegenContext.settings().getService().getNamespace()); + codegenContext + .writerDelegator() + .useFileWriter( + moduleName + "/shim.py", + "", + writer -> { + writer.write( + """ + import Wrappers + from botocore.exceptions import ClientError + import $L + + def _sdk_error_to_dafny_error(e: ClientError): + ""\" + Converts the provided native Smithy-modelled error + into the corresponding Dafny error. + ""\" + ${C|} + + class $L: + def __init__(self, _impl, _region): + self._impl = _impl + self._region = _region + + ${C|} + + """, + typesModulePrelude, + writer.consumer( + w -> generateAwsSdkErrorToDafnyErrorBlock(codegenContext, serviceShape, w)), + AwsSdkNameResolver.shimNameForService(serviceShape), + writer.consumer(w -> generateOperationsBlock(codegenContext, serviceShape, w))); + }); + } + + /** + * Generate the method body for the `_sdk_error_to_dafny_error` method. This writes out a block to + * convert a boto3 ClientError modelled in JSON into a Dafny-modelled error + * + * @param codegenContext + * @param serviceShape + * @param writer + */ + private void generateAwsSdkErrorToDafnyErrorBlock( + GenerationContext codegenContext, ServiceShape serviceShape, PythonWriter writer) { + + // Get modelled error converters for this service + TreeSet errorShapeSet = + new TreeSet( + codegenContext.model().getStructureShapesWithTrait(ErrorTrait.class).stream() + .filter( + structureShape -> + structureShape + .getId() + .getNamespace() + .equals(codegenContext.settings().getService().getNamespace())) + .map(Shape::getId) + .collect(Collectors.toSet())); + + // First error case opens a new `if` block; others do not need to, and write `elif` + boolean hasOpenedIfBlock = false; + + for (ShapeId errorShapeId : errorShapeSet) { + // ex. for KMS.InvalidImportTokenException: + // if e.response['Error']['Code'] == 'InvalidImportTokenException': + // return + // software_amazon_cryptography_services_kms_internaldafny_types.Error_InvalidImportTokenException(message=e.response['Error']['Code']) + Shape errorShape = codegenContext.model().expectShape(errorShapeId); + writer.openBlock( + "$L e.response['Error']['Code'] == '$L':", + "", + hasOpenedIfBlock ? "elif" : "if", + errorShapeId.getName(), + () -> { + writer.write( + "return %1$s" + .formatted( + errorShape.accept( + new AwsSdkToDafnyShapeVisitor(codegenContext, "e.response", writer)))); + }); + hasOpenedIfBlock = true; + } + + if (hasOpenedIfBlock) { + // If `hasOpenedIfBlock` is false, then codegen never wrote any errors, + // and this function should only cast to Opaque errors + writer.write( + """ + return $L.Error_Opaque(obj=e) + """, + DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(serviceShape.getId())); + } else { + // If `hasOpenedIfBlock` is true, then codegen wrote at least one error, + // and this function should only cast to Opaque error via `else` + writer.write( + """ + else: + return $L.Error_Opaque(obj=e) + """, + DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(serviceShape.getId())); + } + } + + /** + * Generate shim methods for all operations in the SDK service shape. Each method will take in a + * Dafny type as input and return a Dafny type as output. Internally, each method will convert the + * Dafny input into a dictionary whose keys are boto3 API request parameters, call the boto3 + * client with the request dictionary mapped to its kwargs representation, receive a boto3 + * response, convert the response into its corresponding Dafny type, and return the Dafny type. + * + * @param codegenContext + * @param serviceShape + * @param writer + */ + private void generateOperationsBlock( + GenerationContext codegenContext, ServiceShape serviceShape, PythonWriter writer) { + + for (ShapeId operationShapeId : serviceShape.getOperations()) { + OperationShape operationShape = + codegenContext.model().expectShape(operationShapeId, OperationShape.class); + + ShapeId inputShape = operationShape.getInputShape(); + ShapeId outputShape = operationShape.getOutputShape(); + + // Import input/output shape types + DafnyNameResolver.importDafnyTypeForShape(writer, inputShape, codegenContext); + DafnyNameResolver.importDafnyTypeForShape(writer, outputShape, codegenContext); + + // No 'native' type to import; native AWS SDK types are modelled in dictionaries + + // Write the Shim operation block. + // This takes in a Dafny input and returns a Dafny output. + // This operation will: + // 1) Convert the Dafny input to a dictionary with keys as AWS SDK API input names; + // 2) Call boto3 client with the input transformed to kwargs; and + // 3) Convert the Smithy output to the Dafny type. + writer.openBlock( + "def $L(self, $L) -> $L:", + "", + operationShape.getId().getName(), + // Do not generate an `input` parameter if the operation does not take in an input + Utils.isUnitShape(inputShape) + ? "" + : "input: " + DafnyNameResolver.getDafnyTypeForShape(inputShape), + // Return `None` type if the operation does not return an output + Utils.isUnitShape(outputShape) + ? "None" + : DafnyNameResolver.getDafnyTypeForShape(outputShape), + () -> { + Shape targetShapeInput = + codegenContext.model().expectShape(operationShape.getInputShape()); + // Generate code that converts the input from the Dafny type to the corresponding Smithy + // type + // `input` will hold a string that converts the Dafny `input` to the Smithy-modelled + // output. + // If this is a type that allows for self-recursion (unions or sets: can contain + // themselves as a member), + // this will delegate to DafnyToAwsSdkConversionFunctionWriter + String input = + targetShapeInput.accept( + new DafnyToAwsSdkShapeVisitor(codegenContext, "input", writer)); + writer.addImport(".", "dafny_to_aws_sdk"); + + // Generate code that: + // 1) "unwraps" the request (converts from the Dafny type to the AWS SDK type); + // 2) calls boto3; + // 3) wraps boto3 ClientErrors as Dafny failures + writer.write( + """ + boto_request_dict = $L + try: + boto_response_dict = self._impl.$L(**boto_request_dict) + except ClientError as e: + return Wrappers.Result_Failure(_sdk_error_to_dafny_error(e)) + """, + input, + codegenContext.symbolProvider().toSymbol(operationShape).getName()); + + Shape targetShape = codegenContext.model().expectShape(operationShape.getOutputShape()); + // Generate code that converts the output from SDK type to the corresponding Dafny type + String output = + targetShape.accept( + new AwsSdkToDafnyShapeVisitor(codegenContext, "boto_response_dict", writer)); + + // Generate code that wraps SDK success shapes as Dafny success shapes + writer.write( + """ + return Wrappers.Result_Success($L) + """, + Utils.isUnitShape(outputShape) ? "None" : output); + }); + } + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/extensions/DafnyPythonAwsSdkClientCodegenPlugin.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/extensions/DafnyPythonAwsSdkClientCodegenPlugin.java new file mode 100644 index 0000000000..0fea017065 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/extensions/DafnyPythonAwsSdkClientCodegenPlugin.java @@ -0,0 +1,91 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.awssdk.extensions; + +import software.amazon.polymorph.smithypython.localservice.extensions.DafnyPythonLocalServiceClientCodegenPlugin; +import software.amazon.polymorph.traits.DafnyAwsSdkProtocolTrait; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.smithy.build.PluginContext; +import software.amazon.smithy.build.SmithyBuildPlugin; +import software.amazon.smithy.codegen.core.directed.CodegenDirector; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.transform.ModelTransformer; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.utils.SmithyUnstableApi; + +import java.util.Map; + +/** + * Plugin to trigger Smithy-Dafny Python code generation for AWS SDK services. This Plugin differs + * from the PythonClientCodegenPlugin by not calling runner.performDefaultCodegenTransforms(); and + * runner.createDedicatedInputsAndOutputs(); These methods transform the model in ways that the + * model does not align with the generated Dafny code. This Plugin also attaches a + * DafnyAwsSdkProtocolTrait to the ServiceShape provided in settings. AWS SDKs do not consistently + * label a protocol, and Smithy-Python requires that a protocol is assigned. Rather than declare + * that we are using some protocol (e.g. `restJson1`) then not use that in practice, it is more + * proper to define some custom protocol and use that. + */ +@SmithyUnstableApi +public final class DafnyPythonAwsSdkClientCodegenPlugin implements SmithyBuildPlugin { + + public DafnyPythonAwsSdkClientCodegenPlugin(Map smithyNamespaceToPythonModuleNameMap) { + super(); + SmithyNameResolver.setSmithyNamespaceToPythonModuleNameMap(smithyNamespaceToPythonModuleNameMap); + } + + public static Model addAwsSdkProtocolTrait(Model model, ServiceShape serviceShape) { + return ModelTransformer.create() + .mapShapes( + model, + shape -> { + if (shape instanceof ServiceShape && shape.hasTrait(LocalServiceTrait.class)) { + return serviceShape.toBuilder() + .addTrait(DafnyAwsSdkProtocolTrait.builder().build()) + .build(); + } else { + return shape; + } + }); + } + + @Override + public String getName() { + return "dafny-python-aws-sdk-client-codegen"; + } + + @Override + public void execute(PluginContext context) { + CodegenDirector runner = + new CodegenDirector<>(); + + PythonSettings settings = PythonSettings.from(context.getSettings()); + settings.setProtocol(DafnyAwsSdkProtocolTrait.ID); + runner.settings(settings); + runner.directedCodegen(new DirectedDafnyPythonAwsSdkCodegen()); + runner.fileManifest(context.getFileManifest()); + runner.service(settings.getService()); + runner.model(context.getModel()); + runner.integrationClass(PythonIntegration.class); + + // Add a DafnyAwsSdkProtocolTrait to the service as a contextual indicator highlighting + // that the DafnyPythonAwsSdk protocol should be used. + ServiceShape serviceShape = + context.getModel().expectShape(settings.getService()).asServiceShape().get(); + runner.model(addAwsSdkProtocolTrait(context.getModel(), serviceShape)); + + runner.run(); + } + + public static Model transformModelForAwsSdkService(Model model, ServiceShape serviceShape) { + Model transformedModel = model; + transformedModel = addAwsSdkProtocolTrait(transformedModel, serviceShape); + transformedModel = DafnyPythonLocalServiceClientCodegenPlugin.transformStringEnumShapesToEnumShapes(transformedModel, serviceShape); + return transformedModel; + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/extensions/DafnyPythonAwsSdkSymbolVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/extensions/DafnyPythonAwsSdkSymbolVisitor.java new file mode 100644 index 0000000000..90c663b3c3 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/extensions/DafnyPythonAwsSdkSymbolVisitor.java @@ -0,0 +1,50 @@ +package software.amazon.polymorph.smithypython.awssdk.extensions; + +import static java.lang.String.format; + +import software.amazon.polymorph.smithypython.awssdk.AwsSdkCodegenConstants; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.localservice.extensions.DafnyPythonLocalServiceSymbolVisitor; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.python.codegen.PythonSettings; + +/** + * SymbolVisitor for wrapped localService codegen. Overrides the generated file for codegen to + * something that is immediately deleted. Smithy ALWAYS writes visited symbols to a file. For AWS + * SDK codegen, we do NOT want to write visited symbols to a file, since boto3 does not use these + * visited symbols. It is very, very difficult to change this writing behavior without rewriting + * Smithy logic in addition to Smithy-Python specific logic. I have tried some workarounds like + * deleting writers or writing to /dev/null but these were not fruitful. This workaround dumps any + * visited symbols into a file whose name will never be used and deletes this file as part of its + * Smithy codegen plugin. + */ +public class DafnyPythonAwsSdkSymbolVisitor extends DafnyPythonLocalServiceSymbolVisitor { + + public DafnyPythonAwsSdkSymbolVisitor(Model model, PythonSettings settings) { + super(model, settings); + } + + /** + * Path to the overridden file that is deleted for wrapped services. + * + * @param namespace + * @return + */ + @Override + protected String getSymbolDefinitionFilePathForNamespaceAndFilename( + String namespace, String filename) { + String directoryFilePath; + if ("smithy.api".equals(namespace)) { + directoryFilePath = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + settings.getService().getNamespace()); + } else { + directoryFilePath = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(namespace); + } + + return format( + "%s/%s.py", + directoryFilePath, AwsSdkCodegenConstants.AWS_SDK_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/extensions/DirectedDafnyPythonAwsSdkCodegen.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/extensions/DirectedDafnyPythonAwsSdkCodegen.java new file mode 100644 index 0000000000..291fcc72e9 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/extensions/DirectedDafnyPythonAwsSdkCodegen.java @@ -0,0 +1,112 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.awssdk.extensions; + +import static java.lang.String.format; + +import java.nio.file.Path; +import java.util.logging.Logger; +import software.amazon.polymorph.smithypython.awssdk.AwsSdkCodegenConstants; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.smithy.build.FileManifest; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.codegen.core.directed.CreateSymbolProviderDirective; +import software.amazon.smithy.codegen.core.directed.CustomizeDirective; +import software.amazon.smithy.codegen.core.directed.GenerateServiceDirective; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.DirectedPythonCodegen; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; + +/** + * DirectedCodegen for Dafny Python AWS SDK models. This overrides DirectedPythonCodegen to 1) Not + * generate a Smithy client (nor its serialize/deserialize bodies, client config, etc.), and 2) + * Remove extraneous generated files. AWS SDK generation does NOT involve generating a Smithy + * client; it will only generate a shim wrapping boto3. + */ +public class DirectedDafnyPythonAwsSdkCodegen extends DirectedPythonCodegen { + + private static final Logger LOGGER = + Logger.getLogger(DirectedDafnyPythonAwsSdkCodegen.class.getName()); + + @Override + public SymbolProvider createSymbolProvider( + CreateSymbolProviderDirective directive) { + return new DafnyPythonAwsSdkSymbolVisitor(directive.model(), directive.settings()); + } + + /** + * Do NOT generate any service config code for Dafny Python AWS SDKs (i.e. `config.py`). Override + * DirectedPythonCodegen to block any service config code generation. + * + * @param directive Directive to perform. + */ + @Override + public void customizeBeforeShapeGeneration( + CustomizeDirective directive) {} + + /** + * Do NOT generate any service code for Dafny Python AWS SDKs. Override DirectedPythonCodegen to + * block any service code generation. In addition to not writing any service code (i.e. not + * writing `client.py`), this also blocks writing `serialize.py` and `deserialize.py`. + * + * @param directive Directive to perform. + */ + @Override + public void generateService( + GenerateServiceDirective directive) {} + + /** + * Call `DirectedPythonCodegen.customizeAfterIntegrations`, then remove `models.py` and + * `errors.py`. The CodegenDirector will invoke this method after shape generation. + * + * @param directive Directive to perform. + */ + @Override + public void customizeAfterIntegrations( + CustomizeDirective directive) { + // DirectedPythonCodegen's customizeAfterIntegrations implementation SHOULD run first; + // its implementation writes all files by flushing its writers; + // this implementation removes some of those files. + super.customizeAfterIntegrations(directive); + + FileManifest fileManifest = directive.fileManifest(); + Path generationPath = + Path.of( + fileManifest.getBaseDir() + + "/" + + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + directive.context().settings().getService().getNamespace())); + + /** + * Smithy ALWAYS writes visited symbols to a file. For AWS SDK codegen, we do NOT want to write + * visited symbols to a file, since boto3 does not use these visited symbols. It is very, very + * difficult to change this writing behavior without rewriting Smithy logic in addition to + * Smithy-Python specific logic. I have tried some workarounds like deleting writers or writing + * to /dev/null but these were not fruitful. This workaround dumps any visited symbols into a + * file whose name will never be used and deletes this file as part of its Smithy codegen + * plugin. + */ + try { + LOGGER.info( + format( + "Attempting to remove %s.py", + AwsSdkCodegenConstants.AWS_SDK_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME)); + CodegenUtils.runCommand( + format( + "rm -f %s.py", + AwsSdkCodegenConstants.AWS_SDK_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME), + generationPath) + .strip(); + } catch (CodegenException e) { + // Fail loudly. We do not want to accidentally distribute this file. + throw new RuntimeException( + format( + "Unable to remove %s.py", + AwsSdkCodegenConstants.AWS_SDK_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME), + e); + } + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/nameresolver/AwsSdkNameResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/nameresolver/AwsSdkNameResolver.java new file mode 100644 index 0000000000..2bc8af8175 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/nameresolver/AwsSdkNameResolver.java @@ -0,0 +1,116 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.awssdk.nameresolver; + +import com.google.common.base.CaseFormat; +import software.amazon.polymorph.smithydafny.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.utils.CaseUtils; + +/** + * Contains utility functions that map Smithy shapes to useful strings used in Smithy-Python + * generated AWS SDK code. + */ +public class AwsSdkNameResolver { + + public static boolean isAwsSdkShape(Shape shape) { + return isAwsSdkShape(shape.getId()); + } + + public static boolean isAwsSdkShape(ShapeId shapeId) { + // If the shape namespace is not in our list of known SDK namespaces, + // it is not a (known) SDK namespace + return shapeId.getNamespace().startsWith("com.amazonaws"); + } + + /** + * Returns the type name of the client for the provided AWS SDK serviceShape. The + * serviceShape SHOULD be an AWS SDK. This also standardizes some "legacy" service names. + * + * @param serviceShape + * @return + */ + public static String clientNameForService(ServiceShape serviceShape) { + return switch (serviceShape.getId().getName()) { + case "TrentService" -> "KMSClient"; + case "DynamoDB_20120810" -> "DynamoDBClient"; + default -> serviceShape.getId().getName(); + }; + } + + /** + * Returns the name of the Smithy-generated shim for the provided AWS SDK serviceShape. The + * serviceShape SHOULD be an AWS SDK. + * + * @param serviceShape + * @return + */ + public static String shimNameForService(ServiceShape serviceShape) { + return clientNameForService(serviceShape) + "Shim"; + } + + /** + * Returns the name of the error type that wraps AWS SDK errors. + * Customers may see this error type, so it should be reasonably informative. + * + * @param serviceShape + * @return + */ + public static String dependencyErrorNameForService(ServiceShape serviceShape) { + return DafnyNameResolver.dafnyBaseModuleName(serviceShape.getId().getNamespace()); + } + + /** + * Resolve the provided smithyNamespace to its corresponding Dafny Extern namespace. Our Dafny + * code declares an extern namespace independent of the Smithy namespace; this function maps the + * two namespaces. + * + * @param smithyNamespace + * @return + */ + public static String resolveAwsSdkSmithyModelNamespaceToDafnyExternNamespace( + String smithyNamespace) { + String rtn = smithyNamespace.toLowerCase(); + if (smithyNamespace.startsWith("aws")) { + rtn = rtn.replaceFirst("aws", "software.amazon"); + } else if (smithyNamespace.startsWith("com.amazonaws")) { + rtn = rtn.replaceFirst("com.amazonaws", "software.amazon.cryptography.services"); + } + return rtn; + } + + /** + * Returns the name of the function that converts the provided shape's Dafny-modelled type to the + * corresponding AWS SDK-modelled type. This function will be defined in the `dafny_to_aws_sdk.py` + * file. ex. example.namespace.ExampleShape -> "example_namespace_ExampleShape" + * + * @param shape + * @return + */ + public static String getDafnyToAwsSdkFunctionNameForShape(Shape shape) { + return SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + shape.getId().getNamespace()) + + "_" + + shape.getId().getName(); + } + + /** + * Returns the name of the function that converts the provided shape's AWS SDK-modelled type to + * the corresponding Dafny-modelled type. This function will be defined in the + * `aws_sdk_to_dafny.py` file. ex. example.namespace.ExampleShape -> + * "example_namespace_ExampleShape" + * + * @param shape + * @return + */ + public static String getAwsSdkToDafnyFunctionNameForShape(Shape shape) { + return SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + shape.getId().getNamespace()) + + "_" + + shape.getId().getName(); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/AwsSdkToDafnyShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/AwsSdkToDafnyShapeVisitor.java new file mode 100644 index 0000000000..e9483d313b --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/AwsSdkToDafnyShapeVisitor.java @@ -0,0 +1,291 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.awssdk.shapevisitor; + +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.awssdk.shapevisitor.conversionwriters.AwsSdkToDafnyConversionFunctionWriter; +import software.amazon.polymorph.smithypython.awssdk.shapevisitor.conversionwriters.DafnyToAwsSdkConversionFunctionWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.Utils; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.shapes.BigDecimalShape; +import software.amazon.smithy.model.shapes.BigIntegerShape; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.BooleanShape; +import software.amazon.smithy.model.shapes.ByteShape; +import software.amazon.smithy.model.shapes.DoubleShape; +import software.amazon.smithy.model.shapes.EnumShape; +import software.amazon.smithy.model.shapes.FloatShape; +import software.amazon.smithy.model.shapes.IntegerShape; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.LongShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.ShortShape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.TimestampShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * ShapeVisitor that should be dispatched from a shape to generate code that maps a AWS SDK + * kwarg-indexed dictionary to the corresponding Dafny shape's internal attributes. + */ +public class AwsSdkToDafnyShapeVisitor extends ShapeVisitor.Default { + private final GenerationContext context; + private final PythonWriter writer; + private final String dataSource; + + /** + * @param context The generation context. + * @param dataSource The in-code location of the data to provide an output of ({@code output.foo}, + * {@code entry}, etc.) + * @param writer A PythonWriter pointing to the in-code location where the ShapeVisitor was called + * from + */ + public AwsSdkToDafnyShapeVisitor( + GenerationContext context, String dataSource, PythonWriter writer) { + this.context = context; + this.dataSource = dataSource; + this.writer = writer; + } + + @Override + protected String getDefault(Shape shape) { + String protocolName = context.protocolGenerator().getName(); + throw new CodegenException( + String.format( + "Unsupported conversion of %s to %s using the %s protocol", + shape, shape.getType(), protocolName)); + } + + @Override + public String blobShape(BlobShape shape) { + writer.addStdlibImport("_dafny", "Seq"); + return "Seq(" + dataSource + ")"; + } + + @Override + public String structureShape(StructureShape structureShape) { + // Dafny does not generate a type for Unit shape + if (Utils.isUnitShape(structureShape.getId())) { + return "None"; + } + + // Conditionally write to/from conversion functions for structureShape + AwsSdkToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers( + structureShape, context, writer); + DafnyToAwsSdkConversionFunctionWriter.writeConverterForShapeAndMembers( + structureShape, context, writer); + + // Import the conversion function module from where the ShapeVisitor was called + String pythonModuleName = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + structureShape.getId().getNamespace(), context); + writer.addStdlibImport(pythonModuleName + ".aws_sdk_to_dafny"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.aws_sdk_to_dafny.example_namespace_ExampleShape(input)` + return "%1$s.aws_sdk_to_dafny.%2$s(%3$s)" + .formatted( + pythonModuleName, + AwsSdkNameResolver.getAwsSdkToDafnyFunctionNameForShape(structureShape), + dataSource); + } + + @Override + public String listShape(ListShape shape) { + writer.addStdlibImport("_dafny", "Seq"); + + StringBuilder builder = new StringBuilder(); + + // Open Dafny sequence: + // `Seq([` + builder.append("Seq(["); + MemberShape memberShape = shape.getMember(); + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + + // Add converted list elements into the list: + // `Seq([`SmithyToDafny(list_element)`` + builder.append( + "%1$s" + .formatted( + targetShape.accept( + new AwsSdkToDafnyShapeVisitor(context, "list_element", writer)))); + + // Close structure + // `Seq([`SmithyToDafny(list_element)` for list_element in `dataSource`])`` + return builder.append(" for list_element in %1$s])".formatted(dataSource)).toString(); + } + + @Override + public String mapShape(MapShape shape) { + writer.addStdlibImport("_dafny", "Map"); + + StringBuilder builder = new StringBuilder(); + + // Open Dafny map: + // `Map({` + builder.append("Map({"); + MemberShape keyMemberShape = shape.getKey(); + final Shape keyTargetShape = context.model().expectShape(keyMemberShape.getTarget()); + MemberShape valueMemberShape = shape.getValue(); + final Shape valueTargetShape = context.model().expectShape(valueMemberShape.getTarget()); + + // Write converted map keys into the map: + // `{`SmithyToDafny(key)`:` + builder.append( + "%1$s: " + .formatted( + keyTargetShape.accept(new AwsSdkToDafnyShapeVisitor(context, "key", writer)))); + + // Write converted map values into the map: + // `{`SmithyToDafny(key)`: `SmithyToDafny(value)`` + builder.append( + "%1$s" + .formatted( + valueTargetShape.accept(new AwsSdkToDafnyShapeVisitor(context, "value", writer)))); + + // Complete map comprehension and close map + // `{`SmithyToDafny(key)`: `SmithyToDafny(value)`` for (key, value) in `dataSource`.items() }` + return builder.append(" for (key, value) in %1$s.items() })".formatted(dataSource)).toString(); + } + + @Override + public String booleanShape(BooleanShape shape) { + return dataSource; + } + + @Override + public String stringShape(StringShape shape) { + writer.addStdlibImport("_dafny", "Seq"); + + // String shapes with the enum trait have their own converters + if (shape.hasTrait(EnumTrait.class)) { + DafnyToAwsSdkConversionFunctionWriter.writeConverterForShapeAndMembers( + shape, context, writer); + AwsSdkToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers( + shape, context, writer); + + // Import the dafny_to_aws_sdk converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + shape.getId().getNamespace(), context); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".aws_sdk_to_dafny"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.dafny_to_aws_sdk.DafnyToAwsSdk_example_namespace_ExampleShape(input)` + return "%1$s.aws_sdk_to_dafny.%2$s(%3$s)" + .formatted( + pythonModuleSmithygeneratedPath, + AwsSdkNameResolver.getAwsSdkToDafnyFunctionNameForShape(shape), + dataSource); + } + + return "Seq(" + dataSource + ")"; + } + + @Override + public String byteShape(ByteShape shape) { + return getDefault(shape); + } + + @Override + public String shortShape(ShortShape shape) { + return getDefault(shape); + } + + @Override + public String integerShape(IntegerShape shape) { + return dataSource; + } + + @Override + public String longShape(LongShape shape) { + return dataSource; + } + + @Override + public String bigIntegerShape(BigIntegerShape shape) { + return getDefault(shape); + } + + @Override + public String floatShape(FloatShape shape) { + return getDefault(shape); + } + + @Override + public String doubleShape(DoubleShape shape) { + return dataSource; + } + + @Override + public String bigDecimalShape(BigDecimalShape shape) { + return getDefault(shape); + } + + @Override + public String enumShape(EnumShape shape) { + DafnyToAwsSdkConversionFunctionWriter.writeConverterForShapeAndMembers( + shape, context, writer); + AwsSdkToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers( + shape, context, writer); + + // Import the dafny_to_aws_sdk converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + shape.getId().getNamespace(), context); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".aws_sdk_to_dafny"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.dafny_to_aws_sdk.DafnyToAwsSdk_example_namespace_ExampleShape(input)` + return "%1$s.aws_sdk_to_dafny.%2$s(%3$s)" + .formatted( + pythonModuleSmithygeneratedPath, + AwsSdkNameResolver.getAwsSdkToDafnyFunctionNameForShape(shape), + dataSource); +} + + @Override + public String timestampShape(TimestampShape shape) { + // TODO-Python BLOCKING: This lets code generate, but will fail when code uses it + return "TypeError(\"TimestampShape not supported\")"; + } + + @Override + public String unionShape(UnionShape unionShape) { + DafnyToAwsSdkConversionFunctionWriter.writeConverterForShapeAndMembers( + unionShape, context, writer); + AwsSdkToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers( + unionShape, context, writer); + + // Import the converter from where the ShapeVisitor was called + String pythonModuleName = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + unionShape.getId().getNamespace(), context); + writer.addStdlibImport(pythonModuleName + ".aws_sdk_to_dafny"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.aws_sdk_to_dafny.example_namespace_ExampleShape(input)` + return "%1$s.aws_sdk_to_dafny.%2$s(%3$s)" + .formatted( + pythonModuleName, + AwsSdkNameResolver.getAwsSdkToDafnyFunctionNameForShape(unionShape), + dataSource); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java new file mode 100644 index 0000000000..cb2bc36578 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java @@ -0,0 +1,277 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.awssdk.shapevisitor; + +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.awssdk.shapevisitor.conversionwriters.AwsSdkToDafnyConversionFunctionWriter; +import software.amazon.polymorph.smithypython.awssdk.shapevisitor.conversionwriters.DafnyToAwsSdkConversionFunctionWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.shapes.BigDecimalShape; +import software.amazon.smithy.model.shapes.BigIntegerShape; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.BooleanShape; +import software.amazon.smithy.model.shapes.ByteShape; +import software.amazon.smithy.model.shapes.DoubleShape; +import software.amazon.smithy.model.shapes.EnumShape; +import software.amazon.smithy.model.shapes.FloatShape; +import software.amazon.smithy.model.shapes.IntegerShape; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.LongShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.ShortShape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.TimestampShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * ShapeVisitor that should be dispatched from a shape to generate code that maps a Dafny shape's + * internal attributes to the corresponding AWS SDK kwarg-indexed dictionary. + */ +public class DafnyToAwsSdkShapeVisitor extends ShapeVisitor.Default { + private final GenerationContext context; + private final PythonWriter writer; + private final String dataSource; + + /** + * @param context The generation context. + * @param dataSource The in-code location of the data to provide an output of ({@code output.foo}, + * {@code entry}, etc.) + * @param writer A PythonWriter pointing to the in-code location where the ShapeVisitor was called + * from + */ + public DafnyToAwsSdkShapeVisitor( + GenerationContext context, String dataSource, PythonWriter writer) { + this.context = context; + this.dataSource = dataSource; + this.writer = writer; + } + + @Override + protected String getDefault(Shape shape) { + String protocolName = context.protocolGenerator().getName(); + throw new CodegenException( + String.format( + "Unsupported conversion of %s to %s using the %s protocol", + shape, shape.getType(), protocolName)); + } + + @Override + public String blobShape(BlobShape shape) { + return "bytes(%1$s)".formatted(dataSource); + } + + @Override + public String structureShape(StructureShape structureShape) { + AwsSdkToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers( + structureShape, context, writer); + DafnyToAwsSdkConversionFunctionWriter.writeConverterForShapeAndMembers( + structureShape, context, writer); + + // Import the converter from where the ShapeVisitor was called + String pythonModuleName = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + structureShape.getId().getNamespace(), context); + writer.addStdlibImport(pythonModuleName + ".dafny_to_aws_sdk"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.dafny_to_aws_sdk.example_namespace_ExampleShape(input)` + return "%1$s.dafny_to_aws_sdk.%2$s(%3$s)" + .formatted( + pythonModuleName, + AwsSdkNameResolver.getDafnyToAwsSdkFunctionNameForShape(structureShape), + dataSource); + } + + @Override + public String listShape(ListShape shape) { + StringBuilder builder = new StringBuilder(); + + // Open list: + // `[` + builder.append("["); + MemberShape memberShape = shape.getMember(); + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + + // Add converted list elements into the list: + // `[list_element for list_element in `DafnyToSmithy(targetShape)`` + builder.append( + "%1$s" + .formatted( + targetShape.accept( + new DafnyToAwsSdkShapeVisitor(context, "list_element", writer)))); + + // Close structure: + // `[list_element for list_element in `DafnyToSmithy(targetShape)`]` + return builder.append(" for list_element in %1$s]".formatted(dataSource)).toString(); + } + + @Override + public String mapShape(MapShape shape) { + StringBuilder builder = new StringBuilder(); + + // Open map: + // `{` + builder.append("{"); + MemberShape keyMemberShape = shape.getKey(); + final Shape keyTargetShape = context.model().expectShape(keyMemberShape.getTarget()); + MemberShape valueMemberShape = shape.getValue(); + final Shape valueTargetShape = context.model().expectShape(valueMemberShape.getTarget()); + + // Write converted map keys into the map: + // `{`DafnyToSmithy(key)`:` + builder.append( + "%1$s: " + .formatted( + keyTargetShape.accept(new DafnyToAwsSdkShapeVisitor(context, "key", writer)))); + + // Write converted map values into the map: + // `{`DafnyToSmithy(key)`: `DafnyToSmithy(value)`` + builder.append( + "%1$s" + .formatted( + valueTargetShape.accept(new DafnyToAwsSdkShapeVisitor(context, "value", writer)))); + + // Complete map comprehension and close map + // `{`DafnyToSmithy(key)`: `DafnyToSmithy(value)`` for (key, value) in `dataSource`.items }` + // No () on items call; `dataSource` is a Dafny map, where `items` is a @property and not a + // method. + return builder.append(" for (key, value) in %1$s.items }".formatted(dataSource)).toString(); + } + + @Override + public String booleanShape(BooleanShape shape) { + return dataSource; + } + + @Override + public String stringShape(StringShape shape) { + if (shape.hasTrait(EnumTrait.class)) { + DafnyToAwsSdkConversionFunctionWriter.writeConverterForShapeAndMembers( + shape, context, writer); + AwsSdkToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers( + shape, context, writer); + + // Import the dafny_to_aws_sdk converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + shape.getId().getNamespace(), context); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".dafny_to_aws_sdk"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.dafny_to_aws_sdk.example_namespace_ExampleShape(input)` + return "%1$s.dafny_to_aws_sdk.%2$s(%3$s)" + .formatted( + pythonModuleSmithygeneratedPath, + AwsSdkNameResolver.getDafnyToAwsSdkFunctionNameForShape(shape), + dataSource); + } + return dataSource + ".VerbatimString(False)"; + } + + @Override + public String byteShape(ByteShape shape) { + return getDefault(shape); + } + + @Override + public String shortShape(ShortShape shape) { + return getDefault(shape); + } + + @Override + public String integerShape(IntegerShape shape) { + return dataSource; + } + + @Override + public String longShape(LongShape shape) { + return dataSource; + } + + @Override + public String bigIntegerShape(BigIntegerShape shape) { + return getDefault(shape); + } + + @Override + public String floatShape(FloatShape shape) { + return getDefault(shape); + } + + @Override + public String doubleShape(DoubleShape shape) { + return dataSource; + } + + @Override + public String bigDecimalShape(BigDecimalShape shape) { + return getDefault(shape); + } + + @Override + public String enumShape(EnumShape shape) { + DafnyToAwsSdkConversionFunctionWriter.writeConverterForShapeAndMembers( + shape, context, writer); + AwsSdkToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers( + shape, context, writer); + + // Import the dafny_to_aws_sdk converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + shape.getId().getNamespace(), context); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".dafny_to_aws_sdk"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.dafny_to_aws_sdk.example_namespace_ExampleShape(input)` + return "%1$s.dafny_to_aws_sdk.%2$s(%3$s)" + .formatted( + pythonModuleSmithygeneratedPath, + AwsSdkNameResolver.getDafnyToAwsSdkFunctionNameForShape(shape), + dataSource); + } + + @Override + public String timestampShape(TimestampShape shape) { + // TODO-Python BLOCKING: This lets code generate, but will fail when code hits it + return "TypeError(\"TimestampShape not supported\")"; + } + + @Override + public String unionShape(UnionShape unionShape) { + DafnyToAwsSdkConversionFunctionWriter.writeConverterForShapeAndMembers( + unionShape, context, writer); + AwsSdkToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers( + unionShape, context, writer); + + // Import the dafny_to_aws_sdk converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + unionShape.getId().getNamespace(), context); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".dafny_to_aws_sdk"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.dafny_to_aws_sdk.example_namespace_ExampleShape(input)` + return "%1$s.dafny_to_aws_sdk.%2$s(%3$s)" + .formatted( + pythonModuleSmithygeneratedPath, + AwsSdkNameResolver.getDafnyToAwsSdkFunctionNameForShape(unionShape), + dataSource); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/conversionwriters/AwsSdkToDafnyConversionFunctionWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/conversionwriters/AwsSdkToDafnyConversionFunctionWriter.java new file mode 100644 index 0000000000..b9e0f4b199 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/conversionwriters/AwsSdkToDafnyConversionFunctionWriter.java @@ -0,0 +1,398 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.awssdk.shapevisitor.conversionwriters; + +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.awssdk.shapevisitor.AwsSdkToDafnyShapeVisitor; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.shapevisitor.conversionwriter.BaseConversionWriter; +import software.amazon.smithy.codegen.core.WriterDelegator; +import software.amazon.smithy.model.shapes.*; +import software.amazon.smithy.model.traits.EnumDefinition; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +import java.util.Map.Entry; + +/** Writes the aws_sdk_to_dafny.py file via the BaseConversionWriter implementation. */ +public class AwsSdkToDafnyConversionFunctionWriter extends BaseConversionWriter { + + // Use a singleton to preserve generatedShapes through multiple generations + static AwsSdkToDafnyConversionFunctionWriter singleton; + + // Instantiate singleton at class-load time + static { + singleton = new AwsSdkToDafnyConversionFunctionWriter(); + } + + private AwsSdkToDafnyConversionFunctionWriter() {} + + /** + * Delegate writing conversion methods for the provided shape and its member shapes + * + * @param shape + * @param context + * @param writer + */ + public static void writeConverterForShapeAndMembers( + Shape shape, GenerationContext context, PythonWriter writer) { + singleton.baseWriteConverterForShapeAndMembers(shape, context, writer); + } + + protected void writeStructureShapeConverter(StructureShape structureShape) { + if (structureShape.hasTrait(ErrorTrait.class)) { + writeErrorStructureShapeConverter(structureShape); + return; + } + + WriterDelegator delegator = context.writerDelegator(); + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()); + + delegator.useFileWriter( + moduleName + "/aws_sdk_to_dafny.py", + "", + conversionWriter -> { + // Within the conversion function, the dataSource becomes the function's input + // This hardcodes the input parameter neme for a conversion function to always be "native_input" + + DafnyNameResolver.importDafnyTypeForShape( + conversionWriter, structureShape.getId(), context); + + conversionWriter.openBlock( + "def $L($L):", + "", + AwsSdkNameResolver.getAwsSdkToDafnyFunctionNameForShape(structureShape), + "native_input", + () -> { + // Open Dafny structure shape + // e.g. + // DafnyStructureName(... + String dataSourceInsideConversionFunction = "native_input"; + conversionWriter.openBlock( + "return $L(", + ")", + structureShape.hasTrait(ErrorTrait.class) + ? DafnyNameResolver.getDafnyTypeForError(structureShape) + : DafnyNameResolver.getDafnyTypeForShape(structureShape), + () -> { + for (Entry memberShapeEntry : + structureShape.getAllMembers().entrySet()) { + String memberName = memberShapeEntry.getKey(); + MemberShape memberShape = memberShapeEntry.getValue(); + + if (structureShape.hasTrait(ErrorTrait.class)) { + writeErrorStructureShapeMemberConverter( + conversionWriter, + dataSourceInsideConversionFunction, + memberName, + memberShape); + } else { + + writeStructureShapeMemberConverter( + conversionWriter, + dataSourceInsideConversionFunction, + memberName, + memberShape); + } + } + }); + }); + }); + } + + protected void writeErrorStructureShapeConverter(StructureShape structureShape) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()); + + delegator.useFileWriter( + moduleName + "/aws_sdk_to_dafny.py", + "", + conversionWriter -> { + // Within the conversion function, the dataSource becomes the function's input + // This hardcodes the input parameter name for a conversion function to always be "native_input" + String dataSourceInsideConversionFunction = "native_input"; + + DafnyNameResolver.importDafnyTypeForShape( + conversionWriter, structureShape.getId(), context); + + conversionWriter.openBlock( + "def $L($L):", + "", + AwsSdkNameResolver.getAwsSdkToDafnyFunctionNameForShape(structureShape), + dataSourceInsideConversionFunction, + () -> { + // Open Dafny structure shape + // e.g. + // DafnyStructureName(... + conversionWriter.openBlock( + "return $L(", + ")", + structureShape.hasTrait(ErrorTrait.class) + ? DafnyNameResolver.getDafnyTypeForError(structureShape) + : DafnyNameResolver.getDafnyTypeForShape(structureShape), + () -> { + for (Entry memberShapeEntry : + structureShape.getAllMembers().entrySet()) { + String memberName = memberShapeEntry.getKey(); + MemberShape memberShape = memberShapeEntry.getValue(); + String memberShapeDataSource; + if (shapeMemberIsOnResponseRoot(memberShape, structureShape)) { + memberShapeDataSource = dataSourceInsideConversionFunction; + } else { + memberShapeDataSource = dataSourceInsideConversionFunction + "['Error']"; + } + writeErrorStructureShapeMemberConverter( + conversionWriter, + memberShapeDataSource, + memberName, + memberShape); + } + }); + }); + }); + } + + private boolean shapeMemberIsOnResponseRoot(MemberShape memberShape, StructureShape structureShape) { + // Case: TransactionCanceledException.CancellationReasons + if ("CancellationReasons".equals(memberShape.getMemberName()) + && "TransactionCanceledException".equals(structureShape.getId().getName()) + && "com.amazonaws.dynamodb".equals(structureShape.getId().getNamespace())) { + return true; + } + return false; + } + + private void writeErrorStructureShapeMemberConverter( + PythonWriter conversionWriter, + String dataSourceInsideConversionFunction, + String memberName, + MemberShape memberShape) { + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + + // Adds `DafnyStructureMember=smithy_structure_member(...)` + // e.g. + // DafnyStructureName(DafnyStructureMember=smithy_structure_member(...), ...) + // The nature of the `smithy_structure_member` conversion depends on the properties of the + // shape, + // as described below + conversionWriter.writeInline("$L=", memberName); + + // `message` members on Error shapes are actually accessed as `Message` + if ("message".equals(memberName)) { + memberName = "Message"; + } + + // Error structure members are always required + conversionWriter.write( + "$L,", + targetShape.accept( + new AwsSdkToDafnyShapeVisitor( + context, + dataSourceInsideConversionFunction + "[\"" + memberName + "\"]", + conversionWriter))); + } + + private void writeStructureShapeMemberConverter( + PythonWriter conversionWriter, + String dataSourceInsideConversionFunction, + String memberName, + MemberShape memberShape) { + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + + // Adds `DafnyStructureMember=smithy_structure_member(...)` + // e.g. + // DafnyStructureName(DafnyStructureMember=smithy_structure_member(...), ...) + // The nature of the `smithy_structure_member` conversion depends on the properties of the + // shape, + // as described below + conversionWriter.writeInline("$L=", memberName); + + // If this shape is optional, write conversion logic to detect and possibly pass + // an empty optional at runtime + if (memberShape.isOptional()) { + conversionWriter.addStdlibImport("Wrappers", "Option_Some"); + conversionWriter.addStdlibImport("Wrappers", "Option_None"); + conversionWriter.write( + "Option_Some($L) if \"$L\" in $L.keys() else Option_None(),", + targetShape.accept( + new AwsSdkToDafnyShapeVisitor( + context, + dataSourceInsideConversionFunction + "[\"" + memberName + "\"]", + conversionWriter)), + memberName, + dataSourceInsideConversionFunction); + } + + // If this shape is required, pass in the shape for conversion without any optional-checking + else { + conversionWriter.write( + "$L,", + targetShape.accept( + new AwsSdkToDafnyShapeVisitor( + context, + dataSourceInsideConversionFunction + "[\"" + memberName + "\"]", + conversionWriter))); + } + } + + /** + * Writes a function definition to convert a Smithy-modelled union shape into the corresponding + * Dafny-modelled union shape. The function definition is written into `aws_sdk_to_dafny.py`. This + * SHOULD only be called once so only one function definition is written. + * + * @param unionShape + */ + protected void writeUnionShapeConverter(UnionShape unionShape) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()); + + delegator.useFileWriter( + moduleName + "/aws_sdk_to_dafny.py", + "", + conversionWriter -> { + + // Within the conversion function, the dataSource becomes the function's input + // This hardcodes the input parameter name for a conversion function to always be "native_input" + String dataSourceInsideConversionFunction = "native_input"; + + // ex. shape: simple.union.ExampleUnion + // Writes `def SmithyToDafny_simple_union_ExampleUnion(input):` + // and wraps inner code inside function definition + conversionWriter.openBlock( + "def $L($L):", + "", + AwsSdkNameResolver.getAwsSdkToDafnyFunctionNameForShape(unionShape), + dataSourceInsideConversionFunction, + () -> { + + // First union value opens a new `if` block; others do not need to and write `elif` + boolean shouldOpenNewIfBlock = true; + for (MemberShape memberShape : unionShape.getAllMembers().values()) { + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + // Write out conversion: + // ex. if ExampleUnion can take on either of (IntegerValue, StringValue), write: + // if isinstance(input, ExampleUnion.IntegerValue): + // example_union_union_value = DafnyExampleUnionIntegerValue(input.member.value) + // elif isinstance(input, ExampleUnion.StringValue): + // example_union_union_value = DafnyExampleUnionIntegerValue(input.member.value) + conversionWriter.write( + """ + $L "$L" in $L.keys(): + $L_union_value = $L($L)""", + // If we need a new `if` block, open one; otherwise, expand on existing one + // with `elif` + shouldOpenNewIfBlock ? "if" : "elif", + memberShape.getMemberName(), + dataSourceInsideConversionFunction, + unionShape.getId().getName(), + DafnyNameResolver.getDafnyTypeForUnion(unionShape, memberShape), + targetShape.accept( + new AwsSdkToDafnyShapeVisitor( + context, + dataSourceInsideConversionFunction + + "[\"" + + memberShape.getMemberName() + + "\"]", + conversionWriter))); + shouldOpenNewIfBlock = false; + + DafnyNameResolver.importDafnyTypeForUnion( + conversionWriter, unionShape, memberShape); + } + + // Write case to handle if union member does not match any of the above cases + conversionWriter.write( + """ + else: + raise ValueError("No recognized union value in union type: " + str($L)) + """, + dataSourceInsideConversionFunction); + + // Return the result of the union conversion + // `return example_union_union_value` + conversionWriter.write( + "return %1$s_union_value".formatted(unionShape.getId().getName())); + }); + }); + } + + /** + * @param stringShapeWithEnumTrait + */ + public void writeStringEnumShapeConverter(StringShape stringShapeWithEnumTrait) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()); + + // Write out common conversion function inside dafny_to_aws_sdk + delegator.useFileWriter( + moduleName + "/aws_sdk_to_dafny.py", + "", + conversionWriter -> { + + // Within the conversion function, the dataSource becomes the function's input + String dataSourceInsideConversionFunction = "native_input"; + + // ex. shape: simple.union.ExampleUnion + // Writes `def DafnyToSmithy_simple_union_ExampleUnion(input):` + // and wraps inner code inside function definition + conversionWriter.openBlock( + "def $L($L):", + "", + AwsSdkNameResolver.getAwsSdkToDafnyFunctionNameForShape(stringShapeWithEnumTrait), + dataSourceInsideConversionFunction, + () -> { + conversionWriter.writeComment( + "Convert %1$s".formatted(stringShapeWithEnumTrait.getId().getName())); + + // First union value opens a new `if` block; others do not need to and write `elif` + boolean shouldOpenNewIfBlock = true; + // Write out conversion: + // ex. if ExampleUnion can take on either of (IntegerValue, StringValue), write: + // if isinstance(input, ExampleUnion_IntegerValue): + // ExampleUnion_union_value = ExampleUnionIntegerValue(input.IntegerValue) + // elif isinstance(input, ExampleUnion_StringValue): + // ExampleUnion_union_value = ExampleUnionStringValue(input.StringValue) + + for (EnumDefinition enumDefinition : + stringShapeWithEnumTrait.getTrait(EnumTrait.class).get().getValues()) { + String value = enumDefinition.getValue(); + conversionWriter.write( + """ + $L $L == "$L": + return $L()""", + // If we need a new `if` block, open one; otherwise, expand on existing one + // with `elif` + shouldOpenNewIfBlock ? "if" : "elif", + dataSourceInsideConversionFunction, + value, + DafnyNameResolver.getDafnyTypeForStringShapeWithEnumTrait( + stringShapeWithEnumTrait, value)); + shouldOpenNewIfBlock = false; + + DafnyNameResolver.importDafnyTypeForStringShapeWithEnumTrait( + conversionWriter, stringShapeWithEnumTrait, value); + } + + // Write case to handle if union member does not match any of the above cases + conversionWriter.write( + """ + else: + raise ValueError("No recognized enum value in enum type: " + $L) + """, + dataSourceInsideConversionFunction); + }); + }); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/conversionwriters/DafnyToAwsSdkConversionFunctionWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/conversionwriters/DafnyToAwsSdkConversionFunctionWriter.java new file mode 100644 index 0000000000..eddeda234c --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/conversionwriters/DafnyToAwsSdkConversionFunctionWriter.java @@ -0,0 +1,295 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.awssdk.shapevisitor.conversionwriters; + +import java.util.Map.Entry; +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.awssdk.shapevisitor.DafnyToAwsSdkShapeVisitor; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.shapevisitor.conversionwriter.BaseConversionWriter; +import software.amazon.smithy.codegen.core.WriterDelegator; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.EnumDefinition; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** Writes the dafny_to_aws_sdk.py file via the BaseConversionWriter implementation. */ +public class DafnyToAwsSdkConversionFunctionWriter extends BaseConversionWriter { + + // Use a singleton to preserve generatedShapes through multiple generations + static DafnyToAwsSdkConversionFunctionWriter singleton; + + // Instantiate singleton at class-load time + static { + singleton = new DafnyToAwsSdkConversionFunctionWriter(); + } + + private DafnyToAwsSdkConversionFunctionWriter() {} + + /** + * Delegate writing conversion methods for the provided shape and its member shapes + * + * @param shape + * @param context + * @param writer + */ + public static void writeConverterForShapeAndMembers( + Shape shape, GenerationContext context, PythonWriter writer) { + singleton.baseWriteConverterForShapeAndMembers(shape, context, writer); + } + + protected void writeStructureShapeConverter(StructureShape structureShape) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()); + // Create a new writer. + // The `writer` on this object points to the location in the code where this converter was + // called. + // This new writer to writes the dafny_to_aws_sdk converter function. + delegator.useFileWriter( + moduleName + "/dafny_to_aws_sdk.py", + "", + conversionWriter -> { + + // Within the conversion function, the dataSource becomes the function's input + // This hardcodes the input parameter name for a conversion function to always be "dafny_input" + String dataSourceInsideConversionFunction = "dafny_input"; + + conversionWriter.openBlock( + "def $L($L):", + "", + AwsSdkNameResolver.getDafnyToAwsSdkFunctionNameForShape(structureShape), + dataSourceInsideConversionFunction, + () -> { + // boto3 takes in kwargs + // Create a dictionary indexed by keys, then cast to kwargs in API call + conversionWriter.write("output = {}"); + + // Recursively dispatch a new ShapeVisitor for each member of the structure + for (Entry memberShapeEntry : + structureShape.getAllMembers().entrySet()) { + String memberName = memberShapeEntry.getKey(); + MemberShape memberShape = memberShapeEntry.getValue(); + writeStructureShapeMemberConverter( + conversionWriter, + dataSourceInsideConversionFunction, + memberName, + memberShape); + } + + conversionWriter.write("return output"); + }); + }); + } + + private void writeStructureShapeMemberConverter( + PythonWriter conversionWriter, + String dataSourceInsideConversionFunction, + String memberName, + MemberShape memberShape) { + + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + + // Optional shapes require an `is_Some` check and their value is accessed via `.value` + // ex. kms.KeyId in DecryptRequest (optional parameter): + // if input.KeyId.is_Some: + // output["KeyId"] = input.KeyId.value.VerbatimString(False) + // (`VerbatimString(False)` comes from the DafnyToAwsSdkShapeVisitor) + if (memberShape.isOptional()) { + conversionWriter.openBlock( + "if $L.$L.is_Some:", + "", + dataSourceInsideConversionFunction, + memberName, + () -> { + conversionWriter.write( + "output[\"$L\"] = $L", + memberName, + targetShape.accept( + new DafnyToAwsSdkShapeVisitor( + context, + dataSourceInsideConversionFunction + "." + memberName + ".value", + conversionWriter))); + }); + // Required shapes are assigned directly + // ex. kms.CiphertextBlob in DecryptRequest (required parameter): + // output["CiphertextBlob"] = bytes(input.CiphertextBlob) + // (`bytes()` comes from the DafnyToAwsSdkShapeVisitor) + } else { + conversionWriter.write( + "output[\"$L\"] = $L", + memberName, + targetShape.accept( + new DafnyToAwsSdkShapeVisitor( + context, + dataSourceInsideConversionFunction + "." + memberName, + conversionWriter))); + } + } + + /** + * Writes a function definition to convert a Dafny-modelled union shape into the corresponding + * Smithy-modelled union shape. The function definition is written into `dafny_to_aws_sdk.py`. + * This SHOULD only be called once so only one function definition is written. + * + * @param unionShape + */ + public void writeUnionShapeConverter(UnionShape unionShape) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()); + + // Write out common conversion function inside dafny_to_aws_sdk + delegator.useFileWriter( + moduleName + "/dafny_to_aws_sdk.py", + "", + conversionWriter -> { + + // Within the conversion function, the dataSource becomes the function's input + String dataSourceInsideConversionFunction = "dafny_input"; + + // ex. shape: simple.union.ExampleUnion + // Writes `def DafnyToSmithy_simple_union_ExampleUnion(input):` + // and wraps inner code inside function definition + conversionWriter.openBlock( + "def $L($L):", + "", + AwsSdkNameResolver.getDafnyToAwsSdkFunctionNameForShape(unionShape), + dataSourceInsideConversionFunction, + () -> { + conversionWriter.writeComment( + "Convert %1$s".formatted(unionShape.getId().getName())); + + // First union value opens a new `if` block; others do not need to and write `elif` + boolean shouldOpenNewIfBlock = true; + // Write out conversion: + // ex. if ExampleUnion can take on either of (IntegerValue, StringValue), write: + // if isinstance(input, ExampleUnion_IntegerValue): + // ExampleUnion_union_value = ExampleUnionIntegerValue(input.IntegerValue) + // elif isinstance(input, ExampleUnion_StringValue): + // ExampleUnion_union_value = ExampleUnionStringValue(input.StringValue) + for (MemberShape memberShape : unionShape.getAllMembers().values()) { + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + conversionWriter.write( + """ + $L isinstance($L, $L): + $L_union_value = {"$L": $L}""", + // If we need a new `if` block, open one; otherwise, expand on existing one + // with `elif` + shouldOpenNewIfBlock ? "if" : "elif", + dataSourceInsideConversionFunction, + DafnyNameResolver.getDafnyTypeForUnion(unionShape, memberShape), + unionShape.getId().getName(), + memberShape.getMemberName(), + targetShape.accept( + new DafnyToAwsSdkShapeVisitor( + context, + dataSourceInsideConversionFunction + + "." + + memberShape.getMemberName(), + conversionWriter))); + shouldOpenNewIfBlock = false; + + DafnyNameResolver.importDafnyTypeForUnion( + conversionWriter, unionShape, memberShape); + } + + // Write case to handle if union member does not match any of the above cases + conversionWriter.write( + """ + else: + raise ValueError("No recognized union value in union type: " + str($L)) + """, + dataSourceInsideConversionFunction); + + // Write return value: + // `return ExampleUnion_union_value` + conversionWriter.write( + """ + return $L_union_value + """, + unionShape.getId().getName()); + }); + }); + } + + /** + * @param stringShapeWithEnumTrait + */ + public void writeStringEnumShapeConverter(StringShape stringShapeWithEnumTrait) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()); + + // Write out common conversion function inside dafny_to_aws_sdk + delegator.useFileWriter( + moduleName + "/dafny_to_aws_sdk.py", + "", + conversionWriter -> { + + // Within the conversion function, the dataSource becomes the function's input + String dataSourceInsideConversionFunction = "dafny_input"; + + // ex. shape: simple.union.ExampleUnion + // Writes `def DafnyToSmithy_simple_union_ExampleUnion(input):` + // and wraps inner code inside function definition + conversionWriter.openBlock( + "def $L($L):", + "", + AwsSdkNameResolver.getDafnyToAwsSdkFunctionNameForShape(stringShapeWithEnumTrait), + dataSourceInsideConversionFunction, + () -> { + conversionWriter.writeComment( + "Convert %1$s".formatted(stringShapeWithEnumTrait.getId().getName())); + + // First union value opens a new `if` block; others do not need to and write `elif` + boolean shouldOpenNewIfBlock = true; + // Write out conversion: + // ex. if ExampleUnion can take on either of (IntegerValue, StringValue), write: + // if isinstance(input, ExampleUnion_IntegerValue): + // ExampleUnion_union_value = ExampleUnionIntegerValue(input.IntegerValue) + // elif isinstance(input, ExampleUnion_StringValue): + // ExampleUnion_union_value = ExampleUnionStringValue(input.StringValue) + + for (EnumDefinition enumDefinition : + stringShapeWithEnumTrait.getTrait(EnumTrait.class).get().getValues()) { + String value = enumDefinition.getValue(); + conversionWriter.write( + """ + $L isinstance($L, $L): + return "$L" + """, + // If we need a new `if` block, open one; otherwise, expand on existing one + // with `elif` + shouldOpenNewIfBlock ? "if" : "elif", + dataSourceInsideConversionFunction, + DafnyNameResolver.getDafnyTypeForStringShapeWithEnumTrait( + stringShapeWithEnumTrait, value), + value); + shouldOpenNewIfBlock = false; + + DafnyNameResolver.importDafnyTypeForStringShapeWithEnumTrait( + conversionWriter, stringShapeWithEnumTrait, value); + } + + // Write case to handle if union member does not match any of the above cases + conversionWriter.write( + """ + else: + raise ValueError("No recognized enum value in enum type: " + $L) + """, + dataSourceInsideConversionFunction); + }); + }); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/customize/CustomFileWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/customize/CustomFileWriter.java new file mode 100644 index 0000000000..50755d65f3 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/customize/CustomFileWriter.java @@ -0,0 +1,24 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.common.customize; + +import java.util.Set; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.GenerationContext; + +/** Interface for writing custom Dafny-Python code to files. */ +public interface CustomFileWriter { + + /** + * Writes code specific to the file targeted by the CustomFileWriter for the provided + * ServiceShape. The ServiceShape SHOULD be a shape annotated with the + * `aws.polymorph#localService` trait. + * + * @param serviceShape + * @param codegenContext + */ + default void customizeFileForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) {} +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/nameresolver/DafnyNameResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/nameresolver/DafnyNameResolver.java new file mode 100644 index 0000000000..bbd3d3d351 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/nameresolver/DafnyNameResolver.java @@ -0,0 +1,386 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.common.nameresolver; + +import java.util.Locale; +import java.util.Optional; +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.traits.PositionalTrait; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.ResourceShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * Contains utility functions that map Smithy shapes to an expected corresponding string generated + * by Dafny's Python compiler. i.e. strings in this file match behavior of Dafny-generated code. + */ +public class DafnyNameResolver { + + /** + * Returns the name of the Python module containing Dafny-generated Python code from the `types` + * module from the same Dafny project for the provided Shape. ex. example.namespace.ExampleShape + * -> "example_namespace_internaldafny_types" + * + * @param shape + * @return + */ + public static String getDafnyPythonTypesModuleNameForShape(Shape shape) { + return getDafnyPythonTypesModuleNameForShape(shape.getId()); + } + + /** + * Returns the name of the Python module containing Dafny-generated Python code from the `types` + * module from the same Dafny project for the provided Shape. ex. example.namespace.ExampleShape + * -> "example_namespace_internaldafny_types" + * + * @param shapeId + * @return + */ + public static String getDafnyPythonTypesModuleNameForShape(ShapeId shapeId) { + return getDafnyTypesModuleNameForSmithyNamespace(shapeId.getNamespace()); + } + + /** + * Returns the name of the Python module containing Dafny-generated Python code from the `index` + * module from the same Dafny project for the provided Shape. ex. example.namespace.ExampleShape + * -> "example_namespace_internaldafny" + * + * @param shape + * @return + */ + public static String getDafnyPythonIndexModuleNameForShape(Shape shape) { + return getDafnyPythonIndexModuleNameForShape(shape.getId()); + } + + /** + * Returns the name of the Python module containing Dafny-generated Python code from the `index` + * module from the same Dafny project for the provided Shape. ex. example.namespace.ExampleShape + * -> "example_namespace_internaldafny" + * + * @param shapeId + * @return + */ + public static String getDafnyPythonIndexModuleNameForShape(ShapeId shapeId) { + return getDafnyIndexModuleNameForSmithyNamespace(shapeId.getNamespace()); + } + + /** + * Returns the name of the Python module containing Dafny-generated Python code from the `index` + * module from the same Dafny project for the provided smithyNamespace. ex. + * example.namespace.ExampleShape -> "example_namespace_internaldafny" + * + * @param smithyNamespace + * @return + */ + public static String getDafnyIndexModuleNameForSmithyNamespace(String smithyNamespace) { + // If this is an AWS SDK shape, rewrite its namespace to match the Dafny extern namespace + String resolvedSmithyNamespace = + AwsSdkNameResolver.resolveAwsSdkSmithyModelNamespaceToDafnyExternNamespace(smithyNamespace); + return resolvedSmithyNamespace.toLowerCase(Locale.ROOT).replace(".", "_") + "_internaldafny"; + } + + /** + * Returns the name of the Python module containing Dafny-generated Python code from the `types` + * module from the same Dafny project for the provided smithyNamespace. ex. + * example.namespace.ExampleShape -> "example_namespace_internaldafny_types" + * + * @param smithyNamespace + * @return + */ + public static String getDafnyTypesModuleNameForSmithyNamespace(String smithyNamespace) { + return getDafnyIndexModuleNameForSmithyNamespace(smithyNamespace) + "_types"; + } + + /** + * Returns a String representing the corresponding Dafny type for the provided shape. This MUST + * NOT be used for errors; for errors use `getDafnyTypeForError`. ex. + * example.namespace.ExampleShape -> "DafnyExampleShape" + * + * @param shapeId + * @return + */ + public static String getDafnyTypeForShape(ShapeId shapeId) { + if (Utils.isUnitShape(shapeId)) { + // Dafny models Unit shapes as the Python `None` type + return "None"; + } else { + // Catch-all: Return `Dafny[shapeName]` + return "Dafny" + shapeId.getName(); + } + } + + /** + * Returns a String representing the Dafny-generated Python type corresponding to the provided + * Shape. ex. example.namespace.ExampleShape -> "DafnyExampleShape" + * + * @param shape + * @return + */ + public static String getDafnyTypeForShape(Shape shape) { + return getDafnyTypeForShape(shape.getId()); + } + + /** + * Returns a String representing the Dafny-generated Python type corresponding to the provided + * Shape. ex. example.namespace.ExampleShape -> "DafnyExampleShape" + * + * @param shape + * @return + */ + public static String getDafnyTypeForStringShapeWithEnumTrait( + StringShape stringShape, String enumValue) { + if (!stringShape.hasTrait(EnumTrait.class) || !stringShape.isStringShape()) { + throw new IllegalArgumentException( + "Argument is not a StringShape with EnumTrait: " + stringShape.getId()); + } + + return stringShape.getId().getName() + "_" + enumValue.replace("_", "__"); + } + + public static void importDafnyTypeForStringShapeWithEnumTrait( + PythonWriter writer, StringShape stringShape, String enumValue) { + if (!stringShape.hasTrait(EnumTrait.class) || !stringShape.isStringShape()) { + throw new IllegalArgumentException( + "Argument is not a StringShape with EnumTrait: " + stringShape.getId()); + } + + // When generating a Dafny import, must ALWAYS first import module_ to avoid circular + // dependencies + writer.addStdlibImport("module_"); + writer.addStdlibImport( + getDafnyTypesModuleNameForSmithyNamespace(stringShape.getId().getNamespace()), + getDafnyTypeForStringShapeWithEnumTrait(stringShape, enumValue)); + } + + /** + * Imports the Dafny-generated Python type corresponding to the provided shape. ex. + * example.namespace.ExampleShape -> "from example_namespace_internaldafny_types import + * DafnyExampleShape" + * + * @param shape + * @return + */ + private static void importDafnyTypeForShape( + PythonWriter writer, Shape shape, GenerationContext context) { + importDafnyTypeForShape(writer, shape.getId(), context); + } + + /** + * Calls writer.addImport to import the corresponding Dafny type for the provided Smithy ShapeId. + * This MUST NOT be used to import errors; use `importDafnyTypeForError`. ex. + * example.namespace.ExampleShape -> "from example_namespace_internaldafny_types import + * DafnyExampleShape" + * + * @param writer + * @param shapeId + */ + public static void importDafnyTypeForShape( + PythonWriter writer, ShapeId shapeId, GenerationContext context) { + if (context.model().expectShape(shapeId).hasTrait(ErrorTrait.class)) { + importDafnyTypeForError(writer, shapeId, context); + } else if (context.model().expectShape(shapeId).hasTrait(PositionalTrait.class)) { + Optional maybeStructureShape = + context.model().expectShape(shapeId).asStructureShape(); + if (maybeStructureShape.isEmpty()) { + throw new IllegalArgumentException( + "PositionalShapes can only be applied to StructureShapes; was applied to " + shapeId); + } + final MemberShape onlyMember = PositionalTrait.onlyMember(maybeStructureShape.get()); + // writer.addStdlibImport(getDafnyPythonTypesModuleNameForShape(onlyMember.getId()), + // onlyMember.getMemberName() + "_" + onlyMember.getMemberName(), + // getDafnyTypeForShape(onlyMember.getId()) ); + // TODO Positional + } else { + // When generating a Dafny import, must ALWAYS first import module_ to avoid circular + // dependencies + writer.addStdlibImport("module_"); + String name = shapeId.getName(); + if (!Utils.isUnitShape(shapeId)) { + writer.addStdlibImport( + getDafnyPythonTypesModuleNameForShape(shapeId), + name.replace("_", "__") + "_" + name.replace("_", "__"), + getDafnyTypeForShape(shapeId)); + } + } + } + + /** + * Returns a String representing the client interface type for the provided serviceShape as Dafny + * models the interface type. ex. example.namespace.ExampleService -> "IExampleServiceClient" + * + * @param serviceShape + * @return + */ + public static String getDafnyClientInterfaceTypeForServiceShape(ServiceShape serviceShape) { + if (AwsSdkNameResolver.isAwsSdkShape(serviceShape)) { + return "I" + AwsSdkNameResolver.clientNameForService(serviceShape); + } else { + return "I" + getDafnyClientTypeForServiceShape(serviceShape); + } + } + + /** + * Returns a String representing the client interface type for the provided serviceShape as Dafny + * models the interface type. ex. example.namespace.ExampleService -> "ExampleServiceClient" + * + * @param serviceShape + * @return + */ + public static String getDafnyClientTypeForServiceShape(ServiceShape serviceShape) { + return serviceShape.getId().getName() + "Client"; + } + + /** + * Returns a String representing the interface type for the provided resourceShape as Dafny models + * the interface type. ex. example.namespace.ExampleResource -> "IExampleResource" + * + * @param resourceShape + * @return + */ + public static String getDafnyInterfaceTypeForResourceShape(ResourceShape resourceShape) { + return "I" + resourceShape.getId().getName(); + } + + /** + * Imports the Dafny-generated Python type corresponding to the provided resourceShape. + * + * @param resourceShape ex. example.namespace.ExampleResource -> "from + * example_namespace_internaldafny_types import IExampleResource" + * @return + */ + public static void importDafnyTypeForResourceShape( + PythonWriter writer, ResourceShape resourceShape) { + // When generating a Dafny import, must ALWAYS first import module_ to avoid circular + // dependencies + writer.addStdlibImport("module_"); + writer.addStdlibImport( + getDafnyPythonTypesModuleNameForShape(resourceShape.getId()), + getDafnyInterfaceTypeForResourceShape(resourceShape)); + } + + /** + * Imports the Dafny-generated Python type corresponding to the provided serviceShape. ex. + * example.namespace.ExampleService -> "from example_namespace_internaldafny_types import + * IExampleServiceClient" + * + * @param serviceShape + * @return + */ + public static void importDafnyTypeForServiceShape( + PythonWriter writer, ServiceShape serviceShape) { + // When generating a Dafny import, must ALWAYS first import module_ to avoid circular + // dependencies + writer.addStdlibImport("module_"); + writer.addStdlibImport( + getDafnyPythonTypesModuleNameForShape(serviceShape.getId()), + getDafnyClientInterfaceTypeForServiceShape(serviceShape)); + } + + /** + * Returns a String representing the corresponding Dafny type for the provided Error shape. This + * MUST ONLY be used for errors; for other shapes use `getDafnyTypeForShape`. ex. + * example.namespace.ExampleError -> "Error_ExampleError" + * + * @param shape + * @return + */ + public static String getDafnyTypeForError(Shape shape) { + return getDafnyTypeForError(shape.getId()); + } + + /** + * Returns a String representing the Dafny-generated Python type corresponding to the provided + * error shape. ex. example.namespace.ExampleError -> "Error_ExampleError" + * + * @param shapeId + * @return + */ + public static String getDafnyTypeForError(ShapeId shapeId) { + return "Error_" + shapeId.getName(); + } + + public static String escapeShapeName(String name) { + if ("none".equalsIgnoreCase(name)) { + return name + "_"; + } + return name.replace("_", "__"); + } + + /** + * Returns a String representing the corresponding Dafny type for the provided UnionShape and one + * of its MemberShapes. This MUST ONLY be used for unions and their members; for other shapes use + * `getDafnyTypeForShape`. ex. example.namespace.ExampleUnion:IntegerValue -> + * "ExampleUnion_IntegerValue" + * + * @param unionShape + * @param memberShape + * @return + */ + public static String getDafnyTypeForUnion(UnionShape unionShape, MemberShape memberShape) { + return unionShape.getId().getName().replace("_", "__") + + "_" + + memberShape.getMemberName().replace("_", "__"); + } + + /** + * Imports the Dafny-generated Python type corresponding to the provided unionShape. ex. + * example.namespace.ExampleUnion:IntegerValue -> "from example_namespace_internaldafny_types + * import ExampleUnion_IntegerValue" + * + * @param unionShape + * @return + */ + public static void importDafnyTypeForUnion( + PythonWriter writer, UnionShape unionShape, MemberShape memberShape) { + writer.addStdlibImport( + getDafnyPythonTypesModuleNameForShape(unionShape), + getDafnyTypeForUnion(unionShape, memberShape)); + } + + /** + * Calls writer.addImport to import the corresponding Dafny type for the provided Smithy ShapeId. + * This MUST ONLY be used for errors; for other shapes use `importDafnyTypeForShape`. ex. + * example.namespace.ExampleUnion:IntegerValue -> "from example_namespace_internaldafny_types + * import ExampleUnion_IntegerValue" + * + * @param writer + * @param shapeId + */ + public static void importDafnyTypeForError( + PythonWriter writer, ShapeId shapeId, GenerationContext context) { + if (!context.model().expectShape(shapeId).hasTrait(ErrorTrait.class)) { + throw new IllegalArgumentException( + "Must provide an error shape to importDafnyTypeForError. Provided " + shapeId); + } + // When generating a Dafny import, must ALWAYS first import module_ to avoid circular + // dependencies + writer.addStdlibImport("module_"); + writer.addStdlibImport( + getDafnyPythonTypesModuleNameForShape(shapeId), getDafnyTypeForError(shapeId)); + } + + /** + * Imports the generic Dafny error type for the provided namespace. ex. example.namespace -> "from + * example_namespace_internaldafny_types import Error" + * + * @param writer + * @param namespace + */ + public static void importGenericDafnyErrorTypeForNamespace( + PythonWriter writer, String namespace) { + // When generating a Dafny import, must ALWAYS first import module_ to avoid circular + // dependencies + writer.addStdlibImport("module_"); + writer.addStdlibImport(getDafnyTypesModuleNameForSmithyNamespace(namespace), "Error"); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/nameresolver/SmithyNameResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/nameresolver/SmithyNameResolver.java new file mode 100644 index 0000000000..56ff7276d2 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/nameresolver/SmithyNameResolver.java @@ -0,0 +1,332 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.common.nameresolver; + +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import software.amazon.awssdk.utils.StringUtils; +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.polymorph.traits.ReferenceTrait; +import software.amazon.smithy.codegen.core.CodegenContext; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * Contains utility functions that map Smithy shapes to useful strings used in Smithy-Python + * generated code. i.e. strings in this file match behavior of Smithy-Python- (or + * Smithy-Dafny-Python-) generated code + */ +public class SmithyNameResolver { + + // Cached set of + static Set localServiceConfigShapes = new HashSet<>(); + // Map from a Smithy service namespace (as a string) to its wrapping Python module name. + // This is passed into Smithy-Dafny, which passes it into Python codegen. + private static Map smithyNamespaceToPythonModuleNameMap; + + /** + * Returns a set of serviceShapes in the model that have the `@aws.polymorph#localService` trait. + * + * @param codegenContext + * @return + */ + public static Set getLocalServiceConfigShapes(CodegenContext codegenContext) { + + return getLocalServiceConfigShapes(codegenContext.model()); + } + + /** + * Retrieve set of localService config shapes; if the set has not been retrieved yet, search the + * model to populate the set. + * + * @param model + * @return + */ + public static Set getLocalServiceConfigShapes(Model model) { + + if (localServiceConfigShapes.isEmpty()) { + localServiceConfigShapes = + model.getServiceShapes().stream() + .filter(serviceShape -> serviceShape.hasTrait(LocalServiceTrait.class)) + .map(serviceShape -> serviceShape.expectTrait(LocalServiceTrait.class)) + .map(localServiceTrait -> localServiceTrait.getConfigId()) + .collect(Collectors.toSet()); + } + return localServiceConfigShapes; + } + + public static void setSmithyNamespaceToPythonModuleNameMap( + Map smithyNamespaceToPythonModuleNameMap) { + SmithyNameResolver.smithyNamespaceToPythonModuleNameMap = smithyNamespaceToPythonModuleNameMap; + } + + public static String getPythonModuleNameForSmithyNamespace(String smithyNamespace) { + if (!smithyNamespaceToPythonModuleNameMap.containsKey(smithyNamespace)) { + throw new IllegalArgumentException("Python module name not found for Smithy namespace: " + smithyNamespace); + } + return smithyNamespaceToPythonModuleNameMap.get(smithyNamespace); + } + + /** + * Returns the name of the Smithy-generated client for the provided serviceShape. The serviceShape + * SHOULD be a localService. ex. example.namespace.ExampleService -> "ExampleServiceClient" + * + * @param serviceShape + * @return + */ + public static String clientNameForService(ServiceShape serviceShape) { + if (serviceShape.hasTrait(LocalServiceTrait.class)) { + return serviceShape.expectTrait(LocalServiceTrait.class).getSdkId() + "Client"; + } else { + throw new UnsupportedOperationException("Non-local services not supported"); + } + } + + /** + * Returns the name of the Smithy-generated shim for the provided serviceShape. The serviceShape + * SHOULD be a localService. ex. example.namespace.ExampleService -> "ExampleServiceShim" + * + * @param serviceShape + * @return + */ + public static String shimNameForService(ServiceShape serviceShape) { + if (serviceShape.hasTrait(LocalServiceTrait.class)) { + return serviceShape.expectTrait(LocalServiceTrait.class).getSdkId() + "Shim"; + } else { + throw new UnsupportedOperationException("Non-local services not supported"); + } + } + + /** + * Returns the name of the Python module containing Smithy code for the provided smithyNamespace. + * ex. example.namespace -> "example_namespace" + * + * @param smithyNamespace + * @return + */ + public static String getServiceSmithygeneratedDirectoryNameForNamespace(String smithyNamespace) { + return smithyNamespace.toLowerCase(Locale.ROOT).replace(".", "_"); + } + + /** + * For a given ShapeId, returns a String representing the path where that shape is generated. The + * return value can be directly used to import that shape; e.g. `from {returnValue} import + * {my_shape.getId()}` + * + * @param shape + * @param codegenContext + * @return + */ + public static String getSmithyGeneratedModelLocationForShape( + Shape shape, GenerationContext codegenContext) { + return getSmithyGeneratedModelLocationForShape(shape.getId(), codegenContext); + } + + /** + * For a given ShapeId, returns a String representing the path where that shape is generated. The + * return value can be directly used to import that shape; e.g. `from {returnValue} import + * {my_shape.getId()}` + * + * @param shapeId + * @param codegenContext + * @return + */ + public static String getSmithyGeneratedModelLocationForShape( + ShapeId shapeId, GenerationContext codegenContext) { + String moduleNamespace = + getPythonModuleSmithygeneratedPathForSmithyNamespace( + shapeId.getNamespace(), codegenContext); + String moduleFilename = getSmithyGeneratedModuleFilenameForSmithyShape(shapeId, codegenContext); + return moduleNamespace + moduleFilename; + } + + /** + * For a given ShapeId and PythonWriter, writes an import for the corresponding generated shape. + * ex. example.namespace.ExampleShape -> "from example_namespace.smithygenerated.[file] import + * ExampleShape" + * + * @param shape + * @param codegenContext + * @param writer + */ + public static void importSmithyGeneratedTypeForShape( + PythonWriter writer, Shape shape, GenerationContext codegenContext) { + importSmithyGeneratedTypeForShape(writer, shape.getId(), codegenContext); + } + + /** + * For a given ShapeId and PythonWriter, writes an import for the corresponding generated shape. + * ex. example.namespace.ExampleShape -> "from example_namespace.smithygenerated.[file] import + * ExampleShape" + * + * @param shapeId + * @param codegenContext + * @param writer + */ + public static void importSmithyGeneratedTypeForShape( + PythonWriter writer, ShapeId shapeId, GenerationContext codegenContext) { + writer.addStdlibImport( + SmithyNameResolver.getSmithyGeneratedModelLocationForShape(shapeId, codegenContext)); + } + + /** + * For any ShapeId, returns the filename inside `.smithygenerated` where that Shape is generated. + * + * @param shape + * @param codegenContext + * @return + */ + public static String getSmithyGeneratedModuleFilenameForSmithyShape( + Shape shape, GenerationContext codegenContext) { + return getSmithyGeneratedModuleFilenameForSmithyShape(shape.getId(), codegenContext); + } + + /** + * For any ShapeId, returns the filename inside `.smithygenerated` where that Shape is generated. + * + * @param shapeId + * @param codegenContext + * @return + */ + public static String getSmithyGeneratedModuleFilenameForSmithyShape( + ShapeId shapeId, GenerationContext codegenContext) { + Shape shape = codegenContext.model().expectShape(shapeId); + if (shape.hasTrait(ReferenceTrait.class) + && shape.isServiceShape() + && shape.hasTrait(LocalServiceTrait.class)) { + // LocalService clients are generated at `my_module.smithygenerated.client` + return ".client"; + } else if (shape.hasTrait(ErrorTrait.class)) { + return ".errors"; + } else if (getLocalServiceConfigShapes(codegenContext).contains(shapeId)) { + return ".config"; + } else if (shape.hasTrait(ReferenceTrait.class)) { + return ".references"; + } else { + return ".models"; + } + } + + /** + * Returns the name of the Smithy-generated type for the provided UnionShape and corresponding + * union value as its MemberShape. ex. example.namespace.ExampleUnion:ExampleMember -> + * "ExampleUnionExampleMember" + * + * @param unionShape + * @param memberShape + * @return + */ + public static String getSmithyGeneratedTypeForUnion( + UnionShape unionShape, MemberShape memberShape, GenerationContext context) { + return unionShape.getId().getName() + StringUtils.capitalize(memberShape.getMemberName()); + } + + /** + * Returns the name of the Smithy-generated type for a service's error. + * This error shape wraps errors from this service if this service is used as a dependency. + * + * @param serviceShape + * @return + */ + public static String getSmithyGeneratedTypeForServiceError( + ServiceShape serviceShape) { + if (serviceShape.hasTrait(LocalServiceTrait.class)) { + return serviceShape.getId().getName(); + } else if (AwsSdkNameResolver.isAwsSdkShape(serviceShape)) { + return AwsSdkNameResolver.dependencyErrorNameForService(serviceShape); + } else { + throw new IllegalArgumentException("Dependency MUST be a local service or AWS SDK shape: " + serviceShape); + } + } + + /** + * Given the namespace of a Smithy shape, returns a Pythonic access path to the namespace that can + * be used to import shapes from its `smithygenerated` namespace. + * + * @param smithyNamespace + * @param codegenContext + * @return + */ + public static String getPythonModuleSmithygeneratedPathForSmithyNamespace( + String smithyNamespace, GenerationContext codegenContext) { + return getPythonModuleSmithygeneratedPathForSmithyNamespace( + smithyNamespace, codegenContext.settings()); + } + + /** + * Given the namespace of a Smithy shape, returns a Pythonic access path to the namespace that can + * be used to import shapes from its Smithy-generated namespace. + * + * @param smithyNamespace + * @param settings + * @return + */ + public static String getPythonModuleSmithygeneratedPathForSmithyNamespace( + String smithyNamespace, PythonSettings settings) { + String pythonModuleName; + String namespace; + // `smithy.api.Unit:` + // Smithy-Dafny will generate a stand-in shape in the service + if ("smithy.api".equals(smithyNamespace)) { + pythonModuleName = settings.getModuleName(); + namespace = settings.getService().getNamespace(); + } else { + pythonModuleName = getPythonModuleNameForSmithyNamespace(smithyNamespace); + namespace = smithyNamespace; + } + return pythonModuleName + ".smithygenerated." + getServiceSmithygeneratedDirectoryNameForNamespace(namespace); + } + + /** + * Returns the name of the function that converts the provided shape's Dafny-modelled type to the + * corresponding Smithy-modelled type. This function will be defined in the `dafny_to_smithy.py` + * file. ex. example.namespace.ExampleShape -> "DafnyToSmithy_example_namespace_ExampleShape" + * + * This is the same as getSmithyToDafnyFunctionNameForShape below. These used to be different. + * There may be some value in preserving these separately if we want to make them different again in the future. + * + * @param shape + * @return + */ + public static String getDafnyToSmithyFunctionNameForShape( + Shape shape, GenerationContext context) { + return SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + shape.getId().getNamespace()) + + "_" + + shape.getId().getName(); + } + + /** + * Returns the name of the function that converts the provided shape's Smithy-modelled type to the + * corresponding Dafny-modelled type. This function will be defined in the `smithy_to_dafny.py` + * file. ex. example.namespace.ExampleShape -> "SmithyToDafny_example_namespace_ExampleShape" + * + * This is the same as getDafnyToSmithyFunctionNameForShape above. These used to be different. + * There may be some value in preserving these separately if we want to make them different again in the future. + * + * @param shape + * @return + */ + public static String getSmithyToDafnyFunctionNameForShape( + Shape shape, GenerationContext context) { + return SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + shape.getId().getNamespace()) + + "_" + + shape.getId().getName(); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/nameresolver/Utils.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/nameresolver/Utils.java new file mode 100644 index 0000000000..d3e425d499 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/nameresolver/Utils.java @@ -0,0 +1,29 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.common.nameresolver; + +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; + +/** + * Utils class containing NameResolver utility functions. This should contain helper methods common + * to >1 name resolver. TODO-Python: Once Utils class has a more clearly defined scope, refactor + * such that it is not a generic Utils class + */ +public class Utils { + + /** + * Returns true if `shapeId` is a Smithy Unit shape. + * + * @param shapeId + * @return + */ + public static boolean isUnitShape(ShapeId shapeId) { + return shapeId.getNamespace().equals("smithy.api") && shapeId.getName().equals("Unit"); + } + + private static boolean isUnitShape(Shape shape) { + return isUnitShape(shape.getId()); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/shapevisitor/ShapeVisitorResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/shapevisitor/ShapeVisitorResolver.java new file mode 100644 index 0000000000..56f9ba83c6 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/shapevisitor/ShapeVisitorResolver.java @@ -0,0 +1,74 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.common.shapevisitor; + +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.awssdk.shapevisitor.AwsSdkToDafnyShapeVisitor; +import software.amazon.polymorph.smithypython.awssdk.shapevisitor.DafnyToAwsSdkShapeVisitor; +import software.amazon.polymorph.smithypython.localservice.shapevisitor.DafnyToLocalServiceShapeVisitor; +import software.amazon.polymorph.smithypython.localservice.shapevisitor.LocalServiceToDafnyShapeVisitor; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * Utility class to return the correct ShapeVisitor for the provided Shape. If the shape is an AWS + * SDK shape, returns an AWS SDK ShapeVisitor; otherwise, returns a LocalService ShapeVisitor. Two + * usage notes: + * + *

1) LocalService ShapeVisitor MUST NOT directly defer to another LocalService ShapeVisitor. + * LocalService shapes can depend on AWS SDK shapes, and this class is responsible for determining + * which ShapeVisitor to defer to. AWS SDK ShapeVisitors CAN defer directly to an AWS SDK + * ShapeVisitor, since AWS SDK shapes do not defer to LocalService shapes. + * + *

2) LocalService ShapeVisitor CAN defer directly to a LocalService ShapeVisitor and not use + * this class if the targetShape is a LocalService Config shape. + */ +public class ShapeVisitorResolver { + + public static ShapeVisitor.Default getToNativeShapeVisitorForShape( + Shape shape, + GenerationContext context, + String dataSource, + PythonWriter writer, + String filename) { + if (AwsSdkNameResolver.isAwsSdkShape(shape)) { + return new DafnyToAwsSdkShapeVisitor(context, dataSource, writer); + } else { + return new DafnyToLocalServiceShapeVisitor(context, dataSource, writer, filename); + } + } + + public static ShapeVisitor.Default getToNativeShapeVisitorForShape( + Shape shape, GenerationContext context, String dataSource, PythonWriter writer) { + if (AwsSdkNameResolver.isAwsSdkShape(shape)) { + return new DafnyToAwsSdkShapeVisitor(context, dataSource, writer); + } else { + return new DafnyToLocalServiceShapeVisitor(context, dataSource, writer); + } + } + + public static ShapeVisitor.Default getToDafnyShapeVisitorForShape( + Shape shape, + GenerationContext context, + String dataSource, + PythonWriter writer, + String filename) { + if (AwsSdkNameResolver.isAwsSdkShape(shape)) { + return new AwsSdkToDafnyShapeVisitor(context, dataSource, writer); + } else { + return new LocalServiceToDafnyShapeVisitor(context, dataSource, writer, filename); + } + } + + public static ShapeVisitor.Default getToDafnyShapeVisitorForShape( + Shape shape, GenerationContext context, String dataSource, PythonWriter writer) { + if (AwsSdkNameResolver.isAwsSdkShape(shape)) { + return new AwsSdkToDafnyShapeVisitor(context, dataSource, writer); + } else { + return new LocalServiceToDafnyShapeVisitor(context, dataSource, writer); + } + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/shapevisitor/conversionwriter/BaseConversionWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/shapevisitor/conversionwriter/BaseConversionWriter.java new file mode 100644 index 0000000000..556bfea106 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/common/shapevisitor/conversionwriter/BaseConversionWriter.java @@ -0,0 +1,107 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.common.shapevisitor.conversionwriter; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import software.amazon.polymorph.smithypython.wrappedlocalservice.DafnyPythonWrappedLocalServiceProtocolGenerator; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * Abstract class for writing Dafny-to-X and X-to-Dafny conversion functions. (X = AWS-SDK or + * LocalService.) Subclasses of this class generate files that contain methods that convert from AWS + * SDK shapes (i.e. boto3 request/response dictionaries) or native Python Smithy shapes to Dafny + * shapes (Smithy-Dafny generated Dafny code transpiled into Python). + * + *

ShapeVisitors should call out to subclasses of this class to write conversions for aggregate + * shapes that can contain themselves as members (Unions and Structures). These shapes require + * delegating conversions to functions that recurse at runtime, and not at code generation time. + */ +public abstract class BaseConversionWriter { + // Store the set of shapes for which the subclass has already generated conversion methods + final Set generatedShapes = new HashSet<>(); + // Queue of shapes to generate + final List shapesToGenerate = new ArrayList<>(); + // Flag to block generating inside a file while already generating another function + // (prevents generating a conversion function inside another conversion function) + boolean generating = false; + + protected GenerationContext context; + // Writer pointing to the in-code location where the ShapeVisitor calling the subclass is writing + protected PythonWriter writer; + + /** + * Writes a function that converts from the AWS SDK dict to the Dafny-modelled shape. If the shape + * has members that also require writing a conversion function, this function will also write + * conversion methods for those shapes (recursively). If this method is called for the same shape + * multiple times, subsequent calls will not write a conversion method for the shape. This is the + * ONLY interface by which clients of this class' subclasses can request writing a new shape. + * + * @param shape + * @param context + * @param writer + */ + public void baseWriteConverterForShapeAndMembers( + Shape shape, GenerationContext context, PythonWriter writer) { + + this.context = context; + // Store where this is being written from where the original ShapeVisitor was dispatched (e.g. + // serialize, shim); + // This allows us to write imports in the original file + this.writer = writer; + // Do NOT write any converters for wrapped localServices. + // The wrapped localService should ONLY generate a Shim class. + // The Shim will use the converters generated as part of the localService. + if (context + .applicationProtocol() + .equals( + DafnyPythonWrappedLocalServiceProtocolGenerator + .DAFNY_PYTHON_WRAPPED_LOCAL_SERVICE_PROTOCOL)) { + return; + } + + // Enqueue this shape if this class has not generated a converter for it or is not already in + // queue + if (!generatedShapes.contains(shape) && !shapesToGenerate.contains(shape)) { + shapesToGenerate.add(shape); + } + + // If `writeConverterForShapeAndMembers` is called while already writing another converter, + // do NOT write the new converter inside the definition of the in-progress converter. + // `shape` will be picked up once the in-progress converter is finished. + while (!generating && !shapesToGenerate.isEmpty()) { + // Indicate to recursive calls that this class is writing a converter to + // prevent writing multiple conversion methods at once + generating = true; + + Shape toGenerate = shapesToGenerate.remove(0); + generatedShapes.add(toGenerate); + + if (toGenerate.isStructureShape()) { + writeStructureShapeConverter(toGenerate.asStructureShape().get()); + } else if (toGenerate.isUnionShape()) { + writeUnionShapeConverter(toGenerate.asUnionShape().get()); + } else if (toGenerate.isStringShape() && toGenerate.hasTrait(EnumTrait.class)) { + writeStringEnumShapeConverter(toGenerate.asStringShape().get()); + } else { + throw new IllegalArgumentException("Unsupported shape passed to ConversionWriter: " + toGenerate); + } + generating = false; + } + } + + protected abstract void writeStructureShapeConverter(StructureShape structureShape); + + protected abstract void writeUnionShapeConverter(UnionShape unionShape); + + protected abstract void writeStringEnumShapeConverter(StringShape stringShapeWithEnumTrait); +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/DafnyLocalServiceCodegenConstants.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/DafnyLocalServiceCodegenConstants.java new file mode 100644 index 0000000000..fa3b52d4aa --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/DafnyLocalServiceCodegenConstants.java @@ -0,0 +1,16 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice; + +public class DafnyLocalServiceCodegenConstants { + + // Dafny ApplicationProtocol constants + public static String DAFNY_PYTHON_LOCAL_SERVICE_APPLICATION_PROTOCOL_NAME = + "dafny_python_local_service"; + public static String DAFNY_PROTOCOL_PYTHON_FILENAME = ".dafny_protocol"; + public static String DAFNY_PROTOCOL_REQUEST = "DafnyRequest"; + public static String DAFNY_PROTOCOL_RESPONSE = "DafnyResponse"; + public static String LOCAL_SERVICE_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME = "localservice_codegen_todelete"; + +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/DafnyPythonLocalServiceIntegration.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/DafnyPythonLocalServiceIntegration.java new file mode 100644 index 0000000000..5b1d2eb957 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/DafnyPythonLocalServiceIntegration.java @@ -0,0 +1,214 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.localservice.customize.ConfigFileWriter; +import software.amazon.polymorph.smithypython.localservice.customize.DafnyImplInterfaceFileWriter; +import software.amazon.polymorph.smithypython.localservice.customize.DafnyProtocolFileWriter; +import software.amazon.polymorph.smithypython.localservice.customize.ErrorsFileWriter; +import software.amazon.polymorph.smithypython.localservice.customize.ModelsFileWriter; +import software.amazon.polymorph.smithypython.localservice.customize.PluginFileWriter; +import software.amazon.polymorph.smithypython.localservice.customize.ReferencesFileWriter; +import software.amazon.polymorph.traits.ReferenceTrait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.model.shapes.*; +import software.amazon.smithy.python.codegen.ConfigProperty; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.integration.ProtocolGenerator; +import software.amazon.smithy.python.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.utils.CodeInterceptor; +import software.amazon.smithy.utils.CodeSection; + +public final class DafnyPythonLocalServiceIntegration implements PythonIntegration { + + private final RuntimeClientPlugin dafnyImplRuntimeClientPlugin = RuntimeClientPlugin.builder() + .configProperties( + // Adds a new field in the client class' config. + // `dafnyImplInterface` is a static interface for accessing Dafny implementation code. + // The Smithy-Dafny Python plugin generates a dafnyImplInterface file + // and populates it with the relevant information from the model + // to interact with the Dafny implementation. + // We use a static interface as we cannot plug the model into this RuntimeClientPlugin + // definition, so this class cannot be aware of model shapes. + // To work around this, we can point the RuntimeClientPlugin to a static interface + // that IS aware of model shapes, and plug the model in there. + Collections.singletonList(ConfigProperty.builder() + .name("dafnyImplInterface") + .type( + Symbol.builder() + .name("DafnyImplInterface") + .namespace(".dafnyImplInterface", ".") + .build() + ) + // nullable is marked as true here. + // This allows the Config to be instantiated without providing a plugin, which + // is required because of how Smithy-Python generates the code. + // However, this plugin MUST be present before using the client. + // Immediately after the Config is instantiated, the Dafny plugin + // will add our plugin to the Config. + .nullable(true) + .documentation("") + .build() + ) + ).pythonPlugin( + SymbolReference.builder() + .symbol( + Symbol.builder() + .name("set_config_impl") + .namespace(".plugin", ".") + .build()) + .build() + ) + .build(); + + @Override + public List> + interceptors(GenerationContext codegenContext) { + return List.of(); + } + + /** + * Generate all Smithy-Dafny custom Python code. + * + * @param codegenContext Code generation context that can be queried when writing additional + * files. + */ + @Override + public void customize(GenerationContext codegenContext) { + // Only perform customizations if generating using the + // DAFNY_PYTHON_LOCAL_SERVICE_APPLICATION_PROTOCOL + if (!codegenContext.applicationProtocol().equals( + DafnyPythonLocalServiceProtocolGenerator.DAFNY_PYTHON_LOCAL_SERVICE_APPLICATION_PROTOCOL)) { + return; + } + + // Generate customizations for service shapes with localService trait + Set serviceShapes = Set.of( + codegenContext.model().expectShape(codegenContext.settings().getService()) + .asServiceShape().get()); + + ServiceShape serviceShape = codegenContext.model() + .expectShape(codegenContext.settings().getService()).asServiceShape().get(); + + customizeForServiceShape(serviceShape, codegenContext); + + // Get set(non-service operation shapes) = set(model operation shapes) - set(service operation shapes) + // This is related to forking Smithy-Python. TODO-Python: resolve when resolving fork. + // Smithy-Python will only generate code for shapes which are used by the protocol. + // Polymorph has a requirement to generate code for all shapes in the model, + // even if the service does not use those shapes. + // (The use case is that other models may depend on shapes that are defined in this model, + // though not used in this model.) + Set serviceOperationShapes = serviceShapes.stream() + .map(EntityShape::getOperations) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + Set nonServiceOperationShapes = codegenContext.model().getOperationShapes() + .stream() + .map(Shape::getId) + .filter(operationShapeId -> operationShapeId.getNamespace() + .equals(serviceShape.getId().getNamespace())) + .collect(Collectors.toSet()); + nonServiceOperationShapes.removeAll(serviceOperationShapes); + + +// nonServiceOperationShapes.addAll(SmithyNameResolver.getLocalServiceConfigShapes(codegenContext)); + +// customizeForNonServiceOperationShapes(nonServiceOperationShapes, codegenContext); + + Set referenceShapes = codegenContext.model() + .getStructureShapesWithTrait(ReferenceTrait.class) + .stream() + .map(structureShape -> structureShape.expectTrait(ReferenceTrait.class)) + .map(ReferenceTrait::getReferentId) + .map(shapeId -> codegenContext.model().expectShape(shapeId)) + .collect(Collectors.toSet()); + + String moduleName = SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(codegenContext.settings().getService().getNamespace()); + + // Only open a writer if there are reference shapes; otherwise this will write an empty file. + if (!referenceShapes.isEmpty()) { + + for (Shape referenceShape : referenceShapes) { + if (referenceShape.isResourceShape()) { + ResourceShape resourceShape = referenceShape.asResourceShape().get(); + + if (ReferencesFileWriter.shouldGenerateResourceForShape(resourceShape, codegenContext)) { + codegenContext + .writerDelegator() + .useFileWriter( + moduleName + "/references.py", + "", + writer -> { + new ReferencesFileWriter() + .generateResourceInterfaceAndImplementation( + resourceShape, codegenContext, writer); + }); + } + + + + } + } + } + } + +// /** +// * Generate any code for operation shapes that are NOT part of the localService. +// * +// * @param operationShapeIds +// * @param codegenContext +// */ +// private void customizeForNonServiceOperationShapes(Set operationShapeIds, +// GenerationContext codegenContext) { +// new ReferencesFileWriter().generateResourceInterfaceAndImplementation(operationShapeIds, +// codegenContext); +// new ReferencesFileWriter().customizeFileForNonServiceShapes(operationShapeIds, +// codegenContext); +// } + + + /** + * Generate any code for the localService ServiceShape. + * + * @param serviceShape + * @param codegenContext + */ + private void customizeForServiceShape(ServiceShape serviceShape, GenerationContext codegenContext) { + new PluginFileWriter().customizeFileForServiceShape(serviceShape, codegenContext); + new DafnyImplInterfaceFileWriter().customizeFileForServiceShape(serviceShape, + codegenContext); + new DafnyProtocolFileWriter().customizeFileForServiceShape(serviceShape, + codegenContext); + new ErrorsFileWriter().customizeFileForServiceShape(serviceShape, codegenContext); + new ModelsFileWriter().customizeFileForServiceShape(serviceShape, codegenContext); + new ConfigFileWriter().customizeFileForServiceShape(serviceShape, codegenContext); + new ReferencesFileWriter().customizeFileForServiceShape(serviceShape, codegenContext); + } + + @Override + public List getProtocolGenerators() { + return Collections.singletonList(new DafnyPythonLocalServiceProtocolGenerator() { + @Override + public ShapeId getProtocol() { + return ShapeId.from("aws.polymorph#localService"); + } + }); + } + + @Override + public List getClientPlugins() { + return Collections.singletonList(dafnyImplRuntimeClientPlugin); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/DafnyPythonLocalServiceProtocolGenerator.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/DafnyPythonLocalServiceProtocolGenerator.java new file mode 100644 index 0000000000..29a9497233 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/DafnyPythonLocalServiceProtocolGenerator.java @@ -0,0 +1,600 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice; + +import static java.lang.String.format; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.Utils; +import software.amazon.polymorph.smithypython.common.shapevisitor.ShapeVisitorResolver; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.polymorph.traits.PositionalTrait; +import software.amazon.polymorph.utils.ModelUtils; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.codegen.core.WriterDelegator; +import software.amazon.smithy.model.shapes.*; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.python.codegen.ApplicationProtocol; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.python.codegen.integration.ProtocolGenerator; +import software.amazon.smithy.utils.CaseUtils; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** This will implement any handling of components outside the request body and error handling. */ +@SmithyUnstableApi +public abstract class DafnyPythonLocalServiceProtocolGenerator implements ProtocolGenerator { + + public static ApplicationProtocol DAFNY_PYTHON_LOCAL_SERVICE_APPLICATION_PROTOCOL = + new ApplicationProtocol( + // Dafny localService ApplicationProtocol for Smithy clients. + DafnyLocalServiceCodegenConstants.DAFNY_PYTHON_LOCAL_SERVICE_APPLICATION_PROTOCOL_NAME, + SymbolReference.builder() + .symbol(createDafnyApplicationProtocolSymbol(DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_REQUEST)) + .build(), + SymbolReference.builder() + .symbol(createDafnyApplicationProtocolSymbol(DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_RESPONSE)) + .build()); + + /** + * Create a Symbol representing shapes inside the generated .dafny_protocol file. + * + * @param symbolName + * @return + */ + private static Symbol createDafnyApplicationProtocolSymbol(String symbolName) { + return Symbol.builder() + .namespace(DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_PYTHON_FILENAME, ".") + .name(symbolName) + .build(); + } + + /** + * Creates the Dafny ApplicationProtocol object. Smithy-Python requests this object as part of the + * ProtocolGenerator implementation. + * + * @return Returns the created application protocol. + */ + @Override + public ApplicationProtocol getApplicationProtocol() { + return DAFNY_PYTHON_LOCAL_SERVICE_APPLICATION_PROTOCOL; + } + + /** + * For all operations in the model, generate a conversion method that takes in a Smithy shape and + * converts it to a DafnyRequest. + * + * @param context + */ + @Override + public void generateRequestSerializers(GenerationContext context) { + WriterDelegator delegator = context.writerDelegator(); + Symbol configSymbol = + Symbol.builder() + .name("Config") + .namespace( + format( + "%s.config", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + context.settings().getService().getNamespace(), context)), + ".") + .definitionFile( + format( + "./%s/config.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()))) + .build(); + + // For each operation in the model, generate a `serialize_{operation input}` method + for (ShapeId operationShapeId : + context + .model() + .expectShape(context.settings().getService()) + .asServiceShape() + .get() + .getAllOperations()) { + OperationShape operationShape = + context.model().expectShape(operationShapeId).asOperationShape().get(); + Symbol serFunction = getSerializationFunction(context, operationShape); + + // Write out the serialization operation + delegator.useFileWriter( + serFunction.getDefinitionFile(), + serFunction.getNamespace(), + writer -> { + writer.addImport( + DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_PYTHON_FILENAME, DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_REQUEST); + writer.pushState(new RequestSerializerSection(operationShape)); + + writer.write( + """ + async def $L(input, config: $T) -> $L: + ${C|} + """, + serFunction.getName(), + configSymbol, + DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_REQUEST, + writer.consumer(w -> generateRequestSerializer(context, operationShape, w))); + writer.popState(); + }); + } + } + + /** + * Generates the symbol for a serializer function for shapes of a service. + * + * @param context The code generation context. + * @param shapeId The shape the serializer function is being generated for. + * @return Returns the generated symbol. + */ + @Override + public Symbol getSerializationFunction(GenerationContext context, ToShapeId shapeId) { + return Symbol.builder() + .name(getSerializationFunctionName(context, shapeId)) + .namespace( + format( + "%s.serialize", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + shapeId.toShapeId().getNamespace(), context)), + "") + .definitionFile( + format( + "./%s/serialize.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + shapeId.toShapeId().getNamespace()))) + .build(); + } + + /** + * Generates the content of the operation request serializer. + * + *

Smithy-Python uses the word 'serialize' in this part of the code. This name stems from its + * default HTTP-style application protocol as this code would, by default, transform + * Smithy-modelled Python objects into serialized HTTP objects. + * + *

The Dafny plugin will not 'serialize' here, but will instead transform Smithy-modelled + * Python objects into native Python code modelling Dafny-compiled objects. + */ + private void generateRequestSerializer( + GenerationContext context, OperationShape operation, PythonWriter writer) { + + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + // Import the Dafny type being converted to + Shape targetShape = context.model().expectShape(operation.getInputShape()); +// if (targetShape.isStructureShape() +// && targetShape +// .asStructureShape() +// .get() +// .hasTrait(PositionalTrait.class)) { +// ShapeId positionalShapeId = ModelUtils.getPositionalStructureMember(targetShape.asStructureShape().get()).orElseThrow(); +// DafnyNameResolver.importDafnyTypeForShape(writer, positionalShapeId, context); +// } else { +// DafnyNameResolver.importDafnyTypeForShape(writer, operation.getInputShape(), context); +// } + + // Determine conversion code from Smithy to Dafny + String input = + targetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape( + targetShape, context, "input", writer)); + + // Write conversion method body + writer.write( + """ + return DafnyRequest(operation_name="$L", dafny_operation_input=$L) + """, + operation.getId().getName(), + Utils.isUnitShape(operation.getInputShape()) ? "None" : input); + } + + @Override + public void generateResponseDeserializers(GenerationContext context) { + WriterDelegator delegator = context.writerDelegator(); + Symbol configSymbol = + Symbol.builder() + .name("Config") + .namespace( + format( + "%s.config", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + context.settings().getService().getNamespace(), context)), + ".") + .definitionFile( + format( + "./%s/config.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()))) + .build(); + + // For each operation in the model, generate a `deserialize_{operation input}` method + for (ShapeId operationShapeId : + context + .model() + .expectShape(context.settings().getService()) + .asServiceShape() + .get() + .getAllOperations()) { + OperationShape operationShape = + context.model().expectShape(operationShapeId).asOperationShape().get(); + Symbol deserFunction = getDeserializationFunction(context, operationShape); + Shape output = context.model().expectShape(operationShape.getOutputShape()); + + delegator.useFileWriter( + deserFunction.getDefinitionFile(), + deserFunction.getNamespace(), + writer -> { + writer.addImport( + DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_PYTHON_FILENAME, DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_RESPONSE); + + writer.pushState(new RequestDeserializerSection(operationShape)); + + writer.write( + """ + async def $L(input: $L, config: $T): + ${C|} + """, + deserFunction.getName(), + DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_RESPONSE, + configSymbol, + writer.consumer( + w -> generateOperationResponseDeserializer(context, operationShape))); + + writer.popState(); + }); + } + + generateErrorResponseDeserializerSection(context); + } + + /** + * Generates the content of the operation response deserializer. + * + *

Smithy-Python uses the word 'deserialize' in this part of the code. This name stems from its + * default HTTP-style application protocol as this code would, by default, transform serialized + * HTTP objects into POJOs of Smithy-modelled objects. + * + *

The Dafny plugin will not 'deserialize' here, but will instead transform POJOs of + * Dafny-compiled objects into POJOs of Smithy-modelled objects. + */ + private void generateOperationResponseDeserializer( + GenerationContext context, OperationShape operation) { + WriterDelegator delegator = context.writerDelegator(); + Symbol deserFunction = getDeserializationFunction(context, operation); + + delegator.useFileWriter( + deserFunction.getDefinitionFile(), + deserFunction.getNamespace(), + writer -> { + writer.pushState(new ResponseDeserializerSection(operation)); + + ShapeId outputShape = operation.getOutputShape(); + + if (context.model().expectShape(operation.getInputShape()).isStructureShape() + && context + .model() + .expectShape(operation.getInputShape()) + .asStructureShape() + .get() + .hasTrait(PositionalTrait.class)) { + // TODO: Typing positionals, and somehow typing the base classes + // Blob, boolean... + } else { + DafnyNameResolver.importDafnyTypeForShape(writer, outputShape, context); + } + + // Smithy Unit shapes have no data, and do not need deserialization + if (Utils.isUnitShape(outputShape)) { + writer.write(""" + return None + """); + } else { + // Determine the deserialization function + Shape targetShape = context.model().expectShape(outputShape); + String output = + targetShape.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape( + targetShape, context, "input.value", writer)); + + writer.write( + """ + if input.IsFailure(): + return await _deserialize_error(input.error) + return $L + """, + output); + } + writer.popState(); + }); + } + + @Override + public Symbol getDeserializationFunction(GenerationContext context, ToShapeId shapeId) { + return Symbol.builder() + .name(getDeserializationFunctionName(context, shapeId)) + .namespace( + format( + "%s.deserialize", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + shapeId.toShapeId().getNamespace(), context)), + "") + .definitionFile( + format( + "./%s/deserialize.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + shapeId.toShapeId().getNamespace()))) + .build(); + } + + private void generateErrorResponseDeserializerSection(GenerationContext context) { + ShapeId serviceShapeId = context.settings().getService(); + ServiceShape serviceShape = context.model().expectShape(serviceShapeId).asServiceShape().get(); + WriterDelegator writerDelegator = context.writerDelegator(); + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()); + + writerDelegator.useFileWriter( + moduleName + "/deserialize.py", + ".", + writer -> { + writer.addStdlibImport("typing", "Any"); + DafnyNameResolver.importGenericDafnyErrorTypeForNamespace( + writer, serviceShape.getId().getNamespace()); + writer.addImport(".errors", "ServiceError"); + writer.addImport(".errors", "OpaqueError"); + writer.addImport(".errors", "CollectionOfErrors"); + writer.addStdlibImport("_dafny"); + writer.openBlock( + "async def _deserialize_error(error: Error) -> ServiceError:", + "", + () -> { + writer.write( + """ + if error.is_Opaque: + return OpaqueError(obj=error.obj) + elif error.is_CollectionOfErrors: + return CollectionOfErrors( + message=_dafny.string_of(error.message), + list=[await _deserialize_error(dafny_e) for dafny_e in error.list], + )"""); + + // Write converters for errors modelled on this local service + generateErrorResponseDeserializerSectionForLocalServiceErrors( + context, serviceShape, writer); + + // Write delegators to dependency services' `_deserialize_error` for dependency + // services + generateErrorResponseDeserializerSectionForLocalServiceDependencyErrors( + context, serviceShape, writer); + + // Write delegators to dependency AWS services' `_sdk_error_to_dafny_error` for dependency + // services + generateErrorResponseDeserializerSectionForAwsSdkDependencyErrors( + context, serviceShape, writer); + + // Generate handler for unmatched Dafny Error. + // Since we don't know anything about this Dafny Error object, + // just pass the Dafny Error object into the Smithy object. + writer.write( + """ + else: + return OpaqueError(obj=error)"""); + }); + }); + } + + private void generateErrorResponseDeserializerSectionForLocalServiceErrors( + GenerationContext context, ServiceShape serviceShape, PythonWriter writer) { + + // Get all of this service's modelled errors + TreeSet deserializingErrorShapes = + new TreeSet( + context.model().getStructureShapesWithTrait(ErrorTrait.class).stream() + .filter( + structureShape -> + structureShape + .getId() + .getNamespace() + .equals(context.settings().getService().getNamespace())) + .map(Shape::getId) + .collect(Collectors.toSet())); + + // Write out deserializers for this service's modelled errors + for (ShapeId errorId : deserializingErrorShapes) { + StructureShape error = context.model().expectShape(errorId, StructureShape.class); + writer.pushState(new ErrorDeserializerSection(error)); + + // Import Smithy-Python modelled-error + writer.addImport(".errors", errorId.getName()); + // Import Dafny-modelled error + DafnyNameResolver.importDafnyTypeForError(writer, errorId, context); + // Import generic Dafny error type + DafnyNameResolver.importGenericDafnyErrorTypeForNamespace(writer, errorId.getNamespace()); + writer.write( + """ + elif error.is_$L: + return $L(message=_dafny.string_of(error.message))""", + errorId.getName(), + errorId.getName()); + writer.addStdlibImport("_dafny"); + writer.popState(); + } + } + + private void generateErrorResponseDeserializerSectionForLocalServiceDependencyErrors( + GenerationContext context, ServiceShape serviceShape, PythonWriter writer) { + + // Generate converters for dependency services that defer to their `_deserialize_error` + Optional maybeLocalServiceTrait = + serviceShape.getTrait(LocalServiceTrait.class); + if (maybeLocalServiceTrait.isPresent()) { + LocalServiceTrait localServiceTrait = maybeLocalServiceTrait.get(); + if (localServiceTrait.getDependencies() == null + || localServiceTrait.getDependencies().isEmpty()) { + return; + } + Set serviceDependencyShapeIds = + localServiceTrait.getDependencies().stream() + .filter( + shapeId -> context.model().expectShape(shapeId).hasTrait(LocalServiceTrait.class)) + .collect(Collectors.toSet()); + + for (ShapeId serviceDependencyShapeId : serviceDependencyShapeIds) { + writer.addImport(".errors", serviceDependencyShapeId.getName()); + + // Import dependency `_deserialize_error` function so this service can defer to it: + // `from dependency.smithygenerated.deserialize import _deserialize_error as + // dependency_deserialize_error` + writer.addImport( + // `from dependency.smithygenerated.deserialize` + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + serviceDependencyShapeId.getNamespace(), context) + + ".deserialize", + // `import _deserialize_error` + "_deserialize_error", + // `as dependency_deserialize_error` + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + serviceDependencyShapeId.getNamespace()) + + "_deserialize_error"); + // Generate deserializer for dependency that defers to its `_deserialize_error` + // TODO: Refactor, this is not specific to AWS SDKs + String serviceDependencyErrorName = AwsSdkNameResolver.dependencyErrorNameForService( + context.model().expectShape(serviceDependencyShapeId).asServiceShape().get() + ); + + // Import this service's Dafny error + ServiceShape dependencyServiceShape = context + .model() + .expectShape(serviceDependencyShapeId) + .asServiceShape() + .get(); + List serviceDependencyErrors = dependencyServiceShape.getErrors(); + if (serviceDependencyErrors.size() > 1) { + throw new IllegalArgumentException("Only 1 service-modelled error per service supported at this time"); + } + + ShapeId serviceDependencyError = serviceDependencyErrors.get(0); + + DafnyNameResolver.importDafnyTypeForError(writer, serviceDependencyError, context); + + writer.write( + """ + elif error.is_$L: + return $L(await $L($L(message=error.$L)))""", + serviceDependencyErrorName, + serviceDependencyShapeId.getName(), + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + serviceDependencyShapeId.getNamespace()) + + "_deserialize_error", + DafnyNameResolver.getDafnyTypeForError(serviceDependencyError), + serviceDependencyErrorName); + } + } + } + + private void generateErrorResponseDeserializerSectionForAwsSdkDependencyErrors( + GenerationContext context, ServiceShape serviceShape, PythonWriter writer) { + + // Generate converters for dependency services that defer to their `_deserialize_error` + Optional maybeLocalServiceTrait = + serviceShape.getTrait(LocalServiceTrait.class); + if (maybeLocalServiceTrait.isPresent()) { + LocalServiceTrait localServiceTrait = maybeLocalServiceTrait.get(); + if (localServiceTrait.getDependencies() == null + || localServiceTrait.getDependencies().isEmpty()) { + return; + } + Set serviceDependencyShapeIds = + localServiceTrait.getDependencies().stream() + .filter( + shapeId -> AwsSdkNameResolver.isAwsSdkShape(shapeId)) + .collect(Collectors.toSet()); + + for (ShapeId serviceDependencyShapeId : serviceDependencyShapeIds) { + String code = AwsSdkNameResolver.dependencyErrorNameForService( + context.model().expectShape(serviceDependencyShapeId).asServiceShape().get() + ); + writer.addImport(".errors", code); + + // Import dependency `_deserialize_error` function so this service can defer to it: + // `from dependency.smithygenerated.deserialize import _deserialize_error as + // dependency_deserialize_error` + writer.addImport( + // `from dependency.smithygenerated.deserialize` + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + serviceDependencyShapeId.getNamespace(), context) + + ".shim", + // `import _deserialize_error` + "_sdk_error_to_dafny_error", + // `as dependency_deserialize_error` + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + serviceDependencyShapeId.getNamespace()) + + "_sdk_error_to_dafny_error"); + // Generate deserializer for dependency that defers to its `_deserialize_error` +// writer.write( +// """ +// elif error.is_$L: +// return $L($L(error.$L.obj))""", +// code, +// code, +// SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( +// serviceDependencyShapeId.getNamespace()) +// + "_sdk_error_to_dafny_error", +// code); + writer.write( + """ + elif error.is_$L: + return $L(message=_dafny.string_of(error.$L.message))""", + code, + code, + code); + writer.addStdlibImport("_dafny"); + } + } + } + + /** + * A section that controls writing out the entire serialization function. + * + *

By pushing and popping CodeSections, other developers can create plugins that intercept a + * CodeSection and inject their own code here. + * + * @param operation The operation whose serializer is being generated. + */ + public record RequestSerializerSection(OperationShape operation) implements CodeSection {} + + /** + * A section that controls writing out the entire deserialization function. + * + * @param operation The operation whose serializer is being generated. + */ + public record RequestDeserializerSection(OperationShape operation) implements CodeSection {} + + /** + * A section that controls writing out the entire deserialization function for an error. + * + * @param error The error whose deserializer is being generated. + */ + public record ErrorDeserializerSection(StructureShape error) implements CodeSection {} + + /** + * A section that controls writing out the entire deserialization function for an operation. By + * pushing and popping this section, we allow other developers to create plugins that intercept + * this section and inject their own code here. + * + * @param operation The operation whose deserializer is being generated. + */ + public record ResponseDeserializerSection(OperationShape operation) implements CodeSection {} +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ConfigFileWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ConfigFileWriter.java new file mode 100644 index 0000000000..0467e54b4b --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ConfigFileWriter.java @@ -0,0 +1,233 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.customize; + +import java.util.Map; +import java.util.Map.Entry; +import software.amazon.polymorph.smithypython.common.customize.CustomFileWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.shapevisitor.ShapeVisitorResolver; +import software.amazon.polymorph.smithypython.localservice.extensions.DafnyPythonLocalServiceStructureGenerator; +import software.amazon.polymorph.smithypython.localservice.shapevisitor.LocalServiceToDafnyShapeVisitor; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.TopologicalIndex; +import software.amazon.smithy.model.knowledge.NullableIndex; +import software.amazon.smithy.model.shapes.*; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.traits.StringTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.utils.CaseUtils; + +/** + * Extends the Smithy-Python-generated config.py file by writing a shape for the localService config + * shape and adding type conversions between it and the Dafny config shape. + */ +public class ConfigFileWriter implements CustomFileWriter { + + @Override + public void customizeFileForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) { + final LocalServiceTrait localServiceTrait = serviceShape.expectTrait(LocalServiceTrait.class); + final StructureShape configShape = + codegenContext.model().expectShape(localServiceTrait.getConfigId(), StructureShape.class); + + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + codegenContext.settings().getService().getNamespace()); + codegenContext + .writerDelegator() + .useFileWriter( + moduleName + "/config.py", + "", + writer -> { + + DafnyNameResolver.importDafnyTypeForShape( + writer, configShape.getId(), codegenContext); + + writer.write( + """ + class $L(Config): + ""\" + Smithy-modelled localService Config shape for this localService. + ""\" + ${C|} + + def __init__( + self, + ${C|} + ): + ${C|} + super().__init__() + ${C|} + + def dafny_config_to_smithy_config(dafny_config) -> $L: + ""\" + Converts the provided Dafny shape for this localService's config + into the corresponding Smithy-modelled shape. + ""\" + ${C|} + + def smithy_config_to_dafny_config(smithy_config) -> $L: + ""\" + Converts the provided Smithy-modelled shape for this localService's config + into the corresponding Dafny shape. + ""\" + ${C|} + """, + configShape.getId().getName(), + writer.consumer(w -> generateConfigClassFields(configShape, codegenContext, w)), + writer.consumer( + w -> generateConfigConstructorParameters(configShape, codegenContext, w)), + writer.consumer( + w -> generateConfigConstructorDocumentation(configShape, codegenContext, w)), + writer.consumer( + w -> + generateConfigConstructorFieldAssignments( + configShape, codegenContext, w)), + configShape.getId().getName(), + writer.consumer( + w -> + generateDafnyConfigToSmithyConfigFunctionBody( + configShape, codegenContext, w)), + DafnyNameResolver.getDafnyTypeForShape(configShape.getId()), + writer.consumer( + w -> + generateSmithyConfigToDafnyConfigFunctionBody( + configShape, codegenContext, w))); + }); + } + + /** + * Generates the members of the Smithy-modelled localService Config shape's class. Called when + * writing the class. + * + * @param configShape + * @param codegenContext + * @param writer + */ + private void generateConfigClassFields( + StructureShape configShape, GenerationContext codegenContext, PythonWriter writer) { + Map memberShapeSet = configShape.getAllMembers(); + NullableIndex index = NullableIndex.of(codegenContext.model()); + for (Entry memberShapeEntry : memberShapeSet.entrySet()) { + String memberName = memberShapeEntry.getKey(); + MemberShape memberShape = memberShapeEntry.getValue(); + final Shape targetShape = codegenContext.model().expectShape(memberShape.getTarget()); + Symbol targetShapeSymbol = codegenContext.symbolProvider().toSymbol(targetShape); + if (index.isMemberNullable(memberShape)) { + writer.addStdlibImport("typing", "Optional"); + writer.write("$L: Optional[$T]", CaseUtils.toSnakeCase(memberShape.getMemberName()), targetShapeSymbol); + } else { + writer.write("$L: $T", CaseUtils.toSnakeCase(memberShape.getMemberName()), targetShapeSymbol); + } + } + } + + /** + * Generates constructor parameters for the localService's Config class. Called when writing + * parameters for the Config class' constructor (__init__ method). + * + * @param configShape + * @param codegenContext + * @param writer + */ + private void generateConfigConstructorParameters( + StructureShape configShape, GenerationContext codegenContext, PythonWriter writer) { + Map memberShapeSet = configShape.getAllMembers(); + NullableIndex index = NullableIndex.of(codegenContext.model()); + for (MemberShape memberShape : memberShapeSet.values()) { + final Shape targetShape = codegenContext.model().expectShape(memberShape.getTarget()); + Symbol targetShapeSymbol = codegenContext.symbolProvider().toSymbol(targetShape); + if (index.isMemberNullable(memberShape)) { + writer.addStdlibImport("typing", "Optional"); + writer.write("$L: Optional[$T] = None,", CaseUtils.toSnakeCase(memberShape.getMemberName()), targetShapeSymbol); + } else { + writer.write("$L: $T,", CaseUtils.toSnakeCase(memberShape.getMemberName()), targetShapeSymbol); + } + } + } + + /** + * Generates constructor parameters for the localService's Config class. Called when writing + * parameters for the Config class' constructor (__init__ method). + * + * @param configShape + * @param codegenContext + * @param writer + */ + private void generateConfigConstructorDocumentation( + StructureShape configShape, GenerationContext codegenContext, PythonWriter writer) { + Map memberShapeSet = configShape.getAllMembers(); + writer.writeDocs(() -> { + var constructorDocs = configShape.getTrait(DocumentationTrait.class) + .map(StringTrait::getValue) + .orElse(String.format("Constructor for %s.", configShape.getId().getName())); + writer.write(constructorDocs + "\n"); + for (MemberShape memberShape : memberShapeSet.values()) { + memberShape.getMemberTrait(codegenContext.model(), DocumentationTrait.class).ifPresent(trait -> { + String memberName = codegenContext.symbolProvider().toMemberName(memberShape); + String memberDocs = writer.formatDocs(String.format(":param %s: %s", memberName, trait.getValue())); + writer.write(memberDocs); + }); + } + }); + } + + /** + * Generates assignments to fields for the localService's Config class. Called when writing the + * Config class' constructor. + * + * @param configShape + * @param codegenContext + * @param writer + */ + private void generateConfigConstructorFieldAssignments( + StructureShape configShape, GenerationContext codegenContext, PythonWriter writer) { + Map memberShapeSet = configShape.getAllMembers(); + for (String memberName : memberShapeSet.keySet()) { + // TODO-Python: Instead of `Any`, map the targetShape.getType Smithy type to the Python type + writer.write( + "self.$L = $L", CaseUtils.toSnakeCase(memberName), CaseUtils.toSnakeCase(memberName)); + } + } + + /** + * Generates the body converting the Dafny Config class (from internaldafny code) to the + * Smithy-modelled Config class defined in this file. + * + * @param configShape + * @param codegenContext + * @param writer + */ + private void generateDafnyConfigToSmithyConfigFunctionBody( + StructureShape configShape, GenerationContext codegenContext, PythonWriter writer) { + String output = + configShape.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape( + configShape, codegenContext, "dafny_config", writer)); + writer.write("return " + output); + } + + /** + * Generates the body converting the Smithy-modelled Config class defined in this file to the + * Dafny Config class. + * + * @param configShape + * @param codegenContext + * @param writer + */ + private void generateSmithyConfigToDafnyConfigFunctionBody( + StructureShape configShape, GenerationContext codegenContext, PythonWriter writer) { + // Dafny-generated config shapes contain a piece of unmodelled behavior, + // which is that every config member is required. + // + String output = + configShape.accept( + new LocalServiceToDafnyShapeVisitor(codegenContext, "smithy_config", writer)); + writer.write("return " + output); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/DafnyImplInterfaceFileWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/DafnyImplInterfaceFileWriter.java new file mode 100644 index 0000000000..9795fab552 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/DafnyImplInterfaceFileWriter.java @@ -0,0 +1,104 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.customize; + +import software.amazon.polymorph.smithypython.localservice.DafnyLocalServiceCodegenConstants; +import software.amazon.polymorph.smithypython.common.customize.CustomFileWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * Creates a dafnyImplInterface.py file containing a DafnyImplInterface class. This provides a + * static (meaning "unchanging") interface for the Smithy-Python-generated client.py request handler + * to interact with. + * + *

(We do this because we cannot extensively customize this part of client.py code generation. + * Instead, we plug this interface into the part we can customize, and do the rest of the + * customization in a file we control (this file).) + */ +public class DafnyImplInterfaceFileWriter implements CustomFileWriter { + + @Override + public void customizeFileForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) { + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + codegenContext.settings().getService().getNamespace()); + String clientName = SmithyNameResolver.clientNameForService(serviceShape); + String implModulePrelude = + DafnyNameResolver.getDafnyPythonIndexModuleNameForShape(serviceShape); + + codegenContext + .writerDelegator() + .useFileWriter( + moduleName + "/dafnyImplInterface.py", + "", + writer -> { + writer.write( + """ + from $L import $L + from $L import $L + + class DafnyImplInterface: + $L: $L | None = None + + # operation_map cannot be created at dafnyImplInterface create time, + # as the map's values reference values inside `self.impl`, + # and impl is only populated at runtime. + # Accessing these before impl is populated results in an error. + # At runtime, the map is populated once and cached. + operation_map = None + + def handle_request(self, input: DafnyRequest): + if self.operation_map is None: + self.operation_map = { + ${C|} + } + + # This logic is where a typical Smithy client would expect the "server" to be. + # This code can be thought of as logic our Dafny "server" uses + # to route incoming client requests to the correct request handler code. + if input.dafny_operation_input is None: + return self.operation_map[input.operation_name]() + else: + return self.operation_map[input.operation_name](input.dafny_operation_input) + """, + implModulePrelude, + clientName, + DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_PYTHON_FILENAME, + DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_REQUEST, + "impl", + clientName, + writer.consumer( + w -> generateImplInterfaceOperationMap(serviceShape, codegenContext, w))); + }); + } + + /** + * Generates the map from the operation name to the Dafny implementation operation for the + * provided localService. + * + * @param serviceShape + * @param codegenContext + * @param writer + */ + private void generateImplInterfaceOperationMap( + ServiceShape serviceShape, GenerationContext codegenContext, PythonWriter writer) { + for (ShapeId operationShapeId : serviceShape.getOperations()) { + final OperationShape operationShape = + codegenContext.model().expectShape(operationShapeId, OperationShape.class); + writer.write( + """ + "$L": self.$L.$L,""", + operationShape.getId().getName(), + "impl", + operationShape.getId().getName()); + } + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/DafnyProtocolFileWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/DafnyProtocolFileWriter.java new file mode 100644 index 0000000000..e0a3c0efb9 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/DafnyProtocolFileWriter.java @@ -0,0 +1,108 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.customize; + +import java.util.HashSet; +import java.util.Set; +import software.amazon.polymorph.smithypython.localservice.DafnyLocalServiceCodegenConstants; +import software.amazon.polymorph.smithypython.common.customize.CustomFileWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.traits.PositionalTrait; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * Writes the dafny_protocol.py file. This file defines the types that are sent to and from the + * dafnyImplInterface. + */ +public class DafnyProtocolFileWriter implements CustomFileWriter { + + @Override + public void customizeFileForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) { + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + codegenContext.settings().getService().getNamespace()); + + // Collect all `inputShapeIds` to identify all possible types `dafny_operation_input` can take + // on + Set inputShapeIds = new HashSet<>(); + for (ShapeId operationShapeId : serviceShape.getAllOperations()) { + OperationShape operationShape = + codegenContext.model().expectShape(operationShapeId, OperationShape.class); + inputShapeIds.add(operationShape.getInputShape()); + } + + codegenContext + .writerDelegator() + .useFileWriter( + moduleName + "/dafny_protocol.py", + "", + writer -> { + writer.write( + """ + import Wrappers + from typing import Union + + class $L: + operation_name: str + + # dafny_operation_input can take on any one of the types + # of the input values passed to the Dafny implementation + dafny_operation_input: Union[ + ${C|} + ] + + def __init__(self, operation_name, dafny_operation_input): + self.operation_name = operation_name + self.dafny_operation_input = dafny_operation_input + + class $L(Wrappers.Result): + def __init__(self): + super().__init__(self) + """, + DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_REQUEST, + writer.consumer( + w -> + generateDafnyOperationInputUnionValues(inputShapeIds, w, codegenContext)), + DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_RESPONSE); + }); + } + + /** + * Generates the list of types that compose the Union of types that `dafny_operation_input` can + * take on. + * + * @param inputShapeIds + * @param writer + */ + private void generateDafnyOperationInputUnionValues( + Set inputShapeIds, PythonWriter writer, GenerationContext context) { + // If all operations on the service take no inputs, + // or if the service has no operations, + // write `None` + if (inputShapeIds.size() == 0) { + writer.write("None"); + } + for (ShapeId inputShapeId : inputShapeIds) { + if (context.model().expectShape(inputShapeId).isStructureShape() + && context + .model() + .expectShape(inputShapeId) + .asStructureShape() + .get() + .hasTrait(PositionalTrait.class)) { + // TODO: Typing positionals, and somehow typing the base classes + // Blob, boolean... + } else { + DafnyNameResolver.importDafnyTypeForShape(writer, inputShapeId, context); + writer.write("$L,", DafnyNameResolver.getDafnyTypeForShape(inputShapeId)); + } + } + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ErrorsFileWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ErrorsFileWriter.java new file mode 100644 index 0000000000..a7e19eb71b --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ErrorsFileWriter.java @@ -0,0 +1,418 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.customize; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.common.customize.CustomFileWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +import static java.lang.String.format; + +/** Extends the Smithy-Python-generated errors.py file by adding Dafny plugin errors. */ +public class ErrorsFileWriter implements CustomFileWriter { + + @Override + public void customizeFileForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) { + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + codegenContext.settings().getService().getNamespace()); + + codegenContext + .writerDelegator() + .useFileWriter( + moduleName + "/errors.py", + "", + writer -> { + writer.addStdlibImport("typing", "Dict"); + writer.addStdlibImport("typing", "Any"); + writer.addStdlibImport("typing", "List"); + + // Generate Smithy shapes for each of this service's modelled errors + TreeSet deserializingErrorShapes = + new TreeSet( + codegenContext.model().getStructureShapesWithTrait(ErrorTrait.class).stream() + .filter( + structureShape -> + structureShape + .getId() + .getNamespace() + .equals( + codegenContext.settings().getService().getNamespace())) + .collect(Collectors.toSet())); + for (StructureShape errorShape : deserializingErrorShapes) { + renderError(codegenContext, writer, errorShape); + } + + // Generate Smithy shapes that wrap each dependency service's modelled and un-modelled + // errors + Optional maybeLocalServiceTrait = + serviceShape.getTrait(LocalServiceTrait.class); + if (maybeLocalServiceTrait.isPresent()) { + LocalServiceTrait localServiceTrait = maybeLocalServiceTrait.get(); + Set serviceDependencyShapeIds = localServiceTrait.getDependencies(); + if (serviceDependencyShapeIds != null) { + for (ShapeId serviceDependencyShapeId : serviceDependencyShapeIds) { + renderDependencyWrappingError(codegenContext, writer, serviceDependencyShapeId); + } + } + } + + // Generate Smithy shapes for each of this service's un-modelled errors + writer.write( + """ + class CollectionOfErrors(ApiError[Literal["CollectionOfErrors"]]): + code: Literal["CollectionOfErrors"] = "CollectionOfErrors" + message: str + list: List[ServiceError] + + def __init__( + self, + *, + message: str, + list + ): + super().__init__(message) + self.list = list + + def as_dict(self) -> Dict[str, Any]: + ""\"Converts the CollectionOfErrors to a dictionary. + + The dictionary uses the modeled shape names rather than the parameter names as + keys to be mostly compatible with boto3. + ""\" + return { + 'message': self.message, + 'code': self.code, + 'list': self.list, + } + + @staticmethod + def from_dict(d: Dict[str, Any]) -> "CollectionOfErrors": + ""\"Creates a CollectionOfErrors from a dictionary. + + The dictionary is expected to use the modeled shape names rather than the + parameter names as keys to be mostly compatible with boto3. + ""\" + kwargs: Dict[str, Any] = { + 'message': d['message'], + 'list': d['list'] + } + + return CollectionOfErrors(**kwargs) + + def __repr__(self) -> str: + result = "CollectionOfErrors(" + result += f'message={self.message},' + if self.message is not None: + result += f"message={repr(self.message)}" + result += f'list={self.list}' + result += ")" + return result + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, CollectionOfErrors): + return False + if not (self.list == other.list): + return False + attributes: list[str] = ['message','message'] + return all( + getattr(self, a) == getattr(other, a) + for a in attributes + ) + + class OpaqueError(ApiError[Literal["OpaqueError"]]): + code: Literal["OpaqueError"] = "OpaqueError" + obj: Any # As an OpaqueError, type of obj is unknown + + def __init__( + self, + *, + obj + ): + super().__init__("") + self.obj = obj + + def as_dict(self) -> Dict[str, Any]: + ""\"Converts the OpaqueError to a dictionary. + + The dictionary uses the modeled shape names rather than the parameter names as + keys to be mostly compatible with boto3. + ""\" + return { + 'message': self.message, + 'code': self.code, + 'obj': self.obj, + } + + @staticmethod + def from_dict(d: Dict[str, Any]) -> "OpaqueError": + ""\"Creates a OpaqueError from a dictionary. + + The dictionary is expected to use the modeled shape names rather than the + parameter names as keys to be mostly compatible with boto3. + ""\" + kwargs: Dict[str, Any] = { + 'message': d['message'], + 'obj': d['obj'] + } + + return OpaqueError(**kwargs) + + def __repr__(self) -> str: + result = "OpaqueError(" + result += f'message={self.message},' + if self.message is not None: + result += f"message={repr(self.message)}" + result += f'obj={self.obj}' + result += ")" + return result + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, OpaqueError): + return False + if not (self.obj == other.obj): + return False + attributes: list[str] = ['message','message'] + return all( + getattr(self, a) == getattr(other, a) + for a in attributes + ) + """); + + // Write error serializer function + // "serializer" smithy-to-dafny + writer.write( + """ + def _smithy_error_to_dafny_error(e: ServiceError): + ""\" + Converts the provided native Smithy-modeled error + into the corresponding Dafny error. + ""\" + ${C|} + """, + writer.consumer( + w -> generateSmithyErrorToDafnyErrorBlock(codegenContext, serviceShape, w))); + }); + } + + /** + * Generate the method body for the `_smithy_error_to_dafny_error` method. + * + * @param codegenContext + * @param serviceShape + * @param writer + */ + private void generateSmithyErrorToDafnyErrorBlock( + GenerationContext codegenContext, ServiceShape serviceShape, PythonWriter writer) { + + // Write modelled error converters for this service + TreeSet errorShapeSet = + new TreeSet( + codegenContext.model().getStructureShapesWithTrait(ErrorTrait.class).stream() + .filter( + structureShape -> + structureShape + .getId() + .getNamespace() + .equals(codegenContext.settings().getService().getNamespace())) + .map(Shape::getId) + .collect(Collectors.toSet())); + for (ShapeId errorShapeId : errorShapeSet) { + SmithyNameResolver.importSmithyGeneratedTypeForShape(writer, errorShapeId, codegenContext); + writer.addStdlibImport(DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(errorShapeId)); + writer.write( + """ + if isinstance(e, $L.$L): + return $L.$L(message=e.message) + """, + SmithyNameResolver.getSmithyGeneratedModelLocationForShape(errorShapeId, codegenContext), + errorShapeId.getName(), + DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(errorShapeId), + DafnyNameResolver.getDafnyTypeForError(errorShapeId)); + } + + // Write out wrapping errors for dependencies. + // This service will generate a dependency-specific error for each dependency. + // This dependency-specific error generated for only this service, and not for the dependency + // service; + // this error type will wrap any dependency service's error for processing by this service. + // The Dafny type for this error contains one member: the dependency's name. + // ex. Dafny type for MyDependency: `Error_MyDependency(Error...) = { MyDependency: ... }` + // This member's value can take on any of the error types modelled in the dependency. + // Polymorph will generate a similar error structure in the primary service's errors.py file. + // ex. Smithy type for MyDependency: `MyDependency(ApiError...) = { MyDependency: ... }` + Optional maybeLocalServiceTrait = + serviceShape.getTrait(LocalServiceTrait.class); + if (maybeLocalServiceTrait.isPresent()) { + LocalServiceTrait localServiceTrait = maybeLocalServiceTrait.get(); + Set serviceDependencyShapeIds = localServiceTrait.getDependencies(); + + if (serviceDependencyShapeIds != null) { + for (ShapeId serviceDependencyShapeId : serviceDependencyShapeIds) { + + ServiceShape dependencyServiceShape = codegenContext + .model() + .expectShape(serviceDependencyShapeId) + .asServiceShape() + .get(); + + String nativeToDafnyErrorName; + String conversionFilename; + if (dependencyServiceShape.hasTrait(LocalServiceTrait.class)) { + nativeToDafnyErrorName = "_smithy_error_to_dafny_error"; + } else if (AwsSdkNameResolver.isAwsSdkShape(dependencyServiceShape)) { + nativeToDafnyErrorName = "_sdk_error_to_dafny_error"; + } else { + throw new IllegalArgumentException("Provided serviceShape is neither localService nor AWS SDK shape: " + dependencyServiceShape); + } + + // Import the dependency service's `_smithy_error_to_dafny_error` so this service + // can defer error conversion to the dependency + writer.addStdlibImport( + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + serviceDependencyShapeId.getNamespace(), codegenContext.settings()) + + (AwsSdkNameResolver.isAwsSdkShape(serviceDependencyShapeId) + ? ".shim" + : ".errors"), + nativeToDafnyErrorName, + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + serviceDependencyShapeId.getNamespace()) + + nativeToDafnyErrorName); + + // Import this service's error that wraps the dependency service's errors + ServiceShape serviceDependencyShape = codegenContext.model().expectShape(serviceDependencyShapeId).asServiceShape().get(); + String dependencyErrorName = SmithyNameResolver.getSmithyGeneratedTypeForServiceError(serviceDependencyShape); +// writer.addImport(".errors", dependencyErrorName); + // Generate conversion method that says: + // "If this is a dependency-specific error, defer to the dependency's + // `_smithy_error_to_dafny_error`" + // if isinstance(e, MyDependency): + // return + // MyService.Error_MyDependency(MyDependency_smithy_error_to_dafny_error(e.message)) + writer.write( + """ + if isinstance(e, $L): + return $L.Error_$L($L(e.message)) + """, + dependencyErrorName, + DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(serviceShape), + dependencyErrorName, + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + serviceDependencyShapeId.getNamespace()) + + nativeToDafnyErrorName); + } + } + } + + // Add service-specific CollectionOfErrors + writer.write( + """ + if isinstance(e, CollectionOfErrors): + return $L.Error_CollectionOfErrors(message=e.message, list=e.list) + """, + DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(serviceShape.getId())); + // Add service-specific OpaqueError + writer.write( + """ + if isinstance(e, OpaqueError): + return $L.Error_Opaque(obj=e.obj) + """, + DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(serviceShape.getId())); + } + + + + // This is lifted from Smithy-Python. + // Smithy-Python has no concept of dependencies or other namespaces. + // This allows errors in other namespaces to be rendered correctly. + private void renderError(GenerationContext context, PythonWriter writer, StructureShape shape) { + writer.addStdlibImport("typing", "Dict"); + writer.addStdlibImport("typing", "Literal"); + + String code = shape.getId().getName(); + Symbol symbol = context.symbolProvider().toSymbol(shape); + var apiError = + Symbol.builder() + .name("ApiError") + .namespace( + format( + "%s.errors", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + context.settings().getService().getNamespace(), context)), + ".") + .definitionFile( + format( + "./%s/errors.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()))) + .build(); + writer.openBlock( + "class $L($T[Literal[$S]]):", + "", + symbol.getName(), + apiError, + code, + () -> { + writer.write("code: Literal[$1S] = $1S", code); + writer.write("message: str"); + }); + writer.write(""); + } + + // This is lifted from Smithy-Python. + // Smithy-Python has no concept of dependencies or other namespaces. + // This allows errors in other namespaces to be rendered correctly. + private void renderDependencyWrappingError( + GenerationContext context, PythonWriter writer, ShapeId serviceDependencyShapeId) { + writer.addStdlibImport("typing", "Dict"); + writer.addStdlibImport("typing", "Any"); + writer.addStdlibImport("typing", "Literal"); + + ServiceShape serviceDependencyShape = context.model().expectShape(serviceDependencyShapeId).asServiceShape().get(); + String code = SmithyNameResolver.getSmithyGeneratedTypeForServiceError(serviceDependencyShape); + + var apiError = + Symbol.builder() + .name("ApiError") + .namespace( + format( + "%s.errors", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + context.settings().getService().getNamespace(), context)), + ".") + .definitionFile( + format( + "./%s/errors.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()))) + .build(); + writer.openBlock( + "class $L($T[Literal[$S]]):", + "", + code, + apiError, + code, + () -> { + writer.write("$L: $L", code, "Any"); + }); + writer.write(""); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ModelsFileWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ModelsFileWriter.java new file mode 100644 index 0000000000..dc95a92b74 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ModelsFileWriter.java @@ -0,0 +1,37 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.customize; + +import software.amazon.polymorph.smithypython.common.customize.CustomFileWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.python.codegen.GenerationContext; + +/** Extends the Smithy-Python-generated models.py file by adding Dafny plugin models. */ +public class ModelsFileWriter implements CustomFileWriter { + + @Override + public void customizeFileForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) { + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + codegenContext.settings().getService().getNamespace()); + codegenContext + .writerDelegator() + .useFileWriter( + moduleName + "/models.py", + "", + writer -> { + + // This block defines an empty `Unit` class used by Smithy-Python generated code + // Defining this seems necessary to avoid forking Smithy-Python + // TODO-Python: Find some way to not need this, or decide this is OK. Low priority + writer.write( + """ + class Unit: + pass + """); + }); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/PluginFileWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/PluginFileWriter.java new file mode 100644 index 0000000000..9338bebe19 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/PluginFileWriter.java @@ -0,0 +1,82 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.customize; + +import software.amazon.polymorph.smithypython.common.customize.CustomFileWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.python.codegen.GenerationContext; + +/** + * Writes the plugin.py file. This file contains logic to load the Dafny plugin into the + * Smithy-Python client.py's Config member. It also defines the Config's retry strategy ("never + * retry" -- this is not a service). + */ +public class PluginFileWriter implements CustomFileWriter { + + @Override + public void customizeFileForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) { + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + codegenContext.settings().getService().getNamespace()); + String implModulePrelude = + DafnyNameResolver.getDafnyPythonIndexModuleNameForShape(serviceShape); + final LocalServiceTrait localServiceTrait = serviceShape.expectTrait(LocalServiceTrait.class); + final StructureShape configShape = + codegenContext.model().expectShape(localServiceTrait.getConfigId(), StructureShape.class); + + codegenContext + .writerDelegator() + .useFileWriter( + moduleName + "/plugin.py", + "", + writer -> { + writer.write( + """ + from .config import Config, Plugin, smithy_config_to_dafny_config, $L + from smithy_python.interfaces.retries import RetryStrategy + from smithy_python.exceptions import SmithyRetryException + from .dafnyImplInterface import DafnyImplInterface + + def set_config_impl(config: Config): + ""\" + Set the Dafny-compiled implementation in the Smithy-Python client Config + and load our custom NoRetriesStrategy. + ""\" + config.dafnyImplInterface = DafnyImplInterface() + if isinstance(config, $L): + from $L import default__ + config.dafnyImplInterface.impl = default__.$L(smithy_config_to_dafny_config(config)).value + config.retry_strategy = NoRetriesStrategy() + + class ZeroRetryDelayToken: + ""\" + Placeholder class required by Smithy-Python client implementation. + Do not wait to retry. + ""\" + retry_delay = 0 + + class NoRetriesStrategy(RetryStrategy): + ""\" + Placeholder class required by Smithy-Python client implementation. + Do not retry calling Dafny code. + ""\" + def acquire_initial_retry_token(self): + return ZeroRetryDelayToken() + + def refresh_retry_token_for_retry(self, token_to_renew, error_info): + # Do not retry + raise SmithyRetryException() + """, + configShape.getId().getName(), + configShape.getId().getName(), + implModulePrelude, + localServiceTrait.getSdkId()); + }); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ReferencesFileWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ReferencesFileWriter.java new file mode 100644 index 0000000000..5ed56f880c --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/customize/ReferencesFileWriter.java @@ -0,0 +1,494 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.customize; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import software.amazon.polymorph.smithypython.common.customize.CustomFileWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.shapevisitor.ShapeVisitorResolver; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.shapes.*; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.utils.CaseUtils; + +/** + * Writes references ({@link ResourceShape}s with a {@link + * software.amazon.polymorph.traits.ReferenceTrait}) to a `references.py` file. References are + * separated from the `models.py` file to avoid circular import issues. + */ +public class ReferencesFileWriter implements CustomFileWriter { + + private static Set generatedResourceShapes = new HashSet<>(); + + public static boolean hasGeneratedResourceForShape(ShapeId shapeId) { + return generatedResourceShapes.contains(shapeId); + } + + public static boolean shouldGenerateResourceForShape(ResourceShape resourceShape, GenerationContext codegenContext) { + return !hasGeneratedResourceForShape(resourceShape.getId()) + && resourceShape + .getId() + .getNamespace() + .equals(codegenContext.settings().getService().getNamespace()); + } + + public void generateResourceInterfaceAndImplementation( + ResourceShape resourceShape, GenerationContext codegenContext, PythonWriter writer) { + if (shouldGenerateResourceForShape(resourceShape, codegenContext)) { + generatedResourceShapes.add(resourceShape.getId()); + generateResourceInterface(resourceShape, codegenContext, writer); + generateResourceImplementation(resourceShape, codegenContext, writer); + } + } + + protected void generateResourceInterface( + ResourceShape resourceShape, GenerationContext context, PythonWriter writer) { + // Only generate resources in the service namespace + if (!resourceShape + .getId() + .getNamespace() + .equals(context.settings().getService().getNamespace())) { + return; + } + // Write reference interface. + // We use the `abc` (Abstract Base Class) library to define a stricter interface contract + // for references in Python than a standard Python subclass contract. + // The generated code will use the ABC library to enforce constraints at object-create time. + // In particular, when the object is constructed, the constructor will validate that the + // object's class implements all callable operations defined in the reference's Smithy model. + // This differs from standard Python duck-typing, where classes implementing an "interface" are + // only checked that an "interface" operation is implemented at operation call-time. + // We do this for a number of reasons: + // 1) This is a Smithy-Dafny code generator, and Dafny has this stricter interface + // contract. We decide to generate code that biases toward the Dafny behavior; + // 2) A strict interface contract will detect issues implementing an interface sooner. + // This is opinionated and may change. + writer.addStdlibImport("abc"); + writer.write( + """ + + class I$L(metaclass=abc.ABCMeta): + ${C|} + @classmethod + def __subclasshook__(cls, subclass): + return ( + ${C|} + ) + + ${C|} + + ${C|} + + + """, + resourceShape.getId().getName(), + writer.consumer( + w -> writeDocsForResourceOrInterfaceClass(writer, resourceShape, context)), + writer.consumer( + w -> generateInterfaceSubclasshookExpressionForResource(context, resourceShape, w)), + writer.consumer( + w -> + generateInterfaceOperationFunctionDefinitionForResource( + context, resourceShape, w)), + writer.consumer( + w -> generateNativeWrapperFunctionDefinitionForResource(context, resourceShape, w)) + ); + + writer.addStdlibImport("typing", "Any"); + } + +protected void generateResourceImplementation( + ResourceShape resourceShape, GenerationContext context, PythonWriter writer) { + // Only generate resources in the service namespace + if (!resourceShape + .getId() + .getNamespace() + .equals(context.settings().getService().getNamespace())) { + return; + } + + String dafnyInterfaceTypeName = + DafnyNameResolver.getDafnyInterfaceTypeForResourceShape(resourceShape); + writer.addStdlibImport(DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(resourceShape)); + + // Write implementation for resource shape + writer.write( + """ + class $1L(I$1L): + ${5C|} + _impl: $2L + + def __init__(self, _impl: $2L): + self._impl = _impl + + ${3C|} + + ${4C|} + """, + resourceShape.getId().getName(), + DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(resourceShape) + + "." + + dafnyInterfaceTypeName, + writer.consumer( + w -> generateSmithyOperationFunctionDefinitionForResource(context, resourceShape, w)), + writer.consumer(w -> generateDictConvertersForResource(resourceShape, w)), + writer.consumer( + w -> writeDocsForResourceOrInterfaceClass(w, resourceShape, context)) + ); + } + + /** + * Generates the expression that validates that a class that claims to implement an interface + * implements all required operations. + * + * @param codegenContext + * @param resourceShape + * @param writer + */ + private void generateInterfaceSubclasshookExpressionForResource( + GenerationContext codegenContext, ResourceShape resourceShape, PythonWriter writer) { + + List operationList = resourceShape.getOperations().stream().toList(); + Iterator operationListIterator = operationList.iterator(); + + // For all but the last operation shape, generate `hasattr and callable and` + // For the last shape, generate `hasattr and callable` + while (operationListIterator.hasNext()) { + ShapeId operationShapeId = operationListIterator.next(); + writer.writeInline( + """ + hasattr(subclass, "$L") and callable(subclass.$L)""", + operationShapeId.getName(), + operationShapeId.getName()); + if (operationListIterator.hasNext()) { + writer.write(" and"); + } + } + } + + /** + * Generates abstract methods for all operations on the provided resource. This is called from a + * resource interface to generate its abstract methods. + * + * @param codegenContext + * @param resourceShape + * @param writer + */ + private void generateInterfaceOperationFunctionDefinitionForResource( + GenerationContext codegenContext, ResourceShape resourceShape, PythonWriter writer) { + Set operationShapeIds = resourceShape.getOperations(); + + for (ShapeId operationShapeId : operationShapeIds) { + writer.write("@abc.abstractmethod"); + + OperationShape operationShape = + codegenContext.model().expectShape(operationShapeId, OperationShape.class); + + Shape targetShapeInput = codegenContext.model().expectShape(operationShape.getInputShape()); + SmithyNameResolver.importSmithyGeneratedTypeForShape(writer, targetShapeInput, codegenContext); + Symbol inputSymbol = codegenContext.symbolProvider().toSymbol(targetShapeInput); + + + Shape targetShapeOutput = codegenContext.model().expectShape(operationShape.getOutputShape()); + SmithyNameResolver.importSmithyGeneratedTypeForShape(writer, targetShapeOutput, codegenContext); + Symbol outputSymbol = codegenContext.symbolProvider().toSymbol(targetShapeOutput); + + + writer.openBlock("def $L(self, param: '$L') -> '$L':", + "", + CaseUtils.toSnakeCase(operationShapeId.getName()), + inputSymbol, + outputSymbol, + () -> { + writeDocsForResourceOrInterfaceOperation(writer, codegenContext.model().expectShape(operationShapeId).asOperationShape().get(), codegenContext); + writer.write("raise NotImplementedError"); + }); + + writer.addStdlibImport(inputSymbol.getNamespace()); + writer.addStdlibImport(outputSymbol.getNamespace()); + } + } + + /** + * Generates abstract methods for all operations on the provided resource. This is called from a + * resource interface to generate its abstract methods. + * + * @param codegenContext + * @param resourceShape + * @param writer + */ + private void generateNativeWrapperFunctionDefinitionForResource( + GenerationContext codegenContext, ResourceShape resourceShape, PythonWriter writer) { + Set operationShapeIds = resourceShape.getOperations(); + + for (ShapeId operationShapeId : operationShapeIds) { + OperationShape operationShape = + codegenContext.model().expectShape(operationShapeId, OperationShape.class); + + Shape targetShapeInput = codegenContext.model().expectShape(operationShape.getInputShape()); + SmithyNameResolver.importSmithyGeneratedTypeForShape(writer, targetShapeInput, codegenContext); + Symbol inputSymbol = codegenContext.symbolProvider().toSymbol(targetShapeInput); + + String dafnyInput = DafnyNameResolver.getDafnyTypeForShape(targetShapeInput); + DafnyNameResolver.importDafnyTypeForShape(writer, targetShapeInput.getId(), codegenContext); + + + Shape targetShapeOutput = codegenContext.model().expectShape(operationShape.getOutputShape()); + SmithyNameResolver.importSmithyGeneratedTypeForShape(writer, targetShapeOutput, codegenContext); + Symbol outputSymbol = codegenContext.symbolProvider().toSymbol(targetShapeOutput); + + String dafnyOutput = DafnyNameResolver.getDafnyTypeForShape(targetShapeOutput); + DafnyNameResolver.importDafnyTypeForShape(writer, targetShapeOutput.getId(), codegenContext); + + ServiceShape serviceShape = codegenContext.model().expectShape(codegenContext.settings().getService()).asServiceShape().get(); + // Services don't specify a "default" error. + // Pick the first one off its list of errors. + // Crypto Tools only specifies one error per service, so this works perfectly, but may not extend well. + // This may need to be updated. + String defaultWrappingError = + !serviceShape.getErrors().isEmpty() + ? DafnyNameResolver.getDafnyTypeForError(serviceShape.getErrors().get(0)) + : "Error"; + + writer.addStdlibImport( + DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(serviceShape), + defaultWrappingError + ); + + writer.openBlock("def $L(self, dafny_input: '$L') -> '$L':", + "", + operationShapeId.getName(), + dafnyInput, + dafnyOutput, + () -> { + writer.write( + /* + native_input = aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.dafny_to_smithy.DafnyToSmithy_aws_cryptography_materialproviders_GetBranchKeyIdInput( + dafny_input + ) + try: + native_output = self.get_branch_key_id(native_input) + dafny_output = aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.smithy_to_dafny.SmithyToDafny_aws_cryptography_materialproviders_GetBranchKeyIdOutput( + native_output + ) + return Wrappers.Result_Success(dafny_output) + except Exception as e: + error = software_amazon_cryptography_materialproviders_internaldafny_types.Error_AwsCryptographicMaterialProvidersException( + message=str(e) + ) + return Wrappers.Result_Failure(error) + */ + """ + ""\" + Do not use. + This method allows custom implementations of this interface to interact with generated code. + ""\" + native_input = $L( + dafny_input + ) + try: + native_output = self.$L(native_input) + dafny_output = $L( + native_output + ) + return Wrappers.Result_Success(dafny_output) + except Exception as e: + error = $L( + message=str(e) + ) + return Wrappers.Result_Failure(error) + """, + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + targetShapeInput.getId().getNamespace(), + codegenContext + ) + ".dafny_to_smithy." + + SmithyNameResolver.getDafnyToSmithyFunctionNameForShape(targetShapeInput, codegenContext), + CaseUtils.toSnakeCase(operationShapeId.getName()), + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + targetShapeOutput.getId().getNamespace(), + codegenContext + ) + ".smithy_to_dafny." + + SmithyNameResolver.getSmithyToDafnyFunctionNameForShape(targetShapeOutput, codegenContext), + defaultWrappingError + ); + writer.addStdlibImport("Wrappers"); + }); + } + } + + /** + * Generates methods for all operations on the provided resource. The generated method takes in a + * native type input and returns a native type output. Internally, the method will convert the + * native type to a Dafny type, call the Dafny implementation with the Dafny type, receive a Dafny + * type from the Dafny implementation, convert the Dafny type back to the corresponding native + * type, and then return the native type. This is called from a concrete resource. + * + * @param codegenContext + * @param resourceShape + * @param writer + */ + private void generateSmithyOperationFunctionDefinitionForResource( + GenerationContext codegenContext, ResourceShape resourceShape, PythonWriter writer) { + Set operationShapeIds = resourceShape.getOperations(); + + for (ShapeId operationShapeId : operationShapeIds) { + OperationShape operationShape = + codegenContext.model().expectShape(operationShapeId, OperationShape.class); + Symbol operationSymbol = codegenContext.symbolProvider().toSymbol(operationShape); + + Shape targetShapeInput = codegenContext.model().expectShape(operationShape.getInputShape()); + SmithyNameResolver.importSmithyGeneratedTypeForShape(writer, targetShapeInput, codegenContext); + // Generate code that converts the input from the Dafny type to the corresponding Smithy type + String input = + targetShapeInput.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape( + targetShapeInput, codegenContext, "param", writer)); + Symbol inputSymbol = codegenContext.symbolProvider().toSymbol(targetShapeInput); + + Shape targetShapeOutput = codegenContext.model().expectShape(operationShape.getOutputShape()); + SmithyNameResolver.importSmithyGeneratedTypeForShape(writer, targetShapeOutput, codegenContext); + // Generate output code converting the return value of the Dafny implementation into + // its corresponding native-modelled type. + String output = + targetShapeOutput.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape( + targetShapeOutput, codegenContext, "dafny_output.value", writer)); + + Symbol outputSymbol = codegenContext.symbolProvider().toSymbol(targetShapeOutput); + + + writer.openBlock( + "def $L(self, param: '$L') -> '$L':", + "", + CaseUtils.toSnakeCase(operationShapeId.getName()), + inputSymbol, + outputSymbol, + () -> { + writeDocsForResourceOrInterfaceOperation(writer, operationShape, codegenContext); + + writer.write("dafny_output = self._impl.$L($L)", operationShapeId.getName(), input); + writer.openBlock("if dafny_output.IsFailure():", + "", + () -> { + writer.addStdlibImport("asyncio"); + writer.addStdlibImport( + // `from dependency.smithygenerated.deserialize` + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + targetShapeOutput.getId().getNamespace(), codegenContext) + + ".deserialize", + // `import _deserialize_error` + "_deserialize_error", + // `as dependency_deserialize_error` + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + targetShapeOutput.getId().getNamespace()) + + "_deserialize_error"); + writer.write("raise asyncio.run($L(dafny_output.error))", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + targetShapeOutput.getId().getNamespace()) + + "_deserialize_error" + ); + } + ); + writer.openBlock("else:", + "", + () -> writer.write("return $L", + output) + ); + }); + writer.addStdlibImport(inputSymbol.getNamespace()); + writer.addStdlibImport(outputSymbol.getNamespace()); + } + } + + private void writeDocsForResourceOrInterfaceClass(PythonWriter writer, ResourceShape resourceShape, + GenerationContext context) { + + if (resourceShape.getTrait(DocumentationTrait.class).isPresent()) { + writer.writeDocs( + () -> { + resourceShape + .getTrait(DocumentationTrait.class) + .ifPresent( + trait -> { + writer.write(writer.formatDocs(trait.getValue())); + }); + }); + } + } + + private void writeDocsForResourceOrInterfaceOperation(PythonWriter writer, OperationShape operationShape, + GenerationContext context) { + + Shape inputShape = context.model().expectShape(operationShape.getInputShape()); + Shape outputShape = context.model().expectShape(operationShape.getOutputShape()); + + if (operationShape.getTrait(DocumentationTrait.class).isPresent() + || inputShape.getTrait(DocumentationTrait.class).isPresent() + || outputShape.getTrait(DocumentationTrait.class).isPresent()) { + writer.writeDocs( + () -> { + operationShape + .getTrait(DocumentationTrait.class) + .ifPresent( + trait -> { + writer.write(writer.formatDocs(trait.getValue())); + }); + inputShape + .getTrait(DocumentationTrait.class) + .ifPresent( + trait -> { + String memberDocs = + writer.formatDocs( + String.format(":param param: %s", trait.getValue())); + writer.write(memberDocs); + }); + outputShape + .getTrait(DocumentationTrait.class) + .ifPresent( + trait -> { + String memberDocs = + writer.formatDocs( + String.format(":returns: %s", trait.getValue())); + writer.write(memberDocs); + }); + }); + } + + } + + /** + * Writes `as_dict` and `from_dict` methods on the resource shape. These convert the shape to/from + * a dictionary and help with compatability across other shapes' as/from dict conversions. + * + * @param resource + * @param writer + */ + private void generateDictConvertersForResource(Shape resource, PythonWriter writer) { + + writer.addStdlibImport("typing", "Dict"); + writer.addStdlibImport("typing", "Any"); + + writer.write("@staticmethod"); + writer.openBlock( + "def from_dict(d: Dict[str, Any]) -> '$L':", + "", + resource.getId().getName(), + () -> { + writer.write("return $L(d['_impl'])", resource.getId().getName()); + }); + + writer.openBlock( + "def as_dict(self) -> Dict[str, Any]:", + "", + () -> { + writer.write("return {'_impl': self._impl}"); + }); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceClientCodegenPlugin.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceClientCodegenPlugin.java new file mode 100644 index 0000000000..627f9dd484 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceClientCodegenPlugin.java @@ -0,0 +1,170 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.extensions; + +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.traits.JavaDocTrait; +import software.amazon.polymorph.traits.ReferenceTrait; +import software.amazon.polymorph.utils.ModelUtils; +import software.amazon.smithy.build.PluginContext; +import software.amazon.smithy.build.SmithyBuildPlugin; +import software.amazon.smithy.codegen.core.directed.CodegenDirector; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.AbstractShapeBuilder; +import software.amazon.smithy.model.shapes.EnumShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.model.transform.ModelTransformer; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.utils.SmithyUnstableApi; + +import java.util.Map; + +/** + * Plugin to trigger Smithy-Dafny Python code generation. + * This differs from the PythonClientCodegenPlugin by not calling + * runner.performDefaultCodegenTransforms(); + * and + * runner.createDedicatedInputsAndOutputs(); + * These methods transform the model in ways that the model does not align with + * the generated Dafny code. + * This also transforms the model to bridge the gap between Polymorph-flavored Smithy and standard Smithy + * by mapping {@link JavaDocTrait}s to {@link DocumentationTrait}s + * and by adding {@link software.amazon.smithy.model.shapes.ResourceShape}s from {@link ReferenceTrait}s + * to the service shape so they can be discovered by Smithy plugins. + */ +@SmithyUnstableApi +public final class DafnyPythonLocalServiceClientCodegenPlugin implements SmithyBuildPlugin { + + public DafnyPythonLocalServiceClientCodegenPlugin(Map smithyNamespaceToPythonModuleNameMap) { + super(); + SmithyNameResolver.setSmithyNamespaceToPythonModuleNameMap(smithyNamespaceToPythonModuleNameMap); + } + + @Override + public String getName() { + return "dafny-python-localservice-client-codegen"; + } + + @Override + public void execute(PluginContext context) { + CodegenDirector runner + = new CodegenDirector<>(); + + PythonSettings settings = PythonSettings.from(context.getSettings()); + runner.settings(settings); + runner.directedCodegen(new DirectedDafnyPythonLocalServiceCodegen()); + runner.fileManifest(context.getFileManifest()); + runner.integrationClass(PythonIntegration.class); + + ServiceShape serviceShape = context.getModel().expectShape(settings.getService()).asServiceShape().get(); + Model transformedModel = transformModelForLocalService(context.getModel(), serviceShape); + + runner.model(transformedModel); + runner.service(settings.getService()); + + runner.run(); + } + + /** + * Perform all transformations on the model before running localService codegen. + * @param model + * @param serviceShape + * @return + */ + public static Model transformModelForLocalService(Model model, ServiceShape serviceShape) { + Model transformedModel = model; + transformedModel = transformJavadocTraitsToDocumentationTraits(transformedModel); + transformedModel = transformServiceShapeToAddReferenceResources(transformedModel, serviceShape); + transformedModel = transformStringEnumShapesToEnumShapes(transformedModel, serviceShape); + return transformedModel; + } + + /** + * For each object with a Polymorph {@link JavaDocTrait} containing documentation, + * add a new Smithy {@link DocumentationTrait} with that documentation. + * Smithy plugins will generate docs for DocumentationTraits. + * @param model + * @return + */ + public static Model transformJavadocTraitsToDocumentationTraits(Model model) { + return ModelTransformer.create().mapShapes(model, shape -> { + if (shape.hasTrait(JavaDocTrait.class)) { + JavaDocTrait javaDocTrait = shape.getTrait(JavaDocTrait.class).get(); + DocumentationTrait documentationTrait = new DocumentationTrait(javaDocTrait.getValue()); + + AbstractShapeBuilder builder = ModelUtils.getBuilderForShape(shape); + builder.addTrait(documentationTrait); + return builder.build(); + // We sometimes write out long strings of slashes to mark a "break" in our smithy files + // However, Smithy plugins interpret strings starting with `//` as a comment + // that becomes a DocumentationTrait! + // This leads to poor UX on some pydocs. Remove any DocumentationTraits consisting solely of `/`s. + } else if (shape.hasTrait(DocumentationTrait.class)) { + DocumentationTrait documentationTrait = shape.getTrait(DocumentationTrait.class).get(); + String documentation = documentationTrait.getValue(); + + if (documentation.trim().matches("/+")) { + AbstractShapeBuilder builder = ModelUtils.getBuilderForShape(shape); + builder.removeTrait(DocumentationTrait.ID); + return builder.build(); + } else { + return shape; + } + } else { + return shape; + } + }); + } + + /** + * Smithy plugins require that resource shapes are attached to a ServiceShape. + * Smithy plugins also do not understand Polymorph's {@link ReferenceTrait} and will not + * discover the linked shape. + * This parses Polymorph's ReferenceTrait to attach any referenced resources to the {@param serviceShape} + * so the Smithy plugin's shape discovery can find the shape. + * @param model + * @param serviceShape + * @return + */ + public static Model transformServiceShapeToAddReferenceResources(Model model, ServiceShape serviceShape) { + + ServiceShape.Builder transformedServiceShapeBuilder = serviceShape.toBuilder(); + ModelTransformer.create().mapShapes(model, shape -> { + if (shape.hasTrait(ReferenceTrait.class)) { + ShapeId referenceShapeId = shape.expectTrait(ReferenceTrait.class).getReferentId(); + if (model.expectShape(referenceShapeId).isResourceShape()) { + transformedServiceShapeBuilder.addResource(referenceShapeId); + } + } + return shape; + }); + + return ModelTransformer.create().mapShapes(model, shape -> { + if (shape.getId().equals(serviceShape.getId())) { + return transformedServiceShapeBuilder.build(); + } else { + return shape; + } + }); + } + + public static Model transformStringEnumShapesToEnumShapes( + Model model, ServiceShape serviceShape) { + + return ModelTransformer.create().mapShapes(model, shape -> { + if (shape.hasTrait(EnumTrait.class)) { + EnumShape asEnum = EnumShape.fromStringShape(shape.asStringShape().get()).get(); + return asEnum; + } else { + return shape; + } + }); + } +} \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceConfigGenerator.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceConfigGenerator.java new file mode 100644 index 0000000000..d8c021e021 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceConfigGenerator.java @@ -0,0 +1,54 @@ +package software.amazon.polymorph.smithypython.localservice.extensions; + +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.traits.PositionalTrait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.python.codegen.*; + +import static java.lang.String.format; + +/** + * Override Smithy-Python's ConfigGenerator to support namespaces in other modules. + */ +public class DafnyPythonLocalServiceConfigGenerator extends ConfigGenerator { + + public DafnyPythonLocalServiceConfigGenerator(PythonSettings settings, GenerationContext context) { + super(settings, context); + } + + @Override + public void run() { + var config = Symbol.builder() + .name("Config") + .namespace(format("%s.config", SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace(settings.getService().getNamespace(), context)), ".") + .definitionFile(format("./%s/config.py", SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(settings.getService().getNamespace()))) + .build(); + context.writerDelegator().useFileWriter + (config.getDefinitionFile(), config.getNamespace(), writer -> { + writeInterceptorsType(writer); + generateConfig(context, writer); + }); + + var plugin = Symbol.builder() + .name("Plugin") + .namespace(format("%s.config", SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace(settings.getService().getNamespace(), context)), ".") + .definitionFile(format("./%s/config.py", SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(settings.getService().getNamespace()))) + .build(); + context.writerDelegator().useFileWriter(plugin.getDefinitionFile(), plugin.getNamespace(), writer -> { + writer.addStdlibImport("typing", "Callable"); + writer.addStdlibImport("typing", "TypeAlias"); + writer.writeComment("A callable that allows customizing the config object on each request."); + writer.write("$L: TypeAlias = Callable[[$T], None]", plugin.getName(), config); + }); + } + + // Write `Any`. Typehints here introduce a circular dependency in case of reference shapes. + protected void writeInterceptorsType(PythonWriter writer) { + writer.addStdlibImport("typing", "Any"); + writer.write("_ServiceInterceptor = Any"); + } + +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceStructureGenerator.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceStructureGenerator.java new file mode 100644 index 0000000000..e960b753c4 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceStructureGenerator.java @@ -0,0 +1,499 @@ +package software.amazon.polymorph.smithypython.localservice.extensions; + +import static java.lang.String.format; + +import java.util.Set; + +import org.checkerframework.checker.units.qual.Length; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.traits.PositionalTrait; +import software.amazon.polymorph.traits.ReferenceTrait; +import software.amazon.polymorph.utils.ConstrainTraitUtils; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.NullableIndex; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.model.traits.LengthTrait; +import software.amazon.smithy.model.traits.RangeTrait; +import software.amazon.smithy.model.traits.SensitiveTrait; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.StructureGenerator; + +/** + * Override Smithy-Python's StructureGenerator, almost entirely to support memberShapes with {@link + * ReferenceTrait}. This often defers to Smithy-Python's StructureGenerator if not processing a + * reference shape. + */ +public class DafnyPythonLocalServiceStructureGenerator extends StructureGenerator { + + public DafnyPythonLocalServiceStructureGenerator( + Model model, + PythonSettings settings, + SymbolProvider symbolProvider, + PythonWriter writer, + StructureShape shape, + Set recursiveShapes) { + super(model, settings, symbolProvider, writer, shape, recursiveShapes); + } + + @Override + public void run() { + if (shape.hasTrait(PositionalTrait.class)) { + // Do not need to render shapes with positional trait, their linked shapes are rendered + return; + } + if (!shape.hasTrait(ErrorTrait.class)) { + renderStructure(); + } else { + renderError(); + } + } + + /** + * Override Smithy-Python renderError to use the correct namespace. + */ + @Override + protected void renderError() { + writer.addStdlibImport("typing", "Dict"); + writer.addStdlibImport("typing", "Any"); + writer.addStdlibImport("typing", "Literal"); + var code = shape.getId().getName(); + var symbol = symbolProvider.toSymbol(shape); + var apiError = + Symbol.builder() + .name("ApiError") + .namespace( + format( + "%s.errors", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + settings.getService().getNamespace(), settings)), + ".") + .definitionFile( + format( + "./%s/errors.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + settings.getService().getNamespace()))) + .build(); + writer.openBlock( + "class $L($T[Literal[$S]]):", + "", + symbol.getName(), + apiError, + code, + () -> { + writer.write("code: Literal[$1S] = $1S", code); + writer.write("message: str"); + writeProperties(true); + writeInit(true); + writeAsDict(true); + writeFromDict(true); + writeRepr(true); + writeEq(true); + }); + writer.write(""); + } + + @Override + protected void writeReprMembers(boolean isError) { + var iter = shape.members().iterator(); + while (iter.hasNext()) { + var member = iter.next(); + var memberName = symbolProvider.toMemberName(member); + var trailingComma = iter.hasNext() ? ", " : ""; + if (member.hasTrait(SensitiveTrait.class)) { + // Sensitive members must not be printed + // see: https://smithy.io/2.0/spec/documentation-traits.html#smithy-api-sensitive-trait + writer.write(""" + if self.$1L is not None: + result += f"$1L=...$2L" + """, memberName, trailingComma); + } else { + writer.write(""" + if self.$1L is not None: + result += f"$1L={repr(self.$1L)}$2L" + """, memberName, trailingComma); + } + } + } + + /** + * Override Smithy-Python writePropertyForMember to handle importing reference modules to avoid + * circular imports. If the {@param memberShape} is not a reference shape or a list of reference + * shapes, this defers to Smithy-Python's implementation. Maps containing reference shapes not + * supported yet. (Unused in MPL.) + * + * @param isError + * @param memberShape + */ + @Override + protected void writePropertyForMember(boolean isError, MemberShape memberShape) { + Shape target = model.expectShape(memberShape.getTarget()); + String memberName = symbolProvider.toMemberName(memberShape); + + if (target.isListShape() + && model + .expectShape(target.asListShape().get().getMember().getTarget()) + .hasTrait(ReferenceTrait.class)) { + Shape listMemberShape = model.expectShape(target.asListShape().get().getMember().getTarget()); + Shape referentShape = + model.expectShape(listMemberShape.expectTrait(ReferenceTrait.class).getReferentId()); + Symbol targetSymbol = symbolProvider.toSymbol(referentShape); + + // Use forward reference for reference traits to avoid circular import + // and do not import the symbol to avoid circular import + String formatString = "$L: list['$L']"; + writer.write( + formatString, memberName, targetSymbol.getNamespace() + "." + targetSymbol.getName()); + return; + } + + // We currently don't have any map shapes that have values with reference traits; + // once we do, this needs to be filled in + if (target.isMapShape() + && model + .expectShape(target.asMapShape().get().getValue().getTarget()) + .hasTrait(ReferenceTrait.class)) { + throw new IllegalArgumentException( + "Map shapes containing references currently unsupported: " + target); + } + + if (target.hasTrait(ReferenceTrait.class)) { + Shape referentShape = + model.expectShape(target.expectTrait(ReferenceTrait.class).getReferentId()); + + NullableIndex index = NullableIndex.of(model); + + if (index.isMemberNullable(memberShape)) { + writer.addStdlibImport("typing", "Optional"); + // Use forward reference for reference traits to avoid circular import + // and do not import the symbol to avoid circular import + String formatString = "$L: Optional['$L']"; + writer.write( + formatString, + memberName, + symbolProvider.toSymbol(referentShape).getNamespace() + + "." + + symbolProvider.toSymbol(referentShape).getName()); + writer.addStdlibImport(symbolProvider.toSymbol(referentShape).getNamespace()); + } else { + // Use forward reference for reference traits to avoid circular import, + // and do not import the symbol to avoid circular import + String formatString = "$L: '$L'"; + writer.write( + formatString, + memberName, + symbolProvider.toSymbol(referentShape).getNamespace() + + "." + + symbolProvider.toSymbol(referentShape).getName()); + writer.addStdlibImport(symbolProvider.toSymbol(referentShape).getNamespace()); + } + } else { + super.writePropertyForMember(isError, memberShape); + } + } + + /** + * Override Smithy-Python writeInitMethodParameterForRequiredMember to handle importing reference + * modules to avoid circular imports. If the {@param memberShape} is not a reference shape or a + * list of reference shapes, this defers to Smithy-Python's implementation. Maps containing + * reference shapes not supported yet. (Unused in MPL.) + * + * @param isError + * @param memberShape + */ + @Override + protected void writeInitMethodParameterForRequiredMember( + boolean isError, MemberShape memberShape) { + Shape target = model.expectShape(memberShape.getTarget()); + String memberName = symbolProvider.toMemberName(memberShape); + + if (target.isListShape() + && model + .expectShape(target.asListShape().get().getMember().getTarget()) + .hasTrait(ReferenceTrait.class)) { + Shape listMemberShape = model.expectShape(target.asListShape().get().getMember().getTarget()); + Shape referentShape = + model.expectShape(listMemberShape.expectTrait(ReferenceTrait.class).getReferentId()); + Symbol targetSymbol = symbolProvider.toSymbol(referentShape); + + // Use forward reference for reference traits to avoid circular import + // and do not import the symbol to avoid circular import + String formatString = "$L: list['$L'],"; + writer.write( + formatString, memberName, targetSymbol.getNamespace() + "." + targetSymbol.getName()); + return; + } + + // We currently don't have any map shapes that have values with reference traits; + // once we do, this needs to be filled in + if (target.isMapShape() + && model + .expectShape(target.asMapShape().get().getValue().getTarget()) + .hasTrait(ReferenceTrait.class)) { + throw new IllegalArgumentException( + "Map shapes containing references currently unsupported: " + target); + } + + if (target.hasTrait(ReferenceTrait.class)) { + Shape referentShape = + model.expectShape(target.expectTrait(ReferenceTrait.class).getReferentId()); + // Use forward reference for reference traits to avoid circular import + // and do not import the symbol to avoid circular import + String formatString = "$L: '$L',"; + writer.write( + formatString, + memberName, + symbolProvider.toSymbol(referentShape).getNamespace() + + "." + + symbolProvider.toSymbol(referentShape).getName()); + writer.addStdlibImport(symbolProvider.toSymbol(referentShape).getNamespace()); + } else { + super.writeInitMethodParameterForRequiredMember(isError, memberShape); + } + } + + /** + * Override Smithy-Python writeInitMethodParameterForOptionalMember to handle importing reference + * modules to avoid circular imports. If the {@param memberShape} is not a reference shape, this + * defers to Smithy-Python's implementation. + * + * @param isError + * @param memberShape + */ + @Override + protected void writeInitMethodParameterForOptionalMember( + boolean isError, MemberShape memberShape) { + Shape target = model.expectShape(memberShape.getTarget()); + + if (target.hasTrait(ReferenceTrait.class)) { + Shape referentShape = + model.expectShape(target.expectTrait(ReferenceTrait.class).getReferentId()); + String memberName = symbolProvider.toMemberName(memberShape); + + writer.addStdlibImport("typing", "Optional"); + // Use forward reference for reference traits to avoid circular import + String formatString = "$L: Optional['$L'] = None,"; + writer.write( + formatString, + memberName, + symbolProvider.toSymbol(referentShape).getNamespace() + + "." + + symbolProvider.toSymbol(referentShape).getName()); + writer.addStdlibImport(symbolProvider.toSymbol(referentShape).getNamespace()); + } else { + super.writeInitMethodParameterForOptionalMember(isError, memberShape); + } + } + + /** + * Override Smithy-Python writeFromDict to handle importing reference modules to avoid circular + * imports. Most of this is lifted directly from Smithy-Python; the changed component is + * highlighted. + * + * @param isError + */ + protected void writeFromDict(boolean isError) { + writer.write("@staticmethod"); + var shapeName = symbolProvider.toSymbol(shape).getName(); + writer.openBlock( + "def from_dict(d: Dict[str, Any]) -> $S:", + "", + shapeName, + () -> { + writer.writeDocs( + () -> { + writer.write("Creates a $L from a dictionary.\n", shapeName); + writer.write( + writer.formatDocs( + """ + The dictionary is expected to use the modeled shape names rather \ + than the parameter names as keys to be mostly compatible with boto3.""")); + }); + + if (shape.members().isEmpty() && !isError) { + writer.write("return $L()", shapeName); + return; + } + + // Block below is the only new addition to this function compared to Smithy-Python. + // Import any modules required for reference shapes to convert from_dict. + // Import within function to avoid circular imports from top-level imports + for (MemberShape memberShape : shape.members()) { + var target = model.expectShape(memberShape.getTarget()); + if (target.hasTrait(ReferenceTrait.class)) { + Symbol targetSymbol = symbolProvider.toSymbol(target); + writer.write( + "from $L import $L", targetSymbol.getNamespace(), targetSymbol.getName()); + } + } + + if (requiredMembers.isEmpty() && !isError) { + writer.write("kwargs: Dict[str, Any] = {}"); + } else { + writer.openBlock( + "kwargs: Dict[str, Any] = {", + "}", + () -> { + if (isError) { + writer.write("'message': d['message'],"); + } + for (MemberShape member : requiredMembers) { + var memberName = symbolProvider.toMemberName(member); + var target = model.expectShape(member.getTarget()); + Symbol targetSymbol = symbolProvider.toSymbol(target); + if (target.isStructureShape()) { + writer.write( + "$S: $L.from_dict(d[$S]),", + memberName, + targetSymbol.getName(), + member.getMemberName()); + } else if (targetSymbol.getProperty("fromDict").isPresent()) { + var targetFromDictSymbol = + targetSymbol.expectProperty("fromDict", Symbol.class); + writer.write( + "$S: $T(d[$S]),", + memberName, + targetFromDictSymbol, + member.getMemberName()); + } else { + writer.write("$S: d[$S],", memberName, member.getMemberName()); + } + } + }); + } + writer.write(""); + + for (MemberShape member : optionalMembers) { + var memberName = symbolProvider.toMemberName(member); + var target = model.expectShape(member.getTarget()); + writer.openBlock( + "if $S in d:", + "", + member.getMemberName(), + () -> { + var targetSymbol = symbolProvider.toSymbol(target); + if (target.isStructureShape()) { + writer.write( + "kwargs[$S] = $L.from_dict(d[$S])", + memberName, + targetSymbol.getName(), + member.getMemberName()); + } else if (targetSymbol.getProperty("fromDict").isPresent()) { + var targetFromDictSymbol = + targetSymbol.expectProperty("fromDict", Symbol.class); + writer.write( + "kwargs[$S] = $T(d[$S]),", + memberName, + targetFromDictSymbol, + member.getMemberName()); + } else { + writer.write("kwargs[$S] = d[$S]", memberName, member.getMemberName()); + } + }); + } + + writer.write("return $L(**kwargs)", shapeName); + }); + writer.write(""); + } + + protected void writeInitMethodAssignerForRequiredMember(MemberShape member, String memberName) { + // Smithy-Dafny: Check traits + // LengthTrait, RangeTrait + Shape targetShape = model.expectShape(member.getTarget()); + if (targetShape.hasTrait(RangeTrait.class)) { + RangeTrait rangeTrait = targetShape.getTrait(RangeTrait.class).get(); + if (rangeTrait.getMin().isPresent()) { + writeRangeTraitMinCheckForMember(member, memberName, rangeTrait); + } + if (rangeTrait.getMax().isPresent()) { + writeRangeTraitMaxCheckForMember(member, memberName, rangeTrait); + } + } + + if (targetShape.hasTrait(LengthTrait.class)) { + LengthTrait lengthTrait = targetShape.getTrait(LengthTrait.class).get(); + if (lengthTrait.getMin().isPresent()) { + writeLengthTraitMinCheckForMember(member, memberName, lengthTrait); + } + if (lengthTrait.getMax().isPresent()) { + writeLengthTraitMaxCheckForMember(member, memberName, lengthTrait); + } + } + writer.write("self.$1L = $1L", memberName); + } + + protected void writeInitMethodAssignerForOptionalMember(MemberShape member, String memberName) { + writer.write("self.$1L = $1L if $1L is not None else $2L", + memberName, getDefaultValue(writer, member)); + } + + protected void writeLengthTraitMinCheckForMember(MemberShape member, String memberName, LengthTrait lengthTrait) { + String min = ConstrainTraitUtils.LengthTraitUtils.min(lengthTrait); + writer.openBlock("if ($1L is not None) and (len($1L) < $2L):", + "", + memberName, + min, + () -> { + writer.write(""" + raise ValueError("The size of $1L must be greater than or equal to $2L") + """, + memberName, + min + ); + }); + } + + protected void writeLengthTraitMaxCheckForMember(MemberShape member, String memberName, LengthTrait lengthTrait) { + String max = ConstrainTraitUtils.LengthTraitUtils.max(lengthTrait); + writer.openBlock("if ($1L is not None) and (len($1L) > $2L):", + "", + memberName, + max, + () -> { + writer.write(""" + raise ValueError("The size of $1L must be less than or equal to $2L") + """, + memberName, + max + ); + }); + } + + protected void writeRangeTraitMinCheckForMember(MemberShape member, String memberName, RangeTrait rangeTrait) { + String min = ConstrainTraitUtils.RangeTraitUtils.minAsShapeType(model.expectShape(member.getTarget()), rangeTrait); + writer.openBlock("if ($1L is not None) and ($1L < $2L):", + "", + memberName, + min, + () -> { + writer.write(""" + raise ValueError("$1L must be greater than or equal to $2L") + """, + memberName, + min + ); + }); + } + + protected void writeRangeTraitMaxCheckForMember(MemberShape member, String memberName, RangeTrait rangeTrait) { + String max = ConstrainTraitUtils.RangeTraitUtils.maxAsShapeType(model.expectShape(member.getTarget()), rangeTrait); + writer.openBlock("if ($1L is not None) and ($1L > $2L):", + "", + memberName, + max, + () -> { + writer.write(""" + raise ValueError("$1L must be less than or equal to $2L") + """, + memberName, + max + ); + }); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceSymbolVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceSymbolVisitor.java new file mode 100644 index 0000000000..b0562a5fe5 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DafnyPythonLocalServiceSymbolVisitor.java @@ -0,0 +1,451 @@ +package software.amazon.polymorph.smithypython.localservice.extensions; + +import static java.lang.String.format; + +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.localservice.DafnyLocalServiceCodegenConstants; +import software.amazon.polymorph.smithypython.localservice.customize.ReferencesFileWriter; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.polymorph.traits.PositionalTrait; +import software.amazon.polymorph.traits.ReferenceTrait; +import software.amazon.smithy.codegen.core.*; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.*; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.model.traits.MediaTypeTrait; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.python.codegen.SymbolVisitor; +import software.amazon.smithy.utils.CaseUtils; +import software.amazon.smithy.utils.MediaType; +import software.amazon.smithy.utils.StringUtils; + +/** + * Override Smithy-Python's SymbolVisitor to support namespaces in other modules and + * Smithy-Dafny-specific traits. + */ +public class DafnyPythonLocalServiceSymbolVisitor extends SymbolVisitor { + + public DafnyPythonLocalServiceSymbolVisitor(Model model, PythonSettings settings) { + super(model, settings); + } + + protected String getSymbolNamespacePathForNamespaceAndFilename( + String namespace, String filename) { + return format( + "%s.%s", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + namespace, settings), + filename); + } + + protected String getSymbolDefinitionFilePathForNamespaceAndFilename( + String namespace, String filename) { + String directoryFilePath; + if ("smithy.api".equals(namespace)) { + directoryFilePath = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + settings.getService().getNamespace()); + } else { + directoryFilePath = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(namespace); + } + + return format("./%s/%s.py", directoryFilePath, filename); + } + + /** + * Override Smithy-Python's serviceShape to only generate shapes for this namespace's localService + * and to properly typehint boto3 clients. + * @param serviceShape + * @return + */ + @Override + public Symbol serviceShape(ServiceShape serviceShape) { + String generationPath = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + settings.getService().getNamespace()); + + if (serviceShape.hasTrait(LocalServiceTrait.class)) { + String name = getDefaultShapeName(serviceShape); + String filename = "client"; + String definitionFile = + serviceShape.getId().equals(settings.getService()) + ? getSymbolDefinitionFilePathForNamespaceAndFilename( + serviceShape.getId().getNamespace(), filename) + // Smithy and Smithy-Python will always attempt to write a referenced symbol. + // There is no way to disable writing referenced symbols, even for externally-defined + // symbols. + // We don't want to write a LocalService symbol, since it is either in this project's + // `client.py`, + // or is already written in another project's `client.py`. + // As a workaround, dump dependency localService symbols to a file that will be + // deleted after codegen. + // Typehints will reference the `namespace` and `serviceShape` name and not this file. + : generationPath + + "/" + + DafnyLocalServiceCodegenConstants + .LOCAL_SERVICE_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME + + ".py"; + return createSymbolBuilder( + serviceShape, + name, + getSymbolNamespacePathForNamespaceAndFilename( + serviceShape.getId().getNamespace(), filename)) + .definitionFile(definitionFile) + .build(); + } else if (AwsSdkNameResolver.isAwsSdkShape(serviceShape)) { + return createSymbolBuilder(serviceShape, "BaseClient", "botocore.client") + // Same as above; there is no way to disable writing referenced symbols. + // Dump boto3 client type into a file that will be deleted after codegen. + // Typehints will reference boto3 clients as `botocore.client.BaseClient`. + .definitionFile( + generationPath + + "/" + + DafnyLocalServiceCodegenConstants + .LOCAL_SERVICE_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME + + ".py") + .build(); + } else { + throw new IllegalArgumentException("ServiceShape not supported: " + serviceShape); + } + } + + /** + * Override Smithy-Python's resourceShape to handle resource shapes. + * @param resourceShape + * @return + */ + @Override + public Symbol resourceShape(ResourceShape resourceShape) { + var name = getDefaultShapeName(resourceShape); + String filename = "references"; + + String generationPath = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + settings.getService().getNamespace()); + + // Smithy and Smithy-Python will always attempt to write a referenced symbol. + // There is no way to disable writing referenced symbols, even for externally-defined symbols. + // We don't want to write a resource reference symbol, since it is either in this project's + // `references.py`, + // or is already written in another project's `references.py`. + // As a workaround, dump dependency reference symbols to a file that will be deleted after + // codegen. + // Typehints will reference the `namespace` and `resourceShape` name and not this file. + return createSymbolBuilder( + resourceShape, + name, + getSymbolNamespacePathForNamespaceAndFilename( + resourceShape.getId().getNamespace(), filename)) + .definitionFile( + generationPath + + "/" + + DafnyLocalServiceCodegenConstants + .LOCAL_SERVICE_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME + + ".py") + .putProperty("stdlib", true) + .build(); + +// return createSymbolBuilder( +// resourceShape, +// name, +// getSymbolNamespacePathForNamespaceAndFilename( +// resourceShape.getId().getNamespace(), filename)) +// .definitionFile( +// generationPath +// + "/" +// + DafnyLocalServiceCodegenConstants +// .LOCAL_SERVICE_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME +// + ".py") +// .putProperty("stdlib", true) +// .build(); + } + + /** + * Override Smithy-Python to handle other namespaces and Polymorph custom traits. + * @param shape + * @return + */ + @Override + public Symbol structureShape(StructureShape shape) { + String name = getDefaultShapeName(shape); + if (shape.hasTrait(ErrorTrait.class)) { + String filename = "errors"; + return createSymbolBuilder( + shape, + name, + getSymbolNamespacePathForNamespaceAndFilename(shape.getId().getNamespace(), filename)) + .definitionFile( + getSymbolDefinitionFilePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .build(); + } + + Set localServiceConfigShapes = SmithyNameResolver.getLocalServiceConfigShapes(model); + if (localServiceConfigShapes.contains(shape.getId())) { + String filename = "config"; + return createSymbolBuilder( + shape, + name, + getSymbolNamespacePathForNamespaceAndFilename(shape.getId().getNamespace(), filename)) + .definitionFile( + getSymbolDefinitionFilePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .build(); + } + + if (shape.hasTrait(PositionalTrait.class)) { + String filename = "models"; + String generationPath = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + settings.getService().getNamespace()); + MemberShape memberShape = PositionalTrait.onlyMember(shape); + ShapeId targetShapeId = memberShape.getTarget(); + Shape targetShape = model.expectShape(targetShapeId); + String targetShapeName = getDefaultShapeName(targetShape); + if (targetShape.hasTrait(ReferenceTrait.class)) { + return structureShape(targetShape.asStructureShape().get()); + } + return createSymbolBuilder( + targetShape, + targetShapeName, + getSymbolNamespacePathForNamespaceAndFilename( + targetShapeId.getNamespace(), filename)) + .definitionFile( + generationPath + + "/" + + DafnyLocalServiceCodegenConstants + .LOCAL_SERVICE_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME + + ".py") + .putProperty("stdlib", true) + .build(); +// return memberShape(memberShape); +// Shape referentShape = model.expectShape(referentShapeId); +// if (referentShape.isResourceShape()) { +// return resourceShape(referentShape.asResourceShape().get()); +// } +// if (referentShape.isServiceShape()) { +// return serviceShape(referentShape.asServiceShape().get()); +// } else { +// throw new IllegalArgumentException("Referent shape is not of a supported type: " + shape); +// } + } + + if (shape.hasTrait(ReferenceTrait.class)) { + ShapeId referentShapeId = shape.expectTrait(ReferenceTrait.class).getReferentId(); + Shape referentShape = model.expectShape(referentShapeId); + if (referentShape.isResourceShape()) { + return resourceShape(referentShape.asResourceShape().get()); + } + if (referentShape.isServiceShape()) { + return serviceShape(referentShape.asServiceShape().get()); + } else { + throw new IllegalArgumentException("Referent shape is not of a supported type: " + shape); + } + } + + String filename = "models"; + return createSymbolBuilder( + shape, + name, + getSymbolNamespacePathForNamespaceAndFilename(shape.getId().getNamespace(), filename)) + .definitionFile( + getSymbolDefinitionFilePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .build(); + } + + /** + * Override Smithy-Python to handle other namespaces + * @param target + * @return + */ + @Override + protected boolean targetRequiresDictHelpers(Shape target) { + // Do not generate dict helpers for a shape in another namespace + // (Don't generate *anything* for a shape in another namespace) + if (!target.getId().getNamespace().equals(service.getId().getNamespace())) { + return false; + // Don't generate dict helpers for a shape with a ReferenceTrait + // This can cause circular import issues + } else if (target.hasTrait(ReferenceTrait.class)) { + return false; + } else { + return super.targetRequiresDictHelpers(target); + } + } + + /** + * Override Smithy-Python to handle other namespaces + * @param target + * @return + */ + @Override + public Symbol memberShape(MemberShape shape) { + var container = model.expectShape(shape.getContainer()); + if (container.isUnionShape()) { + // Union members, unlike other shape members, have types generated for them. + var containerSymbol = container.accept(this); + var name = containerSymbol.getName() + StringUtils.capitalize(shape.getMemberName()); + String filename = "models"; + return createSymbolBuilder( + shape, + name, + getSymbolNamespacePathForNamespaceAndFilename(shape.getId().getNamespace(), filename)) + .definitionFile( + getSymbolDefinitionFilePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .build(); + } + Shape targetShape = + model + .getShape(shape.getTarget()) + .orElseThrow(() -> new CodegenException("Shape not found: " + shape.getTarget())); + return toSymbol(targetShape); + } + + /** + * Override Smithy-Python to handle other namespaces + * @param shape + * @return + */ + @Override + public Symbol enumShape(EnumShape shape) { + var builder = createSymbolBuilder(shape, "str"); + String name = getDefaultShapeName(shape); + String filename = "models"; + Symbol enumSymbol = + createSymbolBuilder( + shape, + name, + getSymbolNamespacePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .definitionFile( + getSymbolDefinitionFilePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .build(); + + // We add this enum symbol as a property on a generic string symbol + // rather than returning the enum symbol directly because we only + // generate the enum constants for convenience. We actually want + // to pass around plain strings rather than what is effectively + // a namespace class. + builder.putProperty("enumSymbol", escaper.escapeSymbol(shape, enumSymbol)); + return builder.build(); + } + + /** + * Override Smithy-Python to handle other namespaces + * @param target + * @return + */ + @Override + public Symbol intEnumShape(IntEnumShape shape) { + var builder = createSymbolBuilder(shape, "int"); + String name = getDefaultShapeName(shape); + String filename = "models"; + Symbol enumSymbol = + createSymbolBuilder( + shape, + name, + getSymbolNamespacePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .definitionFile( + getSymbolDefinitionFilePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .build(); + + // Like string enums, int enums are plain ints when used as members. + builder.putProperty("enumSymbol", escaper.escapeSymbol(shape, enumSymbol)); + return builder.build(); + } + + @Override + public Symbol stringShape(StringShape shape) { + if (shape.hasTrait(EnumTrait.class)) { + EnumShape asEnum = EnumShape.fromStringShape(shape).get(); + return enumShape(asEnum); + } else { + return super.stringShape(shape); + } + } + + /** + * Override Smithy-Python to handle other namespaces + * @param target + * @return + */ + @Override + public Symbol unionShape(UnionShape shape) { + String name = getDefaultShapeName(shape); + + var unknownName = name + "Unknown"; + String filename = "models"; + var unknownSymbol = + createSymbolBuilder( + shape, + name, + getSymbolNamespacePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .definitionFile( + getSymbolDefinitionFilePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .build(); + + return createSymbolBuilder( + shape, + name, + getSymbolNamespacePathForNamespaceAndFilename(shape.getId().getNamespace(), filename)) + .definitionFile( + getSymbolDefinitionFilePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .putProperty("fromDict", createFromDictFunctionSymbol(shape)) + .putProperty("unknown", unknownSymbol) + .build(); + } + + /** + * Override Smithy-Python to handle other namespaces + * @param shape + * @return + */ + @Override + protected Symbol createAsDictFunctionSymbol(Shape shape) { + String filename = "models"; + return Symbol.builder() + .name(String.format("_%s_as_dict", CaseUtils.toSnakeCase(shape.getId().getName()))) + .namespace( + getSymbolNamespacePathForNamespaceAndFilename(shape.getId().getNamespace(), filename), + ".") + .definitionFile( + getSymbolDefinitionFilePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .build(); + } + + /** + * Override Smithy-Python to handle other namespaces + * @param shape + * @return + */ + @Override + protected Symbol createFromDictFunctionSymbol(Shape shape) { + String filename = "models"; + return Symbol.builder() + .name(String.format("_%s_from_dict", CaseUtils.toSnakeCase(shape.getId().getName()))) + .namespace( + getSymbolNamespacePathForNamespaceAndFilename(shape.getId().getNamespace(), filename), + ".") + .definitionFile( + getSymbolDefinitionFilePathForNamespaceAndFilename( + shape.getId().getNamespace(), filename)) + .build(); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DirectedDafnyPythonLocalServiceCodegen.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DirectedDafnyPythonLocalServiceCodegen.java new file mode 100644 index 0000000000..6dc5b306fd --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/DirectedDafnyPythonLocalServiceCodegen.java @@ -0,0 +1,378 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.extensions; + +import static java.lang.String.format; + +import java.nio.file.Path; +import java.util.logging.Logger; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.localservice.DafnyLocalServiceCodegenConstants; +import software.amazon.polymorph.smithypython.localservice.customize.ReferencesFileWriter; +import software.amazon.smithy.build.FileManifest; +import software.amazon.smithy.codegen.core.*; +import software.amazon.smithy.codegen.core.directed.*; +import software.amazon.smithy.python.codegen.*; + +/** + * DirectedCodegen for Dafny Python LocalServices. This overrides Smithy-Python's + * DirectedPythonCodegen behavior. Changes include not writing symbols in a different namespace, + * generating a `client.py` file with a synchronous interface, and handling {@link + * software.amazon.smithy.model.shapes.ResourceShape}s. + */ +public class DirectedDafnyPythonLocalServiceCodegen extends DirectedPythonCodegen { + + private static final Logger LOGGER = + Logger.getLogger(DirectedDafnyPythonLocalServiceCodegen.class.getName()); + + @Override + public SymbolProvider createSymbolProvider( + CreateSymbolProviderDirective directive) { + return new DafnyPythonLocalServiceSymbolVisitor(directive.model(), directive.settings()); + } + + @Override + public void customizeBeforeShapeGeneration( + CustomizeDirective directive) { + generateServiceErrors(directive.settings(), directive.context().writerDelegator()); + new DafnyPythonLocalServiceConfigGenerator(directive.settings(), directive.context()).run(); + } + + /** + * Override Smithy-Python's generateServiceErrors to generate symbols in the correct directories. + * + * @param settings + * @param writers + */ + @Override + protected void generateServiceErrors( + PythonSettings settings, WriterDelegator writers) { + var serviceError = + Symbol.builder() + .name("ServiceError") + .namespace( + format( + "%s.errors", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + settings.getService().getNamespace(), settings)), + ".") + .definitionFile( + format( + "./%s/errors.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + settings.getService().getNamespace()))) + .build(); + writers.useFileWriter( + serviceError.getDefinitionFile(), + serviceError.getNamespace(), + writer -> { + // TODO: subclass a shared error + writer.openBlock( + "class $L(Exception):", + "", + serviceError.getName(), + () -> { + writer.writeDocs("Base error for all errors in the service."); + writer.write("pass"); + }); + }); + var apiError = + Symbol.builder() + .name("ApiError") + .namespace( + format( + "%s.errors", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + settings.getService().getNamespace(), settings)), + ".") + .definitionFile( + format( + "./%s/errors.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + settings.getService().getNamespace()))) + .build(); + writers.useFileWriter( + apiError.getDefinitionFile(), + apiError.getNamespace(), + writer -> { + writer.addStdlibImport("typing", "Generic"); + writer.addStdlibImport("typing", "TypeVar"); + writer.write("T = TypeVar('T')"); + writer.openBlock( + "class $L($T, Generic[T]):", + "", + apiError.getName(), + serviceError, + () -> { + writer.writeDocs("Base error for all api errors in the service."); + writer.write("code: T"); + writer.openBlock( + "def __init__(self, message: str):", + "", + () -> { + writer.write("super().__init__(message)"); + writer.write("self.message = message"); + }); + }); + + var unknownApiError = + Symbol.builder() + .name("UnknownApiError") + .namespace( + format( + "%s.errors", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + settings.getService().getNamespace(), settings)), + ".") + .definitionFile( + format( + "./%s/errors.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + settings.getService().getNamespace()))) + .build(); + writer.addStdlibImport("typing", "Literal"); + writer.openBlock( + "class $L($T[Literal['Unknown']]):", + "", + unknownApiError.getName(), + apiError, + () -> { + writer.writeDocs("Error representing any unknown api errors"); + writer.write("code: Literal['Unknown'] = 'Unknown'"); + }); + }); + } + + + /** + * Override Smithy-Python's generateResource to actually generate resources. + * + * @param directive Directive to perform. + */ + @Override + public void generateResource( + GenerateResourceDirective directive) { + if (ReferencesFileWriter.shouldGenerateResourceForShape(directive.shape(), directive.context())) { + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + directive.context().settings().getService().getNamespace()); + directive + .context() + .writerDelegator() + .useFileWriter( + moduleName + "/references.py", + "", + writer -> { + new ReferencesFileWriter() + .generateResourceInterfaceAndImplementation( + directive.shape(), directive.context(), writer); + }); + } + } + + /** + * Override Smithy-Python's generateStructure to not write a symbol in a different namespace. + * + * @param directive Directive to perform. + */ + @Override + public void generateStructure( + GenerateStructureDirective directive) { + if (directive + .shape() + .getId() + .getNamespace() + .equals(directive.context().settings().getService().getNamespace())) { + + directive + .context() + .writerDelegator() + .useShapeWriter( + directive.shape(), + writer -> { + DafnyPythonLocalServiceStructureGenerator generator = + new DafnyPythonLocalServiceStructureGenerator( + directive.model(), + directive.settings(), + directive.symbolProvider(), + writer, + directive.shape(), + TopologicalIndex.of(directive.model()).getRecursiveShapes()); + generator.run(); + }); + } + } + + /** + * Override Smithy-Python's generateError to not write a symbol in a different namespace. + * + * @param directive Directive to perform. + */ + @Override + public void generateError(GenerateErrorDirective directive) { + if (directive + .shape() + .getId() + .getNamespace() + .equals(directive.context().settings().getService().getNamespace())) { + + directive + .context() + .writerDelegator() + .useShapeWriter( + directive.shape(), + writer -> { + DafnyPythonLocalServiceStructureGenerator generator = + new DafnyPythonLocalServiceStructureGenerator( + directive.model(), + directive.settings(), + directive.symbolProvider(), + writer, + directive.shape(), + TopologicalIndex.of(directive.model()).getRecursiveShapes()); + generator.run(); + }); + } + } + + @Override + public void generateEnumShape(GenerateEnumDirective directive) { + if (directive + .shape() + .getId() + .getNamespace() + .equals(directive.context().settings().getService().getNamespace())) { + + if (!directive.shape().isEnumShape()) { + return; + } + + directive + .context() + .writerDelegator() + .useShapeWriter( + directive.shape(), + writer -> { + EnumGenerator generator = + new EnumGenerator( + directive.model(), + directive.symbolProvider(), + writer, + directive.shape().asEnumShape().get()); + generator.run(); + }); + } + } + + /** + * Override Smithy-Python's generateUnion to not write a symbol in a different namespace. + * + * @param directive Directive to perform. + */ + @Override + public void generateUnion(GenerateUnionDirective directive) { + if (directive + .shape() + .getId() + .getNamespace() + .equals(directive.context().settings().getService().getNamespace())) { + + directive + .context() + .writerDelegator() + .useShapeWriter( + directive.shape(), + writer -> { + UnionGenerator generator = + new UnionGenerator( + directive.model(), + directive.symbolProvider(), + writer, + directive.shape(), + TopologicalIndex.of(directive.model()).getRecursiveShapes()); + generator.run(); + }); + } + } + + /** + * Call `DirectedPythonCodegen.customizeAfterIntegrations`, then remove + * `localservice_codegen_todelete.py`. The CodegenDirector will invoke this method after shape + * generation. + * + * @param directive Directive to perform. + */ + @Override + public void customizeAfterIntegrations( + CustomizeDirective directive) { + // DirectedPythonCodegen's customizeAfterIntegrations implementation SHOULD run first; + // its implementation writes all files by flushing its writers; + // this implementation removes some of those files. + super.customizeAfterIntegrations(directive); + + FileManifest fileManifest = directive.fileManifest(); + Path generationPath = + Path.of( + fileManifest.getBaseDir() + + "/" + + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + directive.context().settings().getService().getNamespace())); + + /** + * Smithy ALWAYS writes visited symbols to a file. For AWS SDK codegen, we do NOT want to write + * visited symbols to a file, since boto3 does not use these visited symbols. It is very, very + * difficult to change this writing behavior without rewriting Smithy logic in addition to + * Smithy-Python specific logic. I have tried some workarounds like deleting writers or writing + * to /dev/null but these were not fruitful. This workaround dumps any visited symbols into a + * file whose name will never be used and deletes this file as part of its Smithy codegen + * plugin. + */ + try { + LOGGER.info( + format( + "Attempting to remove %s.py", + DafnyLocalServiceCodegenConstants + .LOCAL_SERVICE_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME)); + CodegenUtils.runCommand( + format( + "rm -f %s.py", + DafnyLocalServiceCodegenConstants + .LOCAL_SERVICE_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME), + generationPath) + .strip(); + } catch (CodegenException e) { + // Fail loudly. We do not want to accidentally distribute this file. + throw new RuntimeException( + format( + "Unable to remove %s.py", + DafnyLocalServiceCodegenConstants + .LOCAL_SERVICE_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME), + e); + } + } + + /** + * Override Smithy-Python's generateService to generate a synchronous client. + * + * @param directive Directive to perform. + */ + @Override + public void generateService( + GenerateServiceDirective directive) { + new SynchronousClientGenerator(directive.context(), directive.service()).run(); + + var protocolGenerator = directive.context().protocolGenerator(); + if (protocolGenerator == null) { + return; + } + + protocolGenerator.generateSharedSerializerComponents(directive.context()); + protocolGenerator.generateRequestSerializers(directive.context()); + + protocolGenerator.generateSharedDeserializerComponents(directive.context()); + protocolGenerator.generateResponseDeserializers(directive.context()); + + protocolGenerator.generateProtocolTests(directive.context()); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/SynchronousClientGenerator.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/SynchronousClientGenerator.java new file mode 100644 index 0000000000..c5ba526c4b --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/extensions/SynchronousClientGenerator.java @@ -0,0 +1,612 @@ +package software.amazon.polymorph.smithypython.localservice.extensions; + +import static java.lang.String.format; + +import java.util.LinkedHashSet; + +import software.amazon.polymorph.smithypython.localservice.DafnyLocalServiceCodegenConstants; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.polymorph.traits.PositionalTrait; +import software.amazon.polymorph.traits.ReferenceTrait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.model.shapes.*; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.traits.StringTrait; +import software.amazon.smithy.python.codegen.*; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.python.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.python.codegen.sections.*; + +/** + * Custom client generator. The generated Smithy-Python client generates all of its methods as + * `async`. To use generated clients, library consumers would need to wrap calls as async. However, + * generated Dafny code is NOT thread safe, and MUST NOT be used in an async environment. To guard + * users against this, make all user-exposed methods on the client synchronous. + * This class exists to provide a synchronous client interface; however, since it exists, + * we also remove some logic that is not used by Smithy-Dafny Python clients + * (primarily HTTP-request-wrapping logic for Smithy-Python clients). + */ +public class SynchronousClientGenerator extends ClientGenerator { + + SynchronousClientGenerator(GenerationContext context, ServiceShape service) { + super(context, service); + } + + /** + * Override Smithy-Python's `generateService` to + * @param writer + */ + @Override + protected void generateService(PythonWriter writer) { + var serviceSymbol = context.symbolProvider().toSymbol(service); + var pluginSymbol = + Symbol.builder() + .name("Plugin") + .namespace( + format( + "%s.config", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + context.settings().getService().getNamespace(), context)), + ".") + .definitionFile( + format( + "./%s/config.py", + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + context.settings().getService().getNamespace()))) + .build(); + + var configShapeId = service.getTrait(LocalServiceTrait.class).get().getConfigId(); + writer.addImport(".config", configShapeId.getName()); + + writer.addStdlibImport("typing", "TypeVar"); + writer.write( + """ + Input = TypeVar("Input") + Output = TypeVar("Output") + """); + + writer.openBlock( + "class $L:", + "", + serviceSymbol.getName(), + () -> { + var docs = + service + .getTrait(DocumentationTrait.class) + .map(StringTrait::getValue) + .orElse("Client for " + serviceSymbol.getName()); + writer.writeDocs( + () -> { + writer.write( + """ + $L + + :param config: Configuration for the client.""", + docs); + }); + + var defaultPlugins = new LinkedHashSet(); + + for (PythonIntegration integration : context.integrations()) { + for (RuntimeClientPlugin runtimeClientPlugin : integration.getClientPlugins()) { + if (runtimeClientPlugin.matchesService(context.model(), service)) { + runtimeClientPlugin.getPythonPlugin().ifPresent(defaultPlugins::add); + } + } + } + + writer.addStdlibImport( + DafnyNameResolver.getDafnyTypesModuleNameForSmithyNamespace( + service.getId().getNamespace()), + DafnyNameResolver.getDafnyClientInterfaceTypeForServiceShape(service)); + + writer.write( + """ + def __init__( + self, + config: $1L | None = None, + dafny_client: $4L | None = None + ): + if config is None: + self._config = Config() + else: + self._config = config + + client_plugins: list[$2T] = [ + $3C + ] + + for plugin in client_plugins: + plugin(self._config) + + if dafny_client is not None: + self._config.dafnyImplInterface.impl = dafny_client + + """, + configShapeId.getName(), + pluginSymbol, + writer.consumer(w -> writeDefaultPlugins(w, defaultPlugins)), + DafnyNameResolver.getDafnyClientInterfaceTypeForServiceShape(service)); + + for (ShapeId operationShapeId : service.getOperations()) { + generateOperation( + writer, context.model().expectShape(operationShapeId).asOperationShape().get()); + } + }); + + if (context.protocolGenerator() != null) { + generateOperationExecutor(writer); + } + } + + /** + * This overrides Smithy-Python's operation generator to generate synchronous user-facing + * operations. + * + * @param writer + * @param operation + */ + @Override + protected void generateOperation(PythonWriter writer, OperationShape operation) { + var operationSymbol = context.symbolProvider().toSymbol(operation); + + var input = context.model().expectShape(operation.getInputShape()); + + var operationInputShape = + context.model().expectShape(operation.getInputShape()).asStructureShape().get(); + + // "visible" in the sense that this the actual input/output shape considering the @positional trait + Shape visibleOperationInputShape; + Symbol interceptorInputSymbol; + + if (operationInputShape.hasTrait(PositionalTrait.class)) { + final MemberShape onlyMember = PositionalTrait.onlyMember(operationInputShape); + visibleOperationInputShape = context.model().expectShape(onlyMember.getTarget()); + } else { + visibleOperationInputShape = operationInputShape; + } + + + interceptorInputSymbol = context.symbolProvider().toSymbol(visibleOperationInputShape); + + + var operationOutputShape = + context.model().expectShape(operation.getOutputShape()).asStructureShape().get(); + // "visible" in the sense that this the actual input/output shape considering the @positional trait + Shape visibleOperationOutputShape; + Symbol interceptorOutputSymbol; + + if (operationOutputShape.hasTrait(PositionalTrait.class)) { + final MemberShape onlyMember = PositionalTrait.onlyMember(operationOutputShape); + visibleOperationOutputShape = context.model().expectShape(onlyMember.getTarget()); + } else { + visibleOperationOutputShape = operationOutputShape; + } + + + interceptorOutputSymbol = context.symbolProvider().toSymbol(visibleOperationOutputShape); +// + +// Symbol interceptorOutputSymbol; +// if (operationOutputShape.hasTrait(PositionalTrait.class)) { +// final MemberShape onlyMember = PositionalTrait.onlyMember(operationOutputShape); +// final Shape targetShape = context.model().expectShape(onlyMember.getTarget()); +// interceptorOutputSymbol = context.symbolProvider().toSymbol(targetShape); +// } else { +// interceptorOutputSymbol = context.symbolProvider().toSymbol(operationOutputShape); +// } + + if (visibleOperationInputShape.hasTrait(ReferenceTrait.class)) { + writer.addStdlibImport(interceptorInputSymbol.getNamespace()); + } + if (visibleOperationOutputShape.hasTrait(ReferenceTrait.class)) { + writer.addStdlibImport(interceptorOutputSymbol.getNamespace()); + } + + writer.openBlock( + "def $L(self, input: %1$s) -> %2$s:".formatted( + visibleOperationInputShape.hasTrait(ReferenceTrait.class) + ? "'$L'" + : "$T", + visibleOperationOutputShape.hasTrait(ReferenceTrait.class) + ? "'$L'" + : "$T" + + ), + "", + operationSymbol.getName(), + visibleOperationInputShape.hasTrait(ReferenceTrait.class) + ? interceptorInputSymbol.getNamespace() + "." + interceptorInputSymbol.getName() + : interceptorInputSymbol, + visibleOperationOutputShape.hasTrait(ReferenceTrait.class) + ? interceptorOutputSymbol.getNamespace() + "." + interceptorOutputSymbol.getName() + : interceptorOutputSymbol, + () -> { + writer.writeDocs( + () -> { + var docs = + operation + .getTrait(DocumentationTrait.class) + .map(StringTrait::getValue) + .orElse( + String.format( + "Invokes the %s operation.", operation.getId().getName())); + + var inputDocs = + input + .getTrait(DocumentationTrait.class) + .map(StringTrait::getValue) + .orElse("The operation's input."); + + writer.write( + """ + $L + + :param input: $L""", + docs, + inputDocs); + }); + + if (context.protocolGenerator() == null) { + writer.write("raise NotImplementedError()"); + } else { + var protocolGenerator = context.protocolGenerator(); + var serSymbol = protocolGenerator.getSerializationFunction(context, operation); + var deserSymbol = protocolGenerator.getDeserializationFunction(context, operation); + writer.addStdlibImport("asyncio"); + writer.write( + """ + return asyncio.run(self._execute_operation( + input=input, + plugins=[], + serialize=$T, + deserialize=$T, + config=self._config, + operation_name=$S, + )) + """, + serSymbol, + deserSymbol, + operation.getId().getName()); + } + }); + } + + /** + * Override Smithy-Python's generateOperationExecutor. + * This MUST be done because Smithy-Python does not let us intercept its SymbolProvider + * from within this function. + * We also remove code that will not be used by Smithy-Dafny Python clients, + * around HTTP request signing and authentication. + * @param writer + */ + protected void generateOperationExecutor(PythonWriter writer) { + var transportRequest = context.applicationProtocol().requestType(); + var transportResponse = context.applicationProtocol().responseType(); + var errorSymbol = + Symbol.builder() + .name("ServiceError") + .namespace( + format( + "%s.errors", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + context.settings().getService().getNamespace(), context)), + ".") + .definitionFile( + format( + "./%s/errors.py", + SmithyNameResolver.getPythonModuleNameForSmithyNamespace( + context.settings().getService().getNamespace()))) + .build(); + var pluginSymbol = + Symbol.builder() + .name("Plugin") + .namespace( + format( + "%s.config", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + context.settings().getService().getNamespace(), context)), + ".") + .definitionFile( + format( + "./%s/config.py", + SmithyNameResolver.getPythonModuleNameForSmithyNamespace( + context.settings().getService().getNamespace()))) + .build(); + + var configSymbol = CodegenUtils.getConfigSymbol(context.settings()); + writer.addImport(".config", configSymbol.getName()); + + writer.addStdlibImport("typing", "Callable"); + writer.addStdlibImport("typing", "Awaitable"); + writer.addStdlibImport("typing", "cast"); + writer.addStdlibImport("asyncio", "sleep"); + + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.exceptions", "SmithyRetryException"); + writer.addImport("smithy_python.interfaces.interceptor", "Interceptor"); + writer.addImport("smithy_python.interfaces.interceptor", "InterceptorContext"); + writer.addImport("smithy_python.interfaces.retries", "RetryErrorInfo"); + writer.addImport("smithy_python.interfaces.retries", "RetryErrorType"); + + writer.indent(); + writer.write( + """ + async def _execute_operation( + self, + input: Input, + plugins: list[$1T], + serialize: Callable[[Input, $5L], Awaitable[$2T]], + deserialize: Callable[[$3T, $5L], Awaitable[Output]], + config: $5L, + operation_name: str, + ) -> Output: + try: + return await self._handle_execution( + input, plugins, serialize, deserialize, config, operation_name + ) + except Exception as e: + # Make sure every exception that we throw is an instance of $4T so + # customers can reliably catch everything we throw. + if not isinstance(e, $4T): + raise $4T(e) from e + raise e + + async def _handle_execution( + self, + input: Input, + plugins: list[$1T], + serialize: Callable[[Input, $5L], Awaitable[$2T]], + deserialize: Callable[[$3T, $5L], Awaitable[Output]], + config: $5L, + operation_name: str, + ) -> Output: + context: InterceptorContext[Input, None, None, None] = InterceptorContext( + request=input, + response=None, + transport_request=None, + transport_response=None, + ) + _client_interceptors = config.interceptors + client_interceptors = cast( + list[Interceptor[Input, Output, $2T, $3T]], _client_interceptors + ) + interceptors = client_interceptors + + try: + # Step 1a: Invoke read_before_execution on client-level interceptors + for interceptor in client_interceptors: + interceptor.read_before_execution(context) + + # Step 1b: Run operation-level plugins + for plugin in plugins: + plugin(config) + + _client_interceptors = config.interceptors + interceptors = cast( + list[Interceptor[Input, Output, $2T, $3T]], + _client_interceptors, + ) + + # Step 1c: Invoke the read_before_execution hooks on newly added + # interceptors. + for interceptor in interceptors: + if interceptor not in client_interceptors: + interceptor.read_before_execution(context) + + # Step 2: Invoke the modify_before_serialization hooks + for interceptor in interceptors: + context._request = interceptor.modify_before_serialization(context) + + # Step 3: Invoke the read_before_serialization hooks + for interceptor in interceptors: + interceptor.read_before_serialization(context) + + # Step 4: Serialize the request + context_with_transport_request = cast( + InterceptorContext[Input, None, $2T, None], context + ) + context_with_transport_request._transport_request = await serialize( + context_with_transport_request.request, config + ) + + # Step 5: Invoke read_after_serialization + for interceptor in interceptors: + interceptor.read_after_serialization(context_with_transport_request) + + # Step 6: Invoke modify_before_retry_loop + for interceptor in interceptors: + context_with_transport_request._transport_request = ( + interceptor.modify_before_retry_loop(context_with_transport_request) + ) + + # Step 7: Acquire the retry token. + retry_strategy = config.retry_strategy + retry_token = retry_strategy.acquire_initial_retry_token() + + while True: + # Make an attempt, creating a copy of the context so we don't pass + # around old data. + context_with_response = await self._handle_attempt( + deserialize, + interceptors, + context_with_transport_request.copy(), + config, + operation_name, + ) + + # We perform this type-ignored re-assignment because `context` needs + # to point at the latest context so it can be generically handled + # later on. This is only an issue here because we've created a copy, + # so we're no longer simply pointing at the same object in memory + # with different names and type hints. It is possible to address this + # without having to fall back to the type ignore, but it would impose + # unnecessary runtime costs. + context = context_with_response # type: ignore + + if isinstance(context_with_response.response, Exception): + # Step 7u: Reacquire retry token if the attempt failed + try: + retry_token = retry_strategy.refresh_retry_token_for_retry( + token_to_renew=retry_token, + error_info=RetryErrorInfo( + # TODO: Determine the error type. + error_type=RetryErrorType.CLIENT_ERROR, + ) + ) + except SmithyRetryException: + raise context_with_response.response + await sleep(retry_token.retry_delay) + else: + # Step 8: Invoke record_success + retry_strategy.record_success(token=retry_token) + break + except Exception as e: + context._response = e + + # At this point, the context's request will have been definitively set, and + # The response will be set either with the modeled output or an exception. The + # transport_request and transport_response may be set or None. + execution_context = cast( + InterceptorContext[Input, Output, $2T | None, $3T | None], context + ) + return await self._finalize_execution(interceptors, execution_context) + + async def _handle_attempt( + self, + deserialize: Callable[[$3T, $5L], Awaitable[Output]], + interceptors: list[Interceptor[Input, Output, $2T, $3T]], + context: InterceptorContext[Input, None, $2T, None], + config: $5L, + operation_name: str, + ) -> InterceptorContext[Input, Output, $2T, $3T | None]: + try: + # Step 7a: Invoke read_before_attempt + for interceptor in interceptors: + interceptor.read_before_attempt(context) + + """, + pluginSymbol, + transportRequest, + transportResponse, + errorSymbol, + configSymbol.getName()); + + writer.pushState(new SendRequestSection()); + writer.write( + """ + # Step 7m: Involve client Dafny impl + if config.dafnyImplInterface.impl is None: + raise Exception("No impl found on the operation config.") + + context_with_response = cast( + InterceptorContext[Input, None, $L, $L], context + ) + + context_with_response._transport_response = config.dafnyImplInterface.handle_request( + input=context_with_response.transport_request + ) + """, + DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_REQUEST, + DafnyLocalServiceCodegenConstants.DAFNY_PROTOCOL_RESPONSE); + writer.popState(); + + writer.write( + """ + # Step 7n: Invoke read_after_transmit + for interceptor in interceptors: + interceptor.read_after_transmit(context_with_response) + + # Step 7o: Invoke modify_before_deserialization + for interceptor in interceptors: + context_with_response._transport_response = ( + interceptor.modify_before_deserialization(context_with_response) + ) + + # Step 7p: Invoke read_before_deserialization + for interceptor in interceptors: + interceptor.read_before_deserialization(context_with_response) + + # Step 7q: deserialize + context_with_output = cast( + InterceptorContext[Input, Output, $1T, $2T], + context_with_response, + ) + context_with_output._response = await deserialize( + context_with_output._transport_response, config + ) + + # Step 7r: Invoke read_after_deserialization + for interceptor in interceptors: + interceptor.read_after_deserialization(context_with_output) + except Exception as e: + context._response = e + + # At this point, the context's request and transport_request have definitively been set, + # the response is either set or an exception, and the transport_resposne is either set or + # None. This will also be true after _finalize_attempt because there is no opportunity + # there to set the transport_response. + attempt_context = cast( + InterceptorContext[Input, Output, $1T, $2T | None], context + ) + return await self._finalize_attempt(interceptors, attempt_context) + + async def _finalize_attempt( + self, + interceptors: list[Interceptor[Input, Output, $1T, $2T]], + context: InterceptorContext[Input, Output, $1T, $2T | None], + ) -> InterceptorContext[Input, Output, $1T, $2T | None]: + # Step 7s: Invoke modify_before_attempt_completion + try: + for interceptor in interceptors: + context._response = interceptor.modify_before_attempt_completion( + context + ) + except Exception as e: + context._response = e + + # Step 7t: Invoke read_after_attempt + for interceptor in interceptors: + try: + interceptor.read_after_attempt(context) + except Exception as e: + context._response = e + + return context + + async def _finalize_execution( + self, + interceptors: list[Interceptor[Input, Output, $1T, $2T]], + context: InterceptorContext[Input, Output, $1T | None, $2T | None], + ) -> Output: + try: + # Step 9: Invoke modify_before_completion + for interceptor in interceptors: + context._response = interceptor.modify_before_completion(context) + + except Exception as e: + context._response = e + + # Step 11: Invoke read_after_execution + for interceptor in interceptors: + try: + interceptor.read_after_execution(context) + except Exception as e: + context._response = e + + # Step 12: Return / throw + if isinstance(context.response, Exception): + raise context.response + + # We may want to add some aspects of this context to the output types so we can + # return it to the end-users. + return context.response + """, + transportRequest, + transportResponse); + writer.dedent(); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java new file mode 100644 index 0000000000..aa5e4fa30c --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java @@ -0,0 +1,327 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.shapevisitor; + +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.Utils; +import software.amazon.polymorph.smithypython.common.shapevisitor.ShapeVisitorResolver; +import software.amazon.polymorph.smithypython.localservice.shapevisitor.conversionwriter.DafnyToLocalServiceConversionFunctionWriter; +import software.amazon.polymorph.smithypython.localservice.shapevisitor.conversionwriter.LocalServiceToDafnyConversionFunctionWriter; +import software.amazon.polymorph.traits.DafnyUtf8BytesTrait; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.shapes.BigDecimalShape; +import software.amazon.smithy.model.shapes.BigIntegerShape; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.BooleanShape; +import software.amazon.smithy.model.shapes.ByteShape; +import software.amazon.smithy.model.shapes.DoubleShape; +import software.amazon.smithy.model.shapes.EnumShape; +import software.amazon.smithy.model.shapes.FloatShape; +import software.amazon.smithy.model.shapes.IntegerShape; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.LongShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.ShortShape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.TimestampShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * ShapeVisitor that should be dispatched from a shape + * to generate code that maps a Dafny shape's internal attributes + * to the corresponding Smithy shape's internal attributes. + * + * This generates code in a `dafny_to_smithy.py` file. + * The generated code consists of methods that convert from a Dafny-modelled shape + * to a Smithy-modelled shape. + * Code that requires these conversions will call out to this file. + */ +public class DafnyToLocalServiceShapeVisitor extends ShapeVisitor.Default { + private final GenerationContext context; + private String dataSource; + private final PythonWriter writer; + private final String filename; + /** + * @param context The generation context. + * @param dataSource The in-code location of the data to provide an output of + * ({@code output.foo}, {@code entry}, etc.) + * @param writer The PythonWriter being used + */ + public DafnyToLocalServiceShapeVisitor( + GenerationContext context, + String dataSource, + PythonWriter writer + ) { + this.context = context; + this.dataSource = dataSource; + this.writer = writer; + this.filename = ""; + } + + public DafnyToLocalServiceShapeVisitor( + GenerationContext context, + String dataSource, + PythonWriter writer, + String filename + ) { + this.context = context; + this.dataSource = dataSource; + this.writer = writer; + this.filename = filename; + } + + @Override + protected String getDefault(Shape shape) { + String protocolName = context.protocolGenerator().getName(); + throw new CodegenException(String.format( + "Unsupported conversion of %s to %s using the %s protocol", + shape, shape.getType(), protocolName)); + } + + @Override + public String blobShape(BlobShape shape) { + return "bytes(%1$s)".formatted(dataSource); + } + + @Override + public String structureShape(StructureShape structureShape) { + // ONLY write converters if the shape under generation is in the current namespace (or Unit shape) + if (structureShape.getId().getNamespace().equals(context.settings().getService().getNamespace()) + || Utils.isUnitShape(structureShape.getId())) { + LocalServiceToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers(structureShape, + context, writer); + DafnyToLocalServiceConversionFunctionWriter.writeConverterForShapeAndMembers(structureShape, + context, writer); + } + + // Import the converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + structureShape.getId().getNamespace(), + context + ); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".dafny_to_smithy"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.dafny_to_smithy.DafnyToSmithy_example_namespace_ExampleShape(input)` + return "%1$s.dafny_to_smithy.%2$s(%3$s)".formatted( + pythonModuleSmithygeneratedPath, + SmithyNameResolver.getDafnyToSmithyFunctionNameForShape(structureShape, context), + Utils.isUnitShape(structureShape.getId()) ? "" : dataSource + ); + } + + @Override + public String listShape(ListShape shape) { + StringBuilder builder = new StringBuilder(); + + // Open list: + // `[` + builder.append("["); + MemberShape memberShape = shape.getMember(); + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + + // Add converted list elements into the list: + // `[list_element for list_element in `DafnyToSmithy(targetShape)`` + builder.append("%1$s".formatted( + targetShape.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape(targetShape, context, "list_element", writer) + ))); + + // Close structure: + // `[list_element for list_element in `DafnyToSmithy(targetShape)`]` + return builder.append(" for list_element in %1$s]".formatted(dataSource)).toString(); + } + + @Override + public String mapShape(MapShape shape) { + StringBuilder builder = new StringBuilder(); + + // Open map: + // `{` + builder.append("{"); + MemberShape keyMemberShape = shape.getKey(); + final Shape keyTargetShape = context.model().expectShape(keyMemberShape.getTarget()); + MemberShape valueMemberShape = shape.getValue(); + final Shape valueTargetShape = context.model().expectShape(valueMemberShape.getTarget()); + + // Write converted map keys into the map: + // `{`DafnyToSmithy(key)`:` + builder.append("%1$s: ".formatted( + keyTargetShape.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape(keyTargetShape, context, "key", writer) + ) + )); + + // Write converted map values into the map: + // `{`DafnyToSmithy(key)`: `DafnyToSmithy(value)`` + builder.append("%1$s".formatted( + valueTargetShape.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape(valueTargetShape, context, "value", writer) + ) + )); + + // Complete map comprehension and close map + // `{`DafnyToSmithy(key)`: `DafnyToSmithy(value)`` for (key, value) in `dataSource`.items }` + // No () on items call; `dataSource` is a Dafny map, where `items` is a @property and not a method. + return builder.append(" for (key, value) in %1$s.items }".formatted(dataSource)).toString(); + } + + @Override + public String booleanShape(BooleanShape shape) { + return dataSource; + } + + @Override + public String stringShape(StringShape shape) { + // If shape has @DafnyUtf8BytesTrait, use bytes converter + if (shape.hasTrait(DafnyUtf8BytesTrait.class)) { + writer.addStdlibImport("UTF8"); + return "bytes(''.join(UTF8.default__.Decode(%1$s).value.Elements), encoding='utf-8')".formatted(dataSource); +// return "%1$s.encode('utf-8')".formatted(dataSource); +// return "''.join([chr(a) for a in %1$s])".formatted(dataSource); +// return "bytes(%1$s)".formatted(dataSource); + // Smithy has deprecated EnumTrait, but Polymorph still uses it to mark enums + } else if (shape.hasTrait(EnumTrait.class)) { + // ONLY write converters if the shape under generation is in the current namespace + if (shape.getId().getNamespace().equals(context.settings().getService().getNamespace())) { + LocalServiceToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers(shape, + context, writer); + DafnyToLocalServiceConversionFunctionWriter.writeConverterForShapeAndMembers(shape, + context, writer); + } + + // Import the smithy_to_dafny converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + shape.getId().getNamespace(), + context + ); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".dafny_to_smithy"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.smithy_to_dafny.SmithyToDafny_example_namespace_ExampleShape(input)` + return "%1$s.dafny_to_smithy.%2$s(%3$s)".formatted( + pythonModuleSmithygeneratedPath, + SmithyNameResolver.getDafnyToSmithyFunctionNameForShape(shape, context), + dataSource + ); + } + return dataSource + ".VerbatimString(False)"; + } + + @Override + public String byteShape(ByteShape shape) { + return getDefault(shape); + } + + @Override + public String shortShape(ShortShape shape) { + return getDefault(shape); + } + + @Override + public String integerShape(IntegerShape shape) { + return dataSource; + } + + @Override + public String longShape(LongShape shape) { + return dataSource; + } + + @Override + public String bigIntegerShape(BigIntegerShape shape) { + return getDefault(shape); + } + + @Override + public String floatShape(FloatShape shape) { + return getDefault(shape); + } + + @Override + public String doubleShape(DoubleShape shape) { + return dataSource; + } + + @Override + public String bigDecimalShape(BigDecimalShape shape) { + return getDefault(shape); + } + + @Override + public String enumShape(EnumShape shape) { + // ONLY write converters if the shape under generation is in the current namespace + if (shape.getId().getNamespace().equals(context.settings().getService().getNamespace())) { + LocalServiceToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers(shape, + context, writer); + DafnyToLocalServiceConversionFunctionWriter.writeConverterForShapeAndMembers(shape, + context, writer); + } + + // Import the smithy_to_dafny converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + shape.getId().getNamespace(), + context + ); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".dafny_to_smithy"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.smithy_to_dafny.SmithyToDafny_example_namespace_ExampleShape(input)` + return "%1$s.dafny_to_smithy.%2$s(%3$s)".formatted( + pythonModuleSmithygeneratedPath, + SmithyNameResolver.getDafnyToSmithyFunctionNameForShape(shape, context), + dataSource + ); + } + + @Override + public String timestampShape(TimestampShape shape) { + return dataSource; + } + + @Override + public String unionShape(UnionShape unionShape) { + // ONLY write converters if the shape under generation is in the current namespace + if (unionShape.getId().getNamespace().equals(context.settings().getService().getNamespace())) { + LocalServiceToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers(unionShape, + context, writer); + DafnyToLocalServiceConversionFunctionWriter.writeConverterForShapeAndMembers(unionShape, + context, writer); + } + + // Import the converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + unionShape.getId().getNamespace(), + context + ); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".dafny_to_smithy"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.dafny_to_smithy.DafnyToSmithy_example_namespace_ExampleShape(input)` + return "%1$s.dafny_to_smithy.%2$s(%3$s)".formatted( + pythonModuleSmithygeneratedPath, + SmithyNameResolver.getDafnyToSmithyFunctionNameForShape(unionShape, context), + dataSource + ); + } + +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/LocalServiceToDafnyShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/LocalServiceToDafnyShapeVisitor.java new file mode 100644 index 0000000000..167240b3a3 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/LocalServiceToDafnyShapeVisitor.java @@ -0,0 +1,331 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.shapevisitor; + +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.Utils; +import software.amazon.polymorph.smithypython.common.shapevisitor.ShapeVisitorResolver; +import software.amazon.polymorph.smithypython.localservice.shapevisitor.conversionwriter.DafnyToLocalServiceConversionFunctionWriter; +import software.amazon.polymorph.smithypython.localservice.shapevisitor.conversionwriter.LocalServiceToDafnyConversionFunctionWriter; +import software.amazon.polymorph.traits.DafnyUtf8BytesTrait; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.model.shapes.BigDecimalShape; +import software.amazon.smithy.model.shapes.BigIntegerShape; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.BooleanShape; +import software.amazon.smithy.model.shapes.ByteShape; +import software.amazon.smithy.model.shapes.DoubleShape; +import software.amazon.smithy.model.shapes.EnumShape; +import software.amazon.smithy.model.shapes.FloatShape; +import software.amazon.smithy.model.shapes.IntegerShape; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.LongShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.ShortShape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.TimestampShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * ShapeVisitor that should be dispatched from a shape + * to generate code that maps a Smithy-modelled shape's internal attributes + * to the corresponding Dafny shape's internal attributes. + * + * This generates code in a `smithy_to_dafny.py` file. + * The generated code consists of methods that convert from a Smithy-modelled shape + * to a Dafny-modelled shape. + * Code that requires these conversions will call out to this file. + */ +public class LocalServiceToDafnyShapeVisitor extends ShapeVisitor.Default { + private final GenerationContext context; + private String dataSource; + private final PythonWriter writer; + private final String filename; + + /** + * @param context The generation context. + * @param dataSource The in-code location of the data to provide an output of + * ({@code input.foo}, {@code entry}, etc.) + */ + public LocalServiceToDafnyShapeVisitor( + GenerationContext context, + String dataSource, + PythonWriter writer, + String filename + ) { + this.context = context; + this.dataSource = dataSource; + this.writer = writer; + this.filename = filename; + } + + public LocalServiceToDafnyShapeVisitor( + GenerationContext context, + String dataSource, + PythonWriter writer + ) { + this.context = context; + this.dataSource = dataSource; + this.writer = writer; + this.filename = ""; + } + + @Override + protected String getDefault(Shape shape) { + String protocolName = context.protocolGenerator().getName(); + throw new CodegenException(String.format( + "Unsupported conversion of %s to %s using the %s protocol", + shape, shape.getType(), protocolName)); + } + + @Override + public String blobShape(BlobShape shape) { + writer.addStdlibImport("_dafny", "Seq"); + return "Seq(" + dataSource + ")"; + } + + @Override + public String structureShape(StructureShape structureShape) { + // ONLY write converters if the shape under generation is in the current namespace (or Unit shape) + if (structureShape.getId().getNamespace().equals(context.settings().getService().getNamespace()) + || Utils.isUnitShape(structureShape.getId())) { + LocalServiceToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers(structureShape, + context, writer); + DafnyToLocalServiceConversionFunctionWriter.writeConverterForShapeAndMembers(structureShape, + context, writer); + } + + // Import the converter from where the ShapeVisitor was called + String pythonModuleName = SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + structureShape.getId().getNamespace(), + context + ); + writer.addStdlibImport(pythonModuleName + ".smithy_to_dafny"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.smithy_to_dafny.SmithyToDafny_example_namespace_ExampleShape(input)` + return "%1$s.smithy_to_dafny.%2$s(%3$s)".formatted( + pythonModuleName, + SmithyNameResolver.getSmithyToDafnyFunctionNameForShape(structureShape, context), + dataSource + ); + } + + @Override + public String listShape(ListShape shape) { + writer.addStdlibImport("_dafny", "Seq"); + + StringBuilder builder = new StringBuilder(); + + // Open Dafny sequence: + // `Seq([` + builder.append("Seq(["); + MemberShape memberShape = shape.getMember(); + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + + // Add converted list elements into the list: + // `Seq([`SmithyToDafny(list_element)`` + builder.append("%1$s".formatted( + targetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape(targetShape, context, "list_element", writer) + ))); + + // Close structure + // `Seq([`SmithyToDafny(list_element)` for list_element in `dataSource`])`` + return builder.append(" for list_element in %1$s])".formatted(dataSource)).toString(); + } + + @Override + public String mapShape(MapShape shape) { + writer.addStdlibImport("_dafny", "Map"); + + StringBuilder builder = new StringBuilder(); + + // Open Dafny map: + // `Map({` + builder.append("Map({"); + MemberShape keyMemberShape = shape.getKey(); + final Shape keyTargetShape = context.model().expectShape(keyMemberShape.getTarget()); + MemberShape valueMemberShape = shape.getValue(); + final Shape valueTargetShape = context.model().expectShape(valueMemberShape.getTarget()); + + // Write converted map keys into the map: + // `{`SmithyToDafny(key)`:` + builder.append("%1$s: ".formatted( + keyTargetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape(keyTargetShape, context, "key", writer) + ) + )); + + // Write converted map values into the map: + // `{`SmithyToDafny(key)`: `SmithyToDafny(value)`` + builder.append("%1$s".formatted( + valueTargetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape(valueTargetShape, context, "value", writer) + ) + )); + + // Complete map comprehension and close map + // `{`SmithyToDafny(key)`: `SmithyToDafny(value)`` for (key, value) in `dataSource`.items() }` + return builder.append(" for (key, value) in %1$s.items() })".formatted(dataSource)).toString(); + } + + @Override + public String booleanShape(BooleanShape shape) { + return dataSource; + } + + @Override + public String stringShape(StringShape shape) { + writer.addStdlibImport("_dafny", "Seq"); + if (shape.hasTrait(DafnyUtf8BytesTrait.class)) { + writer.addStdlibImport("UTF8"); + return "UTF8.default__.Encode(Seq(%1$s)).value".formatted(dataSource); +// return "Seq(list(ord(c) for c in %1$s))".formatted(dataSource); + // Smithy has deprecated EnumTrait, but Polymorph still uses it to mark enums + } + // Smithy has deprecated EnumTrait, but Polymorph still uses it to mark enums + if (shape.hasTrait(EnumTrait.class)) { + // ONLY write converters if the shape under generation is in the current namespace + if (shape.getId().getNamespace().equals(context.settings().getService().getNamespace())) { + LocalServiceToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers(shape, + context, writer); + DafnyToLocalServiceConversionFunctionWriter.writeConverterForShapeAndMembers(shape, + context, writer); + } + + // Import the smithy_to_dafny converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + shape.getId().getNamespace(), + context + ); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".smithy_to_dafny"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.smithy_to_dafny.SmithyToDafny_example_namespace_ExampleShape(input)` + return "%1$s.smithy_to_dafny.%2$s(%3$s)".formatted( + pythonModuleSmithygeneratedPath, + SmithyNameResolver.getSmithyToDafnyFunctionNameForShape(shape, context), + dataSource + ); + } + writer.addStdlibImport("_dafny", "Seq"); + return "Seq(" + dataSource + ")"; + } + + @Override + public String byteShape(ByteShape shape) { + return getDefault(shape); + } + + @Override + public String shortShape(ShortShape shape) { + return getDefault(shape); + } + + @Override + public String integerShape(IntegerShape shape) { + return dataSource; + } + + @Override + public String longShape(LongShape shape) { + return dataSource; + } + + @Override + public String bigIntegerShape(BigIntegerShape shape) { + return getDefault(shape); + } + + @Override + public String floatShape(FloatShape shape) { + return getDefault(shape); + } + + @Override + public String doubleShape(DoubleShape shape) { + return dataSource; + } + + @Override + public String bigDecimalShape(BigDecimalShape shape) { + return getDefault(shape); + } + + @Override + public String enumShape(EnumShape shape) { +// ONLY write converters if the shape under generation is in the current namespace + if (shape.getId().getNamespace().equals(context.settings().getService().getNamespace())) { + LocalServiceToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers(shape, + context, writer); + DafnyToLocalServiceConversionFunctionWriter.writeConverterForShapeAndMembers(shape, + context, writer); + } + + // Import the smithy_to_dafny converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + shape.getId().getNamespace(), + context + ); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".smithy_to_dafny"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.smithy_to_dafny.SmithyToDafny_example_namespace_ExampleShape(input)` + return "%1$s.smithy_to_dafny.%2$s(%3$s)".formatted( + pythonModuleSmithygeneratedPath, + SmithyNameResolver.getSmithyToDafnyFunctionNameForShape(shape, context), + dataSource + ); + } + + @Override + public String timestampShape(TimestampShape shape) { + return dataSource; + } + + @Override + public String unionShape(UnionShape unionShape) { + // ONLY write converters if the shape under generation is in the current namespace + if (unionShape.getId().getNamespace().equals(context.settings().getService().getNamespace())) { + LocalServiceToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers(unionShape, + context, writer); + DafnyToLocalServiceConversionFunctionWriter.writeConverterForShapeAndMembers(unionShape, + context, writer); + } + + // Import the smithy_to_dafny converter from where the ShapeVisitor was called + String pythonModuleSmithygeneratedPath = + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + unionShape.getId().getNamespace(), + context + ); + writer.addStdlibImport(pythonModuleSmithygeneratedPath + ".smithy_to_dafny"); + + // Return a reference to the generated conversion method + // ex. for shape example.namespace.ExampleShape + // returns + // `example_namespace.smithygenerated.smithy_to_dafny.SmithyToDafny_example_namespace_ExampleShape(input)` + return "%1$s.smithy_to_dafny.%2$s(%3$s)".formatted( + pythonModuleSmithygeneratedPath, + SmithyNameResolver.getSmithyToDafnyFunctionNameForShape(unionShape, context), + dataSource + ); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/conversionwriter/DafnyToLocalServiceConversionFunctionWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/conversionwriter/DafnyToLocalServiceConversionFunctionWriter.java new file mode 100644 index 0000000000..6c0c568c32 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/conversionwriter/DafnyToLocalServiceConversionFunctionWriter.java @@ -0,0 +1,417 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.shapevisitor.conversionwriter; + +import java.util.Map.Entry; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.Utils; +import software.amazon.polymorph.smithypython.common.shapevisitor.conversionwriter.BaseConversionWriter; +import software.amazon.polymorph.smithypython.common.shapevisitor.ShapeVisitorResolver; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.polymorph.traits.PositionalTrait; +import software.amazon.polymorph.traits.ReferenceTrait; +import software.amazon.smithy.codegen.core.WriterDelegator; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.ResourceShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.EnumDefinition; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.utils.CaseUtils; + +/** + * Writes the dafny_to_smithy.py file via the BaseConversionWriter implementation. + */ +public class DafnyToLocalServiceConversionFunctionWriter extends BaseConversionWriter { + + // Use a singleton to preserve generatedShapes through multiple invocations of this class + static DafnyToLocalServiceConversionFunctionWriter singleton; + + protected DafnyToLocalServiceConversionFunctionWriter() { + } + + // Instantiate singleton at class-load time + static { + singleton = new DafnyToLocalServiceConversionFunctionWriter(); + } + + /** + * Delegate writing conversion methods for the provided shape and its member shapes + * + * @param shape + * @param context + * @param writer + */ + public static void writeConverterForShapeAndMembers(Shape shape, GenerationContext context, + PythonWriter writer) { + singleton.baseWriteConverterForShapeAndMembers(shape, context, writer); + } + + protected void writeStructureShapeConverter(StructureShape structureShape) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(context.settings().getService().getNamespace()); + + delegator.useFileWriter(moduleName + "/dafny_to_smithy.py", "", conversionWriter -> { + // Within the conversion function, the dataSource becomes the function's input + // This hardcodes the input parameter name for a conversion function to always be "dafny_input" + String dataSourceInsideConversionFunction = "dafny_input"; + + conversionWriter.openBlock( + "def $L($L):", + "", + SmithyNameResolver.getDafnyToSmithyFunctionNameForShape(structureShape, context), + Utils.isUnitShape(structureShape.getId()) ? "" : dataSourceInsideConversionFunction, + () -> { + if (structureShape.hasTrait(ReferenceTrait.class)) { + writeReferenceStructureShapeConverter(structureShape, conversionWriter, dataSourceInsideConversionFunction); + } else if (structureShape.hasTrait(PositionalTrait.class)) { + writePositionalStructureShapeConverter(structureShape, conversionWriter, dataSourceInsideConversionFunction); + } else { + writeTraitlessStructureShapeConverter(structureShape, conversionWriter, dataSourceInsideConversionFunction); + } + } + ); + }); + } + + private void writeTraitlessStructureShapeConverter(StructureShape shape, PythonWriter conversionWriter, + String dataSourceInsideConversionFunction) { + // Only write top-level import for a shape in `.models` to avoid introducing a circular dependency + if (SmithyNameResolver.getSmithyGeneratedModuleFilenameForSmithyShape(shape, context) + .equals(".models")) { + SmithyNameResolver.importSmithyGeneratedTypeForShape(conversionWriter, shape, context); + } else { + conversionWriter.writeComment("Deferred import of %1$s to avoid circular dependency".formatted( + SmithyNameResolver.getSmithyGeneratedModuleFilenameForSmithyShape(shape, context))); + conversionWriter.write("import $L", + SmithyNameResolver.getSmithyGeneratedModelLocationForShape(shape, context)); + } + // Open Smithy structure shape + // e.g. + // smithy_structure_name(... + conversionWriter.openBlock( + "return $L(", + ")", + SmithyNameResolver.getSmithyGeneratedModelLocationForShape( + shape.getId(), context + ) + "." + context.symbolProvider().toSymbol(shape).getName(), + () -> { + + // Recursively dispatch a new ShapeVisitor for each member of the structure + for (Entry memberShapeEntry : shape.getAllMembers().entrySet()) { + String memberName = memberShapeEntry.getKey(); + MemberShape memberShape = memberShapeEntry.getValue(); + writeNonReferenceStructureShapeMemberConverter(conversionWriter, + dataSourceInsideConversionFunction, memberName, memberShape); + } + } + ); + } + + private void writeNonReferenceStructureShapeMemberConverter(PythonWriter conversionWriter, + String dataSourceInsideConversionFunction, String memberName, MemberShape memberShape) { + + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + + conversionWriter.writeInline("$L=", CaseUtils.toSnakeCase(memberName)); + + // Reference member shapes: + // Optional: `member_name=ShapeVisitor(input.MemberName.UnwrapOr(None))` + // Required: `member_name=ShapeVisitor(input.MemberName)` + if (context.model().expectShape(memberShape.getTarget()).hasTrait(ReferenceTrait.class)) { + conversionWriter.write("($1L) if ($2L is not None) else None,", + targetShape.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction + "." + memberName + + (memberShape.isOptional() ? ".UnwrapOr(None)" : ""), + conversionWriter, + "dafny_to_smithy" + ) + ), + dataSourceInsideConversionFunction + "." + memberName + + (memberShape.isOptional() ? ".UnwrapOr(None)" : "") + ); + } + + // Optional non-reference member shapes: + // `(ShapeVisitor(input.value)) if (input.value.is_Some) else None` + else if (memberShape.isOptional()) { + conversionWriter.write("($L) if ($L.is_Some) else None,", + targetShape.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction + "." + memberName + ".value", + conversionWriter, + "dafny_to_smithy" + ) + ), + dataSourceInsideConversionFunction + "." + memberName + ); + + // Required non-reference member shapes: + // `ShapeVisitor(input.value)` + } else { + conversionWriter.write("$L,", + targetShape.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction + "." + memberName, + conversionWriter, + "dafny_to_smithy" + ) + ) + ); + } + } + + /** + * Called from the StructureShape converter when the StructureShape has a Polymorph Positional trait. + * @return + */ + protected void writePositionalStructureShapeConverter(StructureShape structureShape, PythonWriter conversionWriter, + String dataSourceInsideConversionFunction) { + final MemberShape onlyMember = PositionalTrait.onlyMember(structureShape); + final Shape targetShape = context.model().expectShape(onlyMember.getTarget()); + + String returnValue = + targetShape.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction, + conversionWriter, + "dafny_to_smithy" + ) + ); + + if (onlyMember.isOptional()) { + conversionWriter.openBlock( + "if $L.is_Some:", + "", + dataSourceInsideConversionFunction, + () -> { + conversionWriter.write("return $L", returnValue); + } + ); + } else { + conversionWriter.write("return $L", returnValue); + } + } + + /** + * Called from the StructureShape converter when the StructureShape has a Polymorph Reference trait. + * @param structureShape + * @return + */ + private void writeReferenceStructureShapeConverter(StructureShape structureShape, PythonWriter conversionWriter, + String dataSourceInsideConversionFunction) { + ReferenceTrait referenceTrait = structureShape.expectTrait(ReferenceTrait.class); + Shape resourceOrService = context.model().expectShape(referenceTrait.getReferentId()); + + if (resourceOrService.isResourceShape()) { + writeResourceShapeConverter(resourceOrService.asResourceShape().get(), conversionWriter, + dataSourceInsideConversionFunction); + } else if (resourceOrService.isServiceShape()) { + writeServiceShapeConverter(resourceOrService.asServiceShape().get(), conversionWriter, + dataSourceInsideConversionFunction); + } else { + throw new UnsupportedOperationException("Unknown referenceStructureShape type: " + structureShape); + } + } + + private void writeResourceShapeConverter(ResourceShape resourceShape, PythonWriter conversionWriter, + String dataSourceInsideConversionFunction) { + + conversionWriter.openBlock("if hasattr($L, '_native_impl'):", + "", + dataSourceInsideConversionFunction, + () -> conversionWriter.write("return $L._native_impl", dataSourceInsideConversionFunction)); + + conversionWriter.openBlock("else:", + "", + () -> { + // Import resource at runtime to avoid circular dependency + conversionWriter.write("from $L import $L", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + resourceShape.getId().getNamespace(), context + ) + ".references", + resourceShape.getId().getName() + ); + + conversionWriter.write("return $L(_impl=$L)", + resourceShape.getId().getName(), + dataSourceInsideConversionFunction); + }); + } + + private void writeServiceShapeConverter(ServiceShape serviceShape, PythonWriter conversionWriter, + String dataSourceInsideConversionFunction) { + + if (serviceShape.hasTrait(LocalServiceTrait.class)) { + // Import resource at runtime to avoid circular dependency + conversionWriter.write("from $L import $L", + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + serviceShape.getId().getNamespace(), context + ) + ".client", + serviceShape.getId().getName() + ); + + conversionWriter.write("return $L(config=None, dafny_client=$L)", + serviceShape.getId().getName(), dataSourceInsideConversionFunction); + } else { + conversionWriter.write("return dafny_input._impl"); + } + + } + + /** + * Writes a function definition to convert a Dafny-modelled union shape + * into the corresponding Smithy-modelled union shape. + * The function definition is written into `dafny_to_smithy.py`. + * This SHOULD only be called once so only one function definition is written. + * @param unionShape + */ + protected void writeUnionShapeConverter(UnionShape unionShape) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(context.settings().getService().getNamespace()); + + // Write out common conversion function inside dafny_to_smithy + delegator.useFileWriter(moduleName + "/dafny_to_smithy.py", "", conversionWriter -> { + + // Within the conversion function, the dataSource becomes the function's input + String dataSourceInsideConversionFunction = "dafny_input"; + + // ex. shape: simple.union.ExampleUnion + // Writes `def DafnyToSmithy_simple_union_ExampleUnion(input):` + // and wraps inner code inside function definition + conversionWriter.openBlock( + "def $L($L):", + "", + SmithyNameResolver.getDafnyToSmithyFunctionNameForShape(unionShape, context), + dataSourceInsideConversionFunction, + () -> { + conversionWriter.writeComment("Convert %1$s".formatted( + unionShape.getId().getName() + )); + + // First union value opens a new `if` block; others do not need to and write `elif` + boolean shouldOpenNewIfBlock = true; + // Write out conversion: + // ex. if ExampleUnion can take on either of (IntegerValue, StringValue), write: + // if isinstance(input, ExampleUnion_IntegerValue): + // ExampleUnion_union_value = ExampleUnionIntegerValue(input.IntegerValue) + // elif isinstance(input, ExampleUnion_StringValue): + // ExampleUnion_union_value = ExampleUnionStringValue(input.StringValue) + for (MemberShape memberShape : unionShape.getAllMembers().values()) { + Shape targetShape = context.model().expectShape(memberShape.getTarget()); + conversionWriter.write( + """ + $L isinstance($L, $L): + $L_union_value = $L.$L($L)""", + // If we need a new `if` block, open one; otherwise, expand on existing one with `elif` + shouldOpenNewIfBlock ? "if" : "elif", + dataSourceInsideConversionFunction, + DafnyNameResolver.getDafnyTypeForUnion(unionShape, memberShape), + unionShape.getId().getName(), + SmithyNameResolver.getSmithyGeneratedModelLocationForShape(unionShape.getId(), context), + SmithyNameResolver.getSmithyGeneratedTypeForUnion(unionShape, memberShape, context), + targetShape.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction + "." + DafnyNameResolver.escapeShapeName(memberShape.getMemberName()), + conversionWriter, + "dafny_to_smithy" + ) + ) +// dataSourceInsideConversionFunction, +// DafnyNameResolver.escapeShapeName(memberShape.getMemberName()) + ); + shouldOpenNewIfBlock = false; + + DafnyNameResolver.importDafnyTypeForUnion(conversionWriter, unionShape, memberShape); + SmithyNameResolver.importSmithyGeneratedTypeForShape(conversionWriter, unionShape, context); + } + + // Write case to handle if union member does not match any of the above cases + conversionWriter.write(""" + else: + raise ValueError("No recognized union value in union type: " + str($L)) + """, + dataSourceInsideConversionFunction + ); + + // Write return value: + // `return ExampleUnion_union_value` + conversionWriter.write( + """ + return $L_union_value + """, + unionShape.getId().getName() + ); + } + ); + }); + } + + protected void writeStringEnumShapeConverter(StringShape stringShapeWithEnumTrait) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(context.settings().getService().getNamespace()); + + delegator.useFileWriter(moduleName + "/dafny_to_smithy.py", "", conversionWriter -> { + // Within the conversion function, the dataSource becomes the function's input + // This hardcodes the input parameter name for a conversion function to always be "dafny_input" + String dataSourceInsideConversionFunction = "dafny_input"; + + conversionWriter.openBlock( + "def $L($L):", + "", + SmithyNameResolver.getDafnyToSmithyFunctionNameForShape(stringShapeWithEnumTrait, context), + dataSourceInsideConversionFunction, + () -> { + EnumTrait enumTrait = stringShapeWithEnumTrait.getTrait(EnumTrait.class).get(); + + // First enum value opens a new `if` block; others do not need to and write `elif` + boolean shouldOpenNewIfBlock = true; + for (EnumDefinition enumDefinition : + stringShapeWithEnumTrait.getTrait(EnumTrait.class).get().getValues()) { + String name = enumDefinition.getName().isPresent() + ? enumDefinition.getName().get() + : enumDefinition.getValue(); + String value = enumDefinition.getValue(); + conversionWriter.write( + """ + $L isinstance($L, $L): + return '$L' + """, + // If we need a new `if` block, open one; otherwise, expand on existing one + // with `elif` + shouldOpenNewIfBlock ? "if" : "elif", + dataSourceInsideConversionFunction, + DafnyNameResolver.getDafnyTypeForStringShapeWithEnumTrait( + stringShapeWithEnumTrait, name), + value); + shouldOpenNewIfBlock = false; + + DafnyNameResolver.importDafnyTypeForStringShapeWithEnumTrait( + conversionWriter, stringShapeWithEnumTrait, name); + } + conversionWriter.openBlock("else:", + "", + () -> { + conversionWriter.write("raise ValueError(f'No recognized enum value in enum type: {$L=}')", + dataSourceInsideConversionFunction); + } + ); + } + ); + }); + } + +} \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/conversionwriter/LocalServiceToDafnyConversionFunctionWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/conversionwriter/LocalServiceToDafnyConversionFunctionWriter.java new file mode 100644 index 0000000000..235faf8a72 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/conversionwriter/LocalServiceToDafnyConversionFunctionWriter.java @@ -0,0 +1,453 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.localservice.shapevisitor.conversionwriter; + +import java.util.List; +import java.util.Map.Entry; + +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.Utils; +import software.amazon.polymorph.smithypython.common.shapevisitor.conversionwriter.BaseConversionWriter; +import software.amazon.polymorph.smithypython.common.shapevisitor.ShapeVisitorResolver; +import software.amazon.polymorph.smithypython.localservice.shapevisitor.LocalServiceToDafnyShapeVisitor; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.polymorph.traits.PositionalTrait; +import software.amazon.polymorph.traits.ReferenceTrait; +import software.amazon.smithy.codegen.core.WriterDelegator; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.ResourceShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.EnumDefinition; +import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.utils.CaseUtils; + +/** + * Writes the smithy_to_dafny.py file via the BaseConversionWriter implementation + */ +public class LocalServiceToDafnyConversionFunctionWriter extends BaseConversionWriter { + + // Use a singleton to preserve generatedShapes through multiple invocations of this class + static LocalServiceToDafnyConversionFunctionWriter singleton; + + protected LocalServiceToDafnyConversionFunctionWriter() { + } + + // Instantiate singleton at class-load time + static { + singleton = new LocalServiceToDafnyConversionFunctionWriter(); + } + + /** + * Delegate writing conversion methods for the provided shape and its member shapes + * + * @param shape + * @param context + * @param writer + */ + public static void writeConverterForShapeAndMembers(Shape shape, GenerationContext context, + PythonWriter writer) { + singleton.baseWriteConverterForShapeAndMembers(shape, context, writer); + } + + protected void writeStructureShapeConverter(StructureShape structureShape) { + // Defer localService config to the localService config converter +// if (SmithyNameResolver.getLocalServiceConfigShapes(context).contains(structureShape.getId())) { +// LocalServiceConfigToDafnyConversionFunctionWriter.writeConverterForShapeAndMembers(structureShape, +// context, writer); +// return; +// } + + WriterDelegator delegator = context.writerDelegator(); + String moduleName = SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(context.settings().getService().getNamespace()); + + delegator.useFileWriter(moduleName + "/smithy_to_dafny.py", "", conversionWriter -> { + // Within the conversion function, the dataSource becomes the function's input + // This hardcodes the input parameter name for a conversion function to always be "native_input" + String dataSourceInsideConversionFunction = "native_input"; + + conversionWriter.openBlock( + "def $L($L):", + "", + SmithyNameResolver.getSmithyToDafnyFunctionNameForShape(structureShape, context), + dataSourceInsideConversionFunction, + () -> { + if (Utils.isUnitShape(structureShape.getId())) { + conversionWriter.write("return None"); + } else if (structureShape.hasTrait(ReferenceTrait.class)) { + writeReferenceStructureShapeConverter(structureShape, conversionWriter, dataSourceInsideConversionFunction); + } else if (structureShape.hasTrait(PositionalTrait.class)) { + writePositionalStructureShapeConverter(structureShape, conversionWriter, dataSourceInsideConversionFunction); + } else { + writeNonReferenceStructureShapeConverter(structureShape, conversionWriter, dataSourceInsideConversionFunction); + } + } + ); + }); + } + + private void writeNonReferenceStructureShapeConverter(StructureShape shape, PythonWriter conversionWriter, + String dataSourceInsideConversionFunction) { + DafnyNameResolver.importDafnyTypeForShape(conversionWriter, shape.getId(), context); + + // Open Dafny structure shape + // e.g. + // DafnyStructureName(... + conversionWriter.openBlock( + "return $L(", + ")", + DafnyNameResolver.getDafnyTypeForShape(shape), + () -> { + // Recursively dispatch a new ShapeVisitor for each member of the structure + for (Entry memberShapeEntry : shape.getAllMembers().entrySet()) { + String memberName = memberShapeEntry.getKey(); + MemberShape memberShape = memberShapeEntry.getValue(); + writeNonReferenceStructureShapeMemberConverter(conversionWriter, + dataSourceInsideConversionFunction, memberName, memberShape); + } + } + ); + } + + private void writeNonReferenceStructureShapeMemberConverter(PythonWriter conversionWriter, + String dataSourceInsideConversionFunction, String memberName, MemberShape memberShape) { + final Shape targetShape = context.model().expectShape(memberShape.getTarget()); + + // Adds `DafnyStructureMember=smithy_structure_member(...)` + // e.g. + // DafnyStructureName(DafnyStructureMember=smithy_structure_member(...), ...) + // The nature of the `smithy_structure_member` conversion depends on the properties of the shape, + // as described below + conversionWriter.writeInline("$L=", memberName); + + if (context.model().expectShape(memberShape.getTarget()).hasTrait(ReferenceTrait.class)) { + if (memberShape.isOptional()) { + conversionWriter.write( + "((Option_Some($1L)) if (($2L is not None) and ($1L is not None)) else (Option_None())),", + targetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction + "." + CaseUtils.toSnakeCase(memberName), + conversionWriter, + "smithy_to_dafny" + ) + ), + dataSourceInsideConversionFunction + "." + CaseUtils.toSnakeCase(memberName) + ); + } else { + conversionWriter.write( + "$L,", + targetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction + "." + CaseUtils.toSnakeCase(memberName), + conversionWriter, + "smithy_to_dafny" + ) + ) + ); + } + + } + + // If this is a localService config shape, defer conversion to the config ShapeVisitor TODO remove this + else if (SmithyNameResolver.getLocalServiceConfigShapes(context).contains(targetShape.getId())) { + conversionWriter.write("$L,", + targetShape.accept( + new LocalServiceToDafnyShapeVisitor( + context, + dataSourceInsideConversionFunction + "." + CaseUtils.toSnakeCase(memberName), + // Pass the `conversionWriter` as our source writer; + // if we need to add imports, the imports will only be needed + // from the conversionwriter file + conversionWriter, + "smithy_to_dafny" + ) + ) + ); + } + + // If this shape is optional, write conversion logic to detect and possibly pass + // an empty optional at runtime + else if (memberShape.isOptional()) { + conversionWriter.addStdlibImport("Wrappers", "Option_Some"); + conversionWriter.addStdlibImport("Wrappers", "Option_None"); + conversionWriter.write( + "((Option_Some($L)) if ($L is not None) else (Option_None())),", + targetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction + "." + CaseUtils.toSnakeCase(memberName), + conversionWriter, + "smithy_to_dafny" + ) + ), + dataSourceInsideConversionFunction + "." + CaseUtils.toSnakeCase(memberName) + ); + } + + // If this shape is required, pass in the shape for conversion without any optional-checking + else { + conversionWriter.write("$L,", + targetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction + "." + CaseUtils.toSnakeCase(memberName), + conversionWriter, + "smithy_to_dafny" + ) + ) + ); + } + } + + /** + * Called from the StructureShape converter when the StructureShape has a Polymorph Positional trait. + * @return + */ + protected void writePositionalStructureShapeConverter(StructureShape structureShape, PythonWriter conversionWriter, + String dataSourceInsideConversionFunction) { + final MemberShape onlyMember = PositionalTrait.onlyMember(structureShape); + final Shape targetShape = context.model().expectShape(onlyMember.getTarget()); + + conversionWriter.write("return $L", + targetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction, + conversionWriter, + "smithy_to_dafny" + ) + ) + ); + } + + /** + * Called from the StructureShape converter when the StructureShape has a Polymorph Reference trait. + * @param shape + * @return + */ + protected void writeReferenceStructureShapeConverter(StructureShape shape, PythonWriter conversionWriter, + String dataSourceInsideConversionFunction) { + ReferenceTrait referenceTrait = shape.expectTrait(ReferenceTrait.class); + Shape resourceOrService = context.model().expectShape(referenceTrait.getReferentId()); + + if (resourceOrService.isResourceShape()) { + writeResourceShapeConverter(resourceOrService.asResourceShape().get(), + conversionWriter, dataSourceInsideConversionFunction); + } else if (resourceOrService.isServiceShape()) { + writeServiceShapeConverter(resourceOrService.asServiceShape().get(), conversionWriter, + dataSourceInsideConversionFunction); + } else { + throw new UnsupportedOperationException("Unknown reference type: " + shape); + } + } + + protected void writeServiceShapeConverter(ServiceShape serviceShape, + PythonWriter conversionWriter, String dataSourceInsideConversionFunction) { + + if (serviceShape.hasTrait(LocalServiceTrait.class)) { + conversionWriter.write("return $L._config.dafnyImplInterface.impl" + , dataSourceInsideConversionFunction); + +// DafnyNameResolver.importDafnyTypeForServiceShape(conversionWriter, serviceShape); +// +// // Import service inline to avoid circular dependency +// conversionWriter.write("import $L", +// SmithyNameResolver.getSmithyGeneratedConfigModulePathForSmithyNamespace( +// serviceShape.getId().getNamespace(), context) +// ); +// +// conversionWriter.addStdlibImport(DafnyNameResolver.getDafnyPythonIndexModuleNameForShape(serviceShape)); +// +// // `my_module_client = my_module_internaldafny.MyModuleClient()` +// conversionWriter.write("$L_client = $L.$L()", +// SmithyNameResolver.getPythonModuleNamespaceForSmithyNamespace(serviceShape.getId().getNamespace()), +// DafnyNameResolver.getDafnyPythonIndexModuleNameForShape(serviceShape), +// DafnyNameResolver.getDafnyClientTypeForServiceShape(serviceShape) +// ); +// +// // `my_module_client.ctor__(my_module.smithygenerated.config.smithy_config_to_dafny_config(input._config))` +// conversionWriter.write("$L_client.ctor__($L.smithy_config_to_dafny_config($L._config))", +// SmithyNameResolver.getPythonModuleNamespaceForSmithyNamespace(serviceShape.getId().getNamespace()), +// SmithyNameResolver.getSmithyGeneratedConfigModulePathForSmithyNamespace( +// serviceShape.getId().getNamespace(), context), +// dataSourceInsideConversionFunction +// ); +// +// conversionWriter.write("return $L_client", +// SmithyNameResolver.getPythonModuleNamespaceForSmithyNamespace(serviceShape.getId().getNamespace())); + } else { + DafnyNameResolver.importDafnyTypeForServiceShape(conversionWriter, serviceShape); + + conversionWriter.write(""" + import $1L + client = $1L.default__.$2L(boto_client = $3L) + client.value.impl = $3L + return client.value + """, + DafnyNameResolver.getDafnyPythonIndexModuleNameForShape(serviceShape), + AwsSdkNameResolver.clientNameForService(serviceShape), + dataSourceInsideConversionFunction + ); + } + + } + + protected void writeResourceShapeConverter(ResourceShape resourceShape, + PythonWriter conversionWriter, String dataSourceInsideConversionFunction) { + + conversionWriter.openBlock("if hasattr($L, '_impl'):", + "", + dataSourceInsideConversionFunction, + () -> { + conversionWriter.write("return $L._impl", + dataSourceInsideConversionFunction); + }); + + conversionWriter.openBlock("else:", + "", + () -> { + conversionWriter.write("return $L", + dataSourceInsideConversionFunction); + }); + } + + /** + * Writes a function definition to convert a Smithy-modelled union shape + * into the corresponding Dafny-modelled union shape. + * The function definition is written into `smithy_to_dafny.py`. + * This SHOULD only be called once so only one function definition is written. + * @param unionShape + */ + public void writeUnionShapeConverter(UnionShape unionShape) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(context.settings().getService().getNamespace()); + + delegator.useFileWriter(moduleName + "/smithy_to_dafny.py", "", conversionWriter -> { + + // Within the conversion function, the dataSource becomes the function's input + // This hardcodes the input parameter name for a conversion function to always be "native_input" + String dataSourceInsideConversionFunction = "native_input"; + + // ex. shape: simple.union.ExampleUnion + // Writes `def SmithyToDafny_simple_union_ExampleUnion(input):` + // and wraps inner code inside function definition + conversionWriter.openBlock( + "def $L($L):", + "", + SmithyNameResolver.getSmithyToDafnyFunctionNameForShape(unionShape, context), + dataSourceInsideConversionFunction, + () -> { + + // First union value opens a new `if` block; others do not need to and write `elif` + boolean shouldOpenNewIfBlock = true; + for (MemberShape memberShape : unionShape.getAllMembers().values()) { + Shape targetShape = context.model().expectShape(memberShape.getTarget()); + // Write out conversion: + // ex. if ExampleUnion can take on either of (IntegerValue, StringValue), write: + // if isinstance(input, ExampleUnion.IntegerValue): + // example_union_union_value = DafnyExampleUnionIntegerValue(input.member.value) + // elif isinstance(input, ExampleUnion.StringValue): + // example_union_union_value = DafnyExampleUnionIntegerValue(input.member.value) + conversionWriter.write(""" + $L isinstance($L, $L.$L): + $L_union_value = $L($L)""", + // If we need a new `if` block, open one; otherwise, expand on existing one with `elif` + shouldOpenNewIfBlock ? "if" : "elif", + dataSourceInsideConversionFunction, + SmithyNameResolver.getSmithyGeneratedModelLocationForShape(unionShape.getId(), context), + SmithyNameResolver.getSmithyGeneratedTypeForUnion(unionShape, memberShape, context), + unionShape.getId().getName(), + DafnyNameResolver.getDafnyTypeForUnion(unionShape, memberShape), + targetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape(targetShape, + context, + dataSourceInsideConversionFunction + ".value", + conversionWriter, + "smithy_to_dafny" + ) + ) +// DafnyNameResolver.getDafnyTypeForUnion(unionShape, memberShape), +// dataSourceInsideConversionFunction, +// "value" + ); + shouldOpenNewIfBlock = false; + + DafnyNameResolver.importDafnyTypeForUnion(conversionWriter, unionShape, memberShape); + SmithyNameResolver.importSmithyGeneratedTypeForShape(conversionWriter, unionShape, context); + } + + // Write case to handle if union member does not match any of the above cases + conversionWriter.write(""" + else: + raise ValueError("No recognized union value in union type: " + str($L)) + """, + dataSourceInsideConversionFunction + ); + + // Return the result of the union conversion + // `return example_union_union_value` + conversionWriter.write("return %1$s_union_value".formatted(unionShape.getId().getName())); + }); + }); + } + + protected void writeStringEnumShapeConverter(StringShape stringShapeWithEnumTrait) { + WriterDelegator delegator = context.writerDelegator(); + String moduleName = SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(context.settings().getService().getNamespace()); + + delegator.useFileWriter(moduleName + "/smithy_to_dafny.py", "", conversionWriter -> { + // Within the conversion function, the dataSource becomes the function's input + // This hardcodes the input parameter name for a conversion function to always be "native_input" + String dataSourceInsideConversionFunction = "native_input"; + + conversionWriter.openBlock( + "def $L($L):", + "", + SmithyNameResolver.getSmithyToDafnyFunctionNameForShape(stringShapeWithEnumTrait, context), + dataSourceInsideConversionFunction, + () -> { + + // First enum value opens a new `if` block; others do not need to and write `elif` + boolean shouldOpenNewIfBlock = true; + for (EnumDefinition enumDefinition : + stringShapeWithEnumTrait.getTrait(EnumTrait.class).get().getValues()) { + String name = enumDefinition.getName().isPresent() + ? enumDefinition.getName().get() + : enumDefinition.getValue(); + String value = enumDefinition.getValue(); + conversionWriter.openBlock("$L $L == '$L':", + "", + shouldOpenNewIfBlock ? "if" : "elif", + dataSourceInsideConversionFunction, + value, + () -> { + conversionWriter.write("return $L()", + DafnyNameResolver.getDafnyTypeForStringShapeWithEnumTrait(stringShapeWithEnumTrait, name)); + DafnyNameResolver.importDafnyTypeForStringShapeWithEnumTrait(conversionWriter, stringShapeWithEnumTrait, name); + } + ); + shouldOpenNewIfBlock = false; + } + conversionWriter.openBlock("else:", + "", + () -> { + conversionWriter.write("raise ValueError(f'No recognized enum value in enum type: {$L=}')", + dataSourceInsideConversionFunction); + } + ); + } + ); + }); + } + +} \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/DafnyPythonWrappedLocalServiceIntegration.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/DafnyPythonWrappedLocalServiceIntegration.java new file mode 100644 index 0000000000..70ed7d6aca --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/DafnyPythonWrappedLocalServiceIntegration.java @@ -0,0 +1,60 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.wrappedlocalservice; + +import java.util.Collections; +import java.util.List; +import software.amazon.polymorph.smithypython.wrappedlocalservice.customize.ShimFileWriter; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.integration.ProtocolGenerator; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; + +public final class DafnyPythonWrappedLocalServiceIntegration implements PythonIntegration { + + /** + * Generate all custom code for wrapped LocalServices. + * + * @param codegenContext Code generation context that can be queried when writing additional + * files. + */ + @Override + public void customize(GenerationContext codegenContext) { + // ONLY run this integration's customizations if the codegen is using wrapped localService + if (!codegenContext + .applicationProtocol() + .equals( + DafnyPythonWrappedLocalServiceProtocolGenerator + .DAFNY_PYTHON_WRAPPED_LOCAL_SERVICE_PROTOCOL)) { + return; + } + + customizeForServiceShape( + codegenContext.settings().getService(codegenContext.model()), codegenContext); + } + + /** + * Generate any code for the serviceShape. + * + * @param serviceShape + * @param codegenContext + */ + private void customizeForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) { + new ShimFileWriter().customizeFileForServiceShape(serviceShape, codegenContext); + } + + @Override + public List getProtocolGenerators() { + return Collections.singletonList( + new DafnyPythonWrappedLocalServiceProtocolGenerator() { + // See the WrappedLocalServiceTrait class in this directory. + @Override + public ShapeId getProtocol() { + return ShapeId.from("aws.polymorph#wrappedLocalService"); + } + }); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/DafnyPythonWrappedLocalServiceProtocolGenerator.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/DafnyPythonWrappedLocalServiceProtocolGenerator.java new file mode 100644 index 0000000000..26f4a31556 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/DafnyPythonWrappedLocalServiceProtocolGenerator.java @@ -0,0 +1,35 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.wrappedlocalservice; + +import software.amazon.smithy.python.codegen.ApplicationProtocol; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.integration.ProtocolGenerator; +import software.amazon.smithy.utils.SmithyUnstableApi; + +@SmithyUnstableApi +public abstract class DafnyPythonWrappedLocalServiceProtocolGenerator implements ProtocolGenerator { + + public static ApplicationProtocol DAFNY_PYTHON_WRAPPED_LOCAL_SERVICE_PROTOCOL = + new ApplicationProtocol( + // Define the `dafny-python-wrapped-local-service-application-protocol` + // ApplicationProtocol. + // ApplicationProtocol is distinct from the Protocols used in ProtocolGenerators. + // The ApplicationProtocol is used within our code to determine which code should be + // generated. + // The `null`s reflect that this ApplicationProtocol does not have request + // or response object types, since it does not use Smithy-generated clients. + "dafny-python-wrapped-local-service-application-protocol", null, null); + + @Override + public ApplicationProtocol getApplicationProtocol() { + return DAFNY_PYTHON_WRAPPED_LOCAL_SERVICE_PROTOCOL; + } + + @Override + public void generateRequestSerializers(GenerationContext context) {} + + @Override + public void generateResponseDeserializers(GenerationContext context) {} +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/WrappedCodegenConstants.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/WrappedCodegenConstants.java new file mode 100644 index 0000000000..696724e0d4 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/WrappedCodegenConstants.java @@ -0,0 +1,5 @@ +package software.amazon.polymorph.smithypython.wrappedlocalservice; + +public class WrappedCodegenConstants { + public static String WRAPPED_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME = "wrapped_codegen_todelete"; +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/customize/ShimFileWriter.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/customize/ShimFileWriter.java new file mode 100644 index 0000000000..27dbbcd77c --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/customize/ShimFileWriter.java @@ -0,0 +1,187 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.wrappedlocalservice.customize; + +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import software.amazon.polymorph.smithypython.awssdk.nameresolver.AwsSdkNameResolver; +import software.amazon.polymorph.smithypython.common.customize.CustomFileWriter; +import software.amazon.polymorph.smithypython.common.nameresolver.DafnyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.common.nameresolver.Utils; +import software.amazon.polymorph.smithypython.common.shapevisitor.ShapeVisitorResolver; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.polymorph.traits.PositionalTrait; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonWriter; + +/** + * Writes the shim.py file. The shim wraps the client.py implementation (which itself wraps the + * underlying Dafny implementation). Other Dafny-generated Python code may use the shim to interact + * with this project's Dafny implementation through the Polymorph wrapper. + */ +public class ShimFileWriter implements CustomFileWriter { + + @Override + public void customizeFileForServiceShape( + ServiceShape serviceShape, GenerationContext codegenContext) { + String typesModulePrelude = + DafnyNameResolver.getDafnyPythonTypesModuleNameForShape(serviceShape.getId()); + String moduleName = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + codegenContext.settings().getService().getNamespace()); + codegenContext + .writerDelegator() + .useFileWriter( + moduleName + "/shim.py", + "", + writer -> { + writer.addImport(".errors", "ServiceError"); + writer.addImport(".errors", "CollectionOfErrors"); + writer.addImport(".errors", "OpaqueError"); + + writer.write( + """ + import Wrappers + import $L + import $L.client as client_impl + + class $L($L.$L): + def __init__(self, _impl: client_impl) : + self._impl = _impl + + ${C|} + """, + typesModulePrelude, + SmithyNameResolver.getPythonModuleSmithygeneratedPathForSmithyNamespace( + serviceShape.getId().getNamespace(), codegenContext.settings()), + SmithyNameResolver.shimNameForService(serviceShape), + typesModulePrelude, + DafnyNameResolver.getDafnyClientInterfaceTypeForServiceShape(serviceShape), + writer.consumer(w -> generateOperationsBlock(codegenContext, serviceShape, w))); + }); + } + + /** + * Generate shim methods for all operations in the localService. Each method will take in a Dafny + * type as input and return a Dafny type as output. Internally, each method will convert the Dafny + * input into native Smithy-modelled input, call the native Smithy client with the native input, + * receive a native Smithy-modelled output from the client, convert the native output type into + * its corresponding Dafny type, and return the Dafny type. + * + * @param codegenContext + * @param serviceShape + * @param writer + */ + private void generateOperationsBlock( + GenerationContext codegenContext, ServiceShape serviceShape, PythonWriter writer) { + + for (ShapeId operationShapeId : serviceShape.getOperations()) { + OperationShape operationShape = + codegenContext.model().expectShape(operationShapeId, OperationShape.class); + + // Add imports for operation errors + for (ShapeId errorShapeId : operationShape.getErrors(serviceShape)) { + SmithyNameResolver.importSmithyGeneratedTypeForShape(writer, errorShapeId, codegenContext); + } + + ShapeId inputShape = operationShape.getInputShape(); + ShapeId outputShape = operationShape.getOutputShape(); + + // Import Dafny types for inputs and outputs (shim function input and output) + DafnyNameResolver.importDafnyTypeForShape(writer, inputShape, codegenContext); + DafnyNameResolver.importDafnyTypeForShape(writer, outputShape, codegenContext); + // Import Smithy types for inputs and outputs (Smithy client call input and output) + SmithyNameResolver.importSmithyGeneratedTypeForShape(writer, inputShape, codegenContext); + SmithyNameResolver.importSmithyGeneratedTypeForShape(writer, outputShape, codegenContext); + + boolean isInputPositional = + codegenContext + .model() + .expectShape(inputShape) + .asStructureShape() + .get() + .hasTrait(PositionalTrait.class); + boolean isOutputPoitional = + codegenContext + .model() + .expectShape(inputShape) + .asStructureShape() + .get() + .hasTrait(PositionalTrait.class); + + writer.addStdlibImport("typing", "Any"); + + // Write the Shim operation block. + // This takes in a Dafny input and returns a Dafny output. + // This operation will: + // 1) Convert the Dafny input to a Smithy-modelled input, + // 2) Call the Smithy-generated client with the transformed input, and + // 3) Convert the Smithy output to the Dafny type. + writer.openBlock( + "def $L(self, $L):", + "", + operationShape.getId().getName(), + // Do not generate an `input` parameter if the operation does not take in an input + Utils.isUnitShape(inputShape) ? "" : "input", + () -> { + Shape targetShapeInput = + codegenContext.model().expectShape(operationShape.getInputShape()); + // Generate code that converts the input from the Dafny type to the corresponding Smithy + // type. + // `input` will hold a string that converts the Dafny `input` to the Smithy-modelled + // output. + String input = + targetShapeInput.accept( + ShapeVisitorResolver.getToNativeShapeVisitorForShape( + targetShapeInput, codegenContext, "input", writer)); + + // Generate code that: + // 1) "unwraps" the request (converts from the Dafny type to the Smithy type), + // 2) calls Smithy client, + // 3) wraps Smithy failures as Dafny failures + writer.write( + """ + smithy_client_request: $L.$L = $L + try: + smithy_client_response = self._impl.$L(smithy_client_request) + except ServiceError as e: + return Wrappers.Result_Failure(_smithy_error_to_dafny_error(e)) + + """, + SmithyNameResolver.getSmithyGeneratedModelLocationForShape(inputShape, codegenContext), + inputShape.getName(), + input, + codegenContext.symbolProvider().toSymbol(operationShape).getName()); + + writer.addImport(".errors", "_smithy_error_to_dafny_error"); + + Shape targetShape = codegenContext.model().expectShape(operationShape.getOutputShape()); + // Generate code that converts the output from Smithy type to the corresponding Dafny + // type. + // This has a side effect of possibly writing transformation code at the writer's + // current position. + String output = + targetShape.accept( + ShapeVisitorResolver.getToDafnyShapeVisitorForShape( + targetShape, codegenContext, "smithy_client_response", writer)); + + // Generate code that wraps Smithy success shapes as Dafny success shapes + writer.write( + """ + return Wrappers.Result_Success($L) + """, + Utils.isUnitShape(outputShape) ? "None" : output); + }); + } + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/extensions/DafnyPythonWrappedLocalServiceClientCodegenPlugin.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/extensions/DafnyPythonWrappedLocalServiceClientCodegenPlugin.java new file mode 100644 index 0000000000..84282282f3 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/extensions/DafnyPythonWrappedLocalServiceClientCodegenPlugin.java @@ -0,0 +1,96 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.wrappedlocalservice.extensions; + +import java.util.Map; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.traits.WrappedLocalServiceTrait; +import software.amazon.polymorph.traits.LocalServiceTrait; +import software.amazon.smithy.build.PluginContext; +import software.amazon.smithy.build.SmithyBuildPlugin; +import software.amazon.smithy.codegen.core.directed.CodegenDirector; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.transform.ModelTransformer; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.integration.PythonIntegration; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Plugin to trigger Smithy-Dafny Python code generation for a wrapped localService. This differs + * from the PythonClientCodegenPlugin by not calling runner.performDefaultCodegenTransforms(); and + * runner.createDedicatedInputsAndOutputs(); This differs from the non-wrapped plugin by adding a + * WrappedLocalServiceTrait to the service that is being generated. This is stores the context that + * this plugin requires wrapped localService generation, so that we can identify this from within + * code generation. These methods transform the model such that the model used by the code generator + * does not align with the generated Dafny code. + */ +@SmithyUnstableApi +public final class DafnyPythonWrappedLocalServiceClientCodegenPlugin implements SmithyBuildPlugin { + + public DafnyPythonWrappedLocalServiceClientCodegenPlugin( + Map smithyNamespaceToPythonModuleNameMap) { + super(); + SmithyNameResolver.setSmithyNamespaceToPythonModuleNameMap( + smithyNamespaceToPythonModuleNameMap); + } + + /** + * Add the {@link WrappedLocalServiceTrait} to the serviceShape in the model. This is required to + * allow Smithy-Python to function correctly. + * + * @param model + * @param serviceShape + * @return + */ + public static Model addWrappedLocalServiceTrait(Model model, ServiceShape serviceShape) { + return ModelTransformer.create() + .mapShapes( + model, + shape -> { + if (shape.equals(serviceShape)) { + if (!shape.hasTrait(LocalServiceTrait.class)) { + throw new IllegalArgumentException( + "ServiceShape for LocalService test MUST have a LocalServiceTrait: " + + serviceShape); + } + return serviceShape.toBuilder() + .addTrait(WrappedLocalServiceTrait.builder().build()) + .build(); + } else { + return shape; + } + }); + } + + @Override + public String getName() { + return "dafny-python-wrapped-local-service-client-codegen"; + } + + @Override + public void execute(PluginContext context) { + CodegenDirector runner = + new CodegenDirector<>(); + + PythonSettings settings = PythonSettings.from(context.getSettings()); + settings.setProtocol(WrappedLocalServiceTrait.ID); + runner.settings(settings); + runner.directedCodegen(new DirectedDafnyPythonWrappedLocalServiceCodegen()); + runner.fileManifest(context.getFileManifest()); + runner.service(settings.getService()); + runner.integrationClass(PythonIntegration.class); + + // Add a WrappedLocalServiceTrait to the serviceShape to indicate to codegen + // to generate for a wrapped LocalService + ServiceShape serviceShape = + context.getModel().expectShape(settings.getService()).asServiceShape().get(); + Model transformedModel = addWrappedLocalServiceTrait(context.getModel(), serviceShape); + runner.model(transformedModel); + + runner.run(); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/extensions/DafnyPythonWrappedLocalServiceSymbolVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/extensions/DafnyPythonWrappedLocalServiceSymbolVisitor.java new file mode 100644 index 0000000000..ff45bdf9fc --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/extensions/DafnyPythonWrappedLocalServiceSymbolVisitor.java @@ -0,0 +1,49 @@ +package software.amazon.polymorph.smithypython.wrappedlocalservice.extensions; + +import static java.lang.String.format; + +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.localservice.extensions.DafnyPythonLocalServiceSymbolVisitor; +import software.amazon.polymorph.smithypython.wrappedlocalservice.WrappedCodegenConstants; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.python.codegen.PythonSettings; + +/** + * SymbolVisitor for wrapped localService codegen. Overrides the generated file for codegen to + * something that is immediately deleted. Smithy ALWAYS writes visited symbols to a file. For + * wrapped codegen, we do NOT want to write visited symbols to a file. We want to reuse the + * generated files from localService codegen. It is very, very difficult to change this writing + * behavior without rewriting Smithy logic in addition to Smithy-Python specific logic. I have tried + * some workarounds like deleting writers or writing to /dev/null but these were not fruitful. This + * workaround dumps any visited symbols into a file whose name will never be used and deletes this + * file as part of its Smithy codegen plugin. + */ +public class DafnyPythonWrappedLocalServiceSymbolVisitor + extends DafnyPythonLocalServiceSymbolVisitor { + + public DafnyPythonWrappedLocalServiceSymbolVisitor(Model model, PythonSettings settings) { + super(model, settings); + } + + /** + * Path to the overridden file that is deleted for wrapped services. + * + * @param namespace + * @return + */ + @Override + protected String getSymbolDefinitionFilePathForNamespaceAndFilename(String namespace, String filename) { + String directoryFilePath; + if ("smithy.api".equals(namespace)) { + directoryFilePath = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + settings.getService().getNamespace()); + } else { + directoryFilePath = + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace(namespace); + } + + // Ignore the filename! Wrapped codegen deletes this file. + return format("%s/%s.py", directoryFilePath, WrappedCodegenConstants.WRAPPED_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/extensions/DirectedDafnyPythonWrappedLocalServiceCodegen.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/extensions/DirectedDafnyPythonWrappedLocalServiceCodegen.java new file mode 100644 index 0000000000..fc8f8f6edb --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/wrappedlocalservice/extensions/DirectedDafnyPythonWrappedLocalServiceCodegen.java @@ -0,0 +1,118 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.smithypython.wrappedlocalservice.extensions; + +import static java.lang.String.format; + +import java.nio.file.Path; +import java.util.logging.Logger; +import software.amazon.polymorph.smithypython.common.nameresolver.SmithyNameResolver; +import software.amazon.polymorph.smithypython.localservice.extensions.DirectedDafnyPythonLocalServiceCodegen; +import software.amazon.polymorph.smithypython.wrappedlocalservice.WrappedCodegenConstants; +import software.amazon.smithy.build.FileManifest; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.codegen.core.directed.CreateSymbolProviderDirective; +import software.amazon.smithy.codegen.core.directed.CustomizeDirective; +import software.amazon.smithy.codegen.core.directed.GenerateResourceDirective; +import software.amazon.smithy.codegen.core.directed.GenerateServiceDirective; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; + +/** + * DirectedCodegen for Dafny Python wrapped LocalServices. This overrides DirectedPythonCodegen to + * 1) Not generate a Smithy client (nor its serialize/deserialize bodies, client config, etc.) 2) + * Remove extraneous generated files (TODO-Python: Consider rewriting SymbolVisitor to avoid this) + * Wrapped LocalService generation does NOT involve generating a Smithy client; it will only + * generate a shim wrapping the LocalService-generated Smithy client. + */ +public class DirectedDafnyPythonWrappedLocalServiceCodegen + extends DirectedDafnyPythonLocalServiceCodegen { + + private static final Logger LOGGER = + Logger.getLogger(DirectedDafnyPythonWrappedLocalServiceCodegen.class.getName()); + + @Override + public SymbolProvider createSymbolProvider( + CreateSymbolProviderDirective directive) { + return new DafnyPythonWrappedLocalServiceSymbolVisitor(directive.model(), directive.settings()); + } + + @Override + public void generateResource( + GenerateResourceDirective directive) {} + + /** + * Do NOT generate any service config code for Dafny Python AWS SDKs (i.e. `config.py`). Override + * DirectedPythonCodegen to block any service config code generation. + * + * @param directive Directive to perform. + */ + @Override + public void customizeBeforeShapeGeneration( + CustomizeDirective directive) {} + + /** + * Do NOT generate any service code for Dafny Python AWS SDKs. Override DirectedPythonCodegen to + * block any service code generation. In addition to not writing any service code (i.e. not + * writing `client.py`), this also blocks writing `serialize.py` and `deserialize.py`. + * + * @param directive Directive to perform. + */ + @Override + public void generateService( + GenerateServiceDirective directive) {} + + /** + * Call `DirectedPythonCodegen.customizeAfterIntegrations`, then remove `models.py` and + * `errors.py`. The CodegenDirector will invoke this method after shape generation. + * + * @param directive Directive to perform. + */ + @Override + public void customizeAfterIntegrations( + CustomizeDirective directive) { + // DirectedPythonCodegen's customizeAfterIntegrations implementation SHOULD run first; + // its implementation writes all files by flushing its writers; + // this implementation removes some of those files. + super.customizeAfterIntegrations(directive); + + FileManifest fileManifest = directive.fileManifest(); + Path generationPath = + Path.of( + fileManifest.getBaseDir() + + "/" + + SmithyNameResolver.getServiceSmithygeneratedDirectoryNameForNamespace( + directive.context().settings().getService().getNamespace())); + + /** + * Smithy ALWAYS writes visited symbols to a file. For wrapped codegen, we do NOT want to write + * visited symbols to a file. We want to reuse the generated files from localService codegen. It + * is very, very difficult to change this writing behavior without rewriting Smithy logic in + * addition to Smithy-Python specific logic. I have tried some workarounds like deleting writers + * or writing to /dev/null but these were not fruitful. This workaround dumps any visited + * symbols into a file whose name will never be used and deletes this file as part of its Smithy + * codegen plugin. + */ + try { + LOGGER.info( + format( + "Attempting to remove %s.py", + WrappedCodegenConstants.WRAPPED_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME)); + CodegenUtils.runCommand( + format( + "rm -f %s.py", WrappedCodegenConstants.WRAPPED_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME), + generationPath) + .strip(); + } catch (CodegenException e) { + // Fail loudly. We do not want to accidentally distribute this file. + throw new RuntimeException( + format( + "Unable to remove %s.py", + WrappedCodegenConstants.WRAPPED_CODEGEN_SYMBOLWRITER_DUMP_FILE_FILENAME), + e); + } + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/traits/DafnyAwsSdkProtocolTrait.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/traits/DafnyAwsSdkProtocolTrait.java new file mode 100644 index 0000000000..d6779d5ea8 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/traits/DafnyAwsSdkProtocolTrait.java @@ -0,0 +1,77 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.traits; + +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.AbstractTrait; +import software.amazon.smithy.model.traits.AbstractTraitBuilder; +import software.amazon.smithy.model.traits.Trait; +import software.amazon.smithy.utils.SmithyBuilder; +import software.amazon.smithy.utils.ToSmithyBuilder; + +/** + * A trait that signals that a service is an AWS SDK serviceShape. This trait should NOT be added to + * Smithy model files. This is a trait that can be added at runtime by Polymorph code generators + * that implement SmithyBuildPlugins to signal that the code generation process should generate an + * AWS SDK shim. This is needed because SmithyBuildPlugins require that SOME protocol is present on + * the provided ServiceShape. However, the protocols on our AWS SDK Smithy model files are either + * inconsistent or nonexistent; in addition, we should not declare some usage of any provided + * protocol (e.g. `awsJson1_1`) to allow the SmithyBuildPlugin to perform code generation, then + * ignore that protocol entirely. Since Smithy-Dafny CodegenEngine and subclasses of + * DirectedPythonCodegen are aware of the ServiceShape that is under generation, they can attach a + * new protocol at runtime to fulfill requirements from Smithy that SmithyBuildPlugins have a + * protocol. + */ +public class DafnyAwsSdkProtocolTrait extends AbstractTrait + implements ToSmithyBuilder { + public static final ShapeId ID = ShapeId.from("aws.polymorph#awsSdk"); + + private DafnyAwsSdkProtocolTrait(DafnyAwsSdkProtocolTrait.Builder builder) { + super(ID, builder.getSourceLocation()); + } + + public static final class Provider extends AbstractTrait.Provider { + public Provider() { + super(ID); + } + + @Override + public Trait createTrait(ShapeId target, Node value) { + return builder().build(); + } + } + + public static DafnyAwsSdkProtocolTrait.Builder builder() { + return new DafnyAwsSdkProtocolTrait.Builder(); + } + + @Override + protected Node createNode() { + return Node.objectNodeBuilder().sourceLocation(getSourceLocation()).build(); + } + + @Override + public SmithyBuilder toBuilder() { + return builder().sourceLocation(getSourceLocation()); + } + + /** Builder for {@link MutableLocalStateTrait}. */ + public static final class Builder + extends AbstractTraitBuilder { + + private Builder() {} + + @Override + public DafnyAwsSdkProtocolTrait build() { + return new DafnyAwsSdkProtocolTrait(this); + } + } + + public static Shape getDefinition() { + return StructureShape.builder().id(MutableLocalStateTrait.ID).build(); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/traits/WrappedLocalServiceTrait.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/traits/WrappedLocalServiceTrait.java new file mode 100644 index 0000000000..890ae1ce3a --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/traits/WrappedLocalServiceTrait.java @@ -0,0 +1,70 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.amazon.polymorph.traits; + +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.AbstractTrait; +import software.amazon.smithy.model.traits.AbstractTraitBuilder; +import software.amazon.smithy.model.traits.Trait; +import software.amazon.smithy.utils.SmithyBuilder; +import software.amazon.smithy.utils.ToSmithyBuilder; + +/** + * A trait that signals that a service is a wrapped LocalService. This trait should NOT be added to + * Smithy model files; it is intended to ONLY be added at runtime by Polymorph code generators that + * implement SmithyBuildPlugins to signal that the code generation process should generate a wrapped + * LocalService. + */ +public class WrappedLocalServiceTrait extends AbstractTrait + implements ToSmithyBuilder { + public static final ShapeId ID = ShapeId.from("aws.polymorph#wrappedLocalService"); + + private WrappedLocalServiceTrait(WrappedLocalServiceTrait.Builder builder) { + super(ID, builder.getSourceLocation()); + } + + public static WrappedLocalServiceTrait.Builder builder() { + return new WrappedLocalServiceTrait.Builder(); + } + + public static Shape getDefinition() { + return StructureShape.builder().id(MutableLocalStateTrait.ID).build(); + } + + @Override + protected Node createNode() { + return Node.objectNodeBuilder().sourceLocation(getSourceLocation()).build(); + } + + @Override + public SmithyBuilder toBuilder() { + return builder().sourceLocation(getSourceLocation()); + } + + public static final class Provider extends AbstractTrait.Provider { + public Provider() { + super(ID); + } + + @Override + public Trait createTrait(ShapeId target, Node value) { + return builder().build(); + } + } + + /** Builder for {@link WrappedLocalServiceTrait}. */ + public static final class Builder + extends AbstractTraitBuilder { + + private Builder() {} + + @Override + public WrappedLocalServiceTrait build() { + return new WrappedLocalServiceTrait(this); + } + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java index f1a7008ce1..f05ecb7ac1 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java @@ -590,4 +590,69 @@ public static Model addMissingErrorMessageMembers(Model model) { } ); } + + /** + * Return a builder for the provided shape. + * @param shape + * @return + */ + public static AbstractShapeBuilder getBuilderForShape(Shape shape) { + // This is painful, but there is nothing like `shape.getUnderlyingShapeType`... + // instead, check every possible shape for its builder... + AbstractShapeBuilder builder; + if (shape.isBlobShape()) { + builder = shape.asBlobShape().get().toBuilder(); + } else if (shape.isBooleanShape()) { + builder = shape.asBooleanShape().get().toBuilder(); + } else if (shape.isDocumentShape()) { + builder = shape.asDocumentShape().get().toBuilder(); + } else if (shape.isStringShape()) { + builder = shape.asStringShape().get().toBuilder(); + } else if (shape.isTimestampShape()) { + builder = shape.asTimestampShape().get().toBuilder(); + } else if (shape.isByteShape()) { + builder = shape.asByteShape().get().toBuilder(); + } else if (shape.isIntegerShape()) { + builder = shape.asIntegerShape().get().toBuilder(); + } else if (shape.isFloatShape()) { + builder = shape.asFloatShape().get().toBuilder(); + } else if (shape.isBigIntegerShape()) { + builder = shape.asBigIntegerShape().get().toBuilder(); + } else if (shape.isShortShape()) { + builder = shape.asShortShape().get().toBuilder(); + } else if (shape.isLongShape()) { + builder = shape.asLongShape().get().toBuilder(); + } else if (shape.isDoubleShape()) { + builder = shape.asDoubleShape().get().toBuilder(); + } else if (shape.isBigDecimalShape()) { + builder = shape.asBigDecimalShape().get().toBuilder(); + } else if (shape.isListShape()) { + builder = shape.asListShape().get().toBuilder(); + } else if (shape.isSetShape()) { + builder = shape.asSetShape().get().toBuilder(); + } else if (shape.isMapShape()) { + builder = shape.asMapShape().get().toBuilder(); + } else if (shape.isStructureShape()) { + builder = shape.asStructureShape().get().toBuilder(); + } else if (shape.isUnionShape()) { + builder = shape.asUnionShape().get().toBuilder(); + } else if (shape.isServiceShape()) { + builder = shape.asServiceShape().get().toBuilder(); + } else if (shape.isOperationShape()) { + builder = shape.asOperationShape().get().toBuilder(); + } else if (shape.isResourceShape()) { + builder = shape.asResourceShape().get().toBuilder(); + } else if (shape.isMemberShape()) { + builder = shape.asMemberShape().get().toBuilder(); + } else if (shape.isEnumShape()) { + builder = shape.asEnumShape().get().toBuilder(); + } else if (shape.isIntEnumShape()) { + builder = shape.asIntEnumShape().get().toBuilder(); + } else { + // Unfortunately, there is no "default" shape... + // The above should cover all shapes; if not, new shapes need to be added above. + throw new IllegalArgumentException("Unable to process @javadoc trait on unsupported shape type: " + shape); + } + return builder; + } } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java index cbced7c12e..90b800a011 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java @@ -103,6 +103,7 @@ static Optional fromObject( case "DOTNET", "CSHARP", "CS" -> Stream.of( CodegenEngine.TargetLanguage.DOTNET ); + case "PYTHON" -> Stream.of(CodegenEngine.TargetLanguage.PYTHON); case "DAFNY" -> { LOGGER.error( "Dafny code is always generated, and shouldn't be specified explicitly" @@ -164,26 +165,26 @@ static Optional fromObject( ); } - /** - * Traverses up from the given start path, - * searching for a "smithy-build.json" file and returning its path if found. - */ - private static Optional findSmithyBuildJson(final Path start) { - if (start == null || !start.isAbsolute()) { - throw new IllegalArgumentException( - "Start path must be non-null and absolute" - ); - } - Path cursor = start.normalize(); - final Path root = cursor.getRoot(); - // Shouldn't need to traverse more than 100 levels... but don't hang forever - for (int i = 0; !root.equals(cursor) && i < 100; i++) { - final Path config = cursor.resolve("smithy-build.json"); - if (Files.exists(config)) { - return Optional.of(config); - } - cursor = cursor.getParent(); - } - return Optional.empty(); + /** + * Traverses up from the given start path, + * searching for a "smithy-build.json" file and returning its path if found. + */ + private static Optional findSmithyBuildJson(final Path start) { + if (start == null || !start.isAbsolute()) { + throw new IllegalArgumentException( + "Start path must be non-null and absolute" + ); + } + Path cursor = start.normalize(); + final Path root = cursor.getRoot(); + // Shouldn't need to traverse more than 100 levels... but don't hang forever + for (int i = 0; !root.equals(cursor) && i < 100; i++) { + final Path config = cursor.resolve("smithy-build.json"); + if (Files.exists(config)) { + return Optional.of(config); + } + cursor = cursor.getParent(); + } + return Optional.empty(); } } diff --git a/codegen/smithy-dafny-codegen/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration b/codegen/smithy-dafny-codegen/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration new file mode 100644 index 0000000000..4877aa10e4 --- /dev/null +++ b/codegen/smithy-dafny-codegen/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integration.PythonIntegration @@ -0,0 +1,8 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +software.amazon.polymorph.smithypython.localservice.DafnyPythonLocalServiceIntegration +software.amazon.polymorph.smithypython.wrappedlocalservice.DafnyPythonWrappedLocalServiceIntegration +software.amazon.polymorph.smithypython.awssdk.DafnyPythonAwsSdkIntegration \ No newline at end of file