From 792c6daedaac528ed966bafc933817ad06fce02e Mon Sep 17 00:00:00 2001 From: Connor Kirkpatrick Date: Tue, 1 Aug 2023 00:30:35 +0100 Subject: [PATCH 01/32] chore: Update CFN Lint version to support python 3.11 (#5651) * Update CFN Lint version to support python 3.11 * Run make update-reproducible-reqs --------- Co-authored-by: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> --- requirements/base.txt | 2 +- requirements/reproducible-linux.txt | 6 +++--- requirements/reproducible-mac.txt | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 11e1c7f068..80725f827c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -31,7 +31,7 @@ regex!=2021.10.8 tzlocal==3.0 #Adding cfn-lint dependency for SAM validate -cfn-lint~=0.78.1 +cfn-lint~=0.79.5 # Type checking boto3 objects boto3-stubs[apigateway,cloudformation,ecr,iam,lambda,s3,schemas,secretsmanager,signer,stepfunctions,sts,xray]==1.28.2 diff --git a/requirements/reproducible-linux.txt b/requirements/reproducible-linux.txt index 60823b5628..916aa45e7b 100644 --- a/requirements/reproducible-linux.txt +++ b/requirements/reproducible-linux.txt @@ -119,9 +119,9 @@ cffi==1.15.1 \ --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 # via cryptography -cfn-lint==0.78.1 \ - --hash=sha256:2dacb19d5f70c0d49f466302507707cfa4914f65b8fc9310ae3771a273cec044 \ - --hash=sha256:46118362b2e13b79ba3ae6b3c28b7df5fcd437c06f5bcc3384d13a2defdb7d06 +cfn-lint==0.79.5 \ + --hash=sha256:3054a2e407f77b3115c41de49fe2c11d728d53ab7be4ae478c1b5ff0e21331b7 \ + --hash=sha256:7a00dba8f70b8a92c38bfc8f45e2ab20cedad5c49054a46f1ce931e48fc1cd0c # via aws-sam-cli (setup.py) chardet==5.1.0 \ --hash=sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5 \ diff --git a/requirements/reproducible-mac.txt b/requirements/reproducible-mac.txt index 15c74810ca..be1c8f439e 100644 --- a/requirements/reproducible-mac.txt +++ b/requirements/reproducible-mac.txt @@ -137,9 +137,9 @@ cffi==1.15.1 \ --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 # via cryptography -cfn-lint==0.78.1 \ - --hash=sha256:2dacb19d5f70c0d49f466302507707cfa4914f65b8fc9310ae3771a273cec044 \ - --hash=sha256:46118362b2e13b79ba3ae6b3c28b7df5fcd437c06f5bcc3384d13a2defdb7d06 +cfn-lint==0.79.5 \ + --hash=sha256:3054a2e407f77b3115c41de49fe2c11d728d53ab7be4ae478c1b5ff0e21331b7 \ + --hash=sha256:7a00dba8f70b8a92c38bfc8f45e2ab20cedad5c49054a46f1ce931e48fc1cd0c # via aws-sam-cli (setup.py) chardet==5.1.0 \ --hash=sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5 \ From e0de9987f5321f1fb1926aff32f1fa768d9eeb48 Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Mon, 31 Jul 2023 16:37:07 -0700 Subject: [PATCH 02/32] fix: Require file label to run before maintainers (#5656) * Require file label to run before maintainers * Run maintainer label job even if previous job fails --- .github/workflows/pr-labeler.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 16f48ae22f..fa38d437e2 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -13,6 +13,8 @@ jobs: with: repo-token: "${{ secrets.GITHUB_TOKEN }}" apply-internal-external-label: + needs: apply-file-based-labels + if: ${{ always() }} permissions: pull-requests: write runs-on: ubuntu-latest From 275d44e6b616efaf51c22c0b5f8d1d497f4249fa Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Mon, 31 Jul 2023 16:41:19 -0700 Subject: [PATCH 03/32] Add Windows config --- appveyor-windows-binary.yml | 307 ++++++++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 appveyor-windows-binary.yml diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml new file mode 100644 index 0000000000..42aed38854 --- /dev/null +++ b/appveyor-windows-binary.yml @@ -0,0 +1,307 @@ +version: 1.0.{build} +image: Visual Studio 2022 +build: off + +clone_folder: C:\source + +configuration: + - BuildIntegTesting + - LocalZipTerraformBuildIntegTesting + - LocalZipTerraformBuildInContainerIntegTesting + - S3ZipTerraformBuildIntegTesting + - S3ZipTerraformBuildInContainerIntegTesting + - OtherTerraformBuildIntegTesting + - DeployIntegTesting + - PackageIntegTesting + - DeleteIntegTesting + - SyncIntegTesting + - LocalIntegTesting + - EndToEndTesting + # other Integration testing, Dev, regression and smoke testing + - OtherTesting + +environment: + AWS_DEFAULT_REGION: us-east-1 + SAM_CLI_DEV: 1 + CARGO_LAMBDA_VERSION: "v0.17.1" + + # Python uses $TMPDIR envvar to find root of tempdir + TMPDIR: "%TEMP%" + TMP: "%TEMP%" + + # MSI Installers only use Py3.8. It is sufficient to test with this version here. + PYTHON_HOME: "C:\\Python38-x64" + PYTHON_SCRIPTS: "C:\\Python38-x64\\Scripts" + PYTHON_EXE: "C:\\Python38-x64\\python.exe" + PYTHON_ARCH: "64" + HOME: 'C:\Users\appveyor' + HOMEDRIVE: "C:" + HOMEPATH: 'C:\Users\appveyor' + NOSE_PARAMETERIZED_NO_WARN: 1 + AWS_S3: "AWS_S3_TESTING" + AWS_ECR: "AWS_ECR_TESTING" + APPVEYOR_CONSOLE_DISABLE_PTY: true + +init: + # Uncomment this for RDP + # - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + - ps: gcim Win32_Processor | % { "$($_.NumberOfLogicalProcessors) logical CPUs" } + - ps: gcim Win32_OperatingSystem | % { "$([int]($_.TotalVisibleMemorySize/1mb)) Gb" } + - git config --global core.autocrlf false + - ps: New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force + - ps: git config --system core.longpaths true + +cache: + - C:\ProgramData\chocolatey\bin -> appveyor.yml + - C:\ProgramData\chocolatey\lib -> appveyor.yml + +install: + # setup make + - "choco install make" + # setup Java, Maven and Gradle + - "choco install gradle -y --force" + - 'set JAVA_HOME=C:\Program Files\Java\jdk17' + - 'set PATH=%JAVA_HOME%\bin;%PATH%' + - "javac -version" + - "java -version" + - "gradle -v" + - "mvn --version" + + # Make sure the temp directory exists for Python to use. + - ps: "mkdir -Force C:\\tmp" + - 'set PATH=%PYTHON_HOME%;C:\Ruby27-x64\bin;%PATH%;C:\Python37-x64;C:\Python39-x64;C:\Python310-x64' + - "echo %PYTHON_HOME%" + - "echo %PATH%" + - "python --version" + - "docker info" + - "docker version" + + # install Terraform CLI + - "choco install terraform" + - "terraform -version" + + # Upgrade setuptools, wheel and virtualenv + - "python -m pip install --upgrade setuptools wheel virtualenv" + # Install pip for the python versions which is used by the tests + - "C:\\Python37-x64\\python.exe -m pip install --upgrade pip" + - "C:\\Python39-x64\\python.exe -m pip install --upgrade pip" + - "C:\\Python310-x64\\python.exe -m pip install --upgrade pip" + + # Install AWS CLI Globally via pip3 + - "pip install awscli" + + # Switch to Docker Linux containers + - ps: Switch-DockerLinux + + # Check for git executable + - "git --version" + + # Get testing env vars + - ps: " + If (Test-Path env:BY_CANARY){ + python -m virtualenv venv_env_vars; + ./venv_env_vars/Scripts/pip install boto3; + $test_env_var = ./venv_env_vars/Scripts/python tests/get_testing_resources.py; + $test_env_var_json = ConvertFrom-Json $test_env_var; + + $env:CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID = $env:AWS_ACCESS_KEY_ID; + $env:CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY = $env:AWS_SECRET_ACCESS_KEY; + $env:CI_ACCESS_ROLE_AWS_SESSION_TOKEN = $env:AWS_SESSION_TOKEN; + + $env:AWS_ACCESS_KEY_ID = $test_env_var_json.accessKeyID; + $env:AWS_SECRET_ACCESS_KEY = $test_env_var_json.secretAccessKey; + $env:AWS_SESSION_TOKEN = $test_env_var_json.sessionToken; + $env:TASK_TOKEN = $test_env_var_json.taskToken; + $env:AWS_S3_TESTING = $test_env_var_json.TestBucketName; + $env:AWS_ECR_TESTING = $test_env_var_json.TestECRURI; + $env:AWS_KMS_KEY = $test_env_var_json.TestKMSKeyArn; + $env:AWS_SIGNING_PROFILE_NAME = $test_env_var_json.TestSigningProfileName; + $env:AWS_SIGNING_PROFILE_VERSION_ARN = $test_env_var_json.TestSigningProfileARN; + }" + + + # Create new virtual environment with chosen python version and activate it + - "python -m virtualenv venv" + - "venv\\Scripts\\activate" + - "python --version" + + # Install aws-sam-cli MSI binary + - ps: " + Write-Host "Starting installation for MSI" + Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi + $process = Start-Process C:\Windows\System32\msiexec.exe -ArgumentList "/i $HOME\aws-sam-cli-installer.msi /qn /norestart" -wait -PassThru + $process + Set-Alias -Name sam -Value 'C:\Program Files\Amazon\AWSSAMCLI_NIGHTLY\bin\sam-nightly.cmd' + sam --version + " + + # Install aws cli + - "pip install awscli" + + # Echo final Path + - "echo %PATH%" + + # use amazon-ecr-credential-helper + - choco install amazon-ecr-credential-helper + - ps: " + $docker_config = Get-Content $env:HOME/.docker/config.json -raw | ConvertFrom-Json; + $docker_config.credsStore = 'ecr-login'; + $docker_config | ConvertTo-Json | set-content $env:HOME/.docker/config.json; + " + - ps: "get-content $env:HOME/.docker/config.json" + + # claim some disk space before starting the tests + - "docker system prune -a -f" + # activate virtual environment + - "venv\\Scripts\\activate" + + + +# Final clean up no matter success or failure +on_finish: + # Upload test reports as artifacts + - ps: Get-ChildItem .\TEST_REPORT-*.json | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } + - ps: ' + If (Test-Path env:BY_CANARY){ + $env:AWS_ACCESS_KEY_ID = $env:TEST_REPORT_S3_BUCKET_ACCESS_KEY_ID; + $env:AWS_SECRET_ACCESS_KEY = $env:TEST_REPORT_S3_BUCKET_SECRET_ACCESS_KEY; + $env:AWS_SESSION_TOKEN = $env:TEST_REPORT_S3_BUCKET_SESSION_TOKEN; + aws s3 cp ".\" "s3://$env:TEST_REPORT_S3_BUCKET_NAME/appveyor/$env:APPVEYOR_PROJECT_SLUG/$env:APPVEYOR_BUILD_ID/$env:APPVEYOR_JOB_ID/" --recursive --exclude "*" --include "TEST_REPORT-*.json" --region us-west-2 + }' + + # notify success + - ps: " + If (Test-Path env:BY_CANARY){ + $env:AWS_ACCESS_KEY_ID = $env:CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID; + $env:AWS_SECRET_ACCESS_KEY = $env:CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY; + $env:AWS_SESSION_TOKEN = $env:CI_ACCESS_ROLE_AWS_SESSION_TOKEN; + aws stepfunctions send-task-success --task-token \"$env:TASK_TOKEN\" --task-output \"{}\" --region us-west-2; + }" + + +for: + #Integ testing build, regression + - matrix: + only: + - configuration: BuildIntegTesting + + build_script: + # install Rust in build_script to not override the default "install" actions + - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe + - rustup-init.exe -y --default-host x86_64-pc-windows-msvc --default-toolchain stable + - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin + - set RUST_BACKTRACE=1 + - rustup toolchain install stable --profile minimal --no-self-update + - rustup default stable + - rustup target add x86_64-unknown-linux-gnu --toolchain stable + - rustup target add aarch64-unknown-linux-gnu --toolchain stable + - ps: "choco install zig" + - ps: Invoke-WebRequest -Uri https://github.com/cargo-lambda/cargo-lambda/releases/download/$env:CARGO_LAMBDA_VERSION/cargo-lambda-$env:CARGO_LAMBDA_VERSION.windows-x64.zip -OutFile C:\Users\appveyor\cargo-lambda.zip + - ps: Expand-Archive -DestinationPath C:\Users\appveyor\.cargo\bin C:\Users\appveyor\cargo-lambda.zip + - rustc -V + - cargo -V + - cargo lambda -V + + test_script: + - ps: "pytest -vv tests/integration/buildcmd --ignore tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # Local ZIP Terraform Build integ testing + - matrix: + only: + - configuration: LocalZipTerraformBuildIntegTesting + + test_script: + - ps: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # Local ZIP Terraform Build In Container integ testing + - matrix: + only: + - configuration: LocalZipTerraformBuildInContainerIntegTesting + + test_script: + - ps: "pytest -vv tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # S3 ZIP Terraform Build integ testing + - matrix: + only: + - configuration: S3ZipTerraformBuildIntegTesting + + test_script: + - ps: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # S3 ZIP Terraform Build In Container integ testing + - matrix: + only: + - configuration: S3ZipTerraformBuildInContainerIntegTesting + + test_script: + - ps: "pytest -vv tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # Other Terraform Build integ testing + - matrix: + only: + - configuration: OtherTerraformBuildIntegTesting + + test_script: + - ps: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + #Integ testing deploy + - matrix: + only: + - configuration: DeployIntegTesting + + test_script: + - ps: "pytest -vv tests/integration/deploy -n 4 --reruns 4 --dist=loadgroup --json-report --json-report-file=TEST_REPORT-integration-deploy.json" + + # Integ testing package + - matrix: + only: + - configuration: PackageIntegTesting + + test_script: + - ps: "pytest -vv tests/integration/package -n 4 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-package.json" + + # Integ testing delete + - matrix: + only: + - configuration: DeleteIntegTesting + + test_script: + - ps: "pytest -vv tests/integration/delete -n 4 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-delete.json" + + # Integ testing sync + - matrix: + only: + - configuration: SyncIntegTesting + + test_script: + - ps: "pytest -vv tests/integration/sync -n 3 --reruns 3 --dist loadscope --json-report --json-report-file=TEST_REPORT-integration-sync.json" + + #Integ testing local + - matrix: + only: + - configuration: LocalIntegTesting + + test_script: + - ps: "pytest -vv tests/integration/local --json-report --json-report-file=TEST_REPORT-integration-local.json" + + # End-to-end testing + - matrix: + only: + - configuration: EndToEndTesting + + test_script: + - ps: "pytest -vv -n 4 --reruns 5 --dist loadscope tests/end_to_end --json-report --json-report-file=TEST_REPORT-end-to-end.json" + + #Other testing + - matrix: + only: + - configuration: OtherTesting + + test_script: + - ps: "pytest -vv -n 4 --reruns 4 --dist loadgroup tests/integration --ignore=tests/integration/buildcmd --ignore=tests/integration/delete --ignore=tests/integration/deploy --ignore=tests/integration/package --ignore=tests/integration/sync --ignore=tests/integration/local --json-report --json-report-file=TEST_REPORT-integration-others.json" + - ps: "pytest -vv tests/regression --json-report --json-report-file=TEST_REPORT-regression.json" +# Uncomment for RDP +# on_finish: +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + From d0d68daffa2b8bd7f8cea29e1151d4a0f7e3bda6 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Mon, 31 Jul 2023 16:48:51 -0700 Subject: [PATCH 04/32] Fix formatting --- appveyor-windows-binary.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index 42aed38854..d3ffd2a00b 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -127,12 +127,12 @@ install: # Install aws-sam-cli MSI binary - ps: " - Write-Host "Starting installation for MSI" - Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi - $process = Start-Process C:\Windows\System32\msiexec.exe -ArgumentList "/i $HOME\aws-sam-cli-installer.msi /qn /norestart" -wait -PassThru - $process - Set-Alias -Name sam -Value 'C:\Program Files\Amazon\AWSSAMCLI_NIGHTLY\bin\sam-nightly.cmd' - sam --version + Write-Host "Starting installation for MSI" + Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi + $process = Start-Process C:\Windows\System32\msiexec.exe -ArgumentList "/i $HOME\aws-sam-cli-installer.msi /qn /norestart" -wait -PassThru + $process + Set-Alias -Name sam -Value 'C:\Program Files\Amazon\AWSSAMCLI_NIGHTLY\bin\sam-nightly.cmd' + sam --version " # Install aws cli From 0ae07bb4d564cc101661ab995e82be2a1f16ad64 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Mon, 31 Jul 2023 16:55:22 -0700 Subject: [PATCH 05/32] Update config --- appveyor-windows-binary.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index d3ffd2a00b..3038f99515 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -127,7 +127,6 @@ install: # Install aws-sam-cli MSI binary - ps: " - Write-Host "Starting installation for MSI" Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi $process = Start-Process C:\Windows\System32\msiexec.exe -ArgumentList "/i $HOME\aws-sam-cli-installer.msi /qn /norestart" -wait -PassThru $process From eeee79b08788357208abc7b0f2175b0439999baf Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Mon, 31 Jul 2023 17:00:44 -0700 Subject: [PATCH 06/32] test --- appveyor-windows-binary.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index 3038f99515..5a3f0fc127 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -126,13 +126,13 @@ install: - "python --version" # Install aws-sam-cli MSI binary - - ps: " - Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi - $process = Start-Process C:\Windows\System32\msiexec.exe -ArgumentList "/i $HOME\aws-sam-cli-installer.msi /qn /norestart" -wait -PassThru - $process - Set-Alias -Name sam -Value 'C:\Program Files\Amazon\AWSSAMCLI_NIGHTLY\bin\sam-nightly.cmd' - sam --version - " +# - ps: " +# Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi +# $process = Start-Process C:\Windows\System32\msiexec.exe -ArgumentList "/i $HOME\aws-sam-cli-installer.msi /qn /norestart" -wait -PassThru +# $process +# Set-Alias -Name sam -Value 'C:\Program Files\Amazon\AWSSAMCLI_NIGHTLY\bin\sam-nightly.cmd' +# sam --version +# " # Install aws cli - "pip install awscli" From b378f798aed17b2b6381e8e78dadeff7b05bbf3d Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Mon, 31 Jul 2023 17:10:28 -0700 Subject: [PATCH 07/32] Update config --- appveyor-windows-binary.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index 5a3f0fc127..3d131b6ea6 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -126,13 +126,15 @@ install: - "python --version" # Install aws-sam-cli MSI binary -# - ps: " -# Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi -# $process = Start-Process C:\Windows\System32\msiexec.exe -ArgumentList "/i $HOME\aws-sam-cli-installer.msi /qn /norestart" -wait -PassThru -# $process -# Set-Alias -Name sam -Value 'C:\Program Files\Amazon\AWSSAMCLI_NIGHTLY\bin\sam-nightly.cmd' -# sam --version -# " + - ps: " + Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi + $process = Start-Process C:\Windows\System32\msiexec.exe -ArgumentList "/i $HOME\aws-sam-cli-installer.msi /qn /norestart" -wait -PassThru + $process + Set-Alias -Name sam -Value 'C:\Program Files\Amazon\AWSSAMCLI_NIGHTLY\bin\sam-nightly.cmd' + sam --version + " + + - "pip install pytest" # Install aws cli - "pip install awscli" From 88a21e10b81ed1f66ff07d2a514f2d1e393a04c6 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Mon, 31 Jul 2023 17:14:03 -0700 Subject: [PATCH 08/32] Update config --- appveyor-windows-binary.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index 3d131b6ea6..47ee0a57a9 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -127,12 +127,13 @@ install: # Install aws-sam-cli MSI binary - ps: " + If (Test-Path env:BY_CANARY){ Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi $process = Start-Process C:\Windows\System32\msiexec.exe -ArgumentList "/i $HOME\aws-sam-cli-installer.msi /qn /norestart" -wait -PassThru $process Set-Alias -Name sam -Value 'C:\Program Files\Amazon\AWSSAMCLI_NIGHTLY\bin\sam-nightly.cmd' sam --version - " + }" - "pip install pytest" From a5150f2f9cc4e962338782c8c26e1e7403191539 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Mon, 31 Jul 2023 17:18:28 -0700 Subject: [PATCH 09/32] Update config --- appveyor-windows-binary.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index 47ee0a57a9..a6583ae72f 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -127,13 +127,12 @@ install: # Install aws-sam-cli MSI binary - ps: " - If (Test-Path env:BY_CANARY){ Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi - $process = Start-Process C:\Windows\System32\msiexec.exe -ArgumentList "/i $HOME\aws-sam-cli-installer.msi /qn /norestart" -wait -PassThru + $process = Start-Process 'C:\Windows\System32\msiexec.exe' -ArgumentList '/i $HOME\aws-sam-cli-installer.msi /qn /norestart' -wait -PassThru $process Set-Alias -Name sam -Value 'C:\Program Files\Amazon\AWSSAMCLI_NIGHTLY\bin\sam-nightly.cmd' sam --version - }" + " - "pip install pytest" From 98f2cc88008de5891949a66266f6182c54a2d117 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Mon, 31 Jul 2023 17:21:49 -0700 Subject: [PATCH 10/32] Update config --- appveyor-windows-binary.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index a6583ae72f..bf217e3ec4 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -128,9 +128,9 @@ install: # Install aws-sam-cli MSI binary - ps: " Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi - $process = Start-Process 'C:\Windows\System32\msiexec.exe' -ArgumentList '/i $HOME\aws-sam-cli-installer.msi /qn /norestart' -wait -PassThru + $process = Start-Process 'C:\\Windows\\System32\\msiexec.exe' -ArgumentList '/i $HOME\aws-sam-cli-installer.msi /qn /norestart' -wait -PassThru $process - Set-Alias -Name sam -Value 'C:\Program Files\Amazon\AWSSAMCLI_NIGHTLY\bin\sam-nightly.cmd' + Set-Alias -Name sam -Value 'C:\\Program Files\\Amazon\\AWSSAMCLI_NIGHTLY\\bin\\sam-nightly.cmd' sam --version " From d264edd40cc14a8f2f5f80f12c4ce58d17b8eabf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 1 Aug 2023 02:26:09 +0000 Subject: [PATCH 11/32] chore: update aws_lambda_builders to 1.36.0 (#5655) Co-authored-by: GitHub Action Co-authored-by: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> --- requirements/base.txt | 2 +- requirements/reproducible-linux.txt | 6 +++--- requirements/reproducible-mac.txt | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 80725f827c..299e4dbe48 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -14,7 +14,7 @@ aws-sam-translator==1.71.0 docker~=6.1.0 dateparser~=1.1 requests~=2.31.0 -aws_lambda_builders==1.35.0 +aws_lambda_builders==1.36.0 tomlkit==0.11.8 watchdog==2.1.2 rich~=13.4.2 diff --git a/requirements/reproducible-linux.txt b/requirements/reproducible-linux.txt index 916aa45e7b..354b363df0 100644 --- a/requirements/reproducible-linux.txt +++ b/requirements/reproducible-linux.txt @@ -15,9 +15,9 @@ attrs==23.1.0 \ # jschema-to-python # jsonschema # sarif-om -aws-lambda-builders==1.35.0 \ - --hash=sha256:419d766e60ac2a7303a23889b354d108a4244ce8d467dcf9dc71a62461895780 \ - --hash=sha256:cf462961ec8c9d493f82b955d6b76630dc0ed356b166caddd5436719ce599586 +aws-lambda-builders==1.36.0 \ + --hash=sha256:00ce31612a62dc8fa1a2dcd087648d125b6274ff8b18799d3ebe5bb4824f3e96 \ + --hash=sha256:b673b13d72ab9a85523e70a0980a8df86868bcf285d1ae1c032bb747bfc5b5d1 # via aws-sam-cli (setup.py) aws-sam-translator==1.71.0 \ --hash=sha256:17fb87c8137d8d49e7a978396b2b3b279211819dee44618415aab1e99c2cb659 \ diff --git a/requirements/reproducible-mac.txt b/requirements/reproducible-mac.txt index be1c8f439e..08062fd84e 100644 --- a/requirements/reproducible-mac.txt +++ b/requirements/reproducible-mac.txt @@ -15,9 +15,9 @@ attrs==23.1.0 \ # jschema-to-python # jsonschema # sarif-om -aws-lambda-builders==1.35.0 \ - --hash=sha256:419d766e60ac2a7303a23889b354d108a4244ce8d467dcf9dc71a62461895780 \ - --hash=sha256:cf462961ec8c9d493f82b955d6b76630dc0ed356b166caddd5436719ce599586 +aws-lambda-builders==1.36.0 \ + --hash=sha256:00ce31612a62dc8fa1a2dcd087648d125b6274ff8b18799d3ebe5bb4824f3e96 \ + --hash=sha256:b673b13d72ab9a85523e70a0980a8df86868bcf285d1ae1c032bb747bfc5b5d1 # via aws-sam-cli (setup.py) aws-sam-translator==1.71.0 \ --hash=sha256:17fb87c8137d8d49e7a978396b2b3b279211819dee44618415aab1e99c2cb659 \ From c75486739680e25da08f023fe0885b18fbd75807 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Tue, 1 Aug 2023 08:45:24 -0700 Subject: [PATCH 12/32] Update config --- appveyor-windows-binary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index bf217e3ec4..c2132db576 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -127,7 +127,7 @@ install: # Install aws-sam-cli MSI binary - ps: " - Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile $HOME\aws-sam-cli-installer.msi + Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile aws-sam-cli-installer.msi $process = Start-Process 'C:\\Windows\\System32\\msiexec.exe' -ArgumentList '/i $HOME\aws-sam-cli-installer.msi /qn /norestart' -wait -PassThru $process Set-Alias -Name sam -Value 'C:\\Program Files\\Amazon\\AWSSAMCLI_NIGHTLY\\bin\\sam-nightly.cmd' From d854da79b5afb34824d0d24d3c144ede0042b613 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Tue, 1 Aug 2023 09:14:03 -0700 Subject: [PATCH 13/32] Update config --- appveyor-windows-binary.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index c2132db576..79375ec4e6 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -114,7 +114,7 @@ install: $env:TASK_TOKEN = $test_env_var_json.taskToken; $env:AWS_S3_TESTING = $test_env_var_json.TestBucketName; $env:AWS_ECR_TESTING = $test_env_var_json.TestECRURI; - $env:AWS_KMS_KEY = $test_env_var_json.TestKMSKeyArn; + $env:AWS_KMS_KEY = $test_env_var_json.TestKMSKeyArn;en $env:AWS_SIGNING_PROFILE_NAME = $test_env_var_json.TestSigningProfileName; $env:AWS_SIGNING_PROFILE_VERSION_ARN = $test_env_var_json.TestSigningProfileARN; }" @@ -127,7 +127,7 @@ install: # Install aws-sam-cli MSI binary - ps: " - Invoke-WebRequest https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile aws-sam-cli-installer.msi + Invoke-WebRequest -Uri https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile aws-sam-cli-installer.msi $process = Start-Process 'C:\\Windows\\System32\\msiexec.exe' -ArgumentList '/i $HOME\aws-sam-cli-installer.msi /qn /norestart' -wait -PassThru $process Set-Alias -Name sam -Value 'C:\\Program Files\\Amazon\\AWSSAMCLI_NIGHTLY\\bin\\sam-nightly.cmd' From 253852cbd844075321b9d69108db2c9ee6913b5a Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:00:31 -0500 Subject: [PATCH 14/32] fix(invoke): Write in UTF-8 string instead of bytes (#5642) * Revert "fix: Revert UTF-8 fixes #5485 and #5427 (#5512)" This reverts commit 36f8bf970df7a8072eef58ca76405c40e1cc1c90. * Enforce utf8 on stdout/stderr/logfile --------- Co-authored-by: Jacob Fuss --- .../local/cli_common/invoke_context.py | 8 ++-- .../commands/remote/remote_invoke_context.py | 4 +- samcli/lib/docker/log_streamer.py | 28 +++++------ samcli/lib/package/ecr_uploader.py | 4 +- samcli/lib/package/s3_uploader.py | 2 +- samcli/lib/utils/osutils.py | 19 ++++++-- samcli/lib/utils/stream_writer.py | 25 ++++++++-- samcli/lib/utils/subprocess_utils.py | 4 +- samcli/local/apigw/local_apigw_service.py | 4 +- samcli/local/docker/container.py | 47 +++++++++++++++---- samcli/local/docker/lambda_image.py | 11 +++-- samcli/local/docker/manager.py | 6 +-- .../local_lambda_invoke_service.py | 2 +- samcli/local/services/base_local_service.py | 4 +- .../local/invoke/test_integrations_cli.py | 21 +++++++++ .../local/cli_common/test_invoke_context.py | 2 +- tests/unit/lib/utils/test_osutils.py | 8 ---- tests/unit/lib/utils/test_stream_writer.py | 11 +++-- tests/unit/lib/utils/test_subprocess_utils.py | 13 ++--- tests/unit/local/docker/test_container.py | 29 ++++++------ tests/unit/local/docker/test_lambda_image.py | 9 ++-- tests/unit/local/docker/test_manager.py | 24 +++++++--- .../local/services/test_base_local_service.py | 18 +++---- 23 files changed, 191 insertions(+), 112 deletions(-) diff --git a/samcli/commands/local/cli_common/invoke_context.py b/samcli/commands/local/cli_common/invoke_context.py index 837b122558..30dc8b4c03 100644 --- a/samcli/commands/local/cli_common/invoke_context.py +++ b/samcli/commands/local/cli_common/invoke_context.py @@ -7,7 +7,7 @@ import os from enum import Enum from pathlib import Path -from typing import IO, Any, Dict, List, Optional, Tuple, Type, cast +from typing import Any, Dict, List, Optional, TextIO, Tuple, Type, cast from samcli.commands._utils.template import TemplateFailedParsingException, TemplateNotFoundException from samcli.commands.exceptions import ContainersInitializationException @@ -196,7 +196,7 @@ def __init__( self._stacks: List[Stack] = None # type: ignore self._env_vars_value: Optional[Dict] = None self._container_env_vars_value: Optional[Dict] = None - self._log_file_handle: Optional[IO] = None + self._log_file_handle: Optional[TextIO] = None self._debug_context: Optional[DebugContext] = None self._layers_downloader: Optional[LayerDownloader] = None self._container_manager: Optional[ContainerManager] = None @@ -490,7 +490,7 @@ def _get_env_vars_value(filename: Optional[str]) -> Optional[Dict]: ) from ex @staticmethod - def _setup_log_file(log_file: Optional[str]) -> Optional[IO]: + def _setup_log_file(log_file: Optional[str]) -> Optional[TextIO]: """ Open a log file if necessary and return the file handle. This will create a file if it does not exist @@ -500,7 +500,7 @@ def _setup_log_file(log_file: Optional[str]) -> Optional[IO]: if not log_file: return None - return open(log_file, "wb") + return open(log_file, "w", encoding="utf8") @staticmethod def _get_debug_context( diff --git a/samcli/commands/remote/remote_invoke_context.py b/samcli/commands/remote/remote_invoke_context.py index b710df8410..d1294983bc 100644 --- a/samcli/commands/remote/remote_invoke_context.py +++ b/samcli/commands/remote/remote_invoke_context.py @@ -242,7 +242,7 @@ class DefaultRemoteInvokeResponseConsumer(RemoteInvokeConsumer[RemoteInvokeRespo _stream_writer: StreamWriter def consume(self, remote_invoke_response: RemoteInvokeResponse) -> None: - self._stream_writer.write(cast(str, remote_invoke_response.response).encode()) + self._stream_writer.write_bytes(cast(str, remote_invoke_response.response).encode()) @dataclass @@ -254,4 +254,4 @@ class DefaultRemoteInvokeLogConsumer(RemoteInvokeConsumer[RemoteInvokeLogOutput] _stream_writer: StreamWriter def consume(self, remote_invoke_response: RemoteInvokeLogOutput) -> None: - self._stream_writer.write(remote_invoke_response.log_output.encode()) + self._stream_writer.write_bytes(remote_invoke_response.log_output.encode()) diff --git a/samcli/lib/docker/log_streamer.py b/samcli/lib/docker/log_streamer.py index b013459bae..3bb437781a 100644 --- a/samcli/lib/docker/log_streamer.py +++ b/samcli/lib/docker/log_streamer.py @@ -47,23 +47,21 @@ def stream_progress(self, logs: docker.APIClient.logs): else: curr_log_line_id = ids[_id] change_cursor_count = len(ids) - curr_log_line_id - self._stream.write( + self._stream.write_str( self._cursor_up_formatter.cursor_format(change_cursor_count) - + self._cursor_left_formatter.cursor_format(), - encode=True, + + self._cursor_left_formatter.cursor_format() ) self._stream_write(_id, status, stream, progress, error) if _id: - self._stream.write( + self._stream.write_str( self._cursor_down_formatter.cursor_format(change_cursor_count) - + self._cursor_left_formatter.cursor_format(), - encode=True, + + self._cursor_left_formatter.cursor_format() ) - self._stream.write(os.linesep, encode=True) + self._stream.write_str(os.linesep) - def _stream_write(self, _id: str, status: str, stream: bytes, progress: str, error: str): + def _stream_write(self, _id: str, status: str, stream: str, progress: str, error: str): """ Write stream information to stderr, if the stream information contains a log id, use the carriage return character to rewrite that particular line. @@ -80,14 +78,14 @@ def _stream_write(self, _id: str, status: str, stream: bytes, progress: str, err # NOTE(sriram-mv): Required for the purposes of when the cursor overflows existing terminal buffer. if not stream: - self._stream.write(os.linesep, encode=True) - self._stream.write( - self._cursor_up_formatter.cursor_format() + self._cursor_left_formatter.cursor_format(), encode=True + self._stream.write_str(os.linesep) + self._stream.write_str( + self._cursor_up_formatter.cursor_format() + self._cursor_left_formatter.cursor_format() ) - self._stream.write(self._cursor_clear_formatter.cursor_format(), encode=True) + self._stream.write_str(self._cursor_clear_formatter.cursor_format()) if not _id: - self._stream.write(stream, encode=True) - self._stream.write(status, encode=True) + self._stream.write_str(stream) + self._stream.write_str(status) else: - self._stream.write(f"\r{_id}: {status} {progress}", encode=True) + self._stream.write_str(f"\r{_id}: {status} {progress}") diff --git a/samcli/lib/package/ecr_uploader.py b/samcli/lib/package/ecr_uploader.py index f2d4371407..0393596b39 100644 --- a/samcli/lib/package/ecr_uploader.py +++ b/samcli/lib/package/ecr_uploader.py @@ -2,8 +2,8 @@ Client for uploading packaged artifacts to ecr """ import base64 -import io import logging +from io import StringIO from typing import Dict import botocore @@ -94,7 +94,7 @@ def upload(self, image, resource_name): else: # we need to wait till the image got pushed to ecr, without this workaround sam sync for template # contains image always fail, because the provided ecr uri is not exist. - _log_streamer = LogStreamer(stream=StreamWriter(stream=io.BytesIO(), auto_flush=True)) + _log_streamer = LogStreamer(stream=StreamWriter(stream=StringIO(), auto_flush=True)) _log_streamer.stream_progress(push_logs) except (BuildError, APIError, LogStreamError) as ex: diff --git a/samcli/lib/package/s3_uploader.py b/samcli/lib/package/s3_uploader.py index fe141ada51..95981e92ed 100644 --- a/samcli/lib/package/s3_uploader.py +++ b/samcli/lib/package/s3_uploader.py @@ -265,4 +265,4 @@ def on_progress(self, bytes_transferred, **kwargs): ) sys.stderr.flush() if int(percentage) == 100: # noqa: PLR2004 - sys.stderr.write("\n") + sys.stderr.write(os.linesep) diff --git a/samcli/lib/utils/osutils.py b/samcli/lib/utils/osutils.py index d53dc9ffb5..a9a12bf88c 100644 --- a/samcli/lib/utils/osutils.py +++ b/samcli/lib/utils/osutils.py @@ -1,6 +1,7 @@ """ Common OS utilities """ +import io import logging import os import shutil @@ -78,7 +79,7 @@ def rmtree_if_exists(path: Union[str, Path]): shutil.rmtree(path_obj) -def stdout(): +def stdout() -> io.TextIOWrapper: """ Returns the stdout as a byte stream in a Py2/PY3 compatible manner @@ -87,10 +88,15 @@ def stdout(): io.BytesIO Byte stream of Stdout """ - return sys.stdout.buffer + # ensure stdout is utf8 + sys.stdout.reconfigure(encoding="utf-8") # type:ignore[attr-defined] + # Note(jfuss): sys.stdout is a type typing.TextIO but are initialized to + # io.TextIOWrapper. To make mypy and typing play well, tell mypy to ignore. + return sys.stdout # type:ignore[return-value] -def stderr(): + +def stderr() -> io.TextIOWrapper: """ Returns the stderr as a byte stream in a Py2/PY3 compatible manner @@ -99,7 +105,12 @@ def stderr(): io.BytesIO Byte stream of stderr """ - return sys.stderr.buffer + # ensure stderr is utf8 + sys.stderr.reconfigure(encoding="utf-8") # type:ignore[attr-defined] + + # Note(jfuss): sys.stderr is a type typing.TextIO but are initialized to + # io.TextIOWrapper. To make mypy and typing play well, tell mypy to ignore. + return sys.stderr # type:ignore[return-value] def remove(path): diff --git a/samcli/lib/utils/stream_writer.py b/samcli/lib/utils/stream_writer.py index 1fc62fa690..99f72c1036 100644 --- a/samcli/lib/utils/stream_writer.py +++ b/samcli/lib/utils/stream_writer.py @@ -1,10 +1,11 @@ """ This class acts like a wrapper around output streams to provide any flexibility with output we need """ +from typing import TextIO, Union class StreamWriter: - def __init__(self, stream, auto_flush=False): + def __init__(self, stream: TextIO, auto_flush: bool = False): """ Instatiates new StreamWriter to the specified stream @@ -19,19 +20,33 @@ def __init__(self, stream, auto_flush=False): self._auto_flush = auto_flush @property - def stream(self): + def stream(self) -> TextIO: return self._stream - def write(self, output, encode=False): + def write_bytes(self, output: Union[bytes, bytearray]): """ Writes specified text to the underlying stream Parameters ---------- output bytes-like object - Bytes to write + Bytes to write into buffer """ - self._stream.write(output.encode() if encode else output) + self._stream.buffer.write(output) + + if self._auto_flush: + self._stream.flush() + + def write_str(self, output: str): + """ + Writes specified text to the underlying stream + + Parameters + ---------- + output string object + String to write + """ + self._stream.write(output) if self._auto_flush: self._stream.flush() diff --git a/samcli/lib/utils/subprocess_utils.py b/samcli/lib/utils/subprocess_utils.py index e08ec12e49..1937a44eeb 100644 --- a/samcli/lib/utils/subprocess_utils.py +++ b/samcli/lib/utils/subprocess_utils.py @@ -34,7 +34,7 @@ def default_loading_pattern(stream_writer: Optional[StreamWriter] = None, loadin How frequently to generate the pattern """ stream_writer = stream_writer or StreamWriter(sys.stderr) - stream_writer.write(".") + stream_writer.write_str(".") stream_writer.flush() sleep(loading_pattern_rate) @@ -96,7 +96,7 @@ def _print_loading_pattern(): return_code = process.wait() keep_printing = False - stream_writer.write(os.linesep) + stream_writer.write_str(os.linesep) stream_writer.flush() process_stderr = _check_and_convert_stream_to_string(process.stderr) diff --git a/samcli/local/apigw/local_apigw_service.py b/samcli/local/apigw/local_apigw_service.py index f979b2e9a3..b80b1fc2c2 100644 --- a/samcli/local/apigw/local_apigw_service.py +++ b/samcli/local/apigw/local_apigw_service.py @@ -4,7 +4,7 @@ import json import logging from datetime import datetime -from io import BytesIO +from io import StringIO from time import time from typing import Any, Dict, List, Optional @@ -605,7 +605,7 @@ def _invoke_lambda_function(self, lambda_function_name: str, event: dict) -> str str A string containing the output from the Lambda function """ - with BytesIO() as stdout: + with StringIO() as stdout: event_str = json.dumps(event, sort_keys=True) stdout_writer = StreamWriter(stdout, auto_flush=True) diff --git a/samcli/local/docker/container.py b/samcli/local/docker/container.py index fc2a190b53..d4c7c93fef 100644 --- a/samcli/local/docker/container.py +++ b/samcli/local/docker/container.py @@ -1,6 +1,8 @@ """ Representation of a generic Docker container """ +import io +import json import logging import os import pathlib @@ -9,7 +11,7 @@ import tempfile import threading import time -from typing import Optional +from typing import Iterator, Optional, Tuple, Union import docker import requests @@ -17,6 +19,7 @@ from samcli.lib.constants import DOCKER_MIN_API_VERSION from samcli.lib.utils.retry import retry +from samcli.lib.utils.stream_writer import StreamWriter from samcli.lib.utils.tar import extract_tarfile from samcli.local.docker.effective_user import ROOT_USER_ID, EffectiveUser from samcli.local.docker.exceptions import ContainerNotStartableException, PortAlreadyInUse @@ -318,7 +321,7 @@ def start(self, input_data=None): raise ex @retry(exc=requests.exceptions.RequestException, exc_raise=ContainerResponseException) - def wait_for_http_response(self, name, event, stdout): + def wait_for_http_response(self, name, event, stdout) -> str: # TODO(sriram-mv): `aws-lambda-rie` is in a mode where the function_name is always "function" # NOTE(sriram-mv): There is a connection timeout set on the http call to `aws-lambda-rie`, however there is not # a read time out for the response received from the server. @@ -328,7 +331,7 @@ def wait_for_http_response(self, name, event, stdout): data=event.encode("utf-8"), timeout=(self.RAPID_CONNECTION_TIMEOUT, None), ) - stdout.write(resp.content) + return json.dumps(json.loads(resp.content), ensure_ascii=False) def wait_for_result(self, full_path, event, stdout, stderr, start_timer=None): # NOTE(sriram-mv): Let logging happen in its own thread, so that a http request can be sent. @@ -348,11 +351,21 @@ def wait_for_result(self, full_path, event, stdout, stderr, start_timer=None): # start the timer for function timeout right before executing the function, as waiting for the socket # can take some time timer = start_timer() if start_timer else None - self.wait_for_http_response(full_path, event, stdout) + response = self.wait_for_http_response(full_path, event, stdout) if timer: timer.cancel() - def wait_for_logs(self, stdout=None, stderr=None): + # NOTE(jfuss): Adding a sleep after we get a response from the contianer but before we + # we write the response to ensure the last thing written to stdout is the container response + time.sleep(1) + stdout.write_str(response) + stdout.flush() + + def wait_for_logs( + self, + stdout: Optional[Union[StreamWriter, io.BytesIO, io.TextIOWrapper]] = None, + stderr: Optional[Union[StreamWriter, io.BytesIO, io.TextIOWrapper]] = None, + ): # Return instantly if we don't have to fetch any logs if not stdout and not stderr: return @@ -364,7 +377,6 @@ def wait_for_logs(self, stdout=None, stderr=None): # Fetch both stdout and stderr streams from Docker as a single iterator. logs_itr = real_container.attach(stream=True, logs=True, demux=True) - self._write_container_output(logs_itr, stdout=stdout, stderr=stderr) def _wait_for_socket_connection(self) -> None: @@ -415,7 +427,11 @@ def copy(self, from_container_path, to_host_path) -> None: extract_tarfile(file_obj=fp, unpack_dir=to_host_path) @staticmethod - def _write_container_output(output_itr, stdout=None, stderr=None): + def _write_container_output( + output_itr: Iterator[Tuple[bytes, bytes]], + stdout: Optional[Union[StreamWriter, io.BytesIO, io.TextIOWrapper]] = None, + stderr: Optional[Union[StreamWriter, io.BytesIO, io.TextIOWrapper]] = None, + ): """ Based on the data returned from the Container output, via the iterator, write it to the appropriate streams @@ -434,13 +450,26 @@ def _write_container_output(output_itr, stdout=None, stderr=None): # Iterator returns a tuple of (stdout, stderr) for stdout_data, stderr_data in output_itr: if stdout_data and stdout: - stdout.write(stdout_data) + Container._handle_data_writing(stdout, stdout_data) if stderr_data and stderr: - stderr.write(stderr_data) + Container._handle_data_writing(stderr, stderr_data) + except Exception as ex: LOG.debug("Failed to get the logs from the container", exc_info=ex) + @staticmethod + def _handle_data_writing(output_stream: Union[StreamWriter, io.BytesIO, io.TextIOWrapper], output_data: bytes): + if isinstance(output_stream, StreamWriter): + output_stream.write_bytes(output_data) + output_stream.flush() + + if isinstance(output_stream, io.BytesIO): + output_stream.write(output_data) + + if isinstance(output_stream, io.TextIOWrapper): + output_stream.buffer.write(output_data) + @property def network_id(self): """ diff --git a/samcli/local/docker/lambda_image.py b/samcli/local/docker/lambda_image.py index 923b740edc..72ad813de9 100644 --- a/samcli/local/docker/lambda_image.py +++ b/samcli/local/docker/lambda_image.py @@ -3,6 +3,7 @@ """ import hashlib import logging +import os import platform import re import sys @@ -227,7 +228,7 @@ def build(self, runtime, packagetype, image, layers, architecture, stream=None, or not runtime ): stream_writer = stream or StreamWriter(sys.stderr) - stream_writer.write("Building image...") + stream_writer.write_str("Building image...") stream_writer.flush() self._build_image( image if image else base_image, rapid_image, downloaded_layers, architecture, stream=stream_writer @@ -338,15 +339,15 @@ def set_item_permission(tar_info): platform=get_docker_platform(architecture), ) for log in resp_stream: - stream_writer.write(".") + stream_writer.write_str(".") stream_writer.flush() if "error" in log: - stream_writer.write("\n") + stream_writer.write_str(os.linesep) LOG.exception("Failed to build Docker Image") raise ImageBuildException("Error building docker image: {}".format(log["error"])) - stream_writer.write("\n") + stream_writer.write_str(os.linesep) except (docker.errors.BuildError, docker.errors.APIError) as ex: - stream_writer.write("\n") + stream_writer.write_str(os.linesep) LOG.exception("Failed to build Docker Image") raise ImageBuildException("Building Image failed.") from ex finally: diff --git a/samcli/local/docker/manager.py b/samcli/local/docker/manager.py index a035003bb0..6975828cd1 100644 --- a/samcli/local/docker/manager.py +++ b/samcli/local/docker/manager.py @@ -168,16 +168,16 @@ def pull_image(self, image_name, tag=None, stream=None): raise DockerImagePullFailedException(str(ex)) from ex # io streams, especially StringIO, work only with unicode strings - stream_writer.write("\nFetching {}:{} Docker container image...".format(image_name, tag)) + stream_writer.write_str("\nFetching {}:{} Docker container image...".format(image_name, tag)) # Each line contains information on progress of the pull. Each line is a JSON string for _ in result_itr: # For every line, print a dot to show progress - stream_writer.write(".") + stream_writer.write_str(".") stream_writer.flush() # We are done. Go to the next line - stream_writer.write("\n") + stream_writer.write_str("\n") def has_image(self, image_name): """ diff --git a/samcli/local/lambda_service/local_lambda_invoke_service.py b/samcli/local/lambda_service/local_lambda_invoke_service.py index c6d7506fb2..546066449c 100644 --- a/samcli/local/lambda_service/local_lambda_invoke_service.py +++ b/samcli/local/lambda_service/local_lambda_invoke_service.py @@ -162,7 +162,7 @@ def _invoke_request_handler(self, function_name): request_data = request_data.decode("utf-8") - stdout_stream = io.BytesIO() + stdout_stream = io.StringIO() stdout_stream_writer = StreamWriter(stdout_stream, auto_flush=True) try: diff --git a/samcli/local/services/base_local_service.py b/samcli/local/services/base_local_service.py index fcb7cd95ae..671d48888c 100644 --- a/samcli/local/services/base_local_service.py +++ b/samcli/local/services/base_local_service.py @@ -82,7 +82,7 @@ def service_response(body, headers, status_code): class LambdaOutputParser: @staticmethod - def get_lambda_output(stdout_stream: io.BytesIO) -> Tuple[str, bool]: + def get_lambda_output(stdout_stream: io.StringIO) -> Tuple[str, bool]: """ This method will extract read the given stream and return the response from Lambda function separated out from any log statements it might have outputted. Logs end up in the stdout stream if the Lambda function @@ -100,7 +100,7 @@ def get_lambda_output(stdout_stream: io.BytesIO) -> Tuple[str, bool]: bool If the response is an error/exception from the container """ - lambda_response = stdout_stream.getvalue().decode("utf-8") + lambda_response = stdout_stream.getvalue() # When the Lambda Function returns an Error/Exception, the output is added to the stdout of the container. From # our perspective, the container returned some value, which is not always true. Since the output is the only diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py index 3604fc4010..70711459d6 100644 --- a/tests/integration/local/invoke/test_integrations_cli.py +++ b/tests/integration/local/invoke/test_integrations_cli.py @@ -291,6 +291,27 @@ def test_invoke_returns_expected_result_when_no_event_given(self): self.assertEqual(process.returncode, 0) self.assertEqual("{}", process_stdout.decode("utf-8")) + @pytest.mark.flaky(reruns=3) + def test_invoke_returns_utf8(self): + command_list = InvokeIntegBase.get_command_list( + "EchoEventFunction", template_path=self.template_path, event_path=self.event_utf8_path + ) + + process = Popen(command_list, stdout=PIPE) + try: + stdout, _ = process.communicate(timeout=TIMEOUT) + except TimeoutExpired: + process.kill() + raise + + process_stdout = stdout.strip() + + with open(self.event_utf8_path) as f: + expected_output = json.dumps(json.load(f), ensure_ascii=False) + + self.assertEqual(process.returncode, 0) + self.assertEqual(expected_output, process_stdout.decode("utf-8")) + @pytest.mark.flaky(reruns=3) def test_invoke_with_env_using_parameters(self): command_list = InvokeIntegBase.get_command_list( diff --git a/tests/unit/commands/local/cli_common/test_invoke_context.py b/tests/unit/commands/local/cli_common/test_invoke_context.py index 3cab08c82a..b89d5b6115 100644 --- a/tests/unit/commands/local/cli_common/test_invoke_context.py +++ b/tests/unit/commands/local/cli_common/test_invoke_context.py @@ -1106,7 +1106,7 @@ def test_must_open_file_for_writing(self): with patch("samcli.commands.local.cli_common.invoke_context.open", m): InvokeContext._setup_log_file(filename) - m.assert_called_with(filename, "wb") + m.assert_called_with(filename, "w", encoding="utf8") class TestInvokeContext_get_debug_context(TestCase): diff --git a/tests/unit/lib/utils/test_osutils.py b/tests/unit/lib/utils/test_osutils.py index bf4794f2c4..6f7a6cf4df 100644 --- a/tests/unit/lib/utils/test_osutils.py +++ b/tests/unit/lib/utils/test_osutils.py @@ -34,9 +34,7 @@ def test_raises_on_cleanup_failure(self, rmdir_mock): @patch("os.rmdir") def test_handles_ignore_error_case(self, rmdir_mock): rmdir_mock.side_effect = OSError("fail") - dir_name = None with osutils.mkdir_temp(ignore_errors=True) as tempdir: - dir_name = tempdir self.assertTrue(os.path.exists(tempdir)) @@ -44,9 +42,6 @@ class Test_stderr(TestCase): def test_must_return_sys_stderr(self): expected_stderr = sys.stderr - if sys.version_info.major > 2: - expected_stderr = sys.stderr.buffer - self.assertEqual(expected_stderr, osutils.stderr()) @@ -54,9 +49,6 @@ class Test_stdout(TestCase): def test_must_return_sys_stdout(self): expected_stdout = sys.stdout - if sys.version_info.major > 2: - expected_stdout = sys.stdout.buffer - self.assertEqual(expected_stdout, osutils.stdout()) diff --git a/tests/unit/lib/utils/test_stream_writer.py b/tests/unit/lib/utils/test_stream_writer.py index cb48955850..a6875b59da 100644 --- a/tests/unit/lib/utils/test_stream_writer.py +++ b/tests/unit/lib/utils/test_stream_writer.py @@ -1,6 +1,7 @@ """ Tests for StreamWriter """ +import io from unittest import TestCase @@ -11,13 +12,13 @@ class TestStreamWriter(TestCase): def test_must_write_to_stream(self): - buffer = "something" + buffer = b"something" stream_mock = Mock() writer = StreamWriter(stream_mock) - writer.write(buffer) + writer.write_bytes(buffer) - stream_mock.write.assert_called_once_with(buffer) + stream_mock.buffer.write.assert_called_once_with(buffer) def test_must_flush_underlying_stream(self): stream_mock = Mock() @@ -31,7 +32,7 @@ def test_auto_flush_must_be_off_by_default(self): stream_mock = Mock() writer = StreamWriter(stream_mock) - writer.write("something") + writer.write_str("something") stream_mock.flush.assert_not_called() @@ -46,6 +47,6 @@ def test_when_auto_flush_on_flush_after_each_write(self): writer = StreamWriter(stream_mock, True) for line in lines: - writer.write(line) + writer.write_str(line) flush_mock.assert_called_once_with() flush_mock.reset_mock() diff --git a/tests/unit/lib/utils/test_subprocess_utils.py b/tests/unit/lib/utils/test_subprocess_utils.py index 969f06085b..a9d39afdd2 100644 --- a/tests/unit/lib/utils/test_subprocess_utils.py +++ b/tests/unit/lib/utils/test_subprocess_utils.py @@ -11,6 +11,7 @@ from parameterized import parameterized from unittest.mock import patch, Mock, call, ANY +from samcli.lib.utils.stream_writer import StreamWriter from samcli.lib.utils.subprocess_utils import ( default_loading_pattern, invoke_subprocess_with_loading_pattern, @@ -64,7 +65,7 @@ def test_loader_stream_uses_passed_in_stdout( @patch("samcli.lib.utils.subprocess_utils.Popen") def test_loader_raises_exception_non_zero_exit_code(self, patched_Popen): standard_error = "an error has occurred" - mock_stream_writer = Mock() + mock_stream_writer = Mock(spec=StreamWriter) mock_process = Mock() mock_process.returncode = 1 mock_process.stdout = None @@ -74,7 +75,7 @@ def test_loader_raises_exception_non_zero_exit_code(self, patched_Popen): with self.assertRaises(LoadingPatternError) as ex: invoke_subprocess_with_loading_pattern({"args": ["ls"]}, mock_pattern, mock_stream_writer) self.assertIn(standard_error, ex.exception.message) - mock_stream_writer.write.assert_called_once_with(os.linesep) + mock_stream_writer.write_str.assert_called_once_with(os.linesep) mock_stream_writer.flush.assert_called_once_with() @patch("samcli.lib.utils.subprocess_utils.Popen") @@ -95,19 +96,19 @@ def test_loader_raises_exception_bad_process(self, patched_Popen): @patch("samcli.lib.utils.subprocess_utils.StreamWriter") def test_default_pattern_default_stream_writer(self, patched_stream_writer): - stream_writer_mock = Mock() + stream_writer_mock = Mock(spec=StreamWriter) patched_stream_writer.return_value = stream_writer_mock default_loading_pattern(loading_pattern_rate=0.01) patched_stream_writer.assert_called_once_with(sys.stderr) - stream_writer_mock.write.assert_called_once_with(".") + stream_writer_mock.write_str.assert_called_once_with(".") stream_writer_mock.flush.assert_called_once_with() @patch("samcli.lib.utils.subprocess_utils.StreamWriter") def test_default_pattern(self, patched_stream_writer): - stream_writer_mock = Mock() + stream_writer_mock = Mock(spec=StreamWriter) default_loading_pattern(stream_writer_mock, 0.01) patched_stream_writer.assert_not_called() - stream_writer_mock.write.assert_called_once_with(".") + stream_writer_mock.write_str.assert_called_once_with(".") stream_writer_mock.flush.assert_called_once_with() @parameterized.expand([("hello".encode("utf-8"), "hello"), ("hello", "hello")]) diff --git a/tests/unit/local/docker/test_container.py b/tests/unit/local/docker/test_container.py index b7b7311563..064bb845cb 100644 --- a/tests/unit/local/docker/test_container.py +++ b/tests/unit/local/docker/test_container.py @@ -9,6 +9,7 @@ from requests import RequestException from samcli.lib.utils.packagetype import IMAGE +from samcli.lib.utils.stream_writer import StreamWriter from samcli.local.docker.container import ( Container, ContainerResponseException, @@ -721,17 +722,17 @@ def test_wait_for_result_waits_for_socket_before_post_request(self, patched_time self.assertEqual(mock_requests.post.call_count, 0) def test_write_container_output_successful(self): - stdout_mock = Mock() - stderr_mock = Mock() + stdout_mock = Mock(spec=StreamWriter) + stderr_mock = Mock(spec=StreamWriter) def _output_iterator(): - yield "Hello", None - yield None, "World" + yield b"Hello", None + yield None, b"World" raise ValueError("The pipe has been ended.") Container._write_container_output(_output_iterator(), stdout_mock, stderr_mock) - stdout_mock.assert_has_calls([call.write("Hello")]) - stderr_mock.assert_has_calls([call.write("World")]) + stdout_mock.assert_has_calls([call.write_bytes(b"Hello")]) + stderr_mock.assert_has_calls([call.write_bytes(b"World")]) class TestContainer_wait_for_logs(TestCase): @@ -785,33 +786,33 @@ class TestContainer_write_container_output(TestCase): def setUp(self): self.output_itr = [(b"stdout1", None), (None, b"stderr1"), (b"stdout2", b"stderr2"), (None, None)] - self.stdout_mock = Mock() - self.stderr_mock = Mock() + self.stdout_mock = Mock(spec=StreamWriter) + self.stderr_mock = Mock(spec=StreamWriter) def test_must_write_stdout_and_stderr_data(self): # All the invalid frames must be ignored Container._write_container_output(self.output_itr, stdout=self.stdout_mock, stderr=self.stderr_mock) - self.stdout_mock.write.assert_has_calls([call(b"stdout1"), call(b"stdout2")]) + self.stdout_mock.write_bytes.assert_has_calls([call(b"stdout1"), call(b"stdout2")]) - self.stderr_mock.write.assert_has_calls([call(b"stderr1"), call(b"stderr2")]) + self.stderr_mock.write_bytes.assert_has_calls([call(b"stderr1"), call(b"stderr2")]) def test_must_write_only_stderr(self): # All the invalid frames must be ignored Container._write_container_output(self.output_itr, stdout=None, stderr=self.stderr_mock) - self.stdout_mock.write.assert_not_called() + self.stdout_mock.write_bytes.assert_not_called() - self.stderr_mock.write.assert_has_calls([call(b"stderr1"), call(b"stderr2")]) + self.stderr_mock.write_bytes.assert_has_calls([call(b"stderr1"), call(b"stderr2")]) def test_must_write_only_stdout(self): Container._write_container_output(self.output_itr, stdout=self.stdout_mock, stderr=None) - self.stdout_mock.write.assert_has_calls([call(b"stdout1"), call(b"stdout2")]) + self.stdout_mock.write_bytes.assert_has_calls([call(b"stdout1"), call(b"stdout2")]) - self.stderr_mock.write.assert_not_called() # stderr must never be called + self.stderr_mock.write_bytes.assert_not_called() # stderr must never be called class TestContainer_wait_for_socket_connection(TestCase): diff --git a/tests/unit/local/docker/test_lambda_image.py b/tests/unit/local/docker/test_lambda_image.py index 1e8f936d98..03b57be804 100644 --- a/tests/unit/local/docker/test_lambda_image.py +++ b/tests/unit/local/docker/test_lambda_image.py @@ -1,4 +1,3 @@ -import io import tempfile from unittest import TestCase @@ -271,7 +270,7 @@ def test_force_building_image_that_doesnt_already_exists( docker_client_mock.images.get.side_effect = ImageNotFound("image not found") docker_client_mock.images.list.return_value = [] - stream = io.StringIO() + stream = Mock() lambda_image = LambdaImage(layer_downloader_mock, False, True, docker_client=docker_client_mock) actual_image_id = lambda_image.build( @@ -311,7 +310,7 @@ def test_force_building_image_on_daemon_404( docker_client_mock.images.get.side_effect = NotFound("image not found") docker_client_mock.images.list.return_value = [] - stream = io.StringIO() + stream = Mock() lambda_image = LambdaImage(layer_downloader_mock, False, True, docker_client=docker_client_mock) actual_image_id = lambda_image.build( @@ -351,7 +350,7 @@ def test_docker_distribution_api_error_on_daemon_api_error( docker_client_mock.images.get.side_effect = APIError("error from docker daemon") docker_client_mock.images.list.return_value = [] - stream = io.StringIO() + stream = Mock() lambda_image = LambdaImage(layer_downloader_mock, False, True, docker_client=docker_client_mock) with self.assertRaises(DockerDistributionAPIError): @@ -377,7 +376,7 @@ def test_not_force_building_image_that_doesnt_already_exists( docker_client_mock.images.get.side_effect = ImageNotFound("image not found") docker_client_mock.images.list.return_value = [] - stream = io.StringIO() + stream = Mock() lambda_image = LambdaImage(layer_downloader_mock, False, False, docker_client=docker_client_mock) actual_image_id = lambda_image.build( diff --git a/tests/unit/local/docker/test_manager.py b/tests/unit/local/docker/test_manager.py index ada69903ea..4cb42bbd02 100644 --- a/tests/unit/local/docker/test_manager.py +++ b/tests/unit/local/docker/test_manager.py @@ -1,8 +1,6 @@ """ Tests container manager """ - -import io import importlib from unittest import TestCase from unittest.mock import Mock, patch, MagicMock, ANY, call @@ -218,17 +216,29 @@ def setUp(self): self.manager = ContainerManager(docker_client=self.mock_docker_client) def test_must_pull_and_print_progress_dots(self): - stream = io.StringIO() + stream = Mock() pull_result = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] self.mock_docker_client.api.pull.return_value = pull_result - expected_stream_output = "\nFetching {}:latest Docker container image...{}\n".format( - self.image_name, "." * len(pull_result) # Progress bar will print one dot per response from pull API - ) + expected_stream_calls = [ + call(f"\nFetching {self.image_name}:latest Docker container image..."), + call("."), + call("."), + call("."), + call("."), + call("."), + call("."), + call("."), + call("."), + call("."), + call("."), + call("\n"), + ] self.manager.pull_image(self.image_name, stream=stream) self.mock_docker_client.api.pull.assert_called_with(self.image_name, stream=True, decode=True, tag="latest") - self.assertEqual(stream.getvalue(), expected_stream_output) + + stream.write_str.assert_has_calls(expected_stream_calls) def test_must_raise_if_image_not_found(self): msg = "some error" diff --git a/tests/unit/local/services/test_base_local_service.py b/tests/unit/local/services/test_base_local_service.py index fec13e25c9..34bc44c193 100644 --- a/tests/unit/local/services/test_base_local_service.py +++ b/tests/unit/local/services/test_base_local_service.py @@ -66,17 +66,17 @@ def test_create_returns_not_implemented(self): class TestLambdaOutputParser(TestCase): @parameterized.expand( [ - param("with mixed data and json response", b'data\n{"a": "b"}', 'data\n{"a": "b"}'), - param("with response as string", b"response", "response"), - param("with json response only", b'{"a": "b"}', '{"a": "b"}'), - param("with one new line and json", b'\n{"a": "b"}', '\n{"a": "b"}'), - param("with response only as string", b"this is the response line", "this is the response line"), - param("with whitespaces", b'data\n{"a": "b"} \n\n\n', 'data\n{"a": "b"} \n\n\n'), - param("with empty data", b"", ""), - param("with just new lines", b"\n\n", "\n\n"), + param("with mixed data and json response", 'data\n{"a": "b"}', 'data\n{"a": "b"}'), + param("with response as string", "response", "response"), + param("with json response only", '{"a": "b"}', '{"a": "b"}'), + param("with one new line and json", '\n{"a": "b"}', '\n{"a": "b"}'), + param("with response only as string", "this is the response line", "this is the response line"), + param("with whitespaces", 'data\n{"a": "b"} \n\n\n', 'data\n{"a": "b"} \n\n\n'), + param("with empty data", "", ""), + param("with just new lines", "\n\n", "\n\n"), param( "with whitespaces", - b"\n \n \n", + "\n \n \n", "\n \n \n", ), ] From 27eb2e1e35da226236a2ea2eba2c8d6c8dbb31e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 13:42:30 +0300 Subject: [PATCH 15/32] chore(deps): bump cryptography from 41.0.2 to 41.0.3 in /requirements (#5675) Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.2 to 41.0.3. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.2...41.0.3) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/reproducible-linux.txt | 48 ++++++++++++++--------------- requirements/reproducible-mac.txt | 48 ++++++++++++++--------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/requirements/reproducible-linux.txt b/requirements/reproducible-linux.txt index 354b363df0..821e88a926 100644 --- a/requirements/reproducible-linux.txt +++ b/requirements/reproducible-linux.txt @@ -219,30 +219,30 @@ cookiecutter==2.1.1 \ --hash=sha256:9f3ab027cec4f70916e28f03470bdb41e637a3ad354b4d65c765d93aad160022 \ --hash=sha256:f3982be8d9c53dac1261864013fdec7f83afd2e42ede6f6dd069c5e149c540d5 # via aws-sam-cli (setup.py) -cryptography==41.0.2 \ - --hash=sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711 \ - --hash=sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7 \ - --hash=sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd \ - --hash=sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e \ - --hash=sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58 \ - --hash=sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0 \ - --hash=sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d \ - --hash=sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83 \ - --hash=sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831 \ - --hash=sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766 \ - --hash=sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b \ - --hash=sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c \ - --hash=sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182 \ - --hash=sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f \ - --hash=sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa \ - --hash=sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4 \ - --hash=sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a \ - --hash=sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2 \ - --hash=sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76 \ - --hash=sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5 \ - --hash=sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee \ - --hash=sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f \ - --hash=sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14 +cryptography==41.0.3 \ + --hash=sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306 \ + --hash=sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84 \ + --hash=sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47 \ + --hash=sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d \ + --hash=sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116 \ + --hash=sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207 \ + --hash=sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81 \ + --hash=sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087 \ + --hash=sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd \ + --hash=sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507 \ + --hash=sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858 \ + --hash=sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae \ + --hash=sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34 \ + --hash=sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906 \ + --hash=sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd \ + --hash=sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922 \ + --hash=sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7 \ + --hash=sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4 \ + --hash=sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574 \ + --hash=sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1 \ + --hash=sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c \ + --hash=sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e \ + --hash=sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de # via pyopenssl dateparser==1.1.8 \ --hash=sha256:070b29b5bbf4b1ec2cd51c96ea040dc68a614de703910a91ad1abba18f9f379f \ diff --git a/requirements/reproducible-mac.txt b/requirements/reproducible-mac.txt index 08062fd84e..718796b407 100644 --- a/requirements/reproducible-mac.txt +++ b/requirements/reproducible-mac.txt @@ -237,30 +237,30 @@ cookiecutter==2.1.1 \ --hash=sha256:9f3ab027cec4f70916e28f03470bdb41e637a3ad354b4d65c765d93aad160022 \ --hash=sha256:f3982be8d9c53dac1261864013fdec7f83afd2e42ede6f6dd069c5e149c540d5 # via aws-sam-cli (setup.py) -cryptography==41.0.2 \ - --hash=sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711 \ - --hash=sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7 \ - --hash=sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd \ - --hash=sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e \ - --hash=sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58 \ - --hash=sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0 \ - --hash=sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d \ - --hash=sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83 \ - --hash=sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831 \ - --hash=sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766 \ - --hash=sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b \ - --hash=sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c \ - --hash=sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182 \ - --hash=sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f \ - --hash=sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa \ - --hash=sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4 \ - --hash=sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a \ - --hash=sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2 \ - --hash=sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76 \ - --hash=sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5 \ - --hash=sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee \ - --hash=sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f \ - --hash=sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14 +cryptography==41.0.3 \ + --hash=sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306 \ + --hash=sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84 \ + --hash=sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47 \ + --hash=sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d \ + --hash=sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116 \ + --hash=sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207 \ + --hash=sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81 \ + --hash=sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087 \ + --hash=sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd \ + --hash=sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507 \ + --hash=sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858 \ + --hash=sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae \ + --hash=sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34 \ + --hash=sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906 \ + --hash=sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd \ + --hash=sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922 \ + --hash=sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7 \ + --hash=sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4 \ + --hash=sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574 \ + --hash=sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1 \ + --hash=sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c \ + --hash=sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e \ + --hash=sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de # via pyopenssl dateparser==1.1.8 \ --hash=sha256:070b29b5bbf4b1ec2cd51c96ea040dc68a614de703910a91ad1abba18f9f379f \ From e7413aec659ca4ad173ddf3755c7e7de14df7938 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 09:41:20 -0700 Subject: [PATCH 16/32] Add linux config --- appveyor-linux-binary.yml | 296 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 appveyor-linux-binary.yml diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml new file mode 100644 index 0000000000..46d09bf4db --- /dev/null +++ b/appveyor-linux-binary.yml @@ -0,0 +1,296 @@ +version: 1.0.{build} +image: + - Ubuntu2004 + +configuration: + - BuildIntegTesting + - LocalZipTerraformBuildIntegTesting + - LocalZipTerraformBuildInContainerIntegTesting + - S3ZipTerraformBuildIntegTesting + - S3ZipTerraformBuildInContainerIntegTesting + - OtherTerraformBuildIntegTesting + - DeployIntegTesting + - PackageIntegTesting + - DeleteIntegTesting + - SyncIntegTesting + - LocalIntegTesting + - EndToEndTesting + # other Integration testing, Dev, regression and smoke testing + - OtherTesting + +environment: + AWS_DEFAULT_REGION: us-east-1 + SAM_CLI_DEV: 1 + NODE_VERSION: "14.17.6" + AWS_S3: 'AWS_S3_TESTING' + AWS_ECR: 'AWS_ECR_TESTING' + CARGO_LAMBDA_VERSION: "v0.17.1" + PYTHON_ARCH: '64' + NOSE_PARAMETERIZED_NO_WARN: 1 + APPVEYOR_CONSOLE_DISABLE_PTY: false + APPVEYOR_DETAILED_SHELL_LOGGING: true + + matrix: + + - PYTHON_HOME: "$HOME/venv3.7/bin" + PYTHON_VERSION: '3.7' + + - PYTHON_HOME: "$HOME/venv3.8/bin" + PYTHON_VERSION: '3.8' + + - PYTHON_HOME: "$HOME/venv3.11/bin" + PYTHON_VERSION: '3.11' + +install: + # AppVeyor's apt-get cache might be outdated, and the package could potentially be 404. + - sh: "sudo apt-get update --allow-releaseinfo-change" + + - sh: "gvm use go1.19" + - sh: "echo $PATH" + - sh: "ls /usr/" + # install latest maven which is compatible with jdk17 + - sh: "sudo apt-get -y remove maven" + - sh: "wget https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.zip -P /tmp" + - sh: "sudo unzip -d /opt/mvn /tmp/apache-maven-*.zip" + - sh: "PATH=/opt/mvn/apache-maven-3.8.8/bin:$PATH" + - sh: "JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64" + - sh: "PATH=$JAVA_HOME/bin:$PATH" + - sh: "javac -version" + - sh: "java -version" + - sh: "mvn --version" + + - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate" + - sh: "rvm use 2.7.2" + - sh: "docker info" + - sh: "docker version" + - sh: "nvm install ${NODE_VERSION}" + - sh: "npm install npm@7.24.2 -g" + - sh: "npm -v" + + # Install latest gradle + - sh: "sudo apt-get -y remove gradle" + - sh: "wget https://services.gradle.org/distributions/gradle-7.3.1-bin.zip -P /tmp" + - sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip" + - sh: "PATH=/opt/gradle/gradle-7.3.1/bin:$PATH" + + # Install AWS CLI + - sh: "virtualenv aws_cli" + - sh: "./aws_cli/bin/python -m pip install awscli" + - sh: "PATH=$(echo $PWD'/aws_cli/bin'):$PATH" + + - sh: "PATH=$PATH:$HOME/venv3.7/bin:$HOME/venv3.8/bin:$HOME/venv3.9/bin:$HOME/venv3.10/bin" + + # update ca-certificates which causes failures with newest golang library + - sh: "sudo apt-get install --reinstall ca-certificates" + + # get testing env vars + - sh: "sudo apt install -y jq" + + # install Terraform + - sh: "sudo apt update --allow-releaseinfo-change" + - sh: "TER_VER=`curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | cut -d: -f2 | tr -d \\\"\\,\\v | awk '{$1=$1};1'`" + - sh: "wget https://releases.hashicorp.com/terraform/${TER_VER}/terraform_${TER_VER}_linux_amd64.zip -P /tmp" + - sh: "sudo unzip -d /opt/terraform /tmp/terraform_${TER_VER}_linux_amd64.zip" + - sh: "sudo mv /opt/terraform/terraform /usr/local/bin/" + - sh: "terraform -version" + + # install Rust + - sh: "curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL https://sh.rustup.rs | sh -s -- --default-toolchain none -y > /dev/null 2>&1" + - sh: "source $HOME/.cargo/env" + - sh: "rustup toolchain install stable --profile minimal --no-self-update" + - sh: "rustup default stable" + - sh: "rustup target add x86_64-unknown-linux-gnu --toolchain stable" + - sh: "rustup target add aarch64-unknown-linux-gnu --toolchain stable" + - sh: "pip install cargo-lambda==$CARGO_LAMBDA_VERSION" + - sh: "rustc -V" + - sh: "cargo -V" + - sh: "cargo lambda -V" + + - sh: "python3.9 -m venv .venv_env_vars" + - sh: ".venv_env_vars/bin/pip install boto3" + - sh: "test_env_var=$(.venv_env_vars/bin/python tests/get_testing_resources.py)" + - sh: ' + if [ $? -ne 0 ]; then + echo "get_testing_resources failed. Failed to acquire credentials or test resources."; + false; + fi + ' + + - sh: 'export CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID' + - sh: 'export CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY' + - sh: 'export CI_ACCESS_ROLE_AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN' + + - sh: 'export AWS_ACCESS_KEY_ID=$(echo "$test_env_var" | jq -j ".accessKeyID")' + - sh: 'export AWS_SECRET_ACCESS_KEY=$(echo "$test_env_var" | jq -j ".secretAccessKey")' + - sh: 'export AWS_SESSION_TOKEN=$(echo "$test_env_var" | jq -j ".sessionToken")' + - sh: 'export TASK_TOKEN=$(echo "$test_env_var" | jq -j ".taskToken")' + - sh: 'export AWS_S3_TESTING=$(echo "$test_env_var" | jq -j ".TestBucketName")' + - sh: 'export AWS_ECR_TESTING=$(echo "$test_env_var" | jq -j ".TestECRURI")' + - sh: 'export AWS_KMS_KEY=$(echo "$test_env_var" | jq -j ".TestKMSKeyArn")' + - sh: 'export AWS_SIGNING_PROFILE_NAME=$(echo "$test_env_var" | jq -j ".TestSigningProfileName")' + - sh: 'export AWS_SIGNING_PROFILE_VERSION_ARN=$(echo "$test_env_var" | jq -j ".TestSigningProfileARN")' + + # required for RIE with arm64 in linux + - sh: " + if [[ -n $BY_CANARY ]] && [[ -n $DOCKER_USER ]] && [[ -n $DOCKER_PASS ]]; + then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin registry-1.docker.io; + fi" + - sh: " + if [[ -n $BY_CANARY ]] && [[ -n $DOCKER_USER ]] && [[ -n $DOCKER_PASS ]]; + then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin; + fi" + - sh: "docker run --rm --privileged multiarch/qemu-user-static --reset -p yes" + + + # Runs only in Linux, logging Public ECR when running canary and cred is available + - sh: " + if [[ -n $BY_CANARY ]]; + then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; + fi" + + +build_script: + - curl -L https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/aws-sam-cli-linux-x86_64.zip -o aws-sam-cli-linux-x86_64.zip + - unzip aws-sam-cli-linux-x86_64.zip -d sam-installation + - ./sam-installation/install + - sam --version + +# Final clean up no matter success or failure +on_finish: + # Upload test reports as artifacts + - sh: find "$APPVEYOR_BUILD_FOLDER" -type f -name 'TEST_REPORT-*.json' -print0 | xargs -0 -I '{}' appveyor PushArtifact '{}' + - sh: > + AWS_ACCESS_KEY_ID=$TEST_REPORT_S3_BUCKET_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY=$TEST_REPORT_S3_BUCKET_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN=$TEST_REPORT_S3_BUCKET_SESSION_TOKEN + aws s3 cp "$APPVEYOR_BUILD_FOLDER" "s3://$TEST_REPORT_S3_BUCKET_NAME/appveyor/$APPVEYOR_PROJECT_SLUG/$APPVEYOR_BUILD_ID/$APPVEYOR_JOB_ID/" --recursive --exclude "*" --include "TEST_REPORT-*.json" --region us-west-2 + + # notify task success + - sh: 'export AWS_ACCESS_KEY_ID=$CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID' + - sh: 'export AWS_SECRET_ACCESS_KEY=$CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY' + - sh: 'export AWS_SESSION_TOKEN=$CI_ACCESS_ROLE_AWS_SESSION_TOKEN' + + - sh: 'aws stepfunctions send-task-success --task-token "$TASK_TOKEN" --task-output "{}" --region us-west-2' + +for: + # Integ testing build + - + matrix: + only: + - configuration: BuildIntegTesting + + test_script: + - sh: "pytest -vv tests/integration/buildcmd --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # Set JAVA_HOME to java11 + - sh: "JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" + - sh: "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k test_building_java11_in_process --json-report --json-report-file=TEST_REPORT-integration-buildcmd-java11.json" + + # Local ZIP Terraform Build integ testing + - + matrix: + only: + - configuration: LocalZipTerraformBuildIntegTesting + + test_script: + - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # Local ZIP Terraform Build In Container integ testing + - + matrix: + only: + - configuration: LocalZipTerraformBuildInContainerIntegTesting + + test_script: + - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # S3 ZIP Terraform Build integ testing + - + matrix: + only: + - configuration: S3ZipTerraformBuildIntegTesting + + test_script: + - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # S3 ZIP Terraform Build In Container integ testing + - + matrix: + only: + - configuration: S3ZipTerraformBuildInContainerIntegTesting + + test_script: + - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # Other Terraform Build In Container integ testing + - + matrix: + only: + - configuration: OtherTerraformBuildIntegTesting + + test_script: + - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json" + + # Integ testing deploy + - + matrix: + only: + - configuration: DeployIntegTesting + + test_script: + - sh: "pytest -vv tests/integration/deploy -n 4 --reruns 4 --dist=loadgroup --json-report --json-report-file=TEST_REPORT-integration-deploy.json" + + # Integ testing package + - + matrix: + only: + - configuration: PackageIntegTesting + + test_script: + - sh: "pytest -vv tests/integration/package -n 4 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-package.json" + + # Integ testing delete + - + matrix: + only: + - configuration: DeleteIntegTesting + + test_script: + - sh: "pytest -vv tests/integration/delete -n 4 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-delete.json" + + # Integ testing sync + - + matrix: + only: + - configuration: SyncIntegTesting + + test_script: + - sh: "pytest -vv tests/integration/sync -n 3 --reruns 3 --dist loadscope --json-report --json-report-file=TEST_REPORT-integration-sync.json" + + # Integ testing local + - + matrix: + only: + - configuration: LocalIntegTesting + + test_script: + - sh: "pytest -vv tests/integration/local --json-report --json-report-file=TEST_REPORT-integration-local.json" + + # End-to-end testing + - + matrix: + only: + - configuration: EndToEndTesting + + test_script: + - sh: "pytest -vv -n 4 --reruns 5 --dist loadscope tests/end_to_end --json-report --json-report-file=TEST_REPORT-end-to-end.json" + + # Other testing + - + matrix: + only: + - configuration: OtherTesting + + test_script: + - sh: "pytest -vv -n 4 --reruns 4 --dist loadgroup tests/integration --ignore=tests/integration/buildcmd --ignore=tests/integration/delete --ignore=tests/integration/deploy --ignore=tests/integration/package --ignore=tests/integration/sync --ignore=tests/integration/local --json-report --json-report-file=TEST_REPORT-integration-others.json" + - sh: "pytest -vv tests/regression --json-report --json-report-file=TEST_REPORT-regression.json" From 135da3a1c4facb0117fd5d4ef4aebc81a8a1696d Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Wed, 2 Aug 2023 13:02:48 -0500 Subject: [PATCH 17/32] fix(test): Force reading file with utf8 in tests for windows (#5679) Co-authored-by: Jacob Fuss --- tests/integration/local/invoke/test_integrations_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py index 70711459d6..be015469d0 100644 --- a/tests/integration/local/invoke/test_integrations_cli.py +++ b/tests/integration/local/invoke/test_integrations_cli.py @@ -306,7 +306,7 @@ def test_invoke_returns_utf8(self): process_stdout = stdout.strip() - with open(self.event_utf8_path) as f: + with open(self.event_utf8_path, encoding="utf8") as f: expected_output = json.dumps(json.load(f), ensure_ascii=False) self.assertEqual(process.returncode, 0) From b282cebb2c0cab4056dc8ed43def1515901c0bf6 Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Wed, 2 Aug 2023 13:02:58 -0500 Subject: [PATCH 18/32] fix(test): Increase max execution time for timeout tests (#5680) Co-authored-by: Jacob Fuss --- tests/integration/local/invoke/test_integrations_cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py index be015469d0..b295d81f6c 100644 --- a/tests/integration/local/invoke/test_integrations_cli.py +++ b/tests/integration/local/invoke/test_integrations_cli.py @@ -160,7 +160,9 @@ def test_invoke_with_timeout_set(self, function_name): # validate the time of the cli (timeout is set to 5s) self.assertGreater(wall_clock_cli_duration, 5) - self.assertLess(wall_clock_cli_duration, 20) + # validate the the duration is roughly under the timeout (with some additional + # time to take in account time for SAM CLI to do work) + self.assertLess(wall_clock_cli_duration, 25) self.assertEqual(process.returncode, 0) self.assertEqual( From 78fe5a2f12badf780b396a74d345b513750890c6 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 11:39:11 -0700 Subject: [PATCH 19/32] Update install links --- appveyor-linux-binary.yml | 9 +-------- appveyor-windows-binary.yml | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index 46d09bf4db..05dccfed14 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -31,13 +31,6 @@ environment: APPVEYOR_DETAILED_SHELL_LOGGING: true matrix: - - - PYTHON_HOME: "$HOME/venv3.7/bin" - PYTHON_VERSION: '3.7' - - - PYTHON_HOME: "$HOME/venv3.8/bin" - PYTHON_VERSION: '3.8' - - PYTHON_HOME: "$HOME/venv3.11/bin" PYTHON_VERSION: '3.11' @@ -150,7 +143,7 @@ install: build_script: - - curl -L https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/aws-sam-cli-linux-x86_64.zip -o aws-sam-cli-linux-x86_64.zip + - curl -L https://github.com/aws/aws-sam-cli/releases/download/sam-cli-nightly/aws-sam-cli-linux-x86_64.zip -o aws-sam-cli-linux-x86_64.zip - unzip aws-sam-cli-linux-x86_64.zip -d sam-installation - ./sam-installation/install - sam --version diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index 79375ec4e6..ffd0c59519 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -127,7 +127,7 @@ install: # Install aws-sam-cli MSI binary - ps: " - Invoke-WebRequest -Uri https://github.com/aws/aws-sam-cli/releases/downloads/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile aws-sam-cli-installer.msi + Invoke-WebRequest -Uri https://github.com/aws/aws-sam-cli/releases/download/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile aws-sam-cli-installer.msi $process = Start-Process 'C:\\Windows\\System32\\msiexec.exe' -ArgumentList '/i $HOME\aws-sam-cli-installer.msi /qn /norestart' -wait -PassThru $process Set-Alias -Name sam -Value 'C:\\Program Files\\Amazon\\AWSSAMCLI_NIGHTLY\\bin\\sam-nightly.cmd' From 44d4bfa9649866b570a3a1daf1b6b64868992232 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 13:59:00 -0700 Subject: [PATCH 20/32] Update linux config --- appveyor-linux-binary.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index 05dccfed14..f2ad9b76b3 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -20,7 +20,6 @@ configuration: environment: AWS_DEFAULT_REGION: us-east-1 - SAM_CLI_DEV: 1 NODE_VERSION: "14.17.6" AWS_S3: 'AWS_S3_TESTING' AWS_ECR: 'AWS_ECR_TESTING' @@ -145,8 +144,9 @@ install: build_script: - curl -L https://github.com/aws/aws-sam-cli/releases/download/sam-cli-nightly/aws-sam-cli-linux-x86_64.zip -o aws-sam-cli-linux-x86_64.zip - unzip aws-sam-cli-linux-x86_64.zip -d sam-installation - - ./sam-installation/install - - sam --version + - sudo ./sam-installation/install + - sam-nightly --version + - alias sam='sam-nightly' # Final clean up no matter success or failure on_finish: From b22459d406422604e9970683583bc3ce9f94f6c9 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 14:21:08 -0700 Subject: [PATCH 21/32] Update linux config --- appveyor-linux-binary.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index f2ad9b76b3..1deca2768a 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -70,6 +70,11 @@ install: - sh: "./aws_cli/bin/python -m pip install awscli" - sh: "PATH=$(echo $PWD'/aws_cli/bin'):$PATH" + # Install pytest + - sh: "virtualenv pytest" + - sh: "./pytest/bin/python -m pip install pytest" + - sh: "PATH=$(echo $PWD'/pytest/bin'):$PATH" + - sh: "PATH=$PATH:$HOME/venv3.7/bin:$HOME/venv3.8/bin:$HOME/venv3.9/bin:$HOME/venv3.10/bin" # update ca-certificates which causes failures with newest golang library @@ -147,6 +152,7 @@ build_script: - sudo ./sam-installation/install - sam-nightly --version - alias sam='sam-nightly' + - sam --version # Final clean up no matter success or failure on_finish: From 1ba36ac1ed84af8a4cef8d7991bb26a7b996c7e8 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 15:41:58 -0700 Subject: [PATCH 22/32] Update linux config --- appveyor-linux-binary.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index 1deca2768a..74af3486df 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -151,6 +151,7 @@ build_script: - unzip aws-sam-cli-linux-x86_64.zip -d sam-installation - sudo ./sam-installation/install - sam-nightly --version + - shopt -s expand_aliases - alias sam='sam-nightly' - sam --version From 54972abe7eec96da527d42ba9839571e01d55f52 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 15:53:30 -0700 Subject: [PATCH 23/32] Update linux config --- appveyor-linux-binary.yml | 3 --- tests/testing_utils.py | 6 +++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index 74af3486df..c7f9c7b8b9 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -151,9 +151,6 @@ build_script: - unzip aws-sam-cli-linux-x86_64.zip -d sam-installation - sudo ./sam-installation/install - sam-nightly --version - - shopt -s expand_aliases - - alias sam='sam-nightly' - - sam --version # Final clean up no matter success or failure on_finish: diff --git a/tests/testing_utils.py b/tests/testing_utils.py index a52898f90b..25681c5b33 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -44,7 +44,11 @@ def get_sam_command(): - return "samdev" if os.getenv("SAM_CLI_DEV") else "sam" + if os.getenv("SAM_CLI_NIGHTLY"): + return "sam-nightly" + if os.getenv("SAM_CLI_DEV"): + return "samdev" + return "sam" def method_to_stack_name(method_name): From 66a85e6b2cba296c05da9ea037457075e267f82f Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 16:11:38 -0700 Subject: [PATCH 24/32] Update linux config --- appveyor-linux-binary.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index c7f9c7b8b9..e9ce1ab584 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -71,11 +71,10 @@ install: - sh: "PATH=$(echo $PWD'/aws_cli/bin'):$PATH" # Install pytest - - sh: "virtualenv pytest" - - sh: "./pytest/bin/python -m pip install pytest" - - sh: "PATH=$(echo $PWD'/pytest/bin'):$PATH" + - sh: "$HOME/venv3.11/bin/python -m pip install pytest" - - sh: "PATH=$PATH:$HOME/venv3.7/bin:$HOME/venv3.8/bin:$HOME/venv3.9/bin:$HOME/venv3.10/bin" + - sh: "PATH=$PATH:$HOME/venv3.7/bin:$HOME/venv3.8/bin:$HOME/venv3.9/bin:$HOME/venv3.10/bin:$HOME/venv3.11/bin" + - sh: "pytest --version" # update ca-certificates which causes failures with newest golang library - sh: "sudo apt-get install --reinstall ca-certificates" From fad6c46c0081e2c0e2b7d46ca1e3fc07f7845085 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 16:20:29 -0700 Subject: [PATCH 25/32] Update linux config --- appveyor-linux-binary.yml | 6 ++++-- tests/testing_utils.py | 6 +----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index e9ce1ab584..85bee8aab5 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -71,10 +71,10 @@ install: - sh: "PATH=$(echo $PWD'/aws_cli/bin'):$PATH" # Install pytest - - sh: "$HOME/venv3.11/bin/python -m pip install pytest" + - sh: "sudo apt-get install python3-pytest -y" + - sh: "pytest --version" - sh: "PATH=$PATH:$HOME/venv3.7/bin:$HOME/venv3.8/bin:$HOME/venv3.9/bin:$HOME/venv3.10/bin:$HOME/venv3.11/bin" - - sh: "pytest --version" # update ca-certificates which causes failures with newest golang library - sh: "sudo apt-get install --reinstall ca-certificates" @@ -150,6 +150,8 @@ build_script: - unzip aws-sam-cli-linux-x86_64.zip -d sam-installation - sudo ./sam-installation/install - sam-nightly --version + - mv /usr/local/bin/sam-nightly /usr/local/bin/sam + - sam --version # Final clean up no matter success or failure on_finish: diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 25681c5b33..a52898f90b 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -44,11 +44,7 @@ def get_sam_command(): - if os.getenv("SAM_CLI_NIGHTLY"): - return "sam-nightly" - if os.getenv("SAM_CLI_DEV"): - return "samdev" - return "sam" + return "samdev" if os.getenv("SAM_CLI_DEV") else "sam" def method_to_stack_name(method_name): From 748c100da2f6f11c49843b14e082a209806a8fdf Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 16:27:36 -0700 Subject: [PATCH 26/32] Update linux config --- appveyor-linux-binary.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index 85bee8aab5..7e587920e1 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -71,10 +71,10 @@ install: - sh: "PATH=$(echo $PWD'/aws_cli/bin'):$PATH" # Install pytest - - sh: "sudo apt-get install python3-pytest -y" + - sh: "pip3 install pytest" - sh: "pytest --version" - - sh: "PATH=$PATH:$HOME/venv3.7/bin:$HOME/venv3.8/bin:$HOME/venv3.9/bin:$HOME/venv3.10/bin:$HOME/venv3.11/bin" + - sh: "PATH=$PATH:$HOME/venv3.7/bin:$HOME/venv3.8/bin:$HOME/venv3.9/bin:$HOME/venv3.10/bin" # update ca-certificates which causes failures with newest golang library - sh: "sudo apt-get install --reinstall ca-certificates" From 125f695a272784e74870dbcdecfb42922a077979 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 16:31:21 -0700 Subject: [PATCH 27/32] Update linux config --- appveyor-linux-binary.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index 7e587920e1..b64bc19f8f 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -72,6 +72,7 @@ install: # Install pytest - sh: "pip3 install pytest" + - sh: "pip3 install boto3" - sh: "pytest --version" - sh: "PATH=$PATH:$HOME/venv3.7/bin:$HOME/venv3.8/bin:$HOME/venv3.9/bin:$HOME/venv3.10/bin" From 6d093b2b187ca8f211eb1eb34a9c8f58da0d945d Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 16:37:55 -0700 Subject: [PATCH 28/32] Update linux config --- appveyor-linux-binary.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index b64bc19f8f..39eaafe3ee 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -72,6 +72,7 @@ install: # Install pytest - sh: "pip3 install pytest" + # We currently need to install boto3 for pytest to work since we're ignoring a boto3 deprecation warning in the pytest configuration - sh: "pip3 install boto3" - sh: "pytest --version" @@ -150,8 +151,7 @@ build_script: - curl -L https://github.com/aws/aws-sam-cli/releases/download/sam-cli-nightly/aws-sam-cli-linux-x86_64.zip -o aws-sam-cli-linux-x86_64.zip - unzip aws-sam-cli-linux-x86_64.zip -d sam-installation - sudo ./sam-installation/install - - sam-nightly --version - - mv /usr/local/bin/sam-nightly /usr/local/bin/sam + - sudo mv /usr/local/bin/sam-nightly /usr/local/bin/sam - sam --version # Final clean up no matter success or failure From 64ce0d860acfa6ba012b8b6d0925f3b1cca96383 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 16:48:03 -0700 Subject: [PATCH 29/32] Update linux config --- appveyor-linux-binary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index 39eaafe3ee..dc23ebe124 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -71,7 +71,7 @@ install: - sh: "PATH=$(echo $PWD'/aws_cli/bin'):$PATH" # Install pytest - - sh: "pip3 install pytest" + - sh: "pip3 install -r requirements/dev.txt" # We currently need to install boto3 for pytest to work since we're ignoring a boto3 deprecation warning in the pytest configuration - sh: "pip3 install boto3" - sh: "pytest --version" From abe74fb9f1db6ed5dedf39a9ee11e768fc29fb41 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 2 Aug 2023 16:57:19 -0700 Subject: [PATCH 30/32] Update linux config --- appveyor-linux-binary.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor-linux-binary.yml b/appveyor-linux-binary.yml index dc23ebe124..f4a69d548f 100644 --- a/appveyor-linux-binary.yml +++ b/appveyor-linux-binary.yml @@ -71,9 +71,9 @@ install: - sh: "PATH=$(echo $PWD'/aws_cli/bin'):$PATH" # Install pytest + - sh: "pip3 install -r requirements/pre-dev.txt" - sh: "pip3 install -r requirements/dev.txt" - # We currently need to install boto3 for pytest to work since we're ignoring a boto3 deprecation warning in the pytest configuration - - sh: "pip3 install boto3" + - sh: "pip3 install -r requirements/base.txt" - sh: "pytest --version" - sh: "PATH=$PATH:$HOME/venv3.7/bin:$HOME/venv3.8/bin:$HOME/venv3.9/bin:$HOME/venv3.10/bin" From 841b2f49583f6e1be33ebce3a2d42d913c64c00e Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Thu, 3 Aug 2023 10:43:28 -0700 Subject: [PATCH 31/32] Update Windows config --- appveyor-windows-binary.yml | 19 ++++++++++--------- .../telemetry/test_installed_metric.py | 5 +++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index ffd0c59519..032e80d314 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -114,7 +114,7 @@ install: $env:TASK_TOKEN = $test_env_var_json.taskToken; $env:AWS_S3_TESTING = $test_env_var_json.TestBucketName; $env:AWS_ECR_TESTING = $test_env_var_json.TestECRURI; - $env:AWS_KMS_KEY = $test_env_var_json.TestKMSKeyArn;en + $env:AWS_KMS_KEY = $test_env_var_json.TestKMSKeyArn; $env:AWS_SIGNING_PROFILE_NAME = $test_env_var_json.TestSigningProfileName; $env:AWS_SIGNING_PROFILE_VERSION_ARN = $test_env_var_json.TestSigningProfileARN; }" @@ -125,6 +125,15 @@ install: - "venv\\Scripts\\activate" - "python --version" + # Install testing pre-reqs without installing sam-cli from source + - pip install -r requirements/pre-dev.txt" + - pip install -r requirements/dev.txt" + - pip install -r requirements/base.txt + - pytest --version + + # Install aws cli + - "pip install awscli" + # Install aws-sam-cli MSI binary - ps: " Invoke-WebRequest -Uri https://github.com/aws/aws-sam-cli/releases/download/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile aws-sam-cli-installer.msi @@ -134,11 +143,6 @@ install: sam --version " - - "pip install pytest" - - # Install aws cli - - "pip install awscli" - # Echo final Path - "echo %PATH%" @@ -153,9 +157,6 @@ install: # claim some disk space before starting the tests - "docker system prune -a -f" - # activate virtual environment - - "venv\\Scripts\\activate" - # Final clean up no matter success or failure diff --git a/tests/integration/telemetry/test_installed_metric.py b/tests/integration/telemetry/test_installed_metric.py index 725a2d0ff5..7e8853e28d 100644 --- a/tests/integration/telemetry/test_installed_metric.py +++ b/tests/integration/telemetry/test_installed_metric.py @@ -113,6 +113,11 @@ def filter_installed_metric_requests(all_requests): for r in all_requests: data = r["data"] if "metrics" in data and data["metrics"] and "installed" in data["metrics"][0]: + # If it's a nightly release version, it will have a suffix. + # We can strip it for the purpose of testing telemetry + version = data["metrics"][0].get("installed", {}).get("samcliVersion", "") + if version: + data["metrics"][0]["installed"]["samcliVersion"] = version[: len(SAM_CLI_VERSION)] result.append(r) return result From a3472d5530cdd2e7ced012897948b2654f2071e4 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Thu, 3 Aug 2023 12:07:15 -0700 Subject: [PATCH 32/32] Update Windows config --- appveyor-windows-binary.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/appveyor-windows-binary.yml b/appveyor-windows-binary.yml index 032e80d314..918eae2394 100644 --- a/appveyor-windows-binary.yml +++ b/appveyor-windows-binary.yml @@ -136,11 +136,11 @@ install: # Install aws-sam-cli MSI binary - ps: " - Invoke-WebRequest -Uri https://github.com/aws/aws-sam-cli/releases/download/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile aws-sam-cli-installer.msi - $process = Start-Process 'C:\\Windows\\System32\\msiexec.exe' -ArgumentList '/i $HOME\aws-sam-cli-installer.msi /qn /norestart' -wait -PassThru - $process - Set-Alias -Name sam -Value 'C:\\Program Files\\Amazon\\AWSSAMCLI_NIGHTLY\\bin\\sam-nightly.cmd' - sam --version + Invoke-WebRequest -Uri https://github.com/aws/aws-sam-cli/releases/download/sam-cli-nightly/AWS_SAM_CLI_64_PY3.msi -OutFile aws-sam-cli-installer.msi; + $process = Start-Process 'C:\\Windows\\System32\\msiexec.exe' -ArgumentList '/i $HOME\aws-sam-cli-installer.msi /qn /norestart' -wait -PassThru; + $process; + Set-Alias -Name sam -Value 'C:\\Program Files\\Amazon\\AWSSAMCLI_NIGHTLY\\bin\\sam-nightly.cmd'; + sam --version; " # Echo final Path