Add Python Version Input Option for EC2 (#272)

*Issue description:* Updating Python EC2 to have input option for language version. *Description of changes:* - Update Python sample app dependency versions to be compatible with Python 3.8 - Python 3.8, 3.10, 3.12 isn't available through dnf/yum. Therefore, manually build and install python by downloading source code. - Setting default version to 3.9 since downloading through dnf is faster than manually installing it. (It was also the version already being used by ec2 tests) *Rollback procedure:* Revert Test run: https://github.com/harrryr/aws-application-signals-test-framework/actions/runs/11157966893 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
aws-observability · Oct 3, 2024 · 4abcd08 · 4abcd08
1 parent 1d56ff9
commit 4abcd08
Show file tree

Hide file tree

Showing 9 changed files with 166 additions and 27 deletions.
diff --git a/.github/workflows/python-ec2-asg-test.yml b/.github/workflows/python-ec2-asg-test.yml
@@ -14,6 +14,11 @@ on:
       caller-workflow-name:
         required: true
         type: string
+      python-version:
+        description: "Currently support version 3.8, 3.9, 3.10, 3.11, 3.12"
+        required: false
+        type: string
+        default: '3.9'
       cpu-architecture:
         description: "Permitted values: x86_64 or arm64"
         required: false
@@ -36,6 +41,7 @@ permissions:
 env:
   E2E_TEST_AWS_REGION: ${{ inputs.aws-region }}
   CALLER_WORKFLOW_NAME: ${{ inputs.caller-workflow-name }}
+  PYTHON_VERSION: ${{ inputs.python-version }}
   CPU_ARCHITECTURE: ${{ inputs.cpu-architecture }}
   ADOT_WHEEL_NAME: ${{ inputs.staging-wheel-name }}
   SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}/python-sample-app.zip
@@ -102,9 +108,9 @@ jobs:
         run: |
           if [ "${{ github.event.repository.name }}" = "aws-otel-python-instrumentation" ]; then
             # Reusing the adot-main-build-staging-jar bucket to store the python wheel file
-            echo GET_ADOT_WHEEL_COMMAND="aws s3 cp s3://adot-main-build-staging-jar/${{ env.ADOT_WHEEL_NAME }} ./${{ env.ADOT_WHEEL_NAME }} && python3.9 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
+            echo GET_ADOT_WHEEL_COMMAND="aws s3 cp s3://adot-main-build-staging-jar/${{ env.ADOT_WHEEL_NAME }} ./${{ env.ADOT_WHEEL_NAME }} && sudo python${{ env.PYTHON_VERSION }} -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
           else
-            echo GET_ADOT_WHEEL_COMMAND="python3.9 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
+            echo GET_ADOT_WHEEL_COMMAND="sudo python${{ env.PYTHON_VERSION }} -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
           fi
 
       - name: Set Get CW Agent command environment variable
@@ -154,6 +160,7 @@ jobs:
               -var="sample_app_zip=${{ env.SAMPLE_APP_ZIP }}" \
               -var="get_cw_agent_rpm_command=${{ env.GET_CW_AGENT_RPM_COMMAND }}" \
               -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" \
+              -var="language_version=${{ env.PYTHON_VERSION }}" \
               -var="cpu_architecture=${{ env.CPU_ARCHITECTURE }}" \
             || deployment_failed=$?
           

diff --git a/.github/workflows/python-ec2-canary.yml b/.github/workflows/python-ec2-canary.yml
@@ -29,6 +29,7 @@ jobs:
     with:
       aws-region: ${{ matrix.aws-region }}
       caller-workflow-name: 'appsignals-python-e2e-ec2-canary-test'
+      python-version: '3.9'
       cpu-architecture: 'x86_64'
 
   pypi:
@@ -37,5 +38,6 @@ jobs:
     with:
       aws-region: 'us-east-1'
       caller-workflow-name: 'appsignals-python-e2e-ec2-pypi-canary-test'
+      python-version: '3.9'
       otel-source: 'pypi'
       cpu-architecture: 'x86_64'
diff --git a/.github/workflows/python-ec2-default-retry.yml b/.github/workflows/python-ec2-default-retry.yml
@@ -14,6 +14,9 @@ on:
       caller-workflow-name:
         required: true
         type: string
+      python-version:
+        required: true
+        type: string
       staging-wheel-name:
         required: false
         default: 'aws-opentelemetry-distro'
@@ -38,6 +41,7 @@ jobs:
     with:
       aws-region: ${{ inputs.aws-region }}
       caller-workflow-name: ${{ inputs.caller-workflow-name }}
+      python-version: ${{ inputs.python-version }}
       staging-wheel-name: ${{ inputs.staging-wheel-name }}
       otel-source: ${{ inputs.otel-source }}
       cpu-architecture: ${{ inputs.cpu-architecture }}
@@ -50,6 +54,7 @@ jobs:
     with:
       aws-region: ${{ inputs.aws-region }}
       caller-workflow-name: ${{ inputs.caller-workflow-name }}
+      python-version: ${{ inputs.python-version }}
       staging-wheel-name: ${{ inputs.staging-wheel-name }}
       otel-source: ${{ inputs.otel-source }}
       cpu-architecture: ${{ inputs.cpu-architecture }}

diff --git a/.github/workflows/python-ec2-default-test.yml b/.github/workflows/python-ec2-default-test.yml
@@ -14,6 +14,11 @@ on:
       caller-workflow-name:
         required: true
         type: string
+      python-version:
+        description: "Currently support version 3.8, 3.9, 3.10, 3.11, 3.12"
+        required: false
+        type: string
+        default: '3.9'
       cpu-architecture:
         description: "Permitted values: x86_64 or arm64"
         required: false
@@ -40,6 +45,7 @@ permissions:
 env:
   E2E_TEST_AWS_REGION: ${{ inputs.aws-region }}
   CALLER_WORKFLOW_NAME: ${{ inputs.caller-workflow-name }}
+  PYTHON_VERSION: ${{ inputs.python-version }}
   CPU_ARCHITECTURE: ${{ inputs.cpu-architecture }}
   ADOT_WHEEL_NAME: ${{ inputs.staging-wheel-name }}
   OTEL_SOURCE: ${{ inputs.otel-source }}
@@ -106,14 +112,14 @@ jobs:
         run: |
           if [ "${{ github.event.repository.name }}" = "aws-otel-python-instrumentation" ]; then
             # Reusing the adot-main-build-staging-jar bucket to store the python wheel file
-            echo GET_ADOT_WHEEL_COMMAND="aws s3 cp s3://adot-main-build-staging-jar/${{ env.ADOT_WHEEL_NAME }} ./${{ env.ADOT_WHEEL_NAME }} && python3.9 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
+            echo GET_ADOT_WHEEL_COMMAND="aws s3 cp s3://adot-main-build-staging-jar/${{ env.ADOT_WHEEL_NAME }} ./${{ env.ADOT_WHEEL_NAME }} && sudo python${{ env.PYTHON_VERSION }} -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
           elif [ "${{ env.OTEL_SOURCE }}" == "pypi" ]; then
-            echo GET_ADOT_WHEEL_COMMAND="python3.9 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
+            echo GET_ADOT_WHEEL_COMMAND="sudo python${{ env.PYTHON_VERSION }} -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
           else
             latest_release_version=$(curl -sL https://github.com/aws-observability/aws-otel-python-instrumentation/releases/latest | grep -oP '/releases/tag/v\K[0-9]+\.[0-9]+\.[0-9]+' | head -n 1)
             echo "The latest version is $latest_release_version"
             echo GET_ADOT_WHEEL_COMMAND="wget -O ${{ env.ADOT_WHEEL_NAME }} https://github.com/aws-observability/aws-otel-python-instrumentation/releases/latest/download/aws_opentelemetry_distro-$latest_release_version-py3-none-any.whl \
-            && python3.9 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
+            && sudo python${{ env.PYTHON_VERSION }} -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
           fi
 
       - name: Set Get CW Agent command environment variable
@@ -163,6 +169,7 @@ jobs:
               -var="sample_app_zip=${{ env.SAMPLE_APP_ZIP }}" \
               -var="get_cw_agent_rpm_command=${{ env.GET_CW_AGENT_RPM_COMMAND }}" \
               -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" \
+              -var="language_version=${{ env.PYTHON_VERSION }}" \
               -var="cpu_architecture=${{ env.CPU_ARCHITECTURE }}" \
             || deployment_failed=$?
           

diff --git a/sample-apps/python/django_frontend_service/ec2-requirements.txt b/sample-apps/python/django_frontend_service/ec2-requirements.txt
@@ -1,6 +1,8 @@
 Django~=4.2.9
-boto3~=1.34.3
+boto3~=1.34.161
 pymysql==1.1.1
 python-dotenv~=1.0.1
-requests~=2.31.0
+requests~=2.25.1
 schedule~=1.2.1
+opentelemetry-sdk==1.27.0
+opentelemetry-api==1.27.0
diff --git a/terraform/python/ec2/asg/main.tf b/terraform/python/ec2/asg/main.tf
@@ -98,8 +98,31 @@ resource "aws_launch_configuration" "launch_configuration" {
     # Install Python and wget
     sudo yum install wget -y
     sudo yum install unzip -y
-    sudo dnf install -y python3.9
-    sudo dnf install -y python3.9-pip
+
+    # Dnf does not have the module for python 3.10, 3,10, 3.12, therefore we need to manually install it by downloading the package from the python website.
+    # Building and installing the package takes longer then installing it through dnf, so a seperate installation process was made.
+    # The canary should run on a version without the manual installation process
+    if [ "${var.language_version}" == "3.8" ] || [ "${var.language_version}" = "3.10" ] || [ "${var.language_version}" = "3.12" ]; then
+        # Install modules required to compile Python and also run the sample app
+        sudo dnf groupinstall "Development Tools" -y
+        sudo dnf install openssl-devel sqlite-devel libffi-devel -y
+
+        # Download the Python package
+        cd /usr/src
+        sudo wget https://www.python.org/ftp/python/${var.language_version}.0/Python-${var.language_version}.0.tgz
+        sudo tar xzf Python-${var.language_version}.0.tgz
+
+        # Compile and install Python using c++
+        cd Python-${var.language_version}.0
+        sudo ./configure
+        sudo make install
+
+        # Return back to ec2-user directory
+        cd ~
+    else
+      sudo dnf install -y python${var.language_version}
+      sudo dnf install -y python${var.language_version}-pip
+    fi
 
     # Copy in CW Agent configuration
     agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
@@ -110,6 +133,10 @@ resource "aws_launch_configuration" "launch_configuration" {
     sudo rpm -U ./cw-agent.rpm
     sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:./amazon-cloudwatch-agent.json
 
+    # Install modules with specific version so that it doesn't cause errors with Python 3.8
+    sudo python${var.language_version} -m pip install importlib-metadata==8.4.0 "protobuf>=3.19,<5.0"
+    sudo python${var.language_version} -m pip install grpcio --only-binary=:all:
+
     # Get ADOT Wheel and install it
     ${var.get_adot_wheel_command}
 
@@ -122,7 +149,7 @@ resource "aws_launch_configuration" "launch_configuration" {
     # Delete the requests requirement as it is installed already using rpm. Only applicable for ec2 instances
     # created by Auto Scaling Groups
     sudo sed -i '/requests/d' ./ec2-requirements.txt
-    python3.9 -m pip install -r ec2-requirements.txt
+    sudo python${var.language_version} -m pip install -r ec2-requirements.txt
     export DJANGO_SETTINGS_MODULE="django_frontend_service.settings"
     export OTEL_PYTHON_DISTRO="aws_distro"
     export OTEL_PYTHON_CONFIGURATOR="aws_configurator"
@@ -135,8 +162,8 @@ resource "aws_launch_configuration" "launch_configuration" {
     export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=grpc
     export OTEL_SERVICE_NAME=python-sample-application-${var.test_id}
     export OTEL_TRACES_SAMPLER=always_on
-    python3.9 manage.py migrate
-    nohup opentelemetry-instrument python3.9 manage.py runserver 0.0.0.0:8000 --noreload &
+    python${var.language_version} manage.py migrate
+    nohup opentelemetry-instrument python${var.language_version} manage.py runserver 0.0.0.0:8000 --noreload &
 
     # The application needs time to come up and reach a steady state, this should not take longer than 30 seconds
     sleep 30
@@ -203,8 +230,31 @@ resource "null_resource" "remote_service_setup" {
       # Install Python and wget
       sudo yum install wget -y
       sudo yum install unzip -y
-      sudo dnf install -y python3.9
-      sudo dnf install -y python3.9-pip
+
+      # Dnf does not have the module for python 3.10, 3,10, 3.12, therefore we need to manually install it by downloading the package from the python website.
+      # Building and installing the package takes longer then installing it through dnf, so a seperate installation process was made.
+      # The canary should run on a version without the manual installation process
+      if [ "${var.language_version}" == "3.8" ] || [ "${var.language_version}" = "3.10" ] || [ "${var.language_version}" = "3.12" ]; then
+          # Install modules required to compile Python and also run the sample app
+          sudo dnf groupinstall "Development Tools" -y
+          sudo dnf install openssl-devel sqlite-devel libffi-devel -y
+
+          # Download the Python package
+          cd /usr/src
+          sudo wget https://www.python.org/ftp/python/${var.language_version}.0/Python-${var.language_version}.0.tgz
+          sudo tar xzf Python-${var.language_version}.0.tgz
+
+          # Compile and install Python using c++
+          cd Python-${var.language_version}.0
+          sudo ./configure
+          sudo make install
+
+          # Return back to ec2-user directory
+          cd ~
+      else
+        sudo dnf install -y python${var.language_version}
+        sudo dnf install -y python${var.language_version}-pip
+      fi
 
       # Copy in CW Agent configuration
       agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
@@ -215,6 +265,10 @@ resource "null_resource" "remote_service_setup" {
       sudo rpm -U ./cw-agent.rpm
       sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:./amazon-cloudwatch-agent.json
 
+      # Install modules with specific version so that it doesn't cause errors with Python 3.8
+      sudo python${var.language_version} -m pip install importlib-metadata==8.4.0 "protobuf>=3.19,<5.0"
+      sudo python${var.language_version} -m pip install grpcio --only-binary=:all:
+
       # Get ADOT Wheel and install it
       ${var.get_adot_wheel_command}
 
@@ -224,8 +278,8 @@ resource "null_resource" "remote_service_setup" {
 
       # Export environment variables for instrumentation
       cd ./django_remote_service
+      sudo python${var.language_version} -m pip install -r requirements.txt --force-reinstall
       export DJANGO_SETTINGS_MODULE="django_remote_service.settings"
-      python3.9 -m pip install -r requirements.txt --force-reinstall
       export OTEL_PYTHON_DISTRO="aws_distro"
       export OTEL_PYTHON_CONFIGURATOR="aws_configurator"
       export OTEL_METRICS_EXPORTER=none
@@ -237,8 +291,8 @@ resource "null_resource" "remote_service_setup" {
       export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=grpc
       export OTEL_SERVICE_NAME=python-sample-remote-application-${var.test_id}
       export OTEL_TRACES_SAMPLER=always_on
-      python3.9 manage.py migrate
-      nohup opentelemetry-instrument python3.9 manage.py runserver 0.0.0.0:8001 --noreload &
+      python${var.language_version} manage.py migrate
+      nohup opentelemetry-instrument python${var.language_version} manage.py runserver 0.0.0.0:8001 --noreload &
 
       # The application needs time to come up and reach a steady state, this should not take longer than 30 seconds
       sleep 30

diff --git a/terraform/python/ec2/asg/variables.tf b/terraform/python/ec2/asg/variables.tf
@@ -41,6 +41,10 @@ variable "canary_type" {
   default = "python-ec2-asg"
 }
 
+variable "language_version" {
+  default = "3.9"
+}
+
 variable "cpu_architecture" {
   default = "x86_64"
 }