Reapplies #115 with a fix (#124)

This PR reapplies [this
PR](#115),
which was reverted
[here](#123)
with a fix.

.github/workflows/python-eks-e2e-test.yml

@@ -138,6 +138,23 @@ jobs:
           --region ${{ env.E2E_TEST_AWS_REGION }}"
           sleep_time: 60
 
+      - name: Get RDS database cluster metadata
+        continue-on-error: true
+        run: |
+          RDS_MYSQL_CLUSTER_ENDPOINT=$(aws rds describe-db-cluster-endpoints --db-cluster-identifier "rdsaurorapythonclusterformysql" --region ${{inputs.aws-region}} --query "DBClusterEndpoints[?EndpointType=='WRITER'].Endpoint | [0]" --output text)
+          echo RDS_MYSQL_CLUSTER_ENDPOINT="$RDS_MYSQL_CLUSTER_ENDPOINT" >> $GITHUB_ENV
+          RDS_MYSQL_CLUSTER_IDENTIFIER=$(aws rds describe-db-cluster-endpoints --db-cluster-identifier "rdsaurorapythonclusterformysql" --region ${{inputs.aws-region}} --query "DBClusterEndpoints[?EndpointType=='WRITER'].DBClusterIdentifier | [0]" --output text)
+          RDS_MYSQL_CLUSTER_CREDENTIAL_SECRET_NAME=$(aws secretsmanager list-secrets --region ${{inputs.aws-region}} --query "SecretList[?Tags[?Value=='arn:aws:rds:${{inputs.aws-region}}:${{env.ACCOUNT_ID}}:cluster:$RDS_MYSQL_CLUSTER_IDENTIFIER']].Name | [0]" --output text)
+          echo RDS_MYSQL_CLUSTER_CREDENTIAL_SECRET_NAME="$RDS_MYSQL_CLUSTER_CREDENTIAL_SECRET_NAME" >> $GITHUB_ENV
+
+      - name: Get RDS database credentials from SecretsManager
+        continue-on-error: true
+        uses: aws-actions/aws-secretsmanager-get-secrets@v1
+        with:
+          secret-ids:
+            RDS_MYSQL_CLUSTER_SECRETS, ${{env.RDS_MYSQL_CLUSTER_CREDENTIAL_SECRET_NAME}}
+          parse-json-secrets: true
+
       - name: Initiate Terraform
         uses: ./.github/workflows/actions/execute_and_retry
         with:
@@ -165,15 +182,19 @@ jobs:
             echo "Attempt $retry_counter"
             deployment_failed=0
             terraform apply -auto-approve \
-              -var="test_id=${{ env.TESTING_ID }}" \
-              -var="aws_region=${{ env.E2E_TEST_AWS_REGION }}" \
-              -var="kube_directory_path=${{ github.workspace }}/.kube" \
-              -var="eks_cluster_name=${{ env.CLUSTER_NAME }}" \
+              -var='test_id=${{ env.TESTING_ID }}' \
+              -var='aws_region=${{ env.E2E_TEST_AWS_REGION }}' \
+              -var='kube_directory_path=${{ github.workspace }}/.kube' \
+              -var='eks_cluster_name=${{ env.CLUSTER_NAME }}' \
               -var="eks_cluster_context_name=$(kubectl config current-context)" \
-              -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \
-              -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \
-              -var="python_app_image=${{ env.MAIN_SAMPLE_APP_IMAGE_ARN }}" \
-              -var="python_remote_app_image=${{ env.REMOTE_SAMPLE_APP_IMAGE_ARN }}" \
+              -var='test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}' \
+              -var='service_account_aws_access=service-account-${{ env.TESTING_ID }}' \
+              -var='python_app_image=${{ env.MAIN_SAMPLE_APP_IMAGE_ARN }}' \
+              -var='python_remote_app_image=${{ env.REMOTE_SAMPLE_APP_IMAGE_ARN }}' \
+              -var='rds_mysql_cluster_endpoint=${{env.RDS_MYSQL_CLUSTER_ENDPOINT}}' \
+              -var='rds_mysql_cluster_username=${{env.RDS_MYSQL_CLUSTER_SECRETS_USERNAME}}' \
+              -var='rds_mysql_cluster_password=${{env.RDS_MYSQL_CLUSTER_SECRETS_PASSWORD}}' \
+              -var='rds_mysql_cluster_database=information_schema' \
             || deployment_failed=$?
           
             if [ $deployment_failed -eq 1 ]; then
@@ -248,15 +269,15 @@ jobs:
 
               echo "Destroying terraform"
               terraform destroy -auto-approve \
-                -var="test_id=${{ env.TESTING_ID }}" \
-                -var="aws_region=${{ env.E2E_TEST_AWS_REGION }}" \
-                -var="kube_directory_path=${{ github.workspace }}/.kube" \
-                -var="eks_cluster_name=${{ env.CLUSTER_NAME }}" \
+                -var='test_id=${{ env.TESTING_ID }}' \
+                -var='aws_region=${{ env.E2E_TEST_AWS_REGION }}' \
+                -var='kube_directory_path=${{ github.workspace }}/.kube' \
+                -var='eks_cluster_name=${{ env.CLUSTER_NAME }}' \
                 -var="eks_cluster_context_name=$(kubectl config current-context)" \
-                -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \
-                -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \
-                -var="python_app_image=${{ env.MAIN_SAMPLE_APP_IMAGE_ARN }}" \
-                -var="python_remote_app_image=${{ env.REMOTE_SAMPLE_APP_IMAGE_ARN }}"
+                -var='test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}' \
+                -var='service_account_aws_access=service-account-${{ env.TESTING_ID }}' \
+                -var='python_app_image=${{ env.MAIN_SAMPLE_APP_IMAGE_ARN }}' \
+                -var='python_remote_app_image=${{ env.REMOTE_SAMPLE_APP_IMAGE_ARN }}'
           
               retry_counter=$(($retry_counter+1))
             else
@@ -312,6 +333,7 @@ jobs:
           curl -S -s "http://${{ env.APP_ENDPOINT }}/aws-sdk-call?ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}"; echo
           curl -S -s "http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}"; echo
           curl -S -s "http://${{ env.APP_ENDPOINT }}/client-call"; echo
+          curl -S -s "http://${{ env.APP_ENDPOINT }}/mysql"; echo
 
       - name: Initiate Gradlew Daemon
         if: steps.initiate-gradlew == 'failure'
@@ -418,14 +440,14 @@ jobs:
         working-directory: terraform/python/eks
         run: |
           terraform destroy -auto-approve \
-            -var="test_id=${{ env.TESTING_ID }}" \
-            -var="aws_region=${{ env.E2E_TEST_AWS_REGION }}" \
-            -var="kube_directory_path=${{ github.workspace }}/.kube" \
-            -var="eks_cluster_name=${{ env.CLUSTER_NAME }}" \
-            -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \
-            -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \
-            -var="python_app_image=${{ env.MAIN_SAMPLE_APP_IMAGE_ARN }}" \
-            -var="python_remote_app_image=${{ env.REMOTE_SAMPLE_APP_IMAGE_ARN }}"
+            -var='test_id=${{ env.TESTING_ID }}' \
+            -var='aws_region=${{ env.E2E_TEST_AWS_REGION }}' \
+            -var='kube_directory_path=${{ github.workspace }}/.kube' \
+            -var='eks_cluster_name=${{ env.CLUSTER_NAME }}' \
+            -var='test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}' \
+            -var='service_account_aws_access=service-account-${{ env.TESTING_ID }}' \
+            -var='python_app_image=${{ env.MAIN_SAMPLE_APP_IMAGE_ARN }}' \
+            -var='python_remote_app_image=${{ env.REMOTE_SAMPLE_APP_IMAGE_ARN }}'
 
       - name: Remove aws access service account
         if: always()

sample-apps/python/README.md

@@ -35,3 +35,4 @@ The following are the APIs supported:
 2. http://${ FRONTEND_SERVICE_IP }:8000/aws-sdk-call/
 3. http://${ FRONTEND_SERVICE_IP }:8000/remote-service?ip=${ REMOTE_SERVICE_IP }/
 4. http://${ FRONTEND_SERVICE_IP }:8000/client-call/
+5. http://${ FRONTEND_SERVICE_IP }:8000/mysql/

sample-apps/python/django_frontend_service/django_frontend_service/urls.py

@@ -9,4 +9,5 @@
     path('outgoing-http-call', views.http_call),
     path('remote-service', views.downstream_service),
     path('client-call', views.async_service),
+    path('mysql', views.mysql),
 ]

sample-apps/python/django_frontend_service/ec2-requirements.txt

@@ -1,5 +1,6 @@
 Django~=4.2.9
-requests~=2.31.0
 boto3~=1.34.3
+pymysql==1.1.1
+python-dotenv~=1.0.1
+requests~=2.31.0
 schedule~=1.2.1
-python-dotenv~=1.0.1

sample-apps/python/django_frontend_service/frontend_service_app/views.py

@@ -1,10 +1,12 @@
 ## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 ## SPDX-License-Identifier: Apache-2.0
 import logging
+import os
 import threading
 import time
 
 import boto3
+import pymysql
 import requests
 import schedule
 from django.http import HttpResponse, JsonResponse
@@ -105,3 +107,18 @@ def get_xray_trace_id():
     xray_trace_id = f"1-{trace_id[:8]}-{trace_id[8:]}"
 
     return JsonResponse({"traceId": xray_trace_id})
+
+def mysql(request):
+    logger.info("mysql received")
+    try:
+        connection = pymysql.connect(host=os.environ["RDS_MYSQL_CLUSTER_ENDPOINT"],
+                                     user=os.environ["RDS_MYSQL_CLUSTER_USERNAME"],
+                                     password=os.environ["RDS_MYSQL_CLUSTER_PASSWORD"],
+                                     database=os.environ["RDS_MYSQL_CLUSTER_DATABASE"])
+        with connection:
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT * FROM tables LIMIT 1;")
+    except Exception as e:  # pylint: disable=broad-except
+        logger.error("Could not complete http request to RDS database:" + str(e))
+    finally:
+        return get_xray_trace_id()

sample-apps/python/django_frontend_service/requirements.txt

@@ -1,6 +1,7 @@
 Django~=4.2.9
-requests~=2.31.0
 boto3~=1.34.3
-schedule~=1.2.1
+opentelemetry-api~=1.22.0
+pymysql==1.1.1
 python-dotenv~=1.0.1
-opentelemetry-api~=1.22.0
+requests~=2.31.0
+schedule~=1.2.1

terraform/python/eks/main.tf

@@ -119,7 +119,22 @@ resource "kubernetes_deployment" "python_app_deployment" {
               name = "DJANGO_SETTINGS_MODULE"
               value = "django_frontend_service.settings"
             }
-
+          env {
+            name = "RDS_MYSQL_CLUSTER_ENDPOINT"
+            value = var.rds_mysql_cluster_endpoint
+          }
+          env {
+            name = "RDS_MYSQL_CLUSTER_DATABASE"
+            value = var.rds_mysql_cluster_database
+          }
+          env {
+            name = "RDS_MYSQL_CLUSTER_USERNAME"
+            value = var.rds_mysql_cluster_username
+          }
+          env {
+            name = "RDS_MYSQL_CLUSTER_PASSWORD"
+            value = var.rds_mysql_cluster_password
+          }
           port {
             container_port = 8000
           }

terraform/python/eks/variables.tf

@@ -47,4 +47,20 @@ variable "python_app_image" {
 
 variable "python_remote_app_image" {
   default = "<ECR_IMAGE_LINK>:<TAG>"
-}
+}
+
+variable "rds_mysql_cluster_endpoint" {
+  default = "example.cluster-example.eu-west-1.rds.amazonaws.com"
+}
+
+variable "rds_mysql_cluster_database" {
+  default = "example_database"
+}
+
+variable "rds_mysql_cluster_username" {
+  default = "username"
+}
+
+variable "rds_mysql_cluster_password" {
+  default = "password"
+}