Merge pull request #1936 from ASFHyP3/develop

Release v4.3.2

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.3.2]
+### Fixed
+- Added a Lambda function that sets `Private DNS names enabled` to false for VPC endpoint.
+
 ## [4.3.1]
 ### Added
 - The `ESA_USERNAME` and `ESA_PASSWORD` secrets have been added to all of the job specs that require them.

Makefile

@@ -5,10 +5,11 @@ HANDLE_BATCH_EVENT = ${PWD}/apps/handle-batch-event/src
 SCALE_CLUSTER = ${PWD}/apps/scale-cluster/src
 START_EXECUTION_MANAGER = ${PWD}/apps/start-execution-manager/src
 START_EXECUTION_WORKER = ${PWD}/apps/start-execution-worker/src
+DISABLE_PRIVATE_DNS = ${PWD}/apps/disable-private-dns/src
 UPDATE_DB = ${PWD}/apps/update-db/src
 UPLOAD_LOG = ${PWD}/apps/upload-log/src
 DYNAMO = ${PWD}/lib/dynamo
-export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${HANDLE_BATCH_EVENT}:${SCALE_CLUSTER}:${START_EXECUTION_MANAGER}:${START_EXECUTION_WORKER}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO}
+export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${HANDLE_BATCH_EVENT}:${SCALE_CLUSTER}:${START_EXECUTION_MANAGER}:${START_EXECUTION_WORKER}:${DISABLE_PRIVATE_DNS}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO}
 
 
 build: render
@@ -18,6 +19,7 @@ build: render
 	python -m pip install --upgrade -r requirements-apps-scale-cluster.txt -t ${SCALE_CLUSTER}; \
 	python -m pip install --upgrade -r requirements-apps-start-execution-manager.txt -t ${START_EXECUTION_MANAGER}; \
 	python -m pip install --upgrade -r requirements-apps-start-execution-worker.txt -t ${START_EXECUTION_WORKER}; \
+	python -m pip install --upgrade -r requirements-apps-disable-private-dns.txt -t ${DISABLE_PRIVATE_DNS}; \
 	python -m pip install --upgrade -r requirements-apps-update-db.txt -t ${UPDATE_DB}
 
 test_file ?= tests/
@@ -41,7 +43,7 @@ render:
 static: flake8 openapi-validate cfn-lint
 
 flake8:
-	flake8 --ignore=E731 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,handle_batch_event,check_processing_time,start_execution_manager,start_execution_worker,update_db,upload_log,dynamo,lambda_logging,scale_cluster apps tests lib
+	flake8 --ignore=E731 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,handle_batch_event,check_processing_time,start_execution_manager,start_execution_worker,disable_private_dns,update_db,upload_log,dynamo,lambda_logging,scale_cluster apps tests lib
 
 openapi-validate: render
 	openapi-spec-validator apps/api/src/hyp3_api/api-spec/openapi-spec.yml

apps/disable-private-dns/disable-private-dns-cf.yml.j2

@@ -0,0 +1,91 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Parameters:
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+
+  SecurityGroupId:
+    Type: String
+
+  SubnetIds:
+    Type: CommaDelimitedList
+
+Resources:
+
+  LogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Sub "/aws/lambda/${Lambda}"
+      RetentionInDays: 90
+
+  Role:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          Action: sts:AssumeRole
+          Principal:
+            Service: lambda.amazonaws.com
+          Effect: Allow
+      ManagedPolicyArns:
+        - !Ref Policy
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole
+
+  Policy:
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Action:
+              - logs:CreateLogStream
+              - logs:PutLogEvents
+            Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*"
+          - Effect: Allow
+            Action:
+              - ec2:DescribeVpcEndpoints
+              - ec2:ModifyVpcEndpoint
+            Resource: "*"
+
+  Lambda:
+    Type: AWS::Lambda::Function
+    Properties:
+      Code: src/
+      Handler: disable_private_dns.lambda_handler
+      MemorySize: 128
+      Role: !GetAtt Role.Arn
+      Runtime: python3.9
+      Timeout: 5
+      Environment:
+        Variables:
+          VPCID: !Ref VpcId
+          ENDPOINT_NAME: "VPC Endpoint - Consumer"
+      VpcConfig:
+        SecurityGroupIds:
+          - !Ref SecurityGroupId
+        SubnetIds: !Ref SubnetIds
+
+  EventInvokeConfig:
+    Type: AWS::Lambda::EventInvokeConfig
+    Properties:
+      FunctionName: !Ref Lambda
+      Qualifier: $LATEST
+      MaximumRetryAttempts: 0
+
+  Schedule:
+    Type: AWS::Events::Rule
+    Properties:
+      ScheduleExpression: "rate(1 minute)"
+      Targets:
+        - Arn: !GetAtt Lambda.Arn
+          Id: lambda
+
+  EventPermission:
+    Type: AWS::Lambda::Permission
+    Properties:
+      FunctionName: !GetAtt Lambda.Arn
+      Action: lambda:InvokeFunction
+      Principal: events.amazonaws.com
+      SourceArn: !GetAtt Schedule.Arn

apps/disable-private-dns/src/disable_private_dns.py

@@ -0,0 +1,48 @@
+import os
+
+import boto3
+
+
+CLIENT = boto3.client('ec2')
+
+
+def get_endpoint(vpc_id, endpoint_name):
+    response = CLIENT.describe_vpc_endpoints()
+    endpoints = [endpoint for endpoint in response['VpcEndpoints'] if endpoint['VpcId'] == vpc_id]
+    if len(endpoints) == 0:
+        raise ValueError(f'No endpoints in VPC {vpc_id}.')
+
+    desired_endpoint = None
+    for endpoint in endpoints:
+        retrieved_name = [item['Value'] for item in endpoint['Tags'] if item['Key'] == 'Name'][0]
+        if retrieved_name == endpoint_name:
+            desired_endpoint = endpoint
+
+    if desired_endpoint is None:
+        raise ValueError(f'No endpoint in VPC {vpc_id} with name {endpoint_name} exists.')
+
+    return desired_endpoint
+
+
+def set_private_dns_disabled(endpoint_id):
+    response = CLIENT.modify_vpc_endpoint(VpcEndpointId=endpoint_id, PrivateDnsEnabled=False)
+    # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2/client/modify_vpc_endpoint.html
+    assert response['Return'] is True, response
+    print(f"Private DNS disabled for VPC Endpoint: {endpoint_id}.")
+
+
+def disable_private_dns(vpc_id, endpoint_name):
+    endpoint = get_endpoint(vpc_id, endpoint_name)
+    if endpoint['PrivateDnsEnabled']:
+        print(f"Private DNS enabled for VPC Endpoint: {endpoint['VpcEndpointId']}, changing...")
+        set_private_dns_disabled(endpoint['VpcEndpointId'])
+    else:
+        print(f"Private DNS already disabled for VPC Endpoint: {endpoint['VpcEndpointId']}, doing nothing.")
+
+
+def lambda_handler(event, context):
+    vpc_id = os.environ['VPCID']
+    endpoint_name = os.environ['ENDPOINT_NAME']
+    print(f'VPC ID {vpc_id}')
+    print(f'Endpoint Name: {endpoint_name}')
+    disable_private_dns(vpc_id, endpoint_name)

apps/main-cf.yml.j2

@@ -338,3 +338,14 @@ Resources:
       KeySchema:
         - AttributeName: user_id
           KeyType: HASH
+
+  {% if security_environment == 'EDC' %}
+  DisablePrivateDNS:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        VpcId: !Ref VpcId
+        SecurityGroupId: !GetAtt Cluster.Outputs.SecurityGroupId
+        SubnetIds: !Join [",", !Ref SubnetIds]
+      TemplateURL: disable-private-dns/disable-private-dns-cf.yml
+  {% endif %}

requirements-all.txt

@@ -3,6 +3,7 @@
 -r requirements-apps-scale-cluster.txt
 -r requirements-apps-start-execution-manager.txt
 -r requirements-apps-start-execution-worker.txt
+-r requirements-apps-disable-private-dns.txt
 -r requirements-apps-update-db.txt
 boto3==1.28.82
 jinja2==3.1.2

requirements-apps-disable-private-dns.txt

@@ -0,0 +1 @@
+boto3==1.28.82