From 66e66a230d278ea0413a6851c820983d0cac9e86 Mon Sep 17 00:00:00 2001 From: Godwin Rose Samuel Winston Samuel Date: Thu, 21 Nov 2024 21:52:12 +0000 Subject: [PATCH 1/2] feat: support for Provisioned Poller (MSK and SelfManagedKafka) --- .cfnlintrc.yaml | 1 + .../schema_source/aws_serverless_function.py | 2 + samtranslator/model/eventsources/pull.py | 3 + samtranslator/model/lambda_.py | 1 + samtranslator/schema/schema.json | 6 + schema_source/sam.schema.json | 6 + ...nction_with_provisioned_poller_config.yaml | 40 +++++ ...nction_with_provisioned_poller_config.json | 151 ++++++++++++++++++ ...nction_with_provisioned_poller_config.json | 151 ++++++++++++++++++ ...nction_with_provisioned_poller_config.json | 151 ++++++++++++++++++ 10 files changed, 512 insertions(+) create mode 100644 tests/translator/input/function_with_provisioned_poller_config.yaml create mode 100644 tests/translator/output/aws-cn/function_with_provisioned_poller_config.json create mode 100644 tests/translator/output/aws-us-gov/function_with_provisioned_poller_config.json create mode 100644 tests/translator/output/function_with_provisioned_poller_config.json diff --git a/.cfnlintrc.yaml b/.cfnlintrc.yaml index cda80d6fc..0dd2bd002 100644 --- a/.cfnlintrc.yaml +++ b/.cfnlintrc.yaml @@ -136,6 +136,7 @@ ignore_templates: - tests/translator/output/**/function_with_intrinsics_resource_attribute.json # CFN now supports intrinsics in DeletionPolicy - tests/translator/output/**/function_with_snapstart.json # Snapstart intentionally not attached to a lambda version which causes lint issues - tests/translator/output/**/managed_policies_everything.json # intentionally contains wrong arns + - tests/translator/output/**/function_with_provisioned_poller_config.json ignore_checks: - E2531 # Deprecated runtime; not relevant for transform tests - E2533 # Another deprecated runtime; not relevant for transform tests diff --git a/samtranslator/internal/schema_source/aws_serverless_function.py b/samtranslator/internal/schema_source/aws_serverless_function.py index 74c252dce..7a1ac5d0a 100644 --- a/samtranslator/internal/schema_source/aws_serverless_function.py +++ b/samtranslator/internal/schema_source/aws_serverless_function.py @@ -417,6 +417,7 @@ class MSKEventProperties(BaseModel): Topics: PassThroughProp = mskeventproperties("Topics") SourceAccessConfigurations: Optional[PassThroughProp] = mskeventproperties("SourceAccessConfigurations") DestinationConfig: Optional[PassThroughProp] # TODO: add documentation + ProvisionedPollerConfig: Optional[PassThroughProp] class MSKEvent(BaseModel): @@ -455,6 +456,7 @@ class SelfManagedKafkaEventProperties(BaseModel): StartingPosition: Optional[PassThroughProp] # TODO: add documentation StartingPositionTimestamp: Optional[PassThroughProp] # TODO: add documentation Topics: PassThroughProp = selfmanagedkafkaeventproperties("Topics") + ProvisionedPollerConfig: Optional[PassThroughProp] class SelfManagedKafkaEvent(BaseModel): diff --git a/samtranslator/model/eventsources/pull.py b/samtranslator/model/eventsources/pull.py index b5298771c..3ab1b1d6f 100644 --- a/samtranslator/model/eventsources/pull.py +++ b/samtranslator/model/eventsources/pull.py @@ -55,6 +55,7 @@ class PullEventSource(ResourceMacro, metaclass=ABCMeta): "KmsKeyArn": PassThroughProperty(False), "ConsumerGroupId": PropertyType(False, IS_STR), "ScalingConfig": PropertyType(False, IS_DICT), + "ProvisionedPollerConfig": PropertyType(False, IS_DICT), } BatchSize: Optional[Intrinsicable[int]] @@ -78,6 +79,7 @@ class PullEventSource(ResourceMacro, metaclass=ABCMeta): KmsKeyArn: Optional[Intrinsicable[str]] ConsumerGroupId: Optional[Intrinsicable[str]] ScalingConfig: Optional[Dict[str, Any]] + ProvisionedPollerConfig: Optional[Dict[str, Any]] @abstractmethod def get_policy_arn(self) -> Optional[str]: @@ -145,6 +147,7 @@ def to_cloudformation(self, **kwargs): # type: ignore[no-untyped-def] # noqa: P lambda_eventsourcemapping.FilterCriteria = self.FilterCriteria lambda_eventsourcemapping.KmsKeyArn = self.KmsKeyArn lambda_eventsourcemapping.ScalingConfig = self.ScalingConfig + lambda_eventsourcemapping.ProvisionedPollerConfig = self.ProvisionedPollerConfig self._validate_filter_criteria() if self.KafkaBootstrapServers: diff --git a/samtranslator/model/lambda_.py b/samtranslator/model/lambda_.py index e76809735..6a7e155d7 100644 --- a/samtranslator/model/lambda_.py +++ b/samtranslator/model/lambda_.py @@ -122,6 +122,7 @@ class LambdaEventSourceMapping(Resource): "AmazonManagedKafkaEventSourceConfig": GeneratedProperty(), "SelfManagedKafkaEventSourceConfig": GeneratedProperty(), "ScalingConfig": GeneratedProperty(), + "ProvisionedPollerConfig": GeneratedProperty(), } runtime_attrs = {"name": lambda self: ref(self.logical_id)} diff --git a/samtranslator/schema/schema.json b/samtranslator/schema/schema.json index 4cb9191f0..77f31c1e3 100644 --- a/samtranslator/schema/schema.json +++ b/samtranslator/schema/schema.json @@ -276146,6 +276146,9 @@ "markdownDescription": "The maximum amount of time to gather records before invoking the function, in seconds\\. \n*Type*: Integer \n*Required*: No \n*AWS CloudFormation compatibility*: This property is passed directly to the [`MaximumBatchingWindowInSeconds`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-eventsourcemapping.html#cfn-lambda-eventsourcemapping-maximumbatchingwindowinseconds) property of an `AWS::Lambda::EventSourceMapping` resource\\.", "title": "MaximumBatchingWindowInSeconds" }, + "ProvisionedPollerConfig": { + "$ref": "#/definitions/PassThroughProp" + }, "SourceAccessConfigurations": { "allOf": [ { @@ -276985,6 +276988,9 @@ "KmsKeyArn": { "$ref": "#/definitions/PassThroughProp" }, + "ProvisionedPollerConfig": { + "$ref": "#/definitions/PassThroughProp" + }, "SourceAccessConfigurations": { "allOf": [ { diff --git a/schema_source/sam.schema.json b/schema_source/sam.schema.json index e1b638631..86eda8c13 100644 --- a/schema_source/sam.schema.json +++ b/schema_source/sam.schema.json @@ -2413,6 +2413,9 @@ "markdownDescription": "The maximum amount of time to gather records before invoking the function, in seconds\\. \n*Type*: Integer \n*Required*: No \n*AWS CloudFormation compatibility*: This property is passed directly to the [`MaximumBatchingWindowInSeconds`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-eventsourcemapping.html#cfn-lambda-eventsourcemapping-maximumbatchingwindowinseconds) property of an `AWS::Lambda::EventSourceMapping` resource\\.", "title": "MaximumBatchingWindowInSeconds" }, + "ProvisionedPollerConfig": { + "$ref": "#/definitions/PassThroughProp" + }, "SourceAccessConfigurations": { "allOf": [ { @@ -3183,6 +3186,9 @@ "KmsKeyArn": { "$ref": "#/definitions/PassThroughProp" }, + "ProvisionedPollerConfig": { + "$ref": "#/definitions/PassThroughProp" + }, "SourceAccessConfigurations": { "allOf": [ { diff --git a/tests/translator/input/function_with_provisioned_poller_config.yaml b/tests/translator/input/function_with_provisioned_poller_config.yaml new file mode 100644 index 000000000..0b8b92f3c --- /dev/null +++ b/tests/translator/input/function_with_provisioned_poller_config.yaml @@ -0,0 +1,40 @@ +Resources: + KafkaEventsFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: s3://sam-demo-bucket/metricsConfig.zip + Handler: index.handler + Runtime: nodejs16.x + Events: + MyMskEvent: + Type: MSK + Properties: + StartingPosition: LATEST + Stream: !Sub arn:aws:kafka:${AWS::Region}:012345678901:cluster/mycluster/6cc0432b-8618-4f44-bccc-e1fbd8fb7c4d-2 + Topics: + - MyDummyTestTopic + ConsumerGroupId: consumergroup1 + ProvisionedPollerConfig: + MinimumPollers: 5 + MaximumPollers: 10 + MyKafkaCluster: + Type: SelfManagedKafka + Properties: + KafkaBootstrapServers: + - abc.xyz.com:9092 + - 123.45.67.89:9096 + Topics: + - Topic1 + SourceAccessConfigurations: + - Type: SASL_SCRAM_512_AUTH + URI: arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c + - Type: VPC_SUBNET + URI: subnet:subnet-12345 + - Type: VPC_SECURITY_GROUP + URI: security_group:sg-67890 + ConsumerGroupId: consumergroup1 + StartingPosition: AT_TIMESTAMP + StartingPositionTimestamp: 1672560000 + ProvisionedPollerConfig: + MinimumPollers: 2 + MaximumPollers: 100 diff --git a/tests/translator/output/aws-cn/function_with_provisioned_poller_config.json b/tests/translator/output/aws-cn/function_with_provisioned_poller_config.json new file mode 100644 index 000000000..f89bf28e9 --- /dev/null +++ b/tests/translator/output/aws-cn/function_with_provisioned_poller_config.json @@ -0,0 +1,151 @@ +{ + "Resources": { + "KafkaEventsFunction": { + "Properties": { + "Code": { + "S3Bucket": "sam-demo-bucket", + "S3Key": "metricsConfig.zip" + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "KafkaEventsFunctionRole", + "Arn" + ] + }, + "Runtime": "nodejs16.x", + "Tags": [ + { + "Key": "lambda:createdBy", + "Value": "SAM" + } + ] + }, + "Type": "AWS::Lambda::Function" + }, + "KafkaEventsFunctionMyKafkaCluster": { + "Properties": { + "FunctionName": { + "Ref": "KafkaEventsFunction" + }, + "ProvisionedPollerConfig": { + "MaximumPollers": 100, + "MinimumPollers": 2 + }, + "SelfManagedEventSource": { + "Endpoints": { + "KafkaBootstrapServers": [ + "abc.xyz.com:9092", + "123.45.67.89:9096" + ] + } + }, + "SelfManagedKafkaEventSourceConfig": { + "ConsumerGroupId": "consumergroup1" + }, + "SourceAccessConfigurations": [ + { + "Type": "SASL_SCRAM_512_AUTH", + "URI": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Type": "VPC_SUBNET", + "URI": "subnet:subnet-12345" + }, + { + "Type": "VPC_SECURITY_GROUP", + "URI": "security_group:sg-67890" + } + ], + "StartingPosition": "AT_TIMESTAMP", + "StartingPositionTimestamp": 1672560000, + "Topics": [ + "Topic1" + ] + }, + "Type": "AWS::Lambda::EventSourceMapping" + }, + "KafkaEventsFunctionMyMskEvent": { + "Properties": { + "AmazonManagedKafkaEventSourceConfig": { + "ConsumerGroupId": "consumergroup1" + }, + "EventSourceArn": { + "Fn::Sub": "arn:aws:kafka:${AWS::Region}:012345678901:cluster/mycluster/6cc0432b-8618-4f44-bccc-e1fbd8fb7c4d-2" + }, + "FunctionName": { + "Ref": "KafkaEventsFunction" + }, + "ProvisionedPollerConfig": { + "MaximumPollers": 10, + "MinimumPollers": 5 + }, + "StartingPosition": "LATEST", + "Topics": [ + "MyDummyTestTopic" + ] + }, + "Type": "AWS::Lambda::EventSourceMapping" + }, + "KafkaEventsFunctionRole": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaMSKExecutionRole" + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Action": [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeVpcs", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "SelfManagedKafkaExecutionRolePolicy" + } + ], + "Tags": [ + { + "Key": "lambda:createdBy", + "Value": "SAM" + } + ] + }, + "Type": "AWS::IAM::Role" + } + } +} diff --git a/tests/translator/output/aws-us-gov/function_with_provisioned_poller_config.json b/tests/translator/output/aws-us-gov/function_with_provisioned_poller_config.json new file mode 100644 index 000000000..367f70f6f --- /dev/null +++ b/tests/translator/output/aws-us-gov/function_with_provisioned_poller_config.json @@ -0,0 +1,151 @@ +{ + "Resources": { + "KafkaEventsFunction": { + "Properties": { + "Code": { + "S3Bucket": "sam-demo-bucket", + "S3Key": "metricsConfig.zip" + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "KafkaEventsFunctionRole", + "Arn" + ] + }, + "Runtime": "nodejs16.x", + "Tags": [ + { + "Key": "lambda:createdBy", + "Value": "SAM" + } + ] + }, + "Type": "AWS::Lambda::Function" + }, + "KafkaEventsFunctionMyKafkaCluster": { + "Properties": { + "FunctionName": { + "Ref": "KafkaEventsFunction" + }, + "ProvisionedPollerConfig": { + "MaximumPollers": 100, + "MinimumPollers": 2 + }, + "SelfManagedEventSource": { + "Endpoints": { + "KafkaBootstrapServers": [ + "abc.xyz.com:9092", + "123.45.67.89:9096" + ] + } + }, + "SelfManagedKafkaEventSourceConfig": { + "ConsumerGroupId": "consumergroup1" + }, + "SourceAccessConfigurations": [ + { + "Type": "SASL_SCRAM_512_AUTH", + "URI": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Type": "VPC_SUBNET", + "URI": "subnet:subnet-12345" + }, + { + "Type": "VPC_SECURITY_GROUP", + "URI": "security_group:sg-67890" + } + ], + "StartingPosition": "AT_TIMESTAMP", + "StartingPositionTimestamp": 1672560000, + "Topics": [ + "Topic1" + ] + }, + "Type": "AWS::Lambda::EventSourceMapping" + }, + "KafkaEventsFunctionMyMskEvent": { + "Properties": { + "AmazonManagedKafkaEventSourceConfig": { + "ConsumerGroupId": "consumergroup1" + }, + "EventSourceArn": { + "Fn::Sub": "arn:aws:kafka:${AWS::Region}:012345678901:cluster/mycluster/6cc0432b-8618-4f44-bccc-e1fbd8fb7c4d-2" + }, + "FunctionName": { + "Ref": "KafkaEventsFunction" + }, + "ProvisionedPollerConfig": { + "MaximumPollers": 10, + "MinimumPollers": 5 + }, + "StartingPosition": "LATEST", + "Topics": [ + "MyDummyTestTopic" + ] + }, + "Type": "AWS::Lambda::EventSourceMapping" + }, + "KafkaEventsFunctionRole": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaMSKExecutionRole" + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Action": [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeVpcs", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "SelfManagedKafkaExecutionRolePolicy" + } + ], + "Tags": [ + { + "Key": "lambda:createdBy", + "Value": "SAM" + } + ] + }, + "Type": "AWS::IAM::Role" + } + } +} diff --git a/tests/translator/output/function_with_provisioned_poller_config.json b/tests/translator/output/function_with_provisioned_poller_config.json new file mode 100644 index 000000000..35b8cd51e --- /dev/null +++ b/tests/translator/output/function_with_provisioned_poller_config.json @@ -0,0 +1,151 @@ +{ + "Resources": { + "KafkaEventsFunction": { + "Properties": { + "Code": { + "S3Bucket": "sam-demo-bucket", + "S3Key": "metricsConfig.zip" + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "KafkaEventsFunctionRole", + "Arn" + ] + }, + "Runtime": "nodejs16.x", + "Tags": [ + { + "Key": "lambda:createdBy", + "Value": "SAM" + } + ] + }, + "Type": "AWS::Lambda::Function" + }, + "KafkaEventsFunctionMyKafkaCluster": { + "Properties": { + "FunctionName": { + "Ref": "KafkaEventsFunction" + }, + "ProvisionedPollerConfig": { + "MaximumPollers": 100, + "MinimumPollers": 2 + }, + "SelfManagedEventSource": { + "Endpoints": { + "KafkaBootstrapServers": [ + "abc.xyz.com:9092", + "123.45.67.89:9096" + ] + } + }, + "SelfManagedKafkaEventSourceConfig": { + "ConsumerGroupId": "consumergroup1" + }, + "SourceAccessConfigurations": [ + { + "Type": "SASL_SCRAM_512_AUTH", + "URI": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Type": "VPC_SUBNET", + "URI": "subnet:subnet-12345" + }, + { + "Type": "VPC_SECURITY_GROUP", + "URI": "security_group:sg-67890" + } + ], + "StartingPosition": "AT_TIMESTAMP", + "StartingPositionTimestamp": 1672560000, + "Topics": [ + "Topic1" + ] + }, + "Type": "AWS::Lambda::EventSourceMapping" + }, + "KafkaEventsFunctionMyMskEvent": { + "Properties": { + "AmazonManagedKafkaEventSourceConfig": { + "ConsumerGroupId": "consumergroup1" + }, + "EventSourceArn": { + "Fn::Sub": "arn:aws:kafka:${AWS::Region}:012345678901:cluster/mycluster/6cc0432b-8618-4f44-bccc-e1fbd8fb7c4d-2" + }, + "FunctionName": { + "Ref": "KafkaEventsFunction" + }, + "ProvisionedPollerConfig": { + "MaximumPollers": 10, + "MinimumPollers": 5 + }, + "StartingPosition": "LATEST", + "Topics": [ + "MyDummyTestTopic" + ] + }, + "Type": "AWS::Lambda::EventSourceMapping" + }, + "KafkaEventsFunctionRole": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws:iam::aws:policy/service-role/AWSLambdaMSKExecutionRole" + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Action": [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeVpcs", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "SelfManagedKafkaExecutionRolePolicy" + } + ], + "Tags": [ + { + "Key": "lambda:createdBy", + "Value": "SAM" + } + ] + }, + "Type": "AWS::IAM::Role" + } + } +} From caafeeca7ce9da1a1cbcd2b70ae1cd70f72a178b Mon Sep 17 00:00:00 2001 From: aws-sam-cli-bot <46753707+aws-sam-cli-bot@users.noreply.github.com> Date: Thu, 21 Nov 2024 21:52:12 +0000 Subject: [PATCH 2/2] chore: bump version to 1.94.0 --- samtranslator/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samtranslator/__init__.py b/samtranslator/__init__.py index eb97acb8c..b61fd06fb 100644 --- a/samtranslator/__init__.py +++ b/samtranslator/__init__.py @@ -1 +1 @@ -__version__ = "1.91.0" +__version__ = "1.94.0"