Merge pull request #2371 from farrjam/farrjam-feature-eventbridge-pip…

…es-sqs-to-stepfunctions-terraform

Terraform serverless pattern - eventbridge-pipes-sqs-to-stepfunctions

eventbridge-pipes-sqs-to-stepfunctions-terraform/README.md

@@ -0,0 +1,64 @@
+# Invoke AWS Step Functions state machine with Amazon with EventBridge Pipes
+
+This pattern shows how to use [Amazon EventBridge Pipes](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes.html) to launch an AWS Step Functions state machine with a message coming from an Amazon SQS Queue. The pattern is deployed using Terraform.
+
+Learn more about this pattern at Serverless Land Patterns: https://serverlessland.com/patterns/eventbridge-pipes-sqs-to-stepfunctions-terraform.
+
+Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example.
+
+## Requirements
+
+* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources.
+* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured
+* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+* [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli?in=terraform/aws-get-started) installed
+
+## Deployment Instructions
+
+1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository:
+    ``` 
+    git clone https://github.com/aws-samples/serverless-patterns
+    ```
+1. Change directory to the pattern directory:
+    ```
+    cd eventbridge-pipes-sqs-to-stepfunctions-terraform
+    ```
+1. From the command line, initialize Terraform:
+    ```
+    terraform init
+    ```
+1. From the commend line, apply the configuration in the main.tf file and follow the prompts:
+    ```
+    terraform apply
+    ```
+
+## How it works
+
+Amazon EventBridge Pipes connects sources to targets. This Terraform stack creates an EventBridge Pipe which receives message from the source SQS queue and sends it to the target AWS Step Functions state machine. 
+
+## Testing
+
+1. Starts a Live Tail streaming session for StepFunction LogGroup 
+
+```
+aws logs start-live-tail --log-group-identifiers <StepFunction LogGroup ARN>
+```
+
+2. Put a message into the queue
+
+```
+aws sqs send-message --queue-url <SQS Queue URL> --message-body "Test"
+```
+
+3. Observe the logs for the new execution.
+
+## Cleanup
+
+1. Delete all created resources and follow prompts:
+    ```
+    terraform destroy
+    ```
+----
+Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+SPDX-License-Identifier: MIT-0

eventbridge-pipes-sqs-to-stepfunctions-terraform/eventbridge-pipes-sqs-sfn-tf.json

@@ -0,0 +1,84 @@
+{
+    "title": "Invoke AWS Step Functions state machine with Amazon with EventBridge Pipes",
+    "description": "This pattern shows how to use Amazon EventBridge Pipes to launch an AWS Step Functions state machine with a message coming from an Amazon SQS Queue.",
+    "language": "",
+    "level": "200",
+    "framework": "Terraform",
+    "introBox": {
+        "headline": "How it works",
+        "text": [
+            "Amazon EventBridge Pipes connects sources to targets. This Terraform stack creates an EventBridge Pipe which polls for messages from a source SQS queue and sends the messages to the target AWS Step Functions state machine."
+        ]
+    },
+    "gitHub": {
+        "template": {
+            "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/eventbridge-pipes-sqs-to-stepfunctions-terraform",
+            "templateURL": "eventbridge-pipes-sqs-to-stepfunctions-terraform",
+            "projectFolder": "eventbridge-pipes-sqs-to-stepfunctions-terraform",
+            "templateFile": "main.tf"
+        }
+    },
+    "resources": {
+        "bullets": [
+            {
+                "text": "Amazon EventBridge Pipes",
+                "link": "https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes.html"
+            },
+            {
+                "text": "Video explaining EventBridge Pipes",
+                "link": "https://www.youtube.com/watch?v=xXGXCOc3cBs&t=1037s"
+            }
+        ]
+    },
+    "deploy": {
+        "text": [
+            "terraform apply"
+        ]
+    },
+    "testing": {
+        "text": [
+            "See the GitHub repo for detailed testing instructions."
+        ]
+    },
+    "cleanup": {
+        "text": [
+            "Delete the stack: <code>terraform destroy</code>."
+        ]
+    },
+    "authors": [
+        {
+            "name": "Farrukh Jamal",
+            "image": "https://media.licdn.com/dms/image/C5103AQHH_uSeGxOlZw/profile-displayphoto-shrink_400_400/0/1570698220038?e=1726099200&v=beta&t=7HUDKvPyM5GbBA1eOhVGB9POwFhXC3v87-vrqCIk1d0",
+            "bio": "Farrukh is a Cloud Support Engineer at AWS based in the Australia.",
+            "linkedin": "farrukhjamal/"
+        }
+    ],
+    "patternArch": {
+        "icon1": {
+            "x": 20,
+            "y": 50,
+            "service": "sqs",
+            "label": "Amazon SQS"
+        },
+        "icon2": {
+            "x": 50,
+            "y": 50,
+            "service": "eventbridge-pipes",
+            "label": "Amazon EventBridge Pipes"
+        },
+        "icon3": {
+            "x": 80,
+            "y": 50,
+            "service": "sfn",
+            "label": "AWS Step Functions"
+        },
+        "line1": {
+            "from": "icon1",
+            "to": "icon2"
+        },
+        "line2": {
+            "from": "icon2",
+            "to": "icon3"
+        }
+    }
+}

eventbridge-pipes-sqs-to-stepfunctions-terraform/example-pattern.json

@@ -0,0 +1,56 @@
+{
+  "title": "SQS to Step Functions with EventBridge Pipes",
+  "description": "This pattern shows how to use EventBridge Pipes to launch a Step Functions state machine with a message coming from SQS Queue.",
+  "language": "",
+  "level": "200",
+  "framework": "Terraform",
+  "introBox": {
+    "headline": "How it works",
+    "text": [
+      "Amazon EventBridge Pipes connects sources to targets. This Terraform stack creates an EventBridge Pipe which polls message from source SQS queue and sends it to the target AWS Step Functions state machine."
+    ]
+  },
+  "gitHub": {
+    "template": {
+      "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/eventbridge-pipes-sqs-to-stepfunctions-terraform",
+      "templateURL": "serverless-patterns/eventbridge-pipes-sqs-to-stepfunctions-terraform",
+      "projectFolder": "eventbridge-pipes-sqs-to-stepfunctions-terraform",
+      "templateFile": "eventbridge-pipes-sqs-to-stepfunctions-terraform/main.tf"
+    }
+  },
+  "resources": {
+    "bullets": [
+      {
+        "text": "Amazon EventBridge Pipes",
+        "link": "https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes.html"
+      },
+      {
+        "text": "Video explaining EventBridge Pipes",
+        "link": "https://www.youtube.com/watch?v=xXGXCOc3cBs&t=1037s"
+      }
+    ]
+  },
+  "deploy": {
+    "text": [
+      "terraform apply"
+    ]
+  },
+  "testing": {
+    "text": [
+      "See the GitHub repo for detailed testing instructions."
+    ]
+  },
+  "cleanup": {
+    "text": [
+      "Delete the stack: <code>terraform destroy</code>."
+    ]
+  },
+  "authors": [
+    {
+      "name": "Farrukh Jamal",
+      "image": "https://media.licdn.com/dms/image/C5103AQHH_uSeGxOlZw/profile-displayphoto-shrink_400_400/0/1570698220038?e=1726099200&v=beta&t=7HUDKvPyM5GbBA1eOhVGB9POwFhXC3v87-vrqCIk1d0",
+      "bio": "Farrukh is a Cloud Support Engineer at AWS based in the Australia.",
+      "linkedin": "https://www.linkedin.com/in/farrukhjamal/"
+    }
+  ]
+}

eventbridge-pipes-sqs-to-stepfunctions-terraform/main.tf

@@ -0,0 +1,180 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.57"
+    }
+  }
+
+  required_version = ">= 0.14.9"
+}
+
+provider "aws" {
+  profile = "default"
+  region  = "us-east-1"
+}
+
+# Source SQS
+resource "aws_sqs_queue" "source_queue" {
+  name = "eb-pipes-sqs-sf-SourceQueue"
+  redrive_policy = jsonencode({
+    deadLetterTargetArn = aws_sqs_queue.source_queue_dlq.arn
+    maxReceiveCount     = 5
+  })
+}
+
+# DLQ for source
+resource "aws_sqs_queue" "source_queue_dlq" {
+  name = "eb-pipes-sqs-sf-SourceQueueDLQ"
+}
+
+#Log Group for StepFunction
+resource "aws_cloudwatch_log_group" "target_state_machine_log_group" {
+  name              = "sqs-pipes-sf/StateMachine"
+  retention_in_days = 7
+}
+
+#Execution Role for StepFunction
+resource "aws_iam_role" "target_state_machine_role" {
+  name = "eb-pipes-sqs-sf-TargetStateMachineRole"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = {
+      Effect = "Allow"
+      Action = "sts:AssumeRole"
+      Principal = {
+        Service = "states.amazonaws.com"
+      }
+    }
+  })
+
+  inline_policy {
+    name = "CloudWatchLogs"
+    policy = jsonencode({
+      Version = "2012-10-17"
+      Statement = [
+        {
+          Effect = "Allow"
+          Action = [
+            "logs:CreateLogDelivery",
+            "logs:GetLogDelivery",
+            "logs:UpdateLogDelivery",
+            "logs:DeleteLogDelivery",
+            "logs:ListLogDeliveries",
+            "logs:PutResourcePolicy",
+            "logs:DescribeResourcePolicies",
+            "logs:DescribeLogGroups",
+          ],
+          Resource = ["*"]
+        },
+      ]
+    })
+  }
+}
+
+# Target Step Function State Machine
+resource "aws_sfn_state_machine" "target_state_machine" {
+  name       = "eb-pipes-sqs-sf-TargetStateMachine"
+  role_arn   = aws_iam_role.target_state_machine_role.arn
+  type       = "EXPRESS"
+  definition = templatefile("workflow/sqs-pipes-sfn.asl.json", {})
+  logging_configuration {
+    log_destination        = "${aws_cloudwatch_log_group.target_state_machine_log_group.arn}:*"
+    include_execution_data = true
+    level                  = "ALL"
+  }
+}
+
+
+# Role for EventBridge Pipes to read from SQS and launch SFN
+resource "aws_iam_role" "event_bridge_pipes_role" {
+  name = "eb-pipes-sqs-sf-EventBridgePipesRole"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = {
+      Effect = "Allow"
+      Action = "sts:AssumeRole"
+      Principal = {
+        Service = "pipes.amazonaws.com"
+      }
+    }
+  })
+
+  inline_policy {
+    name = "CloudWatchLogs"
+    policy = jsonencode({
+      Version = "2012-10-17"
+      Statement = [
+        {
+          Effect = "Allow"
+          Action = [
+            "logs:CreateLogGroup",
+            "logs:CreateLogStream",
+            "logs:PutLogEvents"
+          ],
+          Resource = ["*"]
+        },
+      ]
+    })
+  }
+  inline_policy {
+    name = "ReadSQS"
+    policy = jsonencode({
+      Version = "2012-10-17"
+      Statement = [
+        {
+          Effect = "Allow"
+          Action = [
+            "sqs:ReceiveMessage",
+            "sqs:DeleteMessage",
+            "sqs:GetQueueAttributes"
+          ],
+          Resource = [aws_sqs_queue.source_queue.arn, ]
+        },
+      ]
+    })
+  }
+  inline_policy {
+    name = "ExecuteSFN"
+    policy = jsonencode({
+      Version = "2012-10-17"
+      Statement = [
+        {
+          Effect = "Allow"
+          Action = [
+            "states:StartExecution"
+          ],
+          Resource = [aws_sfn_state_machine.target_state_machine.arn, ]
+        },
+      ]
+    })
+  }
+}
+
+# EventBridge Pipes
+resource "aws_pipes_pipe" "sqs_to_sfn" {
+  name     = "SqsToSfnPipe"
+  role_arn = aws_iam_role.event_bridge_pipes_role.arn
+  source   = aws_sqs_queue.source_queue.arn
+  source_parameters {
+    sqs_queue_parameters {
+      batch_size = 1
+    }
+  }
+  target = aws_sfn_state_machine.target_state_machine.arn
+  target_parameters {
+    step_function_state_machine_parameters {
+      invocation_type = "FIRE_AND_FORGET"
+    }
+  }
+}
+
+# Outputs
+output "source_queue_url" {
+  description = "Source SQS Queue URL"
+  value       = aws_sqs_queue.source_queue.id
+}
+output "sfn_log_group_arn" {
+  description = "StepFunctions LogGroup ARN"
+  value       = aws_cloudwatch_log_group.target_state_machine_log_group.arn
+}

eventbridge-pipes-sqs-to-stepfunctions-terraform/workflow/sqs-pipes-sfn.asl.json

@@ -0,0 +1,10 @@
+{
+  "Comment": "Sample SF to call with SQS through EventBridge pipes",
+  "StartAt": "Pass",
+  "States": {
+    "Pass": {
+      "Type": "Pass",
+      "End": true
+    }
+  }
+}