Merge pull request #1468 from shankey28/shankey28-feature-cdk-s3-sqs-…

…lambda-dynamodb

cdk-s3-sqs-lambda-dynamodb/.gitignore

@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+# CDK asset staging directory
+.cdk.staging
+cdk.out
+!./data/

cdk-s3-sqs-lambda-dynamodb/README.md

@@ -0,0 +1,76 @@
+# AWS pattern for S3 to SQS to Lambda and DynamoDB
+
+This CDK pattern creates a serverless data processing workflow using Amazon S3, Amazon SQS, AWS Lambda and Amazon DynamoDB.
+
+Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example.
+
+## Requirements
+
+* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources.
+* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured
+* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+* [Getting started with the AWS CDK](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html)
+
+## Deployment Instructions
+
+ 1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository:
+    ```
+    git clone https://github.com/aws-samples/serverless-patterns
+    ```
+ 2. Change directory to the pattern directory:
+     ```
+     cd cdk-s3-sqs-lambda-dynamodb
+     ```
+ 3. Install dependencies:
+     ```
+     npm install
+     ```
+
+ 4. Synthesize CloudFormation template from the AWS CDK app:
+     ```
+     cdk synth
+     ```
+ 5. Deploy the stack to your default AWS account and region. This command should deploy the serverless workflow to your AWS account.
+     ```
+     cdk deploy
+     ```
+ 6. Browse to the AWS cloudformation console to verify successful deployment of the stack
+
+## How it works
+
+1. The AWS CDK  template deploys S3 bucket, SQS queue, lambda and DynamoDB table 
+2. SQS queue is configured to listen to s3 create object events
+3. Lambda function is configured to read messages from the SQS queue, process messages and insert records into the DynamoDB table named EmployeeInfo.
+
+## Testing
+
+Use the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) to invoke the step function. The function name is in the outputs of the AWS CDK deployment
+
+ 1. Upload the sample csv file from data folder to the s3 bucket name from the output of cloudformation stack
+    ```
+    s3 cp ./data/test.csv s3://BUCKET_NAME/test.csv
+    ```
+ 2. View S3 bucket to see files created in the previously empty bucket: You should see a file has been saved to the S3 bucket:
+    ```
+    Verify the Lambda function CloudWatch logs for successfull processing of csv file
+    Verify the DynamoDB table that the expected record was inserted
+    ```
+
+## Cleanup
+
+ 1. Delete the stack
+    ```
+    cdk destroy
+    ```
+ 2. Confirm the stack has been deleted
+    ```
+    aws cloudformation list-stacks --query "StackSummaries[?contains(StackName,'ENTER_STACK_NAME')].StackStatus"
+    ```
+
+## Useful commands
+
+ * `cdk ls`          list all stacks in the app
+ * `cdk synth`       emits the synthesized CloudFormation template
+ * `cdk deploy`      deploy this stack to your default AWS account/region
+ * `cdk diff`        compare deployed stack with current state
+ * `cdk docs`        open CDK documentation

cdk-s3-sqs-lambda-dynamodb/bin/my-cdk-project.ts

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+import 'source-map-support/register';
+import * as cdk from 'aws-cdk-lib';
+import { MyCdkProjectStack } from '../lib/my-cdk-project-stack';
+
+const app = new cdk.App();
+new MyCdkProjectStack(app, 'MyCdkProjectStack', {
+  /* If you don't specify 'env', this stack will be environment-agnostic.
+   * Account/Region-dependent features and context lookups will not work,
+   * but a single synthesized template can be deployed anywhere. */
+
+  /* Uncomment the next line to specialize this stack for the AWS Account
+   * and Region that are implied by the current CLI configuration. */
+  // env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
+
+  /* Uncomment the next line if you know exactly what Account and Region you
+   * want to deploy the stack to. */
+  // env: { account: '123456789012', region: 'us-east-1' },
+
+  /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */
+});

cdk-s3-sqs-lambda-dynamodb/cdk.json

@@ -0,0 +1,42 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/my-cdk-project.ts",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true,
+    "@aws-cdk/core:stackRelativeExports": true,
+    "@aws-cdk/aws-rds:lowercaseDbIdentifier": true,
+    "@aws-cdk/aws-lambda:recognizeVersionProps": true,
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true,
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ]
+  }
+}

cdk-s3-sqs-lambda-dynamodb/data/test.csv

@@ -0,0 +1,3 @@
+id,name,email
+1,shashi,[email protected]
+2,neha,[email protected]

cdk-s3-sqs-lambda-dynamodb/example-pattern.json

@@ -0,0 +1,58 @@
+{
+  "title": "Process files uploaded to S3 with SQS, Lambda and DynamoDB",
+  "description": "Creates a serverless pattern to store and process csv data using Amazon S3, Amazon SQS, AWS Lambda and Amazon DynamoDB services",
+  "language": "Python",
+  "level": "200",
+  "framework": "CDK",
+  "introBox": {
+    "headline": "How it works",
+    "text": [
+       "This AWS CDK template deploys an S3 bucket, SQS queue, Lambda function and DynamoDB table.",
+       "The SQS queue is configured to listen to S3 create object events",
+       "Lambda function is configured to read messages from the SQS queues, process them and insert records into DynamoDB table named EmployeeInfo."
+    ]
+  },
+  "gitHub": {
+    "template": {
+      "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/cdk-s3-sqs-lambda-dynamodb",
+      "templateURL": "serverless-patterns/cdk-s3-sqs-lambda-dynamodb",
+      "projectFolder": "cdk-s3-sqs-lambda-dynamodb",
+      "templateFile": "cdk-s3-sqs-lambda-dynamodb/lib/my-cdk-project-stack.ts"
+    }
+  },
+  "resources": {
+    "bullets": [
+      {
+        "text": "S3 event notification types and detinations",
+        "link": "https://docs.aws.amazon.com/AmazonS3/latest/userguide/notification-how-to-event-types-and-destinations.html"
+      },
+      {
+        "text": "Configuring SQS queue to use with Lambda",
+        "link": "https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#events-sqs-queueconfig"
+      }
+    ]
+  },
+  "deploy": {
+    "text": [
+      "Create Stack: <code>cdk deploy</code>"
+    ]
+  },
+  "testing": {
+    "text": [
+      "See the GitHub repo for detailed testing instructions."
+    ]
+  },
+  "cleanup": {
+    "text": [
+      "Delete the stack: <code>cdk destroy</code>."
+    ]
+  },
+  "authors": [
+    {
+      "name": "Shashi Shankar",
+      "image": "./assets/shankey28.jpg",
+      "bio": "Shashi Shankar is a Sr Cloud Architect at PwC Advisory services",
+      "linkedin": "https://www.linkedin.com/in/shashi-shankar-3a61a0a/"
+    }
+  ]
+}

cdk-s3-sqs-lambda-dynamodb/jest.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  }
+};

cdk-s3-sqs-lambda-dynamodb/lambda/index.py

@@ -0,0 +1,37 @@
+import csv
+import boto3
+import json
+
+s3 = boto3.client('s3')
+dynamodb = boto3.resource('dynamodb')
+table_name = 'employeeInfoNew'
+
+def lambda_handler(event, context):
+    table = dynamodb.Table(table_name)
+    # Iterate through the records in the event
+    for record in event['Records']:
+        body = json.loads(record['body'])
+        bucket = body['Records'][0]['s3']['bucket']['name']
+        key = body['Records'][0]['s3']['object']['key']
+        # Fetch the CSV file from S3
+        response = s3.get_object(Bucket=bucket, Key=key)
+        csv_data = response['Body'].read().decode('utf-8').splitlines()
+
+        # Parse the CSV file
+        csv_reader = csv.DictReader(csv_data)
+        for row in csv_reader:
+
+            name = row['name']
+            email = row['email']
+
+            item = {
+
+                'name': row['name'],
+                'email': row['email']
+            }
+
+            print(f"Received record: name={name}, email={email}")
+                        # Insert the item into DynamoDB table
+            table.put_item(Item=item)
+
+    print('CSV processing completed.')

cdk-s3-sqs-lambda-dynamodb/lib/my-cdk-project-stack.ts

@@ -0,0 +1,75 @@
+import * as cdk from 'aws-cdk-lib';
+import { Construct } from 'constructs';
+import * as s3 from 'aws-cdk-lib/aws-s3';
+import * as sqs from 'aws-cdk-lib/aws-sqs';
+import * as s3n from 'aws-cdk-lib/aws-s3-notifications';
+import * as lambda from 'aws-cdk-lib/aws-lambda';
+import * as sqsevts from 'aws-cdk-lib/aws-lambda-event-sources';
+import * as iam from 'aws-cdk-lib/aws-iam'
+import * as dynamodb from 'aws-cdk-lib/aws-dynamodb';
+
+export class MyCdkProjectStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
+    super(scope, id, props);
+     // Create an S3 bucket
+    const bucket = new s3.Bucket(this, 'MyBucket', {
+      removalPolicy: cdk.RemovalPolicy.DESTROY,
+    });
+
+    // Create an SQS queue
+    const queue = new sqs.Queue(this, 'MyQueue', {
+      visibilityTimeout: cdk.Duration.seconds(300),
+    });
+
+    // Add S3 object created event notification to the SQS queue
+    bucket.addObjectCreatedNotification(new s3n.SqsDestination(queue));
+
+    // Create DynamoDB table
+    const table = new dynamodb.Table(this, 'EmployeeInfoTable', {
+      tableName: 'employeeInfoNew',
+      partitionKey: { name: 'email', type: dynamodb.AttributeType.STRING },
+      removalPolicy: cdk.RemovalPolicy.DESTROY,
+    });
+
+    // Create a Lambda function
+    const lambdaFn = new lambda.Function(this, 'MyLambdaFunction', {
+      runtime: lambda.Runtime.PYTHON_3_8,
+      handler: 'index.lambda_handler',
+      code: lambda.Code.fromAsset('lambda'),
+    });
+
+    // Add SQS queue as an event source for the Lambda function
+    lambdaFn.addEventSource(new sqsevts.SqsEventSource(queue));
+
+        // Grant S3 read permission to the Lambda function
+    const s3ReadWritePolicy = new iam.PolicyStatement({
+      effect: iam.Effect.ALLOW,
+      actions: ['s3:GetObject'],
+      resources: [bucket.bucketArn + '/*'],
+    });
+    lambdaFn.addToRolePolicy(s3ReadWritePolicy);
+
+    // Grant read/write permissions to the DynamoDB table
+    table.grantReadWriteData(lambdaFn);
+
+    // Output the resource names
+    new cdk.CfnOutput(this, 'BucketNameOutput', {
+      value: bucket.bucketName,
+      description: 'S3 Bucket Name',
+    });
+    new cdk.CfnOutput(this, 'QueueNameOutput', {
+      value: queue.queueName,
+      description: 'SQS Queue Name',
+    });
+    new cdk.CfnOutput(this, 'TableNameOutput', {
+      value: table.tableName,
+      description: 'DynamoDB Table Name',
+    });
+    new cdk.CfnOutput(this, 'FunctionNameOutput', {
+      value: lambdaFn.functionName,
+      description: 'Lambda Function Name',
+    });
+
+
+  }
+}

cdk-s3-sqs-lambda-dynamodb/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "my-cdk-project",
+  "version": "0.1.0",
+  "bin": {
+    "my-cdk-project": "bin/my-cdk-project.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "watch": "tsc -w",
+    "test": "jest",
+    "cdk": "cdk"
+  },
+  "devDependencies": {
+    "@types/jest": "^27.5.2",
+    "@types/node": "10.17.27",
+    "@types/prettier": "2.6.0",
+    "aws-cdk": "2.46.0",
+    "jest": "^27.5.1",
+    "ts-jest": "^27.1.4",
+    "ts-node": "^10.9.1",
+    "typescript": "~3.9.7"
+  },
+  "dependencies": {
+    "aws-cdk-lib": "2.46.0",
+    "constructs": "^10.0.0",
+    "csv-parser": "^3.0.0",
+    "source-map-support": "^0.5.21"
+  }
+}

cdk-s3-sqs-lambda-dynamodb/test/my-cdk-project.test.ts

@@ -0,0 +1,17 @@
+// import * as cdk from 'aws-cdk-lib';
+// import { Template } from 'aws-cdk-lib/assertions';
+// import * as MyCdkProject from '../lib/my-cdk-project-stack';
+
+// example test. To run these tests, uncomment this file along with the
+// example resource in lib/my-cdk-project-stack.ts
+test('SQS Queue Created', () => {
+//   const app = new cdk.App();
+//     // WHEN
+//   const stack = new MyCdkProject.MyCdkProjectStack(app, 'MyTestStack');
+//     // THEN
+//   const template = Template.fromStack(stack);
+
+//   template.hasResourceProperties('AWS::SQS::Queue', {
+//     VisibilityTimeout: 300
+//   });
+});