Fix docs and tests

Sage-Bionetworks-IT · Oct 2, 2024 · 2b063fb · 2b063fb
1 parent 361ff8b
commit 2b063fb
Show file tree

Hide file tree

Showing 5 changed files with 126 additions and 66 deletions.
diff --git a/.github/actions/sam-build/action.yaml → ...ub/actions/sam-build-and-lint/action.yaml b/.github/actions/sam-build/action.yaml → ...ub/actions/sam-build-and-lint/action.yaml
@@ -1,4 +1,4 @@
-name: sam-build
+name: sam-build-and-lint
 
 runs:
   # This creates a composite action to be used as a step in a job
@@ -21,6 +21,14 @@ runs:
       with:
         use-installer: true
 
+    # Lint the input template
+    - run: sam validate --lint
+      shell: bash
+
     # Use a lambda-like docker container to build the lambda artifact
     - run: sam build --use-container
       shell: bash
+
+    # Lint the built template
+    - run: sam validate --lint --template .aws-sam/build/template.yaml
+      shell: bash
diff --git a/.github/workflows/post-merge.yaml b/.github/workflows/post-merge.yaml
@@ -24,8 +24,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      # Install sam-cli and run "sam build"
-      - uses: ./.github/actions/sam-build
+      # install sam-cli, build, and lint
+      - uses: ./.github/actions/sam-build-and-lint
 
       # authenticate with AWS via OIDC
       - uses: aws-actions/configure-aws-credentials@v4
@@ -38,5 +38,8 @@ jobs:
       # upload the lambda artifact to s3 and generate a cloudformation template referencing it
       - run: sam package --template-file .aws-sam/build/template.yaml --s3-bucket $ESSENTIALS_BUCKET --s3-prefix ${{ github.event.repository.name }}/${{ github.ref_name }} --output-template-file .aws-sam/build/${{ github.event.repository.name }}.yaml
 
-      # upload the generated cloudformation template to s3
+      # validate final template with cloudformation
+      - run: aws cloudformation validate-template --template-body file://.aws-sam/build/${{ github.event.repository.name }}.yaml
+
+      # upload the final template to s3
       - run: aws s3 cp .aws-sam/build/${{ github.event.repository.name }}.yaml s3://$BOOTSTRAP_BUCKET/${{ github.event.repository.name }}/${{ github.ref_name }}/
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -38,5 +38,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/sam-build
-      - run: sam validate --lint --template .aws-sam/build/template.yaml
+      - uses: ./.github/actions/sam-build-and-lint
diff --git a/README.md b/README.md
@@ -11,23 +11,12 @@ type totals, then email a report of the results to the given recipients.
 
 | Parameter Name     | Allowed Values                          | Default Value         | Description                                  |
 | ------------------ | --------------------------------------- | --------------------- | -------------------------------------------- |
+| Sender             | SES verified identity                   | Required Value        | Value to use for the `From` email <br/>field |
+| Recipients         | Comma-delimited list of email addresses | Required Value        | The list of email recipients                 |
+| OmitCostsLessThan  | Floating-point number                   | `0.01`                | Totals less than this amount will be ignored |
 | ScheduleExpression | EventBridge Schedule Expression         | `cron(30 10 2 * ? *)` | Schedule for running the lambda              |
-| MinimumValue       | Floating-point number                   | `0.01`                | Totals less than this amount will be ignored |
-| SenderEmail        | SES verified identity                   | `''`                  | Value to use for the `From` email <br/>field |
-| Recipients         | Comma-delimited list of email addresses | `''`                  | The list of email recipients                 |
 
-#### ScheduleExpression
-
-[EventBridge schedule expression](https://docs.aws.amazon.com/lambda/latest/dg/services-cloudwatchevents-expressions.html)
-describing how often to run the lambda. By default it runs at 10:30am UTC on the
-2nd of each month.
-
-#### MinimumValue
-
-Don't send an email if the reported monthly total is less than this amount, by
-default $0.01.
-
-#### SenderEmail
+#### Sender
 
 This email address will appear is the `From` field, and must be
 [verified](https://docs.aws.amazon.com/ses/latest/dg/creating-identities.html)
@@ -37,6 +26,16 @@ before emails will successfully send.
 
 The list of email recipients for email reports.
 
+#### OmitCostsLessThan
+
+Don't include totals less than this amount in the report.
+
+#### ScheduleExpression
+
+[EventBridge schedule expression](https://docs.aws.amazon.com/lambda/latest/dg/services-cloudwatchevents-expressions.html)
+describing how often to run the lambda. By default it runs at 10:30am UTC on the
+2nd of each month.
+
 ### Triggering
 
 The lambda is configured to run on a schedule, by default at 10:30am UTC on the
@@ -59,8 +58,8 @@ Install the following applications:
 - [pre-commit](https://github.com/pre-commit/pre-commit)
 - [pipenv](https://github.com/pypa/pipenv)
 
-Check in [.travis.yml](./.travis.yml) to see how they are installed for this
-repo.
+Check in the [lambda-test](.github/workflows/test.yaml) workflow to see how they
+are installed for automated testing.
 
 ### Install Requirements
 
@@ -111,18 +110,67 @@ managed with `pipenv`. Install the development dependencies and run the tests
 using `coverage`.
 
 ```shell script
-$ pipenv run coverage run -m pytest tests/ -svv
+$ coverage run -m pytest tests/ -svv
+```
+
+And to view the coverage report:
+
+```shell script
+$ coverage report -m
 ```
 
 Automated testing will upload coverage results to [Coveralls](coveralls.io).
 
+### Lint and validate Cloudformation templates
+
+## Lint input template with SAM CLI
+
+Lint the SAM input template (`template.yaml`) using the SAM CLI.
+
+```shell script
+$ sam validate --lint
+```
+
+## Validate input template with SAM CLI
+
+Validate the SAM input template using the SAM CLI. Requires AWS authentication.
+
+```shell script
+$ sam validate
+```
+
+## Validate SAM input template with AWS CLI
+
+Validate the SAM input template using the AWS CLI. Requires AWS Authentication.
+
+```shell script
+$ aws cloudformation validate-template --template-body file://template.yaml
+```
+
+## Validate SAM build template with AWS CLI
+
+Validate the SAM build template using the AWS CLI. Requires AWS Authentication.
+
+```shell script
+$ aws cloudformation validate-template --template-body file://.aws-sam/build/template.yaml
+```
+
+## Validate SAM package template with AWS CLI
+
+Validate the SAM package template using the AWS CLI. Requires AWS
+Authentication.
+
+```shell script
+$ aws cloudformation validate-template --template-body file://.aws-sam/build/output.yaml
+```
+
 ### Run integration tests
 
 Running integration tests
 [requires docker](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-cli-command-reference-sam-local-start-api.html)
 
 ```shell script
-$ sam local invoke HelloWorldFunction --event events/event.json
+$ sam local invoke MonthlyS3Usage --event events/event.json
 ```
 
 ## Deployment
@@ -138,9 +186,9 @@ which requires permissions to upload to Sage
 ```shell script
 sam package --template-file .aws-sam/build/template.yaml \
   --s3-bucket essentials-awss3lambdaartifactsbucket-x29ftznj6pqw \
-  --output-template-file .aws-sam/build/lambda-template.yaml
+  --output-template-file .aws-sam/build/output.yaml
 
-aws s3 cp .aws-sam/build/lambda-template.yaml s3://bootstrap-awss3cloudformationbucket-19qromfd235z9/lambda-template/master/
+aws s3 cp .aws-sam/build/output.yaml s3://bootstrap-awss3cloudformationbucket-19qromfd235z9/lambda-finops-s3-cost-report/VERSION/
 ```
 
 ## Publish Lambda
@@ -152,7 +200,7 @@ accessible in the
 [serverless application repository](https://console.aws.amazon.com/serverlessrepo).
 
 ```shell script
-sam publish --template .aws-sam/build/lambda-template.yaml
+sam publish --template .aws-sam/build/output.yaml
 ```
 
 ### Public access
@@ -168,29 +216,32 @@ aws serverlessrepo put-application-policy \
 
 ## Install Lambda into AWS
 
-When using AWS Organizations, the lambda should be deployed once in the master
-account to aggregate all costs from the member accounts. Otherwise it must be
-deployed into each separate account, resulting in a separate email for each
-account total.
+This lambda is intended to run in a stand-alone account. When using AWS
+Organizations, deploying the lambda to the payer account will aggregate costs
+from all member accounts. To get costs for a single member account, deploy the
+lambda to that member account.
 
 ### Sceptre
 
 Create the following [sceptre](https://github.com/Sceptre/sceptre) file
-config/prod/lambda-template.yaml
+config/prod/lambda-finops-s3-cost-report.yaml
 
 ```yaml
 template:
   type: http
-  url: "https://PUBLISH_BUCKET.s3.amazonaws.com/lambda-template/VERSION/lambda-template.yaml"
-stack_name: "lambda-template"
+  url: "https://PUBLISH_BUCKET.s3.amazonaws.com/lambda-finops-s3-cost-report/VERSION/lambda-finops-s3-cost-report.yaml"
+stack_name: "lambda-finops-s3-cost-report"
 stack_tags:
   OwnerEmail: "[email protected]"
+parameters:
+  Sender: "[email protected]"
+  Recipients: "[email protected]"
 ```
 
 Install the lambda using sceptre:
 
 ```shell script
-sceptre --var "profile=my-profile" --var "region=us-east-1" launch prod/lambda-template.yaml
+sceptre --var "profile=my-profile" --var "region=us-east-1" launch prod/lambda-finops-s3-cost-report.yaml
 ```
 
 ### AWS Console
@@ -232,17 +283,17 @@ prior to the first run of the lambda.
 #### Canary Run
 
 Once the needed addresses have been verified, the lambda should be tested with a
-canary run, restricting output to a list of approved canary users by using the
-`RestrictRecipients` and `ApprovedRecipients` parameters.
+canary run by sending to test addresses.
 
 ```yaml
 template:
   type: http
-  url: "https://PUBLISH_BUCKET.s3.amazonaws.com/lambda-template/VERSION/lambda-template.yaml"
-stack_name: "lambda-template"
+  url: "https://PUBLISH_BUCKET.s3.amazonaws.com/lambda-finops-s3-cost-report/VERSION/lambda-finops-s3-cost-report.yaml"
+stack_name: "lambda-finops-s3-cost-report"
 stack_tags:
   OwnerEmail: "[email protected]"
 parameters:
+  Sender: "[email protected]"
   Recipients: "[email protected],[email protected]"
 ```
 
@@ -260,10 +311,11 @@ recipient restrictions and with any other needed parameters.
 ```yaml
 template:
   type: http
-  url: "https://PUBLISH_BUCKET.s3.amazonaws.com/lambda-template/VERSION/lambda-template.yaml"
-stack_name: "lambda-template"
+  url: "https://PUBLISH_BUCKET.s3.amazonaws.com/lambda-finops-s3-cost-report/VERSION/lambda-finops-s3-cost-report.yaml"
+stack_name: "lambda-finops-s3-cost-report"
 stack_tags:
   OwnerEmail: "[email protected]"
 parameters:
+  Sender: "[email protected]"
   Recipients: "[email protected],[email protected]"
 ```
diff --git a/template.yaml b/template.yaml
@@ -4,18 +4,6 @@ Description: >
   A lambda for sending monthly cloud-spend emails for STRIDES accounts
 
 Parameters:
-  ScheduleExpression:
-    Type: String
-    Description: Schedule expressions using cron
-    Default: cron(30 10 2 * ? *)
-
-  MinimumValue:
-    Type: String
-    Description: 'Minimum total to send an email. Default: $0.01'
-    Default: '0.01'
-    AllowedPattern: '^\d+(\.\d+)?$'
-    ConstraintDescription: 'must be a floating point number'
-
   Sender:
     Type: String
     Description: Sender email address
@@ -24,14 +12,24 @@ Parameters:
     Type: String
     Description: Comma-separated list of email recipients
 
+  OmitCostsLessThan:
+    Type: Number
+    Description: 'Totals less than this amount will not be reported. Default: $0.01'
+    Default: '0.01'
+
+  ScheduleExpression:
+    Type: String
+    Description: EventBridge Schedule Expression
+    Default: cron(30 10 2 * ? *)
+
 # More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst
 Globals:
   Function:
     Timeout: 120
 
 Resources:
 #lambda execution role config
-  MonthlyServicesUsageLambdaRole:
+  MonthlyS3UsageLambdaRole:
     Type: AWS::IAM::Role
     Properties:
       AssumeRolePolicyDocument:
@@ -59,20 +57,20 @@ Resources:
               Resource: "*"
               Effect: Allow
 
-#This Lambda Function will fetch Billing details for AWS linked Account and will send it over mail.
-  MonthlyServicesUsage:
+# This Lambda will query Cost Explorer for costs related to S3
+  MonthlyS3Usage:
     Type: AWS::Serverless::Function
     Properties:
       Handler: s3_cost_report/app.lambda_handler
       CodeUri: .
       Runtime: python3.11
       MemorySize: 128
-      Role: !GetAtt MonthlyServicesUsageLambdaRole.Arn
+      Role: !GetAtt MonthlyS3UsageLambdaRole.Arn
       Environment:
         Variables:
           SENDER: !Ref Sender
           RECIPIENTS: !Ref Recipients
-          MINIMUM: !Ref MinimumValue
+          MINIMUM: !Ref OmitCostsLessThan
       Events:
         ScheduledEventTrigger:
           Type: Schedule
@@ -83,13 +81,13 @@ Resources:
     Type: AWS::Lambda::Permission
     Properties:
       Action: 'lambda:InvokeFunction'
-      FunctionName: !Ref MonthlyServicesUsage
+      FunctionName: !Ref MonthlyS3Usage
       Principal: 'events.amazonaws.com'
 
 Outputs:
-  MonthlyServicesUsageFunctionArn:
-    Description: "MonthlyServicesUsage Lambda Function ARN"
-    Value: !GetAtt MonthlyServicesUsage.Arn
-  MonthlyServicesUsageFunctionRoleArn:
-    Description: "IAM Role created for MonthlyServicesUsage function"
-    Value: !GetAtt MonthlyServicesUsageLambdaRole.Arn
+  MonthlySe3UsageFunctionArn:
+    Description: "MonthlyS3Usage Lambda Function ARN"
+    Value: !GetAtt MonthlyS3Usage.Arn
+  MonthlyS3UsageFunctionRoleArn:
+    Description: "IAM Role created for MonthlyS3Usage function"
+    Value: !GetAtt MonthlyS3UsageLambdaRole.Arn