Release/0.1.0

.github/workflows/deploy-generate.yml

@@ -0,0 +1,93 @@
+# Main build pipeline that verifies, builds, and deploys the software
+name: Build and Deploy
+# Events that trigger the workflow
+on:
+  # Trigger based on push to all branches
+  push:
+    branches:
+      - 'development'
+      - 'feature/**'
+      - 'release/**'
+      - 'main'
+    tags-ignore:
+      - '*'
+  # Run workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build and Deploy
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest    
+    steps:
+
+      # SIT environment variables
+      - name: Set Environment Variables
+        if: |
+          startsWith(github.ref, 'refs/heads/development') ||
+          startsWith(github.ref, 'refs/heads/feature')
+        run: |
+          echo "TARGET_ENV=SIT" >> $GITHUB_ENV
+          echo "PREFIX_ENV=service-generate-sit" >> $GITHUB_ENV
+
+      # UAT environment variables
+      - name: Set Environment Variables
+        if: startsWith(github.ref, 'refs/heads/release') 
+        run: |
+          echo "TARGET_ENV=UAT" >> $GITHUB_ENV
+          echo "PREFIX_ENV=service-generate-uat" >> $GITHUB_ENV
+
+      # OPS environment variables
+      - name: Set Environment Variables
+        if: startsWith(github.ref, 'refs/heads/main') 
+        run: |
+          echo "TARGET_ENV=OPS" >> $GITHUB_ENV
+          echo "PREFIX_ENV=service-generate-ops" >> $GITHUB_ENV
+
+      # Check out GitHub repo
+      - uses: actions/checkout@v3
+
+      # SNYK scan and report
+      - name: Run Snyk to test and report
+        uses: snyk/actions/iac@master
+        env:
+            SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          command: test
+          args: >
+              --org=${{ secrets.SNYK_ORG_ID }}
+              --severity-threshold=high
+              --report
+
+      # Set up Terraform
+      - name: Setup Terraform
+        uses: hashicorp/[email protected]
+        with:
+          terraform_version: 1.3.7
+
+      # Validate Terraform file
+      - name: Validate Terraform
+        run: terraform validate -no-color
+
+      # Set up TF_VAR and AWS credentials environment variables
+      - name: TF_VAR and AWS credentials
+        run: |
+          echo "TF_VAR_environment=$TARGET_ENV" >> $GITHUB_ENV
+          echo "TF_VAR_prefix=$PREFIX_ENV" >> $GITHUB_ENV
+          echo "TF_VAR_cross_account_id=${{ secrets[format('CROSS_ACCOUNT_ID_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV
+          echo "TF_VAR_sns_topic_email=${{ secrets[format('SNS_TOPIC_EMAIL_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV
+          echo "TF_VAR_sns_topic_email_alarms=${{ secrets[format('SNS_TOPIC_EMAIL_ALARMS_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV
+          echo "AWS_ACCESS_KEY_ID=${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV
+          echo "AWS_DEFAULT_REGION=us-west-2" >> $GITHUB_ENV
+
+      # Deploy Terraform
+      - name: Deploy Terraform
+        working-directory: terraform/
+        run: |
+          terraform init -reconfigure \
+              -backend-config="bucket=${PREFIX_ENV}-tf-state" \
+              -backend-config="key=generate.tfstate" \
+              -backend-config="region=${AWS_DEFAULT_REGION}"
+
+          terraform apply -auto-approve 

terraform/generate-batch.tf

@@ -30,7 +30,7 @@ resource "aws_batch_compute_environment" "generate_aqua" {
       launch_template_id = aws_launch_template.aws_batch_ce_lt.id
       version            = aws_launch_template.aws_batch_ce_lt.latest_version
     }
-    max_vcpus          = 32
+    max_vcpus          = 128
     min_vcpus          = 0
     security_group_ids = data.aws_security_groups.vpc_default_sg.ids
     subnets            = data.aws_subnets.private_application_subnets.ids
@@ -44,7 +44,9 @@ resource "aws_batch_compute_environment" "generate_aqua" {
   type         = "MANAGED"
 
   depends_on = [
-    aws_iam_role.aws_batch_service_role
+    aws_iam_role.aws_batch_service_role,
+    aws_iam_policy.batch_service_role_policy,
+    aws_iam_role_policy_attachment.aws_batch_service_role_policy_attach
   ]
 }
 
@@ -84,7 +86,7 @@ resource "aws_batch_compute_environment" "generate_terra" {
       launch_template_id = aws_launch_template.aws_batch_ce_lt.id
       version            = aws_launch_template.aws_batch_ce_lt.latest_version
     }
-    max_vcpus          = 32
+    max_vcpus          = 128
     min_vcpus          = 0
     security_group_ids = data.aws_security_groups.vpc_default_sg.ids
     subnets            = data.aws_subnets.private_application_subnets.ids
@@ -98,7 +100,9 @@ resource "aws_batch_compute_environment" "generate_terra" {
   type         = "MANAGED"
 
   depends_on = [
-    aws_iam_role.aws_batch_service_role
+    aws_iam_role.aws_batch_service_role,
+    aws_iam_policy.batch_service_role_policy,
+    aws_iam_role_policy_attachment.aws_batch_service_role_policy_attach
   ]
 }
 
@@ -138,7 +142,7 @@ resource "aws_batch_compute_environment" "generate_viirs" {
       launch_template_id = aws_launch_template.aws_batch_ce_lt.id
       version            = aws_launch_template.aws_batch_ce_lt.latest_version
     }
-    max_vcpus          = 32
+    max_vcpus          = 128
     min_vcpus          = 0
     security_group_ids = data.aws_security_groups.vpc_default_sg.ids
     subnets            = data.aws_subnets.private_application_subnets.ids
@@ -152,7 +156,9 @@ resource "aws_batch_compute_environment" "generate_viirs" {
   type         = "MANAGED"
 
   depends_on = [
-    aws_iam_role.aws_batch_service_role
+    aws_iam_role.aws_batch_service_role,
+    aws_iam_policy.batch_service_role_policy,
+    aws_iam_role_policy_attachment.aws_batch_service_role_policy_attach
   ]
 }
 

terraform/generate-cw.tf

@@ -1,46 +1,77 @@
+# CloudWatch Alarm
+resource "aws_cloudwatch_metric_alarm" "aws_cloudwatch_ec2_vcpu_alarm" {
+  alarm_name          = "${var.prefix}-ec2-vcpu-alarm"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = "1"
+  threshold           = "85"
+  alarm_description   = "Alarm for when EC2 vCPU usage passes the 85% threshold for all available vCPUs in the account."
+  alarm_actions       = [aws_sns_topic.aws_sns_topic_cloudwatch_alarms.arn]
+  metric_query {
+    id          = "e1"
+    expression  = "m1/SERVICE_QUOTA(m1)*100"
+    label       = "Percentage"
+    return_data = "true"
+  }
+  metric_query {
+    id = "m1"
+    metric {
+      metric_name = "ResourceCount"
+      namespace   = "AWS/Usage"
+      period      = "180"
+      stat        = "Average"
+      dimensions = {
+        Type     = "Resource"
+        Service  = "EC2"
+        Resource = "vCPU"
+        Class    = "Standard/OnDemand"
+      }
+    }
+  }
+}
+
 # CloudWatch Logs
 
 # Downloader
 resource "aws_cloudwatch_log_group" "generate_cw_log_group_downloader" {
   name              = "/aws/batch/job/${var.prefix}-downloader/"
-  retention_in_days = 120
+  retention_in_days = 0
 }
 
 resource "aws_cloudwatch_log_group" "generate_cw_log_group_downloader_error" {
   name              = "/aws/batch/job/${var.prefix}-downloader-errors/"
-  retention_in_days = 120
+  retention_in_days = 0
 }
 
 # Combiner
 resource "aws_cloudwatch_log_group" "generate_cw_log_group_combiner" {
   name              = "/aws/batch/job/${var.prefix}-combiner/"
-  retention_in_days = 120
+  retention_in_days = 0
 }
 
 resource "aws_cloudwatch_log_group" "generate_cw_log_group_combiner_error" {
   name              = "/aws/batch/job/${var.prefix}-combiner-errors/"
-  retention_in_days = 120
+  retention_in_days = 0
 }
 
 # Processor
 resource "aws_cloudwatch_log_group" "generate_cw_log_group_processor" {
   name              = "/aws/batch/job/${var.prefix}-processor/"
-  retention_in_days = 120
+  retention_in_days = 0
 }
 
 resource "aws_cloudwatch_log_group" "generate_cw_log_group_processor_error" {
   name              = "/aws/batch/job/${var.prefix}-processor-errors/"
-  retention_in_days = 120
+  retention_in_days = 0
 }
 
 # Uploader
 resource "aws_cloudwatch_log_group" "generate_cw_log_group_uploader" {
   name              = "/aws/batch/job/${var.prefix}-uploader/"
-  retention_in_days = 120
+  retention_in_days = 0
 }
 
 # CloudWatch Logs
 resource "aws_cloudwatch_log_group" "generate_cw_log_group_license_returner" {
   name              = "/aws/batch/job/${var.prefix}-license-returner/"
-  retention_in_days = 120
+  retention_in_days = 0
 }

terraform/generate-ecr.tf

@@ -2,6 +2,7 @@
 resource "aws_ecr_repository" "download_list_creator" {
   name                 = "${var.prefix}-download-list-creator"
   image_tag_mutability = "MUTABLE"
+  force_delete         = true
   image_scanning_configuration {
     scan_on_push = false
   }
@@ -11,6 +12,7 @@ resource "aws_ecr_repository" "download_list_creator" {
 resource "aws_ecr_repository" "partition_submit" {
   name                 = "${var.prefix}-partition-submit"
   image_tag_mutability = "MUTABLE"
+  force_delete         = true
   image_scanning_configuration {
     scan_on_push = false
   }
@@ -20,6 +22,7 @@ resource "aws_ecr_repository" "partition_submit" {
 resource "aws_ecr_repository" "downloader" {
   name                 = "${var.prefix}-downloader"
   image_tag_mutability = "MUTABLE"
+  force_delete         = true
   image_scanning_configuration {
     scan_on_push = false
   }
@@ -29,6 +32,7 @@ resource "aws_ecr_repository" "downloader" {
 resource "aws_ecr_repository" "combiner" {
   name                 = "${var.prefix}-combiner"
   image_tag_mutability = "MUTABLE"
+  force_delete         = true
   image_scanning_configuration {
     scan_on_push = false
   }
@@ -38,6 +42,7 @@ resource "aws_ecr_repository" "combiner" {
 resource "aws_ecr_repository" "processor" {
   name                 = "${var.prefix}-processor"
   image_tag_mutability = "MUTABLE"
+  force_delete         = true
   image_scanning_configuration {
     scan_on_push = false
   }
@@ -47,6 +52,7 @@ resource "aws_ecr_repository" "processor" {
 resource "aws_ecr_repository" "uploader" {
   name                 = "${var.prefix}-uploader"
   image_tag_mutability = "MUTABLE"
+  force_delete         = true
   image_scanning_configuration {
     scan_on_push = false
   }
@@ -56,6 +62,7 @@ resource "aws_ecr_repository" "uploader" {
 resource "aws_ecr_repository" "license_returner" {
   name                 = "${var.prefix}-license-returner"
   image_tag_mutability = "MUTABLE"
+  force_delete         = true
   image_scanning_configuration {
     scan_on_push = false
   }
@@ -65,6 +72,17 @@ resource "aws_ecr_repository" "license_returner" {
 resource "aws_ecr_repository" "reporter" {
   name                 = "${var.prefix}-reporter"
   image_tag_mutability = "MUTABLE"
+  force_delete         = true
+  image_scanning_configuration {
+    scan_on_push = false
+  }
+}
+
+# Purger
+resource "aws_ecr_repository" "purger" {
+  name                 = "${var.prefix}-purger"
+  image_tag_mutability = "MUTABLE"
+  force_delete         = true
   image_scanning_configuration {
     scan_on_push = false
   }