diff --git a/.github/deploy.yml b/.github/deploy.yml deleted file mode 100644 index 79568e5..0000000 --- a/.github/deploy.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This is the main build pipeline that verifies and publishes the software -name: Build -# Controls when the workflow will run -on: - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -env: - TARGET_ENV_UPPERCASE: 'SANDBOX' - -jobs: - build: - name: Deploy - # The type of runner that the job will run on - runs-on: ubuntu-latest - - steps: - - name: Set AWS Env Variables - run: echo "AWS_ACCESS_KEY_ID=${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}" >> $GITHUB_ENV | - echo "AWS_SECRET_ACCESS_KEY=${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}" >> $GITHUB_ENV - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - env: - AWS_ACCESS_KEY_ID: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }} - AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }} - with: - aws-region: us-west-2 - role-session-name: GitHubActions - - run: aws sts get-caller-identity - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 0.12.31 - terraform_wrapper: false - - - name: Deploy - run: ls -al; cd terraform ; terraform init ; terraform plan diff --git a/.github/workflows/deploy-generate.yml b/.github/workflows/deploy-generate.yml new file mode 100644 index 0000000..35c097f --- /dev/null +++ b/.github/workflows/deploy-generate.yml @@ -0,0 +1,93 @@ +# Main build pipeline that verifies, builds, and deploys the software +name: Build and Deploy +# Events that trigger the workflow +on: + # Trigger based on push to all branches + push: + branches: + - 'development' + - 'feature/**' + - 'release/**' + - 'main' + tags-ignore: + - '*' + # Run workflow manually from the Actions tab + workflow_dispatch: + +jobs: + build: + name: Build and Deploy + # The type of runner that the job will run on + runs-on: ubuntu-latest + steps: + + # SIT environment variables + - name: Set Environment Variables + if: | + startsWith(github.ref, 'refs/heads/development') || + startsWith(github.ref, 'refs/heads/feature') + run: | + echo "TARGET_ENV=SIT" >> $GITHUB_ENV + echo "PREFIX_ENV=service-generate-sit" >> $GITHUB_ENV + + # UAT environment variables + - name: Set Environment Variables + if: startsWith(github.ref, 'refs/heads/release') + run: | + echo "TARGET_ENV=UAT" >> $GITHUB_ENV + echo "PREFIX_ENV=service-generate-uat" >> $GITHUB_ENV + + # OPS environment variables + - name: Set Environment Variables + if: startsWith(github.ref, 'refs/heads/main') + run: | + echo "TARGET_ENV=OPS" >> $GITHUB_ENV + echo "PREFIX_ENV=service-generate-ops" >> $GITHUB_ENV + + # Check out GitHub repo + - uses: actions/checkout@v3 + + # SNYK scan and report + - name: Run Snyk to test and report + uses: snyk/actions/iac@master + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + command: test + args: > + --org=${{ secrets.SNYK_ORG_ID }} + --severity-threshold=high + --report + + # Set up Terraform + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2.0.3 + with: + terraform_version: 1.3.7 + + # Validate Terraform file + - name: Validate Terraform + run: terraform validate -no-color + + # Set up TF_VAR and AWS credentials environment variables + - name: TF_VAR and AWS credentials + run: | + echo "TF_VAR_environment=$TARGET_ENV" >> $GITHUB_ENV + echo "TF_VAR_prefix=$PREFIX_ENV" >> $GITHUB_ENV + echo "TF_VAR_cross_account_id=${{ secrets[format('CROSS_ACCOUNT_ID_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV + echo "TF_VAR_sns_topic_email=${{ secrets[format('SNS_TOPIC_EMAIL_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV + echo "TF_VAR_sns_topic_email_alarms=${{ secrets[format('SNS_TOPIC_EMAIL_ALARMS_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV + echo "AWS_ACCESS_KEY_ID=${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV + echo "AWS_SECRET_ACCESS_KEY=${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV)] }}" >> $GITHUB_ENV + echo "AWS_DEFAULT_REGION=us-west-2" >> $GITHUB_ENV + + # Deploy Terraform + - name: Deploy Terraform + working-directory: terraform/ + run: | + terraform init -reconfigure \ + -backend-config="bucket=${PREFIX_ENV}-tf-state" \ + -backend-config="key=generate.tfstate" \ + -backend-config="region=${AWS_DEFAULT_REGION}" + + terraform apply -auto-approve diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 8b047aa..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This is the main build pipeline that verifies and publishes the software -name: Build -# Controls when the workflow will run -on: - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -env: - TARGET_ENV_UPPERCASE: 'SANDBOX' - -jobs: - build: - name: Deploy - # The type of runner that the job will run on - runs-on: ubuntu-latest - - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - - name: Set AWS Env Variables - run: echo "AWS_ACCESS_KEY_ID=${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}" >> $GITHUB_ENV | - echo "AWS_SECRET_ACCESS_KEY=${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}" >> $GITHUB_ENV - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - env: - AWS_ACCESS_KEY_ID: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }} - AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }} - with: - aws-region: us-west-2 - role-session-name: GitHubActions - - run: aws sts get-caller-identity - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 0.12.31 - terraform_wrapper: false - - - name: Deploy - run: cd terraform ; terraform init ; terraform plan diff --git a/terraform/generate-batch.tf b/terraform/generate-batch.tf index 8643e1b..81fdb03 100644 --- a/terraform/generate-batch.tf +++ b/terraform/generate-batch.tf @@ -30,7 +30,7 @@ resource "aws_batch_compute_environment" "generate_aqua" { launch_template_id = aws_launch_template.aws_batch_ce_lt.id version = aws_launch_template.aws_batch_ce_lt.latest_version } - max_vcpus = 32 + max_vcpus = 128 min_vcpus = 0 security_group_ids = data.aws_security_groups.vpc_default_sg.ids subnets = data.aws_subnets.private_application_subnets.ids @@ -44,7 +44,9 @@ resource "aws_batch_compute_environment" "generate_aqua" { type = "MANAGED" depends_on = [ - aws_iam_role.aws_batch_service_role + aws_iam_role.aws_batch_service_role, + aws_iam_policy.batch_service_role_policy, + aws_iam_role_policy_attachment.aws_batch_service_role_policy_attach ] } @@ -84,7 +86,7 @@ resource "aws_batch_compute_environment" "generate_terra" { launch_template_id = aws_launch_template.aws_batch_ce_lt.id version = aws_launch_template.aws_batch_ce_lt.latest_version } - max_vcpus = 32 + max_vcpus = 128 min_vcpus = 0 security_group_ids = data.aws_security_groups.vpc_default_sg.ids subnets = data.aws_subnets.private_application_subnets.ids @@ -98,7 +100,9 @@ resource "aws_batch_compute_environment" "generate_terra" { type = "MANAGED" depends_on = [ - aws_iam_role.aws_batch_service_role + aws_iam_role.aws_batch_service_role, + aws_iam_policy.batch_service_role_policy, + aws_iam_role_policy_attachment.aws_batch_service_role_policy_attach ] } @@ -138,7 +142,7 @@ resource "aws_batch_compute_environment" "generate_viirs" { launch_template_id = aws_launch_template.aws_batch_ce_lt.id version = aws_launch_template.aws_batch_ce_lt.latest_version } - max_vcpus = 32 + max_vcpus = 128 min_vcpus = 0 security_group_ids = data.aws_security_groups.vpc_default_sg.ids subnets = data.aws_subnets.private_application_subnets.ids @@ -152,7 +156,9 @@ resource "aws_batch_compute_environment" "generate_viirs" { type = "MANAGED" depends_on = [ - aws_iam_role.aws_batch_service_role + aws_iam_role.aws_batch_service_role, + aws_iam_policy.batch_service_role_policy, + aws_iam_role_policy_attachment.aws_batch_service_role_policy_attach ] } diff --git a/terraform/generate-cw.tf b/terraform/generate-cw.tf index 5ee864a..1eeddf9 100644 --- a/terraform/generate-cw.tf +++ b/terraform/generate-cw.tf @@ -1,46 +1,77 @@ +# CloudWatch Alarm +resource "aws_cloudwatch_metric_alarm" "aws_cloudwatch_ec2_vcpu_alarm" { + alarm_name = "${var.prefix}-ec2-vcpu-alarm" + comparison_operator = "GreaterThanThreshold" + evaluation_periods = "1" + threshold = "85" + alarm_description = "Alarm for when EC2 vCPU usage passes the 85% threshold for all available vCPUs in the account." + alarm_actions = [aws_sns_topic.aws_sns_topic_cloudwatch_alarms.arn] + metric_query { + id = "e1" + expression = "m1/SERVICE_QUOTA(m1)*100" + label = "Percentage" + return_data = "true" + } + metric_query { + id = "m1" + metric { + metric_name = "ResourceCount" + namespace = "AWS/Usage" + period = "180" + stat = "Average" + dimensions = { + Type = "Resource" + Service = "EC2" + Resource = "vCPU" + Class = "Standard/OnDemand" + } + } + } +} + # CloudWatch Logs # Downloader resource "aws_cloudwatch_log_group" "generate_cw_log_group_downloader" { name = "/aws/batch/job/${var.prefix}-downloader/" - retention_in_days = 120 + retention_in_days = 0 } resource "aws_cloudwatch_log_group" "generate_cw_log_group_downloader_error" { name = "/aws/batch/job/${var.prefix}-downloader-errors/" - retention_in_days = 120 + retention_in_days = 0 } # Combiner resource "aws_cloudwatch_log_group" "generate_cw_log_group_combiner" { name = "/aws/batch/job/${var.prefix}-combiner/" - retention_in_days = 120 + retention_in_days = 0 } resource "aws_cloudwatch_log_group" "generate_cw_log_group_combiner_error" { name = "/aws/batch/job/${var.prefix}-combiner-errors/" - retention_in_days = 120 + retention_in_days = 0 } # Processor resource "aws_cloudwatch_log_group" "generate_cw_log_group_processor" { name = "/aws/batch/job/${var.prefix}-processor/" - retention_in_days = 120 + retention_in_days = 0 } resource "aws_cloudwatch_log_group" "generate_cw_log_group_processor_error" { name = "/aws/batch/job/${var.prefix}-processor-errors/" - retention_in_days = 120 + retention_in_days = 0 } # Uploader resource "aws_cloudwatch_log_group" "generate_cw_log_group_uploader" { name = "/aws/batch/job/${var.prefix}-uploader/" - retention_in_days = 120 + retention_in_days = 0 } # CloudWatch Logs resource "aws_cloudwatch_log_group" "generate_cw_log_group_license_returner" { name = "/aws/batch/job/${var.prefix}-license-returner/" - retention_in_days = 120 + retention_in_days = 0 } \ No newline at end of file diff --git a/terraform/generate-ecr.tf b/terraform/generate-ecr.tf index 2ddbe44..71431c5 100644 --- a/terraform/generate-ecr.tf +++ b/terraform/generate-ecr.tf @@ -2,6 +2,7 @@ resource "aws_ecr_repository" "download_list_creator" { name = "${var.prefix}-download-list-creator" image_tag_mutability = "MUTABLE" + force_delete = true image_scanning_configuration { scan_on_push = false } @@ -11,6 +12,7 @@ resource "aws_ecr_repository" "download_list_creator" { resource "aws_ecr_repository" "partition_submit" { name = "${var.prefix}-partition-submit" image_tag_mutability = "MUTABLE" + force_delete = true image_scanning_configuration { scan_on_push = false } @@ -20,6 +22,7 @@ resource "aws_ecr_repository" "partition_submit" { resource "aws_ecr_repository" "downloader" { name = "${var.prefix}-downloader" image_tag_mutability = "MUTABLE" + force_delete = true image_scanning_configuration { scan_on_push = false } @@ -29,6 +32,7 @@ resource "aws_ecr_repository" "downloader" { resource "aws_ecr_repository" "combiner" { name = "${var.prefix}-combiner" image_tag_mutability = "MUTABLE" + force_delete = true image_scanning_configuration { scan_on_push = false } @@ -38,6 +42,7 @@ resource "aws_ecr_repository" "combiner" { resource "aws_ecr_repository" "processor" { name = "${var.prefix}-processor" image_tag_mutability = "MUTABLE" + force_delete = true image_scanning_configuration { scan_on_push = false } @@ -47,6 +52,7 @@ resource "aws_ecr_repository" "processor" { resource "aws_ecr_repository" "uploader" { name = "${var.prefix}-uploader" image_tag_mutability = "MUTABLE" + force_delete = true image_scanning_configuration { scan_on_push = false } @@ -56,6 +62,7 @@ resource "aws_ecr_repository" "uploader" { resource "aws_ecr_repository" "license_returner" { name = "${var.prefix}-license-returner" image_tag_mutability = "MUTABLE" + force_delete = true image_scanning_configuration { scan_on_push = false } @@ -65,6 +72,17 @@ resource "aws_ecr_repository" "license_returner" { resource "aws_ecr_repository" "reporter" { name = "${var.prefix}-reporter" image_tag_mutability = "MUTABLE" + force_delete = true + image_scanning_configuration { + scan_on_push = false + } +} + +# Purger +resource "aws_ecr_repository" "purger" { + name = "${var.prefix}-purger" + image_tag_mutability = "MUTABLE" + force_delete = true image_scanning_configuration { scan_on_push = false } diff --git a/terraform/generate-efs.tf b/terraform/generate-efs.tf index 1700756..4412ebf 100644 --- a/terraform/generate-efs.tf +++ b/terraform/generate-efs.tf @@ -21,8 +21,8 @@ resource "aws_efs_access_point" "generate_efs_ap_ps" { file_system_id = aws_efs_file_system.generate_efs_fs.id tags = { Name = "${var.prefix}-partition-submit" } posix_user { - gid = 1000 - uid = 1000 + gid = 0 + uid = 0 } root_directory { creation_info { @@ -57,8 +57,8 @@ resource "aws_efs_access_point" "generate_efs_ap_r" { file_system_id = aws_efs_file_system.generate_efs_fs.id tags = { Name = "${var.prefix}-reporter" } posix_user { - gid = 1000 - uid = 1000 + gid = 0 + uid = 0 } root_directory { creation_info { diff --git a/terraform/generate-s3.tf b/terraform/generate-s3.tf index fad137b..8d1d258 100644 --- a/terraform/generate-s3.tf +++ b/terraform/generate-s3.tf @@ -1,7 +1,8 @@ # S3 Bucket to hold final L2P granules resource "aws_s3_bucket" "aws_s3_bucket_final_granules" { - bucket = "${var.prefix}-l2p-granules" - tags = { Name = "${var.prefix}-l2p-granules" } + bucket = "${var.prefix}-l2p-granules" + force_destroy = true + tags = { Name = "${var.prefix}-l2p-granules" } } resource "aws_s3_bucket_public_access_block" "aws_s3_bucket_idl_server_public_block" { @@ -56,33 +57,34 @@ resource "aws_s3_bucket_policy" "allow_access_from_another_account" { }) } -# Bucket to hold download lists -resource "aws_s3_bucket" "aws_s3_bucket_dlc" { - bucket = "${var.prefix}-download-lists" - tags = { Name = "${var.prefix}-download-lists" } +# Bucket to hold Generate data including download lists +resource "aws_s3_bucket" "aws_s3_bucket_gen" { + bucket = var.prefix + force_destroy = true + tags = { Name = "${var.prefix}" } } -resource "aws_s3_bucket_public_access_block" "aws_s3_bucket_dlc_public_block" { - bucket = aws_s3_bucket.aws_s3_bucket_dlc.id +resource "aws_s3_bucket_public_access_block" "aws_s3_bucket_gen_public_block" { + bucket = aws_s3_bucket.aws_s3_bucket_gen.id block_public_acls = true block_public_policy = true ignore_public_acls = true restrict_public_buckets = true } -resource "aws_s3_bucket_ownership_controls" "aws_s3_bucket_dlc_ownership" { - bucket = aws_s3_bucket.aws_s3_bucket_dlc.id +resource "aws_s3_bucket_ownership_controls" "aws_s3_bucket_gen_ownership" { + bucket = aws_s3_bucket.aws_s3_bucket_gen.id rule { object_ownership = "BucketOwnerEnforced" } } -resource "aws_s3_bucket_server_side_encryption_configuration" "aws_s3_bucket_dlc_encryption" { - bucket = aws_s3_bucket.aws_s3_bucket_dlc.bucket +resource "aws_s3_bucket_server_side_encryption_configuration" "aws_s3_bucket_gen_encryption" { + bucket = aws_s3_bucket.aws_s3_bucket_gen.bucket rule { apply_server_side_encryption_by_default { sse_algorithm = "aws:kms" kms_master_key_id = "aws/s3" } } -} \ No newline at end of file +} diff --git a/terraform/generate-sns.tf b/terraform/generate-sns.tf index 9ed6530..df744d8 100644 --- a/terraform/generate-sns.tf +++ b/terraform/generate-sns.tf @@ -65,4 +65,40 @@ resource "aws_sns_topic_subscription" "aws_sns_topic_batch_job_failure_subscript endpoint = var.sns_topic_email protocol = "email" topic_arn = aws_sns_topic.aws_sns_topic_batch_job_failure.arn +} + +# SNS Topic for CloudWatch alarms +resource "aws_sns_topic" "aws_sns_topic_cloudwatch_alarms" { + name = "${var.prefix}-cloudwatch-alarms" + display_name = "${var.prefix}-cloudwatch-alarms" +} + +resource "aws_sns_topic_policy" "aws_sns_topic_cloudwatch_alarms_policy" { + arn = aws_sns_topic.aws_sns_topic_cloudwatch_alarms.arn + policy = jsonencode({ + "Version" : "2008-10-17", + "Id" : "__default_policy_ID", + "Statement" : [ + { + "Sid" : "AllowPublishAlarms", + "Effect" : "Allow", + "Principal" : { + "Service" : "cloudwatch.amazonaws.com" + }, + "Action" : "sns:Publish", + "Resource" : "${aws_sns_topic.aws_sns_topic_cloudwatch_alarms.arn}", + "Condition" : { + "ArnLike" : { + "aws:SourceArn" : "arn:aws:cloudwatch:${var.aws_region}:${local.account_id}:alarm:*" + } + } + } + ] + }) +} + +resource "aws_sns_topic_subscription" "aws_sns_topic_cloudwatch_alarms_subscription" { + endpoint = var.sns_topic_email_alarms + protocol = "email" + topic_arn = aws_sns_topic.aws_sns_topic_cloudwatch_alarms.arn } \ No newline at end of file diff --git a/terraform/generate-ssm.tf b/terraform/generate-ssm.tf index 2b38536..245aeb5 100644 --- a/terraform/generate-ssm.tf +++ b/terraform/generate-ssm.tf @@ -1,30 +1,30 @@ -# SSM Parameter Store parameters for IDL idlenses +# SSM Parameter Store parameters for IDL licenses # MODIS Aqua resource "aws_ssm_parameter" "aws_ssm_parameter_ps_idl_aqua" { name = "${var.prefix}-idl-aqua" type = "String" - value = "4" + value = "6" } # MODIS Terra resource "aws_ssm_parameter" "aws_ssm_parameter_ps_idl_terra" { name = "${var.prefix}-idl-terra" type = "String" - value = "4" + value = "6" } # VIIRS resource "aws_ssm_parameter" "aws_ssm_parameter_ps_idl_viirs" { name = "${var.prefix}-idl-viirs" type = "String" - value = "4" + value = "6" } # Floating resource "aws_ssm_parameter" "aws_ssm_parameter_ps_idl_floating" { name = "${var.prefix}-idl-floating" type = "String" - value = "4" + value = "2" } # Retrieval indicator @@ -32,4 +32,4 @@ resource "aws_ssm_parameter" "aws_ssm_parameter_ps_idl_ret" { name = "${var.prefix}-idl-retrieving-license" type = "String" value = "False" -} \ No newline at end of file +} diff --git a/terraform/main.tf b/terraform/main.tf index 99a1c56..d7dc2d9 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -18,8 +18,7 @@ provider "aws" { ignore_tags { key_prefixes = ["gsfc-ngap"] } - region = var.aws_region - profile = var.profile + region = var.aws_region } # Data sources diff --git a/terraform/variables.tf b/terraform/variables.tf index d63743b..7810d1a 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -47,12 +47,12 @@ variable "prefix" { description = "Prefix to add to all AWS resources as a unique identifier" } -variable "profile" { +variable "sns_topic_email" { type = string - description = "Named profile to build infrastructure with" + description = "Email to send SNS Topic messages to" } -variable "sns_topic_email" { +variable "sns_topic_email_alarms" { type = string - description = "Email to send SNS Topic messages to" + description = "Email to send CloudWatch alarms to" } \ No newline at end of file