From e0d782e3e3140ef37aaaf3abdebe9849f52f6d77 Mon Sep 17 00:00:00 2001 From: Pat Heard Date: Thu, 7 Nov 2024 11:39:09 -0500 Subject: [PATCH] feat: replicate CUR to data lake (#318) Update the Cost and Usage report bucket to replicate objects to the Data Lake Raw bucket. Update the billing extract tag Lambda to write directly to the Data Lake Raw bucket and remove the dedicated bucket in the account. --- .../org_account/cost_usage_report/iam.tf | 79 ++++++++++++-- .../org_account/cost_usage_report/import.tf | 49 --------- .../org_account/cost_usage_report/lambda.tf | 2 +- .../lambdas/billing_extract_tags/main.py | 2 +- .../org_account/cost_usage_report/locals.tf | 2 + .../org_account/cost_usage_report/s3.tf | 101 +++--------------- 6 files changed, 93 insertions(+), 142 deletions(-) delete mode 100644 terragrunt/org_account/cost_usage_report/import.tf diff --git a/terragrunt/org_account/cost_usage_report/iam.tf b/terragrunt/org_account/cost_usage_report/iam.tf index b7864323..40318fe7 100644 --- a/terragrunt/org_account/cost_usage_report/iam.tf +++ b/terragrunt/org_account/cost_usage_report/iam.tf @@ -1,3 +1,6 @@ +# +# Lambda function to extract account tags and write them to the Data Lake +# resource "aws_iam_role" "billing_extract_tags" { name = "BillingExtractTags" assume_role_policy = data.aws_iam_policy_document.billing_extract_tags_assume.json @@ -37,15 +40,10 @@ data "aws_iam_policy_document" "billing_extract_tags" { statement { effect = "Allow" actions = [ - "s3:PutObject*", - "s3:ListBucket", - "s3:GetObject*", - "s3:DeleteObject*", - "s3:GetBucketLocation" + "s3:PutObject" ] resources = [ - module.billing_extract_tags.s3_bucket_arn, - "${module.billing_extract_tags.s3_bucket_arn}/*", + "${local.data_lake_raw_s3_bucket_arn}/operations/aws/organization/account-tags.json", ] } } @@ -78,3 +76,70 @@ resource "aws_iam_role_policy_attachment" "lambda_insights" { role = aws_iam_role.billing_extract_tags.name policy_arn = data.aws_iam_policy.lambda_insights.arn } + +# +# Replicate the Cost and Usage Report data to the Data Lake +# +resource "aws_iam_role" "cur_replicate" { + name = "CostUsageReplicateToDataLake" + assume_role_policy = data.aws_iam_policy_document.cur_replicate_assume.json + tags = local.common_tags +} + +resource "aws_iam_policy" "cur_replicate" { + name = "CostUsageReplicateToDataLake" + policy = data.aws_iam_policy_document.cur_replicate.json + tags = local.common_tags +} + +resource "aws_iam_role_policy_attachment" "cur_replicate" { + role = aws_iam_role.cur_replicate.name + policy_arn = aws_iam_policy.cur_replicate.arn +} + +data "aws_iam_policy_document" "cur_replicate_assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = [ + "s3.amazonaws.com" + ] + } + } +} + +data "aws_iam_policy_document" "cur_replicate" { + statement { + effect = "Allow" + actions = [ + "s3:GetReplicationConfiguration", + "s3:ListBucket" + ] + resources = [ + module.cost_usage_report.s3_bucket_arn + ] + } + statement { + effect = "Allow" + actions = [ + "s3:GetObjectVersion", + "s3:GetObjectVersionAcl" + ] + resources = [ + "${module.cost_usage_report.s3_bucket_arn}/*" + ] + } + statement { + effect = "Allow" + actions = [ + "s3:ObjectOwnerOverrideToBucketOwner", + "s3:ReplicateObject", + "s3:ReplicateDelete" + ] + resources = [ + "${local.data_lake_raw_s3_bucket_arn}/*" + ] + } +} \ No newline at end of file diff --git a/terragrunt/org_account/cost_usage_report/import.tf b/terragrunt/org_account/cost_usage_report/import.tf deleted file mode 100644 index 1d5ca2fa..00000000 --- a/terragrunt/org_account/cost_usage_report/import.tf +++ /dev/null @@ -1,49 +0,0 @@ -import { - to = aws_cloudwatch_event_rule.billing_extract_tags - id = "default/billing_extract_tags_daily" -} - -import { - to = aws_cloudwatch_event_target.billing_extract_tags - id = "billing_extract_tags_daily/terraform-20240305180415483500000002" -} - -import { - to = aws_iam_role.billing_extract_tags - id = "BillingExtractTags" -} - -import { - to = aws_iam_policy.billing_extract_tags - id = "arn:aws:iam::659087519042:policy/BillingExtractTags" -} - -import { - to = aws_iam_role_policy_attachment.billing_extract_tags - id = "BillingExtractTags/arn:aws:iam::659087519042:policy/BillingExtractTags" -} - -import { - to = aws_iam_role_policy_attachment.org_read_only - id = "BillingExtractTags/arn:aws:iam::aws:policy/AWSOrganizationsReadOnlyAccess" -} - -import { - to = aws_iam_role_policy_attachment.lambda_insights - id = "BillingExtractTags/arn:aws:iam::aws:policy/CloudWatchLambdaInsightsExecutionRolePolicy" -} - -import { - to = aws_lambda_function.billing_extract_tags - id = "billing_extract_tags" -} - -import { - to = aws_lambda_permission.billing_extract_tags - id = "billing_extract_tags/AllowBillingExtractTagsDaily" -} - -import { - to = aws_cloudwatch_log_group.billing_extract_tags - id = "/aws/lambda/billing_extract_tags" -} diff --git a/terragrunt/org_account/cost_usage_report/lambda.tf b/terragrunt/org_account/cost_usage_report/lambda.tf index 16812146..602bf86f 100644 --- a/terragrunt/org_account/cost_usage_report/lambda.tf +++ b/terragrunt/org_account/cost_usage_report/lambda.tf @@ -17,7 +17,7 @@ resource "aws_lambda_function" "billing_extract_tags" { environment { variables = { - TARGET_BUCKET = module.billing_extract_tags.s3_bucket_id + TARGET_BUCKET = local.data_lake_raw_s3_bucket_name } } diff --git a/terragrunt/org_account/cost_usage_report/lambdas/billing_extract_tags/main.py b/terragrunt/org_account/cost_usage_report/lambdas/billing_extract_tags/main.py index 381070ec..94cde011 100644 --- a/terragrunt/org_account/cost_usage_report/lambdas/billing_extract_tags/main.py +++ b/terragrunt/org_account/cost_usage_report/lambdas/billing_extract_tags/main.py @@ -68,6 +68,6 @@ def handler(event, context): # save accounts to an s3 bucket logging.info("Saving account tags to s3") - s3.put_object(Bucket=TARGET_BUCKET, Key="account_tags.json", Body=accounts) + s3.put_object(Bucket=TARGET_BUCKET, Key="/operations/aws/organization/account-tags.json", Body=accounts) return {"statusCode": 200} diff --git a/terragrunt/org_account/cost_usage_report/locals.tf b/terragrunt/org_account/cost_usage_report/locals.tf index 9c562e80..54f9d866 100644 --- a/terragrunt/org_account/cost_usage_report/locals.tf +++ b/terragrunt/org_account/cost_usage_report/locals.tf @@ -3,4 +3,6 @@ locals { CostCentre = var.billing_code Terraform = "true" } + data_lake_raw_s3_bucket_arn = "arn:aws:s3:::${local.data_lake_raw_s3_bucket_name}" + data_lake_raw_s3_bucket_name = "cds-data-lake-raw-production" } \ No newline at end of file diff --git a/terragrunt/org_account/cost_usage_report/s3.tf b/terragrunt/org_account/cost_usage_report/s3.tf index 8a1fa713..2833cebb 100644 --- a/terragrunt/org_account/cost_usage_report/s3.tf +++ b/terragrunt/org_account/cost_usage_report/s3.tf @@ -9,6 +9,20 @@ module "cost_usage_report" { enabled = true } + replication_configuration = { + role = aws_iam_role.cur_replicate.arn + + rules = [ + { + id = "send-to-data-lake" + priority = 10 + destination = { + bucket = local.data_lake_raw_s3_bucket_arn + } + } + ] + } + billing_tag_value = var.billing_code } @@ -40,95 +54,14 @@ data "aws_iam_policy_document" "cost_usage_report" { test = "StringLike" variable = "aws:SourceArn" values = [ - "arn:aws:cur:us-east-1:659087519042:definition/*", - "arn:aws:bcm-data-exports:us-east-1:659087519042:export/*" + "arn:aws:cur:us-east-1:${var.account_id}:definition/*", + "arn:aws:bcm-data-exports:us-east-1:${var.account_id}:export/*" ] } condition { test = "StringLike" variable = "aws:SourceAccount" - values = ["659087519042"] - } - } - - statement { - effect = "Allow" - principals { - type = "AWS" - identifiers = ["arn:aws:iam::066023111852:root"] - } - actions = [ - "s3:ListMultipartUploadParts", - "s3:ListBucketMultipartUploads", - "s3:ListBucket", - "s3:GetObject", - "s3:GetBucketLocation", - "s3:AbortMultipartUpload" - ] - resources = [ - module.cost_usage_report.s3_bucket_arn, - "${module.cost_usage_report.s3_bucket_arn}/*" - ] - } -} - -# -# Account billing tags -# -module "billing_extract_tags" { - source = "github.com/cds-snc/terraform-modules//S3?ref=v9.6.8" - bucket_name = "cds-account-billing-extract-tags" - acl = null - - versioning = { - enabled = true - } - - billing_tag_value = var.billing_code -} - -resource "aws_s3_bucket_policy" "billing_extract_tags" { - bucket = module.billing_extract_tags.s3_bucket_id - policy = data.aws_iam_policy_document.billing_extract_tags_bucket.json -} - -data "aws_iam_policy_document" "billing_extract_tags_bucket" { - statement { - effect = "Allow" - principals { - type = "AWS" - identifiers = [aws_iam_role.billing_extract_tags.arn] + values = [var.account_id] } - actions = [ - "s3:PutObject*", - "s3:ListBucket", - "s3:GetObject*", - "s3:DeleteObject*", - "s3:GetBucketLocation" - ] - resources = [ - module.billing_extract_tags.s3_bucket_arn, - "${module.billing_extract_tags.s3_bucket_arn}/*", - ] - } - - statement { - effect = "Allow" - principals { - type = "AWS" - identifiers = ["arn:aws:iam::066023111852:root"] - } - actions = [ - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListMultipartUploadParts", - "s3:AbortMultipartUpload" - ] - resources = [ - module.billing_extract_tags.s3_bucket_arn, - "${module.billing_extract_tags.s3_bucket_arn}/*", - ] } }