Skip to content

Commit

Permalink
feat: replicate CUR to data lake (#318)
Browse files Browse the repository at this point in the history
Update the Cost and Usage report bucket to replicate objects to
the Data Lake Raw bucket.

Update the billing extract tag Lambda to write directly to the Data
Lake Raw bucket and remove the dedicated bucket in the account.
  • Loading branch information
patheard authored Nov 7, 2024
1 parent 8effb74 commit e0d782e
Show file tree
Hide file tree
Showing 6 changed files with 93 additions and 142 deletions.
79 changes: 72 additions & 7 deletions terragrunt/org_account/cost_usage_report/iam.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
#
# Lambda function to extract account tags and write them to the Data Lake
#
resource "aws_iam_role" "billing_extract_tags" {
name = "BillingExtractTags"
assume_role_policy = data.aws_iam_policy_document.billing_extract_tags_assume.json
Expand Down Expand Up @@ -37,15 +40,10 @@ data "aws_iam_policy_document" "billing_extract_tags" {
statement {
effect = "Allow"
actions = [
"s3:PutObject*",
"s3:ListBucket",
"s3:GetObject*",
"s3:DeleteObject*",
"s3:GetBucketLocation"
"s3:PutObject"
]
resources = [
module.billing_extract_tags.s3_bucket_arn,
"${module.billing_extract_tags.s3_bucket_arn}/*",
"${local.data_lake_raw_s3_bucket_arn}/operations/aws/organization/account-tags.json",
]
}
}
Expand Down Expand Up @@ -78,3 +76,70 @@ resource "aws_iam_role_policy_attachment" "lambda_insights" {
role = aws_iam_role.billing_extract_tags.name
policy_arn = data.aws_iam_policy.lambda_insights.arn
}

#
# Replicate the Cost and Usage Report data to the Data Lake
#
resource "aws_iam_role" "cur_replicate" {
name = "CostUsageReplicateToDataLake"
assume_role_policy = data.aws_iam_policy_document.cur_replicate_assume.json
tags = local.common_tags
}

resource "aws_iam_policy" "cur_replicate" {
name = "CostUsageReplicateToDataLake"
policy = data.aws_iam_policy_document.cur_replicate.json
tags = local.common_tags
}

resource "aws_iam_role_policy_attachment" "cur_replicate" {
role = aws_iam_role.cur_replicate.name
policy_arn = aws_iam_policy.cur_replicate.arn
}

data "aws_iam_policy_document" "cur_replicate_assume" {
statement {
effect = "Allow"
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = [
"s3.amazonaws.com"
]
}
}
}

data "aws_iam_policy_document" "cur_replicate" {
statement {
effect = "Allow"
actions = [
"s3:GetReplicationConfiguration",
"s3:ListBucket"
]
resources = [
module.cost_usage_report.s3_bucket_arn
]
}
statement {
effect = "Allow"
actions = [
"s3:GetObjectVersion",
"s3:GetObjectVersionAcl"
]
resources = [
"${module.cost_usage_report.s3_bucket_arn}/*"
]
}
statement {
effect = "Allow"
actions = [
"s3:ObjectOwnerOverrideToBucketOwner",
"s3:ReplicateObject",
"s3:ReplicateDelete"
]
resources = [
"${local.data_lake_raw_s3_bucket_arn}/*"
]
}
}
49 changes: 0 additions & 49 deletions terragrunt/org_account/cost_usage_report/import.tf

This file was deleted.

2 changes: 1 addition & 1 deletion terragrunt/org_account/cost_usage_report/lambda.tf
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ resource "aws_lambda_function" "billing_extract_tags" {

environment {
variables = {
TARGET_BUCKET = module.billing_extract_tags.s3_bucket_id
TARGET_BUCKET = local.data_lake_raw_s3_bucket_name
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,6 @@ def handler(event, context):

# save accounts to an s3 bucket
logging.info("Saving account tags to s3")
s3.put_object(Bucket=TARGET_BUCKET, Key="account_tags.json", Body=accounts)
s3.put_object(Bucket=TARGET_BUCKET, Key="/operations/aws/organization/account-tags.json", Body=accounts)

return {"statusCode": 200}
2 changes: 2 additions & 0 deletions terragrunt/org_account/cost_usage_report/locals.tf
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,6 @@ locals {
CostCentre = var.billing_code
Terraform = "true"
}
data_lake_raw_s3_bucket_arn = "arn:aws:s3:::${local.data_lake_raw_s3_bucket_name}"
data_lake_raw_s3_bucket_name = "cds-data-lake-raw-production"
}
101 changes: 17 additions & 84 deletions terragrunt/org_account/cost_usage_report/s3.tf
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,20 @@ module "cost_usage_report" {
enabled = true
}

replication_configuration = {
role = aws_iam_role.cur_replicate.arn

rules = [
{
id = "send-to-data-lake"
priority = 10
destination = {
bucket = local.data_lake_raw_s3_bucket_arn
}
}
]
}

billing_tag_value = var.billing_code
}

Expand Down Expand Up @@ -40,95 +54,14 @@ data "aws_iam_policy_document" "cost_usage_report" {
test = "StringLike"
variable = "aws:SourceArn"
values = [
"arn:aws:cur:us-east-1:659087519042:definition/*",
"arn:aws:bcm-data-exports:us-east-1:659087519042:export/*"
"arn:aws:cur:us-east-1:${var.account_id}:definition/*",
"arn:aws:bcm-data-exports:us-east-1:${var.account_id}:export/*"
]
}
condition {
test = "StringLike"
variable = "aws:SourceAccount"
values = ["659087519042"]
}
}

statement {
effect = "Allow"
principals {
type = "AWS"
identifiers = ["arn:aws:iam::066023111852:root"]
}
actions = [
"s3:ListMultipartUploadParts",
"s3:ListBucketMultipartUploads",
"s3:ListBucket",
"s3:GetObject",
"s3:GetBucketLocation",
"s3:AbortMultipartUpload"
]
resources = [
module.cost_usage_report.s3_bucket_arn,
"${module.cost_usage_report.s3_bucket_arn}/*"
]
}
}

#
# Account billing tags
#
module "billing_extract_tags" {
source = "github.com/cds-snc/terraform-modules//S3?ref=v9.6.8"
bucket_name = "cds-account-billing-extract-tags"
acl = null

versioning = {
enabled = true
}

billing_tag_value = var.billing_code
}

resource "aws_s3_bucket_policy" "billing_extract_tags" {
bucket = module.billing_extract_tags.s3_bucket_id
policy = data.aws_iam_policy_document.billing_extract_tags_bucket.json
}

data "aws_iam_policy_document" "billing_extract_tags_bucket" {
statement {
effect = "Allow"
principals {
type = "AWS"
identifiers = [aws_iam_role.billing_extract_tags.arn]
values = [var.account_id]
}
actions = [
"s3:PutObject*",
"s3:ListBucket",
"s3:GetObject*",
"s3:DeleteObject*",
"s3:GetBucketLocation"
]
resources = [
module.billing_extract_tags.s3_bucket_arn,
"${module.billing_extract_tags.s3_bucket_arn}/*",
]
}

statement {
effect = "Allow"
principals {
type = "AWS"
identifiers = ["arn:aws:iam::066023111852:root"]
}
actions = [
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
"s3:AbortMultipartUpload"
]
resources = [
module.billing_extract_tags.s3_bucket_arn,
"${module.billing_extract_tags.s3_bucket_arn}/*",
]
}
}

0 comments on commit e0d782e

Please sign in to comment.