Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove s3 arn variable as a requirement #36

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Remove s3 arn variable as a requirement #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e7d6106
Remove s3 arn variable as requirement
fossouo Jul 25, 2023
5408a5a
Remove s3 arn variable as requirement
fossouo Jul 25, 2023
9a04d3a
Update security groups and IAM Roles
fossouo Jul 26, 2023
7a14320
Update README.md
fossouo Jul 26, 2023
ba7aeb7
Update README.md
fossouo Jul 26, 2023
4be2cca
Update README.md
fossouo Jul 26, 2023
4dbe524
Update README.md
fossouo Jul 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,3 +178,12 @@ No modules.
| <a name="output_mwaa_status"></a> [mwaa\_status](#output\_mwaa\_status) | The status of the Amazon MWAA Environment |
| <a name="output_mwaa_webserver_url"></a> [mwaa\_webserver\_url](#output\_mwaa\_webserver\_url) | The webserver URL of the MWAA Environment |
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

## Troubleshooting
Tools and sample code provided by AWS Premium Support for troubleshooting your MWAA Deployments :

```hcl
$ pip3 install boto3 --upgrade --user
$ git clone https://github.com/awslabs/aws-support-tools.git
$ python3 aws-support-tools/MWAA/verify_env/verify_env.py --envname YOUR_ENV_NAME_HERE
```
104 changes: 35 additions & 69 deletions data.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,13 @@ data "aws_region" "current" {}

data "aws_caller_identity" "current" {}

data "aws_s3_bucket" "selected" {
bucket = var.source_bucket_name
}

data "aws_kms_key" "by_alias" {
key_id = var.kms_key
}
# ---------------------------------------------------------------------------------------------------------------------
# MWAA Role
# ---------------------------------------------------------------------------------------------------------------------
Expand All @@ -21,32 +28,14 @@ data "aws_iam_policy_document" "mwaa_assume" {
identifiers = ["airflow-env.amazonaws.com"]
}

principals {
type = "Service"
identifiers = ["batch.amazonaws.com"]
}

principals {
type = "Service"
identifiers = ["ssm.amazonaws.com"]
}
principals {
type = "Service"
identifiers = ["lambda.amazonaws.com"]
}
principals {
type = "Service"
identifiers = ["s3.amazonaws.com"]
}
}
}
#tfsec:ignore:AWS099
data "aws_iam_policy_document" "mwaa" {
statement {
effect = "Allow"
actions = [
"airflow:PublishMetrics",
"airflow:CreateWebLoginToken"
"airflow:PublishMetrics"
]
resources = [
"arn:${data.aws_partition.current.id}:airflow:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:environment/${var.name}"
Expand All @@ -58,11 +47,20 @@ data "aws_iam_policy_document" "mwaa" {
"s3:*"
]
resources = [
local.source_bucket_arn,
"${local.source_bucket_arn}/*"
data.aws_s3_bucket.selected.arn,
"${data.aws_s3_bucket.selected.arn}/*"
]
}
# Restrict Public Access
statement {
effect = "Allow"
actions = [
"s3:GetAccountPublicAccessBlock"
]
resources = [
"*"
]
}

statement {
effect = "Allow"
actions = [
Expand All @@ -83,10 +81,7 @@ data "aws_iam_policy_document" "mwaa" {
effect = "Allow"
actions = [
"logs:DescribeLogGroups",
"cloudwatch:PutMetricData",
"batch:DescribeJobs",
"batch:ListJobs",
"eks:*"
"cloudwatch:PutMetricData"
]
resources = [
"*"
Expand All @@ -109,57 +104,28 @@ data "aws_iam_policy_document" "mwaa" {
}

statement {
effect = "Allow"
actions = [
effect = "Allow"
actions = [
"kms:Decrypt",
"kms:DescribeKey",
"kms:GenerateDataKey*",
"kms:Encrypt"
]
not_resources = [
"arn:${data.aws_partition.current.id}:kms:*:${data.aws_caller_identity.current.account_id}:key/*"
]
"kms:Encrypt",
"kms:PutKeyPolicy"
]
resources = data.aws_kms_key.by_alias.arn != null ? [
data.aws_kms_key.by_alias.arn
] : []
not_resources = data.aws_kms_key.by_alias.arn == null ? [
"arn:aws:kms:*:${data.aws_caller_identity.current.account_id}:key/*"
] : []
condition {
test = "StringLike"
variable = "kms:ViaService"

values = [
values = data.aws_kms_key.by_alias.arn != null ? [
"sqs.${data.aws_region.current.name}.amazonaws.com" ] : [
"sqs.${data.aws_region.current.name}.amazonaws.com"
]
variable = "kms:ViaService"
}
}

statement {
effect = "Allow"
actions = [
"batch:*",
]
resources = [
"arn:${data.aws_partition.current.id}:batch:*:${data.aws_caller_identity.current.account_id}:*"
]
}

statement {
effect = "Allow"
actions = [
"ssm:*"
]
resources = [
"arn:${data.aws_partition.current.id}:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/*"
]
}

statement {
effect = "Allow"
actions = [
"logs:*"
]
resources = ["arn:${data.aws_partition.current.id}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*"]
}

statement {
effect = "Allow"
actions = ["cloudwatch:*"]
resources = ["arn:${data.aws_partition.current.id}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*"]
}
}
2 changes: 1 addition & 1 deletion locals.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ locals {

security_group_ids = var.create_security_group ? concat([aws_security_group.mwaa[0].id], var.security_group_ids) : var.security_group_ids

source_bucket_arn = var.create_s3_bucket ? aws_s3_bucket.mwaa[0].arn : var.source_bucket_arn
source_bucket_arn = var.create_s3_bucket ? aws_s3_bucket.mwaa[0].arn : data.aws_s3_bucket.selected.arn

default_airflow_configuration_options = {
"logging.logging_level" = "INFO"
Expand Down
4 changes: 2 additions & 2 deletions main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ resource "aws_mwaa_environment" "mwaa" {
environment_class = var.environment_class
min_workers = var.min_workers
max_workers = var.max_workers
kms_key = var.kms_key
kms_key = data.aws_kms_key.by_alias.arn

dag_s3_path = var.dag_s3_path
plugins_s3_object_version = var.plugins_s3_object_version
Expand All @@ -20,7 +20,7 @@ resource "aws_mwaa_environment" "mwaa" {
execution_role_arn = local.execution_role_arn
airflow_configuration_options = local.airflow_configuration_options

source_bucket_arn = local.source_bucket_arn
source_bucket_arn = data.aws_s3_bucket.selected.arn
webserver_access_mode = var.webserver_access_mode
weekly_maintenance_window_start = var.weekly_maintenance_window_start

Expand Down
10 changes: 5 additions & 5 deletions variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,11 +206,11 @@ variable "source_bucket_name" {
default = null
}

variable "source_bucket_arn" {
description = "(Required) The Amazon Resource Name (ARN) of your Amazon S3 storage bucket. For example, arn:aws:s3:::airflow-mybucketname"
type = string
default = null
}
#variable "source_bucket_arn" {
# description = "(Required) The Amazon Resource Name (ARN) of your Amazon S3 storage bucket. For example, arn:aws:s3:::airflow-mybucketname"
# type = string
# default = null
#}

#----------------------------------------------------------------
# MWAA Security groups
Expand Down