From 09f598e4aca6e139d02210d43f0697e61f33a983 Mon Sep 17 00:00:00 2001 From: eternaltyro Date: Thu, 4 Apr 2024 13:32:50 +0100 Subject: [PATCH] Fix Terraform CI checks --- .github/workflows/terraform-checks.yml | 73 +-- infra/production/.terraform.lock.hcl | 62 --- infra/production/bootstrap_backend.sh.tftpl | 36 -- infra/production/container.tf | 90 ---- infra/production/main.tf | 416 ------------------ infra/production/outputs.tf | 55 --- infra/production/provider.tf | 42 -- .../tests/compliance/cloudfront.feature | 62 --- infra/production/tests/compliance/s3.feature | 23 - infra/production/variables.tf | 99 ----- infra/staging/main.tf | 201 --------- infra/staging/outputs.tf | 34 -- infra/staging/provider.tf | 42 -- .../tests/compliance/cloudfront.feature | 62 --- infra/staging/tests/compliance/s3.feature | 23 - infra/staging/variables.tf | 63 --- 16 files changed, 4 insertions(+), 1379 deletions(-) delete mode 100644 infra/production/.terraform.lock.hcl delete mode 100644 infra/production/bootstrap_backend.sh.tftpl delete mode 100644 infra/production/container.tf delete mode 100644 infra/production/main.tf delete mode 100644 infra/production/outputs.tf delete mode 100644 infra/production/provider.tf delete mode 100644 infra/production/tests/compliance/cloudfront.feature delete mode 100644 infra/production/tests/compliance/s3.feature delete mode 100644 infra/production/variables.tf delete mode 100644 infra/staging/main.tf delete mode 100644 infra/staging/outputs.tf delete mode 100644 infra/staging/provider.tf delete mode 100644 infra/staging/tests/compliance/cloudfront.feature delete mode 100644 infra/staging/tests/compliance/s3.feature delete mode 100644 infra/staging/variables.tf diff --git a/.github/workflows/terraform-checks.yml b/.github/workflows/terraform-checks.yml index 11f1fe2c..d139de27 100644 --- a/.github/workflows/terraform-checks.yml +++ b/.github/workflows/terraform-checks.yml @@ -3,56 +3,24 @@ name: "Terraform Checks" on: push: branches: - - master - - main - develop - - "deployment/**" paths: - infra/** pull_request: branches: - - master - - main - develop - - "deployment/**" paths: - infra/** jobs: - terraform-CI-checks-staging: - name: "Formatting and validation Checks for Staging" - runs-on: ubuntu-latest - defaults: - run: - working-directory: infra - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - with: - cli_config_credentials_token: ${{ secrets.TF_CLOUD_TOKEN }} - - - name: Check code formating - id: fmt - run: terraform fmt -check - - - name: Initialise modules - id: init - run: terraform init - - - name: Validate template - id: validate - run: terraform validate -no-color terraform-CI-check-production: name: "Formatting and validation Checks for Production" runs-on: ubuntu-latest defaults: run: - working-directory: infra/production + working-directory: infra/prod-aws steps: - name: Checkout uses: actions/checkout@v3 @@ -74,31 +42,12 @@ jobs: id: validate run: terraform validate -no-color - terrascan-staging: - name: "Terrascan Staging Checks" - runs-on: ubuntu-latest - defaults: - run: - working-directory: infra - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Run Terrascan on staging - id: terrascan - uses: tenable/terrascan-action@main - with: - iac_type: "terraform" - iac_dir: "./infra" - iac_version: "v14" - policy_type: "all" - terrascan-production: name: "Terrascan Production Checks" runs-on: ubuntu-latest defaults: run: - working-directory: infra/production + working-directory: infra/prod-aws steps: - name: Checkout uses: actions/checkout@v3 @@ -108,24 +57,10 @@ jobs: uses: tenable/terrascan-action@main with: iac_type: "terraform" - iac_dir: "./infra/production" + iac_dir: "./infra/prod-aws" iac_version: "v14" policy_type: "all" - checkov-staging: - runs-on: ubuntu-latest - name: "Checkov Staging Checks" - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Checkov GitHub Action - uses: bridgecrewio/checkov-action@v12 - with: - directory: infra/ - output_format: cli,sarif - output_file_path: console,results.sarif - checkov-production: runs-on: ubuntu-latest permissions: @@ -138,7 +73,7 @@ jobs: - name: Checkov GitHub Action uses: bridgecrewio/checkov-action@v12 with: - directory: infra/production/ + directory: infra/prod-aws/ output_format: cli,sarif output_file_path: console,results.sarif diff --git a/infra/production/.terraform.lock.hcl b/infra/production/.terraform.lock.hcl deleted file mode 100644 index 127bfa6d..00000000 --- a/infra/production/.terraform.lock.hcl +++ /dev/null @@ -1,62 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.65.0" - constraints = "3.65.0" - hashes = [ - "h1:Wpq9+x8PynJqzfxaI1hnxhFgHSXkCz07UqroUVJCseU=", - "zh:0077d19c1cbd8916a6d96bad17e72f88535ac207fb7f88b714c6fc6da736f80d", - "zh:084f9de2f0f84e6508f81b6578ff195afeed79e5d18a0c8d2348abd7d22611c9", - "zh:0ea05826c0f9d2e4a5a9887e6d182ba1a5db6eba52b22eb45f0b8576d2d5ddb5", - "zh:5142f9cf59f8152bdb9debcdc39c04cb4ca8b26bd50e44f4605b2bcdc4fc514e", - "zh:67af71aa233dbe5e2ce59f8b8aa02a7ce71f55b4389dc6bdd1c85e463f810f37", - "zh:785b2c4845a0e99fc1a00d1c293cee49cf150a4f1a83d86632dd3fcd9e953d9c", - "zh:aae6352ff80d760bebd2148cd86a544cd6df8e1e5abd6d472143e40875983428", - "zh:aff6914ad258d27781ba66a915ef714a3f0d31136eeb06b12ed2220cc6530b4b", - "zh:b21ca9e271db7a57e5f08bf2b47bd8db291faf699fabf14bb38d4a73a9a05c21", - "zh:c8ff94c42249a9fdab87b6c974d6eb59af4c01c955cd76279b7a4f66eacd9754", - "zh:f4053b76a6efd46f79b45098c3e3df06b8e6340532970c91d1a9ead63dcf72b6", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.5.1" - constraints = "3.5.1" - hashes = [ - "h1:VSnd9ZIPyfKHOObuQCaKfnjIHRtR7qTw19Rz8tJxm+k=", - "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", - "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", - "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831", - "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3", - "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b", - "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2", - "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865", - "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03", - "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602", - "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014", - ] -} - -provider "registry.terraform.io/hashicorp/tfe" { - version = "0.46.0" - constraints = "~> 0.46.0" - hashes = [ - "h1:+MpkT+2BwxXwX559GbyubpBQ/Bzlsfvv3q0fcgP45uA=", - "zh:0176ba631987d3e5193f25437f09b30255580f2be9b7fdd38ca4f72f0ec9970f", - "zh:160984fec9483957e2766629eb75e01b18b7ba14804434148fc0da6202e3aba5", - "zh:62ab5618480c0e54903a013fcb692f43e32a2d3460872384b6be28bbdb08e173", - "zh:6589b01880cac29f2f2c891ea94022e906fbb82b7daf36a083fe0ac261b17988", - "zh:686cb76af94e778481541a624b2be52df70742f2d8c94d8bd5090e45352b365c", - "zh:7100cf6731f9f61a45bb3277e77c6b4e714be347e7c2fe739b7ba957cc59b59e", - "zh:741070559939785264bd6a5130ac345b0f06da8847322d5603438fcd4f209346", - "zh:8fe63f043a12df04c43a1ab9322dc8804771cd508d6dd9e182f9ad8a5c8ddeb5", - "zh:9535e2e0f9ce1a579cbda1e3bf854b3ec121459ca519b4fba8bf7fcf6ff0aa61", - "zh:d578db9c5a279adf1b62a66aa395caf635b4700a1cc156a29032bcc997489eca", - "zh:d95ec293fa70e946b6cd657912b33155f8be3413e6128ed2bfa5a493f788e439", - "zh:f83f166807e14b23f843c035860f7a8c07cb0d14c12453dc919d0513b0d3589b", - ] -} diff --git a/infra/production/bootstrap_backend.sh.tftpl b/infra/production/bootstrap_backend.sh.tftpl deleted file mode 100644 index b02d6dc7..00000000 --- a/infra/production/bootstrap_backend.sh.tftpl +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -APP_ADMIN_USER=${APP_ADMIN_USER} -APP_ADMIN_GROUP=$APP_ADMIN_USER -APP_BASE_DIR=/opt/raw-data-api - -apt-get -y update && \ -apt-get -y upgrade - -# Install generic packages -apt-get -y install \ - curl \ - git \ - wget \ - python-is-python3 \ - python3-virtualenv \ - python3-pip \ - python3-dev \ - certbot \ - podman \ - buildah - -# Install app-specific packages -apt-get -y install \ - osm2pgsql \ - libpq-dev \ - gdal-bin \ - python3-gdal \ - redis-tools \ - osmium-tool - -# Clone the application repo and update directory ownership -pushd /opt -git clone https://github.com/hotosm/raw-data-api.git -chown -R $APP_ADMIN_USER:$APP_ADMIN_GROUP $APP_BASE_DIR -chmod -R 640 $APP_BASE_DIR diff --git a/infra/production/container.tf b/infra/production/container.tf deleted file mode 100644 index 1bd247a8..00000000 --- a/infra/production/container.tf +++ /dev/null @@ -1,90 +0,0 @@ -locals { - redis_connection_endpoint = join("", [ - "rediss://", - ":", - azurerm_redis_cache.raw-data-queue.primary_access_key, - "@", - azurerm_redis_cache.raw-data-queue.hostname, - ":", - azurerm_redis_cache.raw-data-queue.ssl_port, - "/0?ssl_cert_reqs=required" - ] - ) -} - -resource "azurerm_container_group" "app" { - name = join("-", [var.project_name, var.deployment_environment]) - resource_group_name = azurerm_resource_group.raw-data.name - location = azurerm_resource_group.raw-data.location - - ip_address_type = "Private" - subnet_ids = [azurerm_subnet.raw-data-containers.id] - os_type = "Linux" - - container { - name = "api" - image = lookup(var.container_images, "api") - cpu = "0.5" - memory = "1.5" - - ports { - port = 8000 - protocol = "TCP" - } - - environment_variables = merge( - var.container_envvar, - { - PGHOST = azurerm_postgresql_flexible_server.raw-data.fqdn - PGPORT = "5432" - PGUSER = lookup(var.admin_usernames, "database") - PGDATABASE = azurerm_postgresql_flexible_server_database.default-db.name - } - ) - - secure_environment_variables = merge( - var.container_sensitive_envvar, - { - PGPASSWORD = azurerm_key_vault_secret.raw-data-db.value - CELERY_BROKER_URL = local.redis_connection_endpoint - CELERY_RESULT_BACKEND = local.redis_connection_endpoint - } - ) - } - - container { - name = "worker" - image = lookup(var.container_images, "worker") - cpu = "0.5" - memory = "1.5" - - commands = ["celery", "--app", "API.api_worker", "worker", "--loglevel=INFO"] - - ports { - port = 8080 - protocol = "TCP" - } - - environment_variables = merge( - var.container_envvar, - { - PGHOST = azurerm_postgresql_flexible_server.raw-data.fqdn - PGPORT = "5432" - PGUSER = lookup(var.admin_usernames, "database") - PGDATABASE = azurerm_postgresql_flexible_server_database.default-db.name - } - ) - - secure_environment_variables = merge( - var.container_sensitive_envvar, - { - PGPASSWORD = azurerm_key_vault_secret.raw-data-db.value - CELERY_BROKER_URL = local.redis_connection_endpoint - CELERY_RESULT_BACKEND = local.redis_connection_endpoint - } - ) - } - - tags = { - } -} diff --git a/infra/production/main.tf b/infra/production/main.tf deleted file mode 100644 index f8ca2ab0..00000000 --- a/infra/production/main.tf +++ /dev/null @@ -1,416 +0,0 @@ -data "tfe_ip_ranges" "addresses" {} - -data "azurerm_client_config" "current" {} - -locals { - required_tags = { - deployment_environment = var.deployment_environment - infrastructure_management = "terraform" - project = var.project_name - } - - conditional_tags = { - _monitor_cloudwatch = "Yes" - _monitor_newrelic_infra = "Yes" - _monitor_apm = "No" - _monitor_sentry = "No" - _patch_management = "No" - } -} - -resource "random_string" "raw_data_db_password" { - length = 20 - override_special = "*()-_=+[]{}<>" -} - -resource "random_string" "raw_data" { - length = 3 - special = false - upper = false -} - -resource "azurerm_resource_group" "raw-data" { - #ts:skip=accurics.azure.NS.272 [TODO] Explore what resource lock is and if it's appropriate here - name = join("-", [var.project_name, random_string.raw_data.id]) - location = var.arm_location -} - -resource "azurerm_storage_account" "raw-data" { - name = "hotosmrawdata" - resource_group_name = azurerm_resource_group.raw-data.name - location = azurerm_resource_group.raw-data.location - account_tier = "Standard" - account_replication_type = "LRS" - - tags = local.required_tags -} - -resource "azurerm_virtual_network" "raw-data" { - name = join("-", [var.project_name, var.deployment_environment]) - resource_group_name = azurerm_resource_group.raw-data.name - address_space = ["10.0.0.0/16"] - location = azurerm_resource_group.raw-data.location - - tags = local.required_tags -} - -resource "azurerm_subnet" "raw-data" { - #ts:skip=accurics.azure.NS.161 [TODO] Give the VNet subnet a network security group - name = join("-", [var.project_name, var.deployment_environment]) - resource_group_name = azurerm_resource_group.raw-data.name - virtual_network_name = azurerm_virtual_network.raw-data.name - address_prefixes = [cidrsubnet(azurerm_virtual_network.raw-data.address_space[0], 8, 0)] - - service_endpoints = ["Microsoft.KeyVault"] -} - -resource "azurerm_subnet" "raw-data-containers" { - #ts:skip=accurics.azure.NS.161 [TODO] Give the VNet subnet a network security group - name = join("-", [var.project_name, "containers", var.deployment_environment]) - resource_group_name = azurerm_resource_group.raw-data.name - virtual_network_name = azurerm_virtual_network.raw-data.name - address_prefixes = [cidrsubnet(azurerm_virtual_network.raw-data.address_space[0], 5, 1)] - - delegation { - name = "containers" - - service_delegation { - name = "Microsoft.ContainerInstance/containerGroups" - actions = [ - "Microsoft.Network/virtualNetworks/subnets/join/action" - ] - } - } - - service_endpoints = [ - "Microsoft.ContainerRegistry", - "Microsoft.KeyVault" - ] -} - -resource "azurerm_subnet" "raw-data-db" { - #ts:skip=accurics.azure.NS.161 [TODO] Give the VNet subnet a network security group - name = join("-", [var.project_name, "database", var.deployment_environment]) - resource_group_name = azurerm_resource_group.raw-data.name - virtual_network_name = azurerm_virtual_network.raw-data.name - address_prefixes = [cidrsubnet(azurerm_virtual_network.raw-data.address_space[0], 8, 1)] - - delegation { - name = "postgres" - - service_delegation { - name = "Microsoft.DBforPostgreSQL/flexibleServers" - actions = [ - "Microsoft.Network/virtualNetworks/subnets/join/action" - ] - } - } - - service_endpoints = [ - "Microsoft.KeyVault", - "Microsoft.Storage" - ] -} - -resource "azurerm_key_vault" "raw-data" { - #checkov:skip=CKV_AZURE_110:[BACKLOG] Purge protection not enabled while developing - #checkov:skip=CKV_AZURE_42:[BACKLOG] Key Vault is not set to be recoverable while developing - #ts:skip=accurics.azure.EKM.20 [TODO] How much does enabling logging cost? - - // [WARNING] Name has a length constraint: 8-24 characters - name = join("-", [ - var.project_name, - var.deployment_environment, - random_string.raw_data.id - ]) - location = azurerm_resource_group.raw-data.location - resource_group_name = azurerm_resource_group.raw-data.name - sku_name = "standard" - tenant_id = data.azurerm_client_config.current.tenant_id - - network_acls { - bypass = "AzureServices" - default_action = "Allow" // "Deny" will cut-off Terraform workers IP - ip_rules = data.tfe_ip_ranges.addresses.api - virtual_network_subnet_ids = [ - azurerm_subnet.raw-data.id, - azurerm_subnet.raw-data-db.id - ] - } - - // [WARNING] Setting this to false will make Terraform unable to access Key Vault - // ... because it would cut off **ALL** public access - public_network_access_enabled = true - - access_policy { - tenant_id = data.azurerm_client_config.current.tenant_id - object_id = data.azurerm_client_config.current.object_id - - secret_permissions = [ - "Set", - "Get", - "List", - "Delete", - "Purge" - ] - - } - - access_policy { - tenant_id = data.azurerm_client_config.current.tenant_id - object_id = var.azuread_admin_group_object_id - - key_permissions = [ - "List", - "Get", - "Delete", - "Purge", - "Rotate" - ] - - secret_permissions = [ - "Set", - "Get", - "List", - "Delete", - "Purge" - ] - - storage_permissions = [ - "Get", - "Set", - "List", - "Purge", - "Update" - ] - } - - // [ACHTUNG | DANGER] DO NOT ENABLE PURGE PROTECTION!! - purge_protection_enabled = false - soft_delete_retention_days = 7 - - tags = local.required_tags -} - -resource "azurerm_key_vault_secret" "raw-data-db" { - #checkov:skip=CKV_AZURE_41:[BACKLOG] Expiration date for secrets can't be set until there's a policy for rotation - name = join("-", [ - var.project_name, - "database", - var.deployment_environment - ]) - value = random_string.raw_data_db_password.result - content_type = "text/plain" - - key_vault_id = azurerm_key_vault.raw-data.id - - tags = local.required_tags -} - -resource "azurerm_public_ip" "raw-data-backend" { - name = join("-", [ - var.project_name, - "backend", - var.deployment_environment - ]) - resource_group_name = azurerm_resource_group.raw-data.name - location = azurerm_resource_group.raw-data.location - allocation_method = "Static" // or "Dynamic" - - tags = local.required_tags -} - -resource "azurerm_network_interface" "raw-data-backend" { - name = join("-", [var.project_name, var.deployment_environment]) - location = azurerm_resource_group.raw-data.location - resource_group_name = azurerm_resource_group.raw-data.name - enable_accelerated_networking = true - - ip_configuration { - name = "internal" - subnet_id = azurerm_subnet.raw-data.id - private_ip_address_allocation = "Dynamic" - primary = true - public_ip_address_id = azurerm_public_ip.raw-data-backend.id - } -} - -resource "azurerm_managed_disk" "backend-data-volume" { - #checkov:skip=CKV_AZURE_93:[WONT DO] Disk encryption unnecessary and benefits negligible - name = join("-", [ - var.project_name, - var.deployment_environment, - "data-volume" - ] - ) - location = azurerm_resource_group.raw-data.location - resource_group_name = azurerm_resource_group.raw-data.name - storage_account_type = "StandardSSD_LRS" - create_option = "Empty" - disk_size_gb = "1000" - - tags = merge(local.required_tags, local.conditional_tags) -} - -resource "azurerm_linux_virtual_machine" "raw-data-backend" { - #checkov:skip=CKV_AZURE_179:[TODO] VM Agent installed by default. - admin_username = lookup(var.admin_usernames, "backend") - location = var.arm_location - name = join("-", [var.project_name, "backend", var.deployment_environment]) - network_interface_ids = [azurerm_network_interface.raw-data-backend.id] - resource_group_name = azurerm_resource_group.raw-data.name - size = lookup(var.server_skus, "backend") - - allow_extension_operations = false - - custom_data = base64encode( - templatefile( - "${path.module}/bootstrap_backend.sh.tftpl", - { - APP_ADMIN_USER = lookup(var.admin_usernames, "backend") - } - ) - ) - - source_image_reference { - publisher = "Canonical" - offer = "0001-com-ubuntu-server-jammy" - sku = "22_04-lts-gen2" - version = "latest" - } - - /* Ref: https://wiki.debian.org/Cloud/MicrosoftAzure - source_image_reference { - publisher = "Debian" - offer = "debian-11" - sku = "11" - version = "latest" - } - */ - - os_disk { - caching = "None" - storage_account_type = "StandardSSD_LRS" // StandardSSD_ZRS - disk_size_gb = lookup(var.disk_size, "backend_os") - name = join("-", [var.project_name, var.deployment_environment]) - - } - - admin_ssh_key { - public_key = var.ssh_public_key - username = lookup(var.admin_usernames, "backend") - } - - tags = merge(local.required_tags, local.conditional_tags) -} - -resource "azurerm_virtual_machine_data_disk_attachment" "backend-volume" { - virtual_machine_id = azurerm_linux_virtual_machine.raw-data-backend.id - managed_disk_id = azurerm_managed_disk.backend-data-volume.id - lun = "10" - caching = "None" -} - -resource "azurerm_private_dns_zone" "raw-data-db" { - name = join("", [ - var.project_name, - random_string.raw_data.id, - ".postgres.database.azure.com" - ] - ) - resource_group_name = azurerm_resource_group.raw-data.name - - tags = local.required_tags -} - -resource "azurerm_private_dns_zone_virtual_network_link" "dns-link" { - name = "priv-dns-vnet-link" - resource_group_name = azurerm_resource_group.raw-data.name - - private_dns_zone_name = azurerm_private_dns_zone.raw-data-db.name - virtual_network_id = azurerm_virtual_network.raw-data.id - - tags = local.required_tags -} - -resource "azurerm_postgresql_flexible_server" "raw-data" { - #checkov:skip=CKV_AZURE_136:[WONT DO] Geo-redundant backups are expensive - depends_on = [ - azurerm_private_dns_zone_virtual_network_link.dns-link - ] - - name = join("-", [ - var.project_name, - var.deployment_environment, - random_string.raw_data.id - ] - ) - resource_group_name = azurerm_resource_group.raw-data.name - location = azurerm_resource_group.raw-data.location - sku_name = lookup(var.server_skus, "database") - delegated_subnet_id = azurerm_subnet.raw-data-db.id - private_dns_zone_id = azurerm_private_dns_zone.raw-data-db.id - - administrator_login = lookup(var.admin_usernames, "database") - administrator_password = azurerm_key_vault_secret.raw-data-db.value - - backup_retention_days = 7 - geo_redundant_backup_enabled = false - storage_mb = 2097152 - - tags = merge(local.required_tags, local.conditional_tags) - - version = 14 - zone = "1" - - lifecycle { - ignore_changes = [ - storage_mb, - ] - } -} - -resource "azurerm_postgresql_flexible_server_configuration" "raw-data-postgis" { - name = "azure.extensions" - server_id = azurerm_postgresql_flexible_server.raw-data.id - value = "BTREE_GIST,INTARRAY,POSTGIS" -} - -// Improve password hashing security for PostgreSQL users -resource "azurerm_postgresql_flexible_server_configuration" "raw-data-password-encryption" { - server_id = azurerm_postgresql_flexible_server.raw-data.id - name = "password_encryption" - value = "scram-sha-256" -} - -resource "azurerm_postgresql_flexible_server_configuration" "raw-data-azure-password-encryption" { - server_id = azurerm_postgresql_flexible_server.raw-data.id - name = "azure.accepted_password_auth_method" - value = "md5,scram-sha-256" -} - -resource "azurerm_postgresql_flexible_server_database" "default-db" { - name = "osm_raw_data" - server_id = azurerm_postgresql_flexible_server.raw-data.id -} - -resource "azurerm_redis_cache" "raw-data-queue" { - #checkov:skip=CKV_AZURE_89:[TODO] Disable public network access for Redis Cache - name = join("-", [ - var.project_name, - var.deployment_environment, - random_string.raw_data.id - ]) - resource_group_name = azurerm_resource_group.raw-data.name - location = azurerm_resource_group.raw-data.location - capacity = 0 - family = "C" - sku_name = "Basic" - - minimum_tls_version = "1.2" - redis_version = 6 - - // public_network_access_enabled = false - - tags = local.required_tags -} diff --git a/infra/production/outputs.tf b/infra/production/outputs.tf deleted file mode 100644 index 86dfd149..00000000 --- a/infra/production/outputs.tf +++ /dev/null @@ -1,55 +0,0 @@ -output "raw-data-backend-public-IP" { - value = azurerm_public_ip.raw-data-backend.ip_address - description = "Public SSH-able IP address for the raw-data backend VM" -} - -output "raw-data-db-endpoint" { - value = azurerm_postgresql_flexible_server.raw-data.fqdn - description = "FQDN to connect to raw-data API PostgreSQL DB" -} - -output "default_backend_ssh_string" { - description = "SSH string to use to connect to the backend VM" - value = join( - "", - [ - "ssh ", - lookup(var.admin_usernames, "backend"), - "@", - azurerm_public_ip.raw-data-backend.ip_address - ] - ) -} - -output "redis-connection-string-default" { - sensitive = true - - description = "Default secure connection string for Redis" - value = join("", - [ - "rediss://:", - azurerm_redis_cache.raw-data-queue.primary_access_key, - "@", - azurerm_redis_cache.raw-data-queue.hostname, - ":", - azurerm_redis_cache.raw-data-queue.ssl_port, - "/0?ssl_cert_reqs=required" - ] - ) -} - -output "raw-data-redis-endpoint" { - description = "Redis cache service endpoint" - value = join( - ":", - [ - azurerm_redis_cache.raw-data-queue.hostname, - azurerm_redis_cache.raw-data-queue.ssl_port - ] - ) -} - -output "container-subnet-id" { - description = "Subnet ID for the container subnet" - value = azurerm_subnet.raw-data-containers.id -} diff --git a/infra/production/provider.tf b/infra/production/provider.tf deleted file mode 100644 index debc9c60..00000000 --- a/infra/production/provider.tf +++ /dev/null @@ -1,42 +0,0 @@ -terraform { - required_version = ">= 1.5.0" - - backend "remote" { - organization = "hotosm" - - workspaces { - name = "raw-data-production" - } - } - - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "=3.65.0" - } - - random = { - source = "hashicorp/random" - version = "=3.5.1" - } - - tfe = { - source = "hashicorp/tfe" - version = "~> 0.46.0" - } - - } -} - -provider "azurerm" { - features {} - - subscription_id = var.azure_subscription_id - client_id = var.azure_client_id - client_secret = var.azure_client_secret - tenant_id = var.azure_tenant_id -} - -provider "random" { - -} diff --git a/infra/production/tests/compliance/cloudfront.feature b/infra/production/tests/compliance/cloudfront.feature deleted file mode 100644 index 8ac5598a..00000000 --- a/infra/production/tests/compliance/cloudfront.feature +++ /dev/null @@ -1,62 +0,0 @@ -Feature: Cloudfront CDN related policies - Cloudfront distribution must be secure by default - - Acceptance criteria - - Cloudfront clients must connect over TLS/SSL - - Cloudfront viewer TLS protocol must be TLSv1.2 minimum - - Cloudfront cache sane defaults must be enabled for Security Headers - - Cloudfront distributions must have sane defaults set for networking - - Acceptance criteria - - Cloudfront distributions must have IPv6 enabled - - Cloudfront distributions should connect over http2 - - Cloudfront distributions must not restrict by geography - - Cloudfront distributions must have reasonable cache settings - - Acceptance criteria - - Minimum cache TTL must be longer than 5 minutes - - Maximum cache TTL must be no longer than one month - - Scenario: Cloudfront distribution ViewerProtocolPolicy must be set to redirect to HTTPS - Given I have aws_cloudfront_distribution defined - Then it must have default_cache_behavior - And it must have viewer_protocol_policy - And its value must be redirect-to-https - - Scenario: Cloudfront distribution must have IPv6 enabled - Given I have aws_cloudfront_distribution defined - Then it must have is_ipv6_enabled - And its value must be true - - Scenario: Cloudfront distribution must have http2 enabled - Given I have aws_cloudfront_distribution defined - Then it must have http_version - And its value must be http2 - - Scenario: Cloudfront clients must use at least TLSv1.2 to connect - Given I have aws_cloudfront_distribution defined - Then it must have viewer_certificate - And it must have minimum_protocol_version - And its value must be TLSv1.2_2021 - - Scenario: Cloudfront must not restrict by geography - Given I have aws_cloudfront_distribution defined - Then it must have restrictions - And it must have geo_restriction - And it must have restriction_type - And its value must be none - - Scenario: Cloudfront cache maximum TTL must be shorter than a month - Given I have aws_cloudfront_distribution defined - Then it must have default_cache_behavior - And it must have max_ttl - And its value must be 2419200 - - Scenario: Cloudfront cache default TTL must be about a week - Given I have aws_cloudfront_distribution defined - Then it must have default_cache_behavior - And it must have default_ttl - And its value must be 604800 - diff --git a/infra/production/tests/compliance/s3.feature b/infra/production/tests/compliance/s3.feature deleted file mode 100644 index acc9c5ab..00000000 --- a/infra/production/tests/compliance/s3.feature +++ /dev/null @@ -1,23 +0,0 @@ -# CHECK SECURITY OF S3 Bucket - -Feature: Ensure that S3 bucket has tags assigned - As an S3 bucket containing sensitive information - It has to contain tags - So that it can be used to classify the bucket - - Acceptance criteria: - - Bucket should have tags - -Scenario Outline: Ensure specific tags are defined for S3 buckets - Given I have aws_s3_bucket defined - When it has tags - Then it must contain - And its value must match the "" regex - -Examples: - |tags |value | - |Name |.+ | - |Environment |^(prod\|dev\|stage\|uat)+| - |Project |.+ | - |Maintainer |.+ | - |Documentation|.+ | diff --git a/infra/production/variables.tf b/infra/production/variables.tf deleted file mode 100644 index 0be95287..00000000 --- a/infra/production/variables.tf +++ /dev/null @@ -1,99 +0,0 @@ -variable "project_name" { - type = string - default = "raw-data" -} - -variable "azure_subscription_id" { - type = string - default = "" -} - -variable "azure_client_id" { - type = string - default = "" -} - -variable "azure_client_secret" { - type = string - default = "" -} - -variable "azure_tenant_id" { - type = string - default = "" -} - -variable "arm_location" { - type = string - default = "West Europe" - -} - -variable "deployment_environment" { - type = string - default = "production" -} - -variable "admin_usernames" { - type = map(any) - default = { - backend = "hotsysadmin" - database = "hotdba" - } -} -variable "disk_size" { - type = map(number) - default = { - backend_os = 32 - } -} - -variable "server_skus" { - type = map(any) - default = { - database = "GP_Standard_D2ds_v4" - backend_old = "Standard_D2ls_v5" - backend = "Standard_E4as_v5" - } -} - -variable "ssh_public_key" { - type = string - default = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDqvEc1ESPkG7z2lBX2xYg1fsjXq+JNlcFJdEYtP2SKaVg8Vt0q4/oPBZSHN4p74oHZLEAF//2uFFbqADSvZYRtXIAdp78KvKGXo0Gkqd1pyf0ZPk4kEsfQfxcGeNYmT2T5I1ciYEdpbNgMv9C+WMdXZg0qZhOrgoAeJ6cudsBMtrJu3TTf6At3VELWqB0wL8fMHAfhDxy2+nojitp2OC20y9vAzwg8Uwpv+hVtjf19pijWT5i3gZspNYh+QxsZm+iPzhsD0E40Yi5UH/sqHmulbRYdlbemybeV4XoPEzcZ9UZHTQNXE3yvM592k7AxKHKdhr0y8qL+YzuO3Q0OCmxLtK9iQSchtBvnjhWqQQFfQoVxe+W/Yz0woKzy6+tixZOaTuTio2c23SQrAozTCIEpUkmDpv38FJXDAPl0Emsd5cUWyI2kcyq642B2jKZaKIZgJ0DBlbo7n1TIFYoBRYSa+wDQJ6VkMhNWVOhOBZ6ugu2wOFUe9BU2Q8Fd68hCSc=" - description = "Content of SSH Key public component" -} - -variable "azuread_admin_group_object_id" { - type = string - default = "" -} - -variable "newrelic_license_key" { - type = string - default = "" -} - -variable "container_images" { - description = "Remote container image URI to pull from" - type = map(string) - - default = { - api = "quay.io/hotosm/raw-data-api:latest" - worker = "quay.io/hotosm/raw-data-api:latest" - } -} - -variable "container_envvar" { - description = "Environment Variables to pass to the container" - type = map(string) - - default = {} -} - -variable "container_sensitive_envvar" { - description = "Environment Variables to pass to the container" - type = map(string) - - default = {} -} - diff --git a/infra/staging/main.tf b/infra/staging/main.tf deleted file mode 100644 index f40bf588..00000000 --- a/infra/staging/main.tf +++ /dev/null @@ -1,201 +0,0 @@ -data "tfe_ip_ranges" "addresses" {} - -data "azurerm_client_config" "current" {} - -locals { - required_tags = { - deployment_environment = var.deployment_environment - infrastructure_management = "terraform" - project = var.project_name - } - - conditional_tags = { - _monitor_cloudwatch = "Yes" - _monitor_newrelic_infra = "Yes" - _monitor_apm = "No" - _monitor_sentry = "No" - _patch_management = "No" - } -} - -resource "azurerm_resource_group" "raw-data" { - name = var.project_name - location = var.arm_location -} - -resource "azurerm_virtual_network" "raw-data" { - name = join("-", [var.project_name, var.deployment_environment]) - resource_group_name = azurerm_resource_group.raw-data.name - address_space = ["10.0.0.0/16"] - location = azurerm_resource_group.raw-data.location - - tags = local.required_tags -} - -resource "azurerm_subnet" "raw-data" { - name = join("-", [var.project_name, var.deployment_environment]) - resource_group_name = azurerm_resource_group.raw-data.name - virtual_network_name = azurerm_virtual_network.raw-data.name - address_prefixes = [cidrsubnet(azurerm_virtual_network.raw-data.address_space[0], 8, 0)] - - service_endpoints = ["Microsoft.KeyVault"] -} - -resource "random_string" "raw_data_db_password" { - length = 20 - override_special = "*()-_=+[]{}<>" -} - -resource "azurerm_key_vault" "raw-data" { - name = join("-", [var.project_name, var.deployment_environment]) - location = azurerm_resource_group.raw-data.location - resource_group_name = azurerm_resource_group.raw-data.name - sku_name = "standard" - tenant_id = data.azurerm_client_config.current.tenant_id - - network_acls { - bypass = "AzureServices" - default_action = "Allow" // Todo: Deny - ip_rules = data.tfe_ip_ranges.addresses.api - virtual_network_subnet_ids = [azurerm_subnet.raw-data.id] - } - - access_policy { - tenant_id = data.azurerm_client_config.current.tenant_id - object_id = data.azurerm_client_config.current.object_id - - key_permissions = [ - "List", - "Get", - ] - - secret_permissions = [ - "Set", - "Get", - "List", - ] - - storage_permissions = [ - ] - } - - purge_protection_enabled = false - soft_delete_retention_days = 7 - - tags = local.required_tags -} - -resource "azurerm_key_vault_secret" "raw-data-db" { - name = join("-", [var.project_name, "database", var.deployment_environment]) - value = random_string.raw_data_db_password.result - key_vault_id = azurerm_key_vault.raw-data.id - - tags = local.required_tags -} - -resource "azurerm_public_ip" "raw-data-backend" { - name = join("-", [var.project_name, "backend", var.deployment_environment]) - resource_group_name = azurerm_resource_group.raw-data.name - location = azurerm_resource_group.raw-data.location - allocation_method = "Static" // or "Dynamic" - - tags = local.required_tags -} - -resource "azurerm_network_interface" "raw-data-backend" { - name = join("-", [var.project_name, var.deployment_environment]) - location = azurerm_resource_group.raw-data.location - resource_group_name = azurerm_resource_group.raw-data.name - - ip_configuration { - name = "internal" - subnet_id = azurerm_subnet.raw-data.id - private_ip_address_allocation = "Dynamic" - primary = true - public_ip_address_id = azurerm_public_ip.raw-data-backend.id - } - - tags = local.required_tags -} - -resource "azurerm_linux_virtual_machine" "raw-data-backend" { - admin_username = lookup(var.admin_usernames, "backend") - location = var.arm_location - name = join("-", [var.project_name, "backend", var.deployment_environment]) - network_interface_ids = [azurerm_network_interface.raw-data-backend.id] - resource_group_name = azurerm_resource_group.raw-data.name - size = lookup(var.server_skus, "backend") - - source_image_reference { - publisher = "Canonical" - offer = "0001-com-ubuntu-server-jammy" - sku = "22_04-lts" - version = "latest" - } - - /* Ref: https://wiki.debian.org/Cloud/MicrosoftAzure - source_image_reference { - publisher = "Debian" - offer = "debian-11" - sku = "11" - version = "latest" - } - */ - - os_disk { - caching = "None" - storage_account_type = "StandardSSD_LRS" // StandardSSD_ZRS - disk_size_gb = lookup(var.disk_size, "backend_os") - name = join("-", [var.project_name, var.deployment_environment]) - - } - - admin_ssh_key { - public_key = var.ssh_public_key - username = lookup(var.admin_usernames, "backend") - } - - tags = merge(local.required_tags, local.conditional_tags) -} - -resource "azurerm_postgresql_flexible_server" "raw-data" { - name = join("-", [var.project_name, var.deployment_environment]) - resource_group_name = azurerm_resource_group.raw-data.name - location = azurerm_resource_group.raw-data.location - sku_name = lookup(var.server_skus, "database") - - administrator_login = lookup(var.admin_usernames, "database") - administrator_password = azurerm_key_vault_secret.raw-data-db.value - - authentication { - } - - backup_retention_days = 7 - geo_redundant_backup_enabled = false - storage_mb = 2097152 - - tags = local.required_tags - - version = 14 - zone = "1" -} - -resource "azurerm_postgresql_flexible_server_configuration" "raw-data-postgis" { - name = "azure.extensions" - server_id = azurerm_postgresql_flexible_server.raw-data.id - value = "BTREE_GIST,INTARRAY,POSTGIS" -} - -resource "azurerm_redis_cache" "raw-data-queue" { - name = join("-", [var.project_name, var.deployment_environment]) - resource_group_name = azurerm_resource_group.raw-data.name - location = azurerm_resource_group.raw-data.location - capacity = 0 - family = "C" - sku_name = "Basic" - - minimum_tls_version = "1.2" - redis_version = 6 - - tags = local.required_tags -} diff --git a/infra/staging/outputs.tf b/infra/staging/outputs.tf deleted file mode 100644 index 624002c0..00000000 --- a/infra/staging/outputs.tf +++ /dev/null @@ -1,34 +0,0 @@ -output "raw-data-backend-public-IP" { - value = azurerm_public_ip.raw-data-backend.ip_address - description = "Public SSH-able IP address for the raw-data backend VM" -} - -output "raw-data-db-endpoint" { - value = azurerm_postgresql_flexible_server.raw-data.fqdn - description = "FQDN to connect to raw-data API PostgreSQL DB" -} - -output "default_backend_ssh_string" { - description = "SSH string to use to connect to the backend VM" - value = join( - "", - [ - "ssh ", - lookup(var.admin_usernames, "backend"), - "@", - azurerm_public_ip.raw-data-backend.ip_address - ] - ) -} - -output "raw-data-redis-endpoint" { - description = "Redis cache service endpoint" - value = join( - ":", - [ - azurerm_redis_cache.raw-data-queue.hostname, - azurerm_redis_cache.raw-data-queue.ssl_port - ] - ) -} - diff --git a/infra/staging/provider.tf b/infra/staging/provider.tf deleted file mode 100644 index 61036fff..00000000 --- a/infra/staging/provider.tf +++ /dev/null @@ -1,42 +0,0 @@ -terraform { - required_version = ">= 1.4.0" - - backend "remote" { - organization = "hotosm" - - workspaces { - name = "raw-data-staging" - } - } - - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "=3.46.0" - } - - random = { - source = "hashicorp/random" - version = "=3.4.3" - } - - tfe = { - source = "hashicorp/tfe" - version = "~> 0.42.0" - } - - } -} - -provider "azurerm" { - subscription_id = var.azure_subscription_id - client_id = var.azure_client_id - client_secret = var.azure_client_secret - tenant_id = var.azure_tenant_id - - features {} -} - -provider "random" { - -} diff --git a/infra/staging/tests/compliance/cloudfront.feature b/infra/staging/tests/compliance/cloudfront.feature deleted file mode 100644 index 8ac5598a..00000000 --- a/infra/staging/tests/compliance/cloudfront.feature +++ /dev/null @@ -1,62 +0,0 @@ -Feature: Cloudfront CDN related policies - Cloudfront distribution must be secure by default - - Acceptance criteria - - Cloudfront clients must connect over TLS/SSL - - Cloudfront viewer TLS protocol must be TLSv1.2 minimum - - Cloudfront cache sane defaults must be enabled for Security Headers - - Cloudfront distributions must have sane defaults set for networking - - Acceptance criteria - - Cloudfront distributions must have IPv6 enabled - - Cloudfront distributions should connect over http2 - - Cloudfront distributions must not restrict by geography - - Cloudfront distributions must have reasonable cache settings - - Acceptance criteria - - Minimum cache TTL must be longer than 5 minutes - - Maximum cache TTL must be no longer than one month - - Scenario: Cloudfront distribution ViewerProtocolPolicy must be set to redirect to HTTPS - Given I have aws_cloudfront_distribution defined - Then it must have default_cache_behavior - And it must have viewer_protocol_policy - And its value must be redirect-to-https - - Scenario: Cloudfront distribution must have IPv6 enabled - Given I have aws_cloudfront_distribution defined - Then it must have is_ipv6_enabled - And its value must be true - - Scenario: Cloudfront distribution must have http2 enabled - Given I have aws_cloudfront_distribution defined - Then it must have http_version - And its value must be http2 - - Scenario: Cloudfront clients must use at least TLSv1.2 to connect - Given I have aws_cloudfront_distribution defined - Then it must have viewer_certificate - And it must have minimum_protocol_version - And its value must be TLSv1.2_2021 - - Scenario: Cloudfront must not restrict by geography - Given I have aws_cloudfront_distribution defined - Then it must have restrictions - And it must have geo_restriction - And it must have restriction_type - And its value must be none - - Scenario: Cloudfront cache maximum TTL must be shorter than a month - Given I have aws_cloudfront_distribution defined - Then it must have default_cache_behavior - And it must have max_ttl - And its value must be 2419200 - - Scenario: Cloudfront cache default TTL must be about a week - Given I have aws_cloudfront_distribution defined - Then it must have default_cache_behavior - And it must have default_ttl - And its value must be 604800 - diff --git a/infra/staging/tests/compliance/s3.feature b/infra/staging/tests/compliance/s3.feature deleted file mode 100644 index acc9c5ab..00000000 --- a/infra/staging/tests/compliance/s3.feature +++ /dev/null @@ -1,23 +0,0 @@ -# CHECK SECURITY OF S3 Bucket - -Feature: Ensure that S3 bucket has tags assigned - As an S3 bucket containing sensitive information - It has to contain tags - So that it can be used to classify the bucket - - Acceptance criteria: - - Bucket should have tags - -Scenario Outline: Ensure specific tags are defined for S3 buckets - Given I have aws_s3_bucket defined - When it has tags - Then it must contain - And its value must match the "" regex - -Examples: - |tags |value | - |Name |.+ | - |Environment |^(prod\|dev\|stage\|uat)+| - |Project |.+ | - |Maintainer |.+ | - |Documentation|.+ | diff --git a/infra/staging/variables.tf b/infra/staging/variables.tf deleted file mode 100644 index fac2ff64..00000000 --- a/infra/staging/variables.tf +++ /dev/null @@ -1,63 +0,0 @@ -variable "project_name" { - type = string - default = "raw-data" -} - -variable "azure_subscription_id" { - type = string - default = "" -} - -variable "azure_client_id" { - type = string - default = "" -} - -variable "azure_client_secret" { - type = string - default = "" -} - -variable "azure_tenant_id" { - type = string - default = "" -} - -variable "arm_location" { - type = string - default = "West Europe" - -} - -variable "deployment_environment" { - type = string - default = "staging" -} - -variable "admin_usernames" { - type = map(any) - default = { - backend = "hotsysadmin" - database = "hotdba" - } -} -variable "disk_size" { - type = map(number) - default = { - backend_os = 30 - } -} - -variable "server_skus" { - type = map(any) - default = { - database = "GP_Standard_D2ds_v4" - backend = "Standard_F2" - } -} - -variable "ssh_public_key" { - type = string - default = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDqvEc1ESPkG7z2lBX2xYg1fsjXq+JNlcFJdEYtP2SKaVg8Vt0q4/oPBZSHN4p74oHZLEAF//2uFFbqADSvZYRtXIAdp78KvKGXo0Gkqd1pyf0ZPk4kEsfQfxcGeNYmT2T5I1ciYEdpbNgMv9C+WMdXZg0qZhOrgoAeJ6cudsBMtrJu3TTf6At3VELWqB0wL8fMHAfhDxy2+nojitp2OC20y9vAzwg8Uwpv+hVtjf19pijWT5i3gZspNYh+QxsZm+iPzhsD0E40Yi5UH/sqHmulbRYdlbemybeV4XoPEzcZ9UZHTQNXE3yvM592k7AxKHKdhr0y8qL+YzuO3Q0OCmxLtK9iQSchtBvnjhWqQQFfQoVxe+W/Yz0woKzy6+tixZOaTuTio2c23SQrAozTCIEpUkmDpv38FJXDAPl0Emsd5cUWyI2kcyq642B2jKZaKIZgJ0DBlbo7n1TIFYoBRYSa+wDQJ6VkMhNWVOhOBZ6ugu2wOFUe9BU2Q8Fd68hCSc=" - description = "Content of SSH Key public component" -}