From b64e7cdb62f2c765114487c819f41906b18603c2 Mon Sep 17 00:00:00 2001
From: Paul Yu <paul.d.yu@gmail.com>
Date: Tue, 13 Feb 2024 14:25:11 -0800
Subject: [PATCH 1/6] feat: modularize tf deployment resolves #112

---
 azd-hooks/predeploy.sh                        | 62 +++++++++++++------
 azd-hooks/preprovision.sh                     |  1 +
 .../aks-store-demo/templates/ai-service.yaml  | 46 +++++++++++---
 charts/aks-store-demo/values.yaml             |  4 +-
 infra/terraform/cosmosdb.tf                   | 31 ++++++----
 infra/terraform/keyvault.tf                   | 56 +++++++++--------
 infra/terraform/kubernetes.tf                 | 29 ++++++---
 infra/terraform/locals.tf                     | 15 +++--
 infra/terraform/main.tfvars.json              |  7 ++-
 infra/terraform/observability.tf              | 43 ++++++++-----
 infra/terraform/openai.tf                     | 16 +++--
 infra/terraform/outputs.tf                    | 35 ++++++-----
 infra/terraform/servicebus.tf                 | 10 ++-
 infra/terraform/variables.tf                  | 36 ++++++++++-
 14 files changed, 264 insertions(+), 127 deletions(-)

diff --git a/azd-hooks/predeploy.sh b/azd-hooks/predeploy.sh
index 1c59adf7..b2423607 100755
--- a/azd-hooks/predeploy.sh
+++ b/azd-hooks/predeploy.sh
@@ -4,31 +4,53 @@ echo "Retrieving cluster credentials"
 az aks get-credentials --resource-group ${AZURE_RESOURCEGROUP_NAME} --name ${AZURE_AKS_CLUSTER_NAME} --overwrite-existing
 
 echo "Deploy Helm chart"
-helm upgrade aks-store-demo ./charts/aks-store-demo \
+cmd="helm upgrade aks-store-demo ./charts/aks-store-demo \
   --install \
-  --set aiService.create=true \
-  --set aiService.modelDeploymentName=${AZURE_OPENAI_MODEL_NAME} \
-  --set aiService.openAiEndpoint=${AZURE_OPENAI_ENDPOINT} \
-  --set aiService.managedIdentityClientId=${AZURE_IDENTITY_CLIENT_ID} \
   --set aiService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/ai-service \
-  --set orderService.useAzureServiceBus=true \
-  --set orderService.queueHost=${AZURE_SERVICE_BUS_HOST} \
-  --set orderService.queuePort=5671 \
-  --set orderService.queueUsername=${AZURE_SERVICE_BUS_SENDER_NAME} \
-  --set orderService.queuePassword=$(az keyvault secret show --name ${AZURE_SERVICE_BUS_SENDER_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv) \
-  --set orderService.queueTransport=tls \
   --set orderService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/order-service \
-  --set makelineService.useAzureCosmosDB=true \
-  --set makelineService.orderQueueUri=${AZURE_SERVICE_BUS_URI} \
-  --set makelineService.orderQueueUsername=${AZURE_SERVICE_BUS_LISTENER_NAME} \
-  --set makelineService.orderQueuePassword=$(az keyvault secret show --name ${AZURE_SERVICE_BUS_LISTENER_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv) \
-  --set makelineService.orderDBUri=${AZURE_COSMOS_DATABASE_URI} \
-  --set makelineService.orderDBUsername=${AZURE_COSMOS_DATABASE_NAME} \
-  --set makelineService.orderDBPassword=$(az keyvault secret show --name ${AZURE_COSMOS_DATABASE_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv) \
   --set makelineService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/makeline-service \
   --set productService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/product-service \
   --set storeAdmin.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/store-admin \
   --set storeFront.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/store-front \
   --set virtualCustomer.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/virtual-customer \
-  --set virtualWorker.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/virtual-worker \
-  $(if [ "${AZURE_DATABASE_API}" == "cosmosdbsql" ]; then echo "--set makelineService.useSqlApi=true"; fi)
+  --set virtualWorker.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/virtual-worker"
+
+if [ -n "${AZURE_OPENAI_ENDPOINT}" ]; then
+  cmd+=" --set aiService.create=true \
+    --set aiService.openAiEndpoint=${AZURE_OPENAI_ENDPOINT} \
+    --set aiService.modelDeploymentName=${AZURE_OPENAI_MODEL_NAME} \
+    --set aiService.useAzureOpenAi=true"
+
+  if [ -n "${AZURE_IDENTITY_CLIENT_ID}" ]; then
+    cmd+=" --set aiService.managedIdentityClientId=${AZURE_IDENTITY_CLIENT_ID} \
+      --set aiService.useAzureAd=true"
+  else
+    cmd+=" --set aiService.openAiKey=$(az keyvault secret show --name ${AZURE_OPENAI_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv)  \
+      --set aiService.useAzureAd=false"
+  fi
+fi
+
+if [ -n "${AZURE_SERVICE_BUS_HOST}" ]; then
+  cmd+=" --set orderService.useAzureServiceBus=true \
+    --set orderService.queueHost=${AZURE_SERVICE_BUS_HOST} \
+    --set orderService.queuePort=5671 \
+    --set orderService.queueUsername=${AZURE_SERVICE_BUS_SENDER_NAME} \
+    --set orderService.queuePassword=$(az keyvault secret show --name ${AZURE_SERVICE_BUS_SENDER_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv) \
+    --set orderService.queueTransport=tls \
+    --set makelineService.orderQueueUri=${AZURE_SERVICE_BUS_URI} \
+    --set makelineService.orderQueueUsername=${AZURE_SERVICE_BUS_LISTENER_NAME} \
+    --set makelineService.orderQueuePassword=$(az keyvault secret show --name ${AZURE_SERVICE_BUS_LISTENER_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv)"
+fi
+
+if [ -n "${AZURE_COSMOS_DATABASE_URI}" ]; then
+  cmd+=" --set makelineService.useAzureCosmosDB=true \
+    --set makelineService.orderDBUri=${AZURE_COSMOS_DATABASE_URI} \
+    --set makelineService.orderDBUsername=${AZURE_COSMOS_DATABASE_NAME} \
+    --set makelineService.orderDBPassword=$(az keyvault secret show --name ${AZURE_COSMOS_DATABASE_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv)"
+
+  if [ "${AZURE_DATABASE_API}" == "cosmosdbsql" ]; then
+    cmd+=" --set makelineService.useSqlApi=true"
+  fi
+fi
+
+eval $cmd
diff --git a/azd-hooks/preprovision.sh b/azd-hooks/preprovision.sh
index 61b9fb43..bdc09024 100755
--- a/azd-hooks/preprovision.sh
+++ b/azd-hooks/preprovision.sh
@@ -6,5 +6,6 @@ az feature register --namespace Microsoft.ContainerService --name AKS-KedaPrevie
 az feature register --namespace Microsoft.ContainerService --name AKS-PrometheusAddonPreview
 az feature register --namespace Microsoft.ContainerService --name EnableWorkloadIdentityPreview
 az feature register --namespace Microsoft.ContainerService --name NetworkObservabilityPreview
+az feature register --namespace "Microsoft.ContainerService" --name "NodeOsUpgradeChannelPreview"
 az extension add --upgrade --name aks-preview
 az extension add --upgrade --name amg
\ No newline at end of file
diff --git a/charts/aks-store-demo/templates/ai-service.yaml b/charts/aks-store-demo/templates/ai-service.yaml
index 697df893..d80e9dee 100644
--- a/charts/aks-store-demo/templates/ai-service.yaml
+++ b/charts/aks-store-demo/templates/ai-service.yaml
@@ -6,19 +6,41 @@ metadata:
 data:
   AZURE_OPENAI_DEPLOYMENT_NAME: "{{ .Values.aiService.modelDeploymentName }}"
   AZURE_OPENAI_ENDPOINT: "{{ .Values.aiService.openAiEndpoint }}"
-  {{- if .Values.aiService.useAzureOpenAi }}
+  {{- /*
+  Use Azure OpenAI or OpenAI  
+  */}}
+  {{- if eq .Values.aiService.useAzureOpenAi true }}
   USE_AZURE_OPENAI: "True"
   {{- else }}
   USE_AZURE_OPENAI: "False"
   OPENAI_ORG_ID: "{{ .Values.aiService.openAiOrgId }}"
   {{- end }}
-  {{- if .Values.aiService.useAzureAd }}
+  
+  {{- /*
+  Use Azure AD or OpenAI API Key  
+  */}}
+  {{- if eq .Values.aiService.useAzureAd true }}
   USE_AZURE_AD: "True"
   {{- else }}
   USE_AZURE_AD: "False"
   {{- end }}
 ---
-{{- if eq .Values.aiService.useAzureAd false }}
+
+{{- /*
+If Azure AD is used, create a service account with managed identity
+*/}}
+{{- if eq .Values.aiService.useAzureAd true }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ai-service-account
+  annotations:
+    azure.workload.identity/client-id: "{{ .Values.aiService.managedIdentityClientId }}"
+---
+{{- /*
+If Azure AD is not used, create a secret with OpenAI API Key
+*/}}
+{{- else }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -27,13 +49,7 @@ data:
   OPENAI_API_KEY: "{{ .Values.aiService.openAiKey | b64enc }}"
 ---
 {{- end }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: ai-service-account
-  annotations:
-    azure.workload.identity/client-id: "{{ .Values.aiService.managedIdentityClientId }}"
----
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -47,9 +63,19 @@ spec:
     metadata:
       labels:
         app: ai-service
+        {{- /*
+        If Azure AD is used, set the label to use workload identity
+        */}}
+        {{- if eq .Values.aiService.useAzureAd true }}
         azure.workload.identity/use: "true"
+        {{- end }}
     spec:
+      {{- /*
+      If Azure AD is used, use the service account
+      */}}
+      {{- if eq .Values.aiService.useAzureAd true }}
       serviceAccount: ai-service-account
+      {{- end }}
       nodeSelector:
         "kubernetes.io/os": linux
       containers:
diff --git a/charts/aks-store-demo/values.yaml b/charts/aks-store-demo/values.yaml
index 90bc469b..9f6ea13b 100644
--- a/charts/aks-store-demo/values.yaml
+++ b/charts/aks-store-demo/values.yaml
@@ -14,8 +14,8 @@ aiService:
   openAiKey: ""
   openAiOrgId: ""
   managedIdentityClientId: ""
-  useAzureOpenAi: true
-  useAzureAd: true
+  useAzureOpenAi: false
+  useAzureAd: false
   image:
     repository: "ghcr.io/azure-samples/aks-store-demo/ai-service"
     tag: "latest"
diff --git a/infra/terraform/cosmosdb.tf b/infra/terraform/cosmosdb.tf
index bf4b20bb..e0c92eb6 100644
--- a/infra/terraform/cosmosdb.tf
+++ b/infra/terraform/cosmosdb.tf
@@ -1,4 +1,5 @@
 resource "azurerm_cosmosdb_account" "example" {
+  count               = local.deploy_azure_cosmosdb ? 1 : 0
   name                = "db-${local.name}"
   location            = azurerm_resource_group.example.location
   resource_group_name = azurerm_resource_group.example.name
@@ -32,39 +33,43 @@ resource "azurerm_cosmosdb_account" "example" {
 }
 
 resource "azurerm_cosmosdb_mongo_database" "example" {
-  count               = local.cosmosdb_account_kind == "MongoDB" ? 1 : 0
+  count               = local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "MongoDB" ? 1 : 0
   name                = "orderdb"
-  resource_group_name = azurerm_cosmosdb_account.example.resource_group_name
-  account_name        = azurerm_cosmosdb_account.example.name
+  resource_group_name = azurerm_cosmosdb_account.example[0].resource_group_name
+  account_name        = azurerm_cosmosdb_account.example[0].name
   throughput          = 400
 }
 
 resource "azurerm_cosmosdb_mongo_collection" "example" {
-  count               = local.cosmosdb_account_kind == "MongoDB" ? 1 : 0
+  count               = local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "MongoDB" ? 1 : 0
   name                = "orders"
-  resource_group_name = azurerm_cosmosdb_account.example.resource_group_name
-  account_name        = azurerm_cosmosdb_account.example.name
+  resource_group_name = azurerm_cosmosdb_account.example[0].resource_group_name
+  account_name        = azurerm_cosmosdb_account.example[0].name
   database_name       = azurerm_cosmosdb_mongo_database.example[0].name
   throughput          = 400
 
   index {
-    keys   = ["_id"]
+    keys = ["_id"]
+  }
+
+  lifecycle {
+    ignore_changes = [index]
   }
 }
 
 resource "azurerm_cosmosdb_sql_database" "example" {
-  count               = local.cosmosdb_account_kind == "GlobalDocumentDB" ? 1 : 0
+  count               = local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "GlobalDocumentDB" ? 1 : 0
   name                = "orderdb"
-  resource_group_name = azurerm_cosmosdb_account.example.resource_group_name
-  account_name        = azurerm_cosmosdb_account.example.name
+  resource_group_name = azurerm_cosmosdb_account.example[0].resource_group_name
+  account_name        = azurerm_cosmosdb_account.example[0].name
   throughput          = 400
 }
 
 resource "azurerm_cosmosdb_sql_container" "example" {
-  count                 = local.cosmosdb_account_kind == "GlobalDocumentDB" ? 1 : 0
+  count                 = local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "GlobalDocumentDB" ? 1 : 0
   name                  = "orders"
-  resource_group_name   = azurerm_cosmosdb_account.example.resource_group_name
-  account_name          = azurerm_cosmosdb_account.example.name
+  resource_group_name   = azurerm_cosmosdb_account.example[0].resource_group_name
+  account_name          = azurerm_cosmosdb_account.example[0].name
   database_name         = azurerm_cosmosdb_sql_database.example[0].name
   partition_key_path    = "/storeId"
   partition_key_version = 1
diff --git a/infra/terraform/keyvault.tf b/infra/terraform/keyvault.tf
index fc24b1fd..b85bb66c 100644
--- a/infra/terraform/keyvault.tf
+++ b/infra/terraform/keyvault.tf
@@ -1,43 +1,47 @@
 resource "azurerm_key_vault" "example" {
-    name                        = "akv-${local.name}"
-    location                    = azurerm_resource_group.example.location
-    resource_group_name         = azurerm_resource_group.example.name
-    enabled_for_disk_encryption = true
-    tenant_id                   = data.azurerm_client_config.current.tenant_id
-    sku_name                    = "standard"
-    enable_rbac_authorization   = true
+  name                        = "akv-${local.name}"
+  location                    = azurerm_resource_group.example.location
+  resource_group_name         = azurerm_resource_group.example.name
+  enabled_for_disk_encryption = true
+  tenant_id                   = data.azurerm_client_config.current.tenant_id
+  sku_name                    = "standard"
+  enable_rbac_authorization   = true
 }
 
 resource "azurerm_role_assignment" "example_akv_rbac" {
-    principal_id         = data.azurerm_client_config.current.object_id
-    role_definition_name = "Key Vault Administrator"
-    scope                = azurerm_key_vault.example.id
+  principal_id         = data.azurerm_client_config.current.object_id
+  role_definition_name = "Key Vault Administrator"
+  scope                = azurerm_key_vault.example.id
 }
 
 resource "azurerm_key_vault_secret" "openai_key" {
-    name         = "AZURE-OPENAI-KEY"
-    value        = azurerm_cognitive_account.example.primary_access_key
-    key_vault_id = azurerm_key_vault.example.id
-    depends_on = [ azurerm_role_assignment.example_akv_rbac ]
+  count        = local.deploy_azure_openai ? 1 : 0
+  name         = "AZURE-OPENAI-KEY"
+  value        = azurerm_cognitive_account.example[0].primary_access_key
+  key_vault_id = azurerm_key_vault.example.id
+  depends_on   = [azurerm_role_assignment.example_akv_rbac]
 }
 
 resource "azurerm_key_vault_secret" "cosmosdb_key" {
-    name         = "AZURE-COSMOS-KEY"
-    value        = azurerm_cosmosdb_account.example.primary_key
-    key_vault_id = azurerm_key_vault.example.id
-    depends_on = [ azurerm_role_assignment.example_akv_rbac ]
+  count        = local.deploy_azure_cosmosdb ? 1 : 0
+  name         = "AZURE-COSMOS-KEY"
+  value        = azurerm_cosmosdb_account.example[0].primary_key
+  key_vault_id = azurerm_key_vault.example.id
+  depends_on   = [azurerm_role_assignment.example_akv_rbac]
 }
 
 resource "azurerm_key_vault_secret" "listener_key" {
-    name         = "AZURE-SERVICE-BUS-LISTENER-KEY"
-    value        = azurerm_servicebus_namespace_authorization_rule.example.primary_key
-    key_vault_id = azurerm_key_vault.example.id
-    depends_on = [ azurerm_role_assignment.example_akv_rbac ]
+  count        = local.deploy_azure_servicebus ? 1 : 0
+  name         = "AZURE-SERVICE-BUS-LISTENER-KEY"
+  value        = azurerm_servicebus_namespace_authorization_rule.example[0].primary_key
+  key_vault_id = azurerm_key_vault.example.id
+  depends_on   = [azurerm_role_assignment.example_akv_rbac]
 }
 
 resource "azurerm_key_vault_secret" "sender_key" {
-    name         = "AZURE-SERVICE-BUS-SENDER-KEY"
-    value        = azurerm_servicebus_queue_authorization_rule.example.primary_key
-    key_vault_id = azurerm_key_vault.example.id
-    depends_on = [ azurerm_role_assignment.example_akv_rbac ]
+  count        = local.deploy_azure_servicebus ? 1 : 0
+  name         = "AZURE-SERVICE-BUS-SENDER-KEY"
+  value        = azurerm_servicebus_queue_authorization_rule.example[0].primary_key
+  key_vault_id = azurerm_key_vault.example.id
+  depends_on   = [azurerm_role_assignment.example_akv_rbac]
 }
diff --git a/infra/terraform/kubernetes.tf b/infra/terraform/kubernetes.tf
index 95010734..5b0d5e03 100644
--- a/infra/terraform/kubernetes.tf
+++ b/infra/terraform/kubernetes.tf
@@ -1,5 +1,5 @@
 resource "azurerm_container_registry" "example" {
-  count               = local.deploy_acr ? 1 : 0
+  count               = local.deploy_azure_container_registry ? 1 : 0
   name                = "acr${local.name}"
   resource_group_name = azurerm_resource_group.example.name
   location            = azurerm_resource_group.example.location
@@ -22,15 +22,26 @@ resource "azurerm_kubernetes_cluster" "example" {
     type = "SystemAssigned"
   }
 
-  oidc_issuer_enabled       = true
-  workload_identity_enabled = true
+  node_os_channel_upgrade   = "SecurityPatch"
+  oidc_issuer_enabled       = local.deploy_azure_workload_identity
+  workload_identity_enabled = local.deploy_azure_workload_identity
 
-  monitor_metrics {
+  key_vault_secrets_provider {
+    secret_rotation_enabled = true
   }
-  
-  oms_agent {
-    log_analytics_workspace_id      = azurerm_log_analytics_workspace.example.id
-    msi_auth_for_monitoring_enabled = true
+
+  dynamic "monitor_metrics" {
+    for_each = local.deploy_observability_tools ? [1] : []
+    content {
+    }
+  }
+
+  dynamic "oms_agent" {
+    for_each = local.deploy_observability_tools ? [1] : []
+    content {
+      log_analytics_workspace_id      = azurerm_log_analytics_workspace.example[0].id
+      msi_auth_for_monitoring_enabled = true
+    }
   }
 
   lifecycle {
@@ -43,7 +54,7 @@ resource "azurerm_kubernetes_cluster" "example" {
 }
 
 resource "azurerm_role_assignment" "example" {
-  count                            = local.deploy_acr ? 1 : 0
+  count                            = local.deploy_azure_container_registry ? 1 : 0
   principal_id                     = azurerm_kubernetes_cluster.example.kubelet_identity[0].object_id
   role_definition_name             = "AcrPull"
   scope                            = azurerm_container_registry.example[0].id
diff --git a/infra/terraform/locals.tf b/infra/terraform/locals.tf
index c8caf81b..3f7d64f9 100644
--- a/infra/terraform/locals.tf
+++ b/infra/terraform/locals.tf
@@ -1,7 +1,12 @@
 locals {
-  name                          = "${random_pet.example.id}${random_integer.example.result}"
-  location                      = var.location
-  default_cosmosdb_account_kind = "MongoDB"
-  cosmosdb_account_kind         = var.cosmosdb_account_kind != "" ? var.cosmosdb_account_kind : local.default_cosmosdb_account_kind
-  deploy_acr                    = var.deploy_acr == "true" ? true : false
+  name                            = "${random_pet.example.id}${random_integer.example.result}"
+  location                        = var.location
+  default_cosmosdb_account_kind   = "MongoDB"
+  cosmosdb_account_kind           = var.cosmosdb_account_kind != "" ? var.cosmosdb_account_kind : local.default_cosmosdb_account_kind
+  deploy_azure_container_registry = var.deploy_azure_container_registry == "true" ? true : false
+  deploy_azure_workload_identity  = var.deploy_azure_workload_identity == "true" ? true : false
+  deploy_azure_openai             = var.deploy_azure_openai == "true" ? true : false
+  deploy_azure_servicebus         = var.deploy_azure_servicebus == "true" ? true : false
+  deploy_azure_cosmosdb           = var.deploy_azure_cosmosdb == "true" ? true : false
+  deploy_observability_tools      = var.deploy_observability_tools == "true" ? true : false
 }
\ No newline at end of file
diff --git a/infra/terraform/main.tfvars.json b/infra/terraform/main.tfvars.json
index f1a7f544..49a45768 100644
--- a/infra/terraform/main.tfvars.json
+++ b/infra/terraform/main.tfvars.json
@@ -2,5 +2,10 @@
   "location": "${AZURE_LOCATION}",
   "ai_location": "${AZURE_LOCATION}",
   "cosmosdb_account_kind": "${AZURE_COSMOSDB_ACCOUNT_KIND}",
-  "deploy_acr": "${DEPLOY_AZURE_CONTAINER_REGISTRY}"
+  "deploy_azure_container_registry": "${DEPLOY_AZURE_CONTAINER_REGISTRY}",
+  "deploy_azure_workload_identity": "${DEPLOY_AZURE_WORKLOAD_IDENTITY}",
+  "deploy_azure_openai": "${DEPLOY_AZURE_OPENAI}",
+  "deploy_azure_servicebus": "${DEPLOY_AZURE_SERVICE_BUS}",
+  "deploy_azure_cosmosdb": "${DEPLOY_AZURE_COSMOSDB}",
+  "deploy_observability_tools": "${DEPLOY_OBSERVABILITY_TOOLS}"
 }
\ No newline at end of file
diff --git a/infra/terraform/observability.tf b/infra/terraform/observability.tf
index 62577d52..850d0efd 100644
--- a/infra/terraform/observability.tf
+++ b/infra/terraform/observability.tf
@@ -1,10 +1,12 @@
 resource "azurerm_monitor_workspace" "example" {
+  count               = local.deploy_observability_tools ? 1 : 0
   name                = "amon-${local.name}"
   resource_group_name = azurerm_resource_group.example.name
   location            = azurerm_resource_group.example.location
 }
 
 resource "azurerm_log_analytics_workspace" "example" {
+  count               = local.deploy_observability_tools ? 1 : 0
   name                = "alog-${local.name}"
   resource_group_name = azurerm_resource_group.example.name
   location            = azurerm_resource_group.example.location
@@ -13,6 +15,7 @@ resource "azurerm_log_analytics_workspace" "example" {
 }
 
 resource "azurerm_dashboard_grafana" "example" {
+  count               = local.deploy_observability_tools ? 1 : 0
   name                = "amg-${local.name}"
   resource_group_name = azurerm_resource_group.example.name
   location            = azurerm_resource_group.example.location
@@ -22,23 +25,26 @@ resource "azurerm_dashboard_grafana" "example" {
   }
 
   azure_monitor_workspace_integrations {
-    resource_id = azurerm_monitor_workspace.example.id
+    resource_id = azurerm_monitor_workspace.example[0].id
   }
 }
 
 resource "azurerm_role_assignment" "example_amg_me" {
-  scope                = azurerm_dashboard_grafana.example.id
+  count                = local.deploy_observability_tools ? 1 : 0
+  scope                = azurerm_dashboard_grafana.example[0].id
   role_definition_name = "Grafana Admin"
   principal_id         = data.azurerm_client_config.current.object_id
 }
 
 resource "azurerm_role_assignment" "example_rg_amg" {
-  principal_id         = azurerm_dashboard_grafana.example.identity[0].principal_id
+  count                = local.deploy_observability_tools ? 1 : 0
+  principal_id         = azurerm_dashboard_grafana.example[0].identity[0].principal_id
   role_definition_name = "Monitoring Data Reader"
   scope                = azurerm_resource_group.example.id
 }
 
 resource "azurerm_monitor_data_collection_endpoint" "example_msprom" {
+  count               = local.deploy_observability_tools ? 1 : 0
   name                = "MSProm-${azurerm_resource_group.example.location}-${azurerm_kubernetes_cluster.example.name}"
   resource_group_name = azurerm_resource_group.example.name
   location            = azurerm_resource_group.example.location
@@ -47,10 +53,11 @@ resource "azurerm_monitor_data_collection_endpoint" "example_msprom" {
 
 
 resource "azurerm_monitor_data_collection_rule" "example_msprom" {
+  count                       = local.deploy_observability_tools ? 1 : 0
   name                        = "MSProm-${azurerm_kubernetes_cluster.example.location}-${azurerm_kubernetes_cluster.example.name}"
   resource_group_name         = azurerm_resource_group.example.name
   location                    = azurerm_resource_group.example.location
-  data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.example_msprom.id
+  data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.example_msprom[0].id
 
   data_sources {
     prometheus_forwarder {
@@ -61,36 +68,39 @@ resource "azurerm_monitor_data_collection_rule" "example_msprom" {
 
   destinations {
     monitor_account {
-      monitor_account_id = azurerm_monitor_workspace.example.id
-      name               = azurerm_monitor_workspace.example.name
+      monitor_account_id = azurerm_monitor_workspace.example[0].id
+      name               = azurerm_monitor_workspace.example[0].name
     }
   }
 
   data_flow {
     streams      = ["Microsoft-PrometheusMetrics"]
-    destinations = [azurerm_monitor_workspace.example.name]
+    destinations = [azurerm_monitor_workspace.example[0].name]
   }
 }
 
 resource "azurerm_monitor_data_collection_rule_association" "example_dcr_to_aks" {
+  count                   = local.deploy_observability_tools ? 1 : 0
   name                    = "dcr-${azurerm_kubernetes_cluster.example.name}"
   target_resource_id      = azurerm_kubernetes_cluster.example.id
-  data_collection_rule_id = azurerm_monitor_data_collection_rule.example_msprom.id
+  data_collection_rule_id = azurerm_monitor_data_collection_rule.example_msprom[0].id
 }
 
 resource "azurerm_monitor_data_collection_rule_association" "example_dce_to_aks" {
+  count                       = local.deploy_observability_tools ? 1 : 0
   target_resource_id          = azurerm_kubernetes_cluster.example.id
-  data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.example_msprom.id
+  data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.example_msprom[0].id
 }
 
 resource "azurerm_monitor_alert_prometheus_rule_group" "example_node" {
+  count               = local.deploy_observability_tools ? 1 : 0
   name                = "NodeRecordingRulesRuleGroup-${azurerm_kubernetes_cluster.example.name}"
   resource_group_name = azurerm_resource_group.example.name
   location            = azurerm_resource_group.example.location
   cluster_name        = azurerm_kubernetes_cluster.example.name
   rule_group_enabled  = true
   interval            = "PT1M"
-  scopes              = [azurerm_monitor_workspace.example.id]
+  scopes              = [azurerm_monitor_workspace.example[0].id]
 
   rule {
     record     = "instance:node_num_cpu:sum"
@@ -149,13 +159,14 @@ resource "azurerm_monitor_alert_prometheus_rule_group" "example_node" {
 }
 
 resource "azurerm_monitor_alert_prometheus_rule_group" "example_k8s" {
+  count               = local.deploy_observability_tools ? 1 : 0
   name                = "KubernetesRecordingRulesRuleGroup-${azurerm_kubernetes_cluster.example.name}"
   resource_group_name = azurerm_resource_group.example.name
   location            = azurerm_resource_group.example.location
   cluster_name        = azurerm_kubernetes_cluster.example.name
   rule_group_enabled  = true
   interval            = "PT1M"
-  scopes              = [azurerm_monitor_workspace.example.id]
+  scopes              = [azurerm_monitor_workspace.example[0].id]
 
   rule {
     record     = "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate"
@@ -267,6 +278,7 @@ resource "azurerm_monitor_alert_prometheus_rule_group" "example_k8s" {
 }
 
 resource "azurerm_monitor_data_collection_rule" "example_msci" {
+  count               = local.deploy_observability_tools ? 1 : 0
   name                = "MSCI-${azurerm_resource_group.example.location}-${azurerm_kubernetes_cluster.example.name}"
   resource_group_name = azurerm_resource_group.example.name
   location            = azurerm_resource_group.example.location
@@ -291,19 +303,20 @@ resource "azurerm_monitor_data_collection_rule" "example_msci" {
 
   destinations {
     log_analytics {
-      workspace_resource_id = azurerm_log_analytics_workspace.example.id
-      name                  = azurerm_log_analytics_workspace.example.name
+      workspace_resource_id = azurerm_log_analytics_workspace.example[0].id
+      name                  = azurerm_log_analytics_workspace.example[0].name
     }
   }
 
   data_flow {
     streams      = ["Microsoft-ContainerInsights-Group-Default"]
-    destinations = [azurerm_log_analytics_workspace.example.name]
+    destinations = [azurerm_log_analytics_workspace.example[0].name]
   }
 }
 
 resource "azurerm_monitor_data_collection_rule_association" "example_msci_to_aks" {
+  count                   = local.deploy_observability_tools ? 1 : 0
   name                    = "msci-${azurerm_kubernetes_cluster.example.name}"
   target_resource_id      = azurerm_kubernetes_cluster.example.id
-  data_collection_rule_id = azurerm_monitor_data_collection_rule.example_msci.id
+  data_collection_rule_id = azurerm_monitor_data_collection_rule.example_msci[0].id
 }
\ No newline at end of file
diff --git a/infra/terraform/openai.tf b/infra/terraform/openai.tf
index 51f976ce..a856d4cd 100644
--- a/infra/terraform/openai.tf
+++ b/infra/terraform/openai.tf
@@ -1,4 +1,5 @@
 resource "azurerm_cognitive_account" "example" {
+  count                 = local.deploy_azure_openai ? 1 : 0
   name                  = "aoai-${local.name}"
   location              = var.ai_location
   resource_group_name   = azurerm_resource_group.example.name
@@ -8,8 +9,9 @@ resource "azurerm_cognitive_account" "example" {
 }
 
 resource "azurerm_cognitive_deployment" "example" {
+  count                = local.deploy_azure_openai ? 1 : 0
   name                 = var.openai_model_name
-  cognitive_account_id = azurerm_cognitive_account.example.id
+  cognitive_account_id = azurerm_cognitive_account.example[0].id
 
   model {
     format  = "OpenAI"
@@ -24,28 +26,32 @@ resource "azurerm_cognitive_deployment" "example" {
 }
 
 resource "azurerm_user_assigned_identity" "example" {
+  count               = local.deploy_azure_openai && local.deploy_azure_workload_identity ? 1 : 0
   location            = var.ai_location
   name                = "aoai-${local.name}"
   resource_group_name = azurerm_resource_group.example.name
 }
 
 resource "azurerm_federated_identity_credential" "example" {
+  count               = local.deploy_azure_openai && local.deploy_azure_workload_identity ? 1 : 0
   name                = "aoai-${local.name}"
   resource_group_name = azurerm_resource_group.example.name
-  parent_id           = azurerm_user_assigned_identity.example.id
+  parent_id           = azurerm_user_assigned_identity.example[0].id
   audience            = ["api://AzureADTokenExchange"]
   issuer              = azurerm_kubernetes_cluster.example.oidc_issuer_url
   subject             = "system:serviceaccount:${var.k8s_namespace}:ai-service-account"
 }
 
 resource "azurerm_role_assignment" "example_aoai_me" {
+  count                = local.deploy_azure_openai ? 1 : 0
   principal_id         = data.azurerm_client_config.current.object_id
   role_definition_name = "Cognitive Services OpenAI User"
-  scope                = azurerm_cognitive_account.example.id
+  scope                = azurerm_cognitive_account.example[0].id
 }
 
 resource "azurerm_role_assignment" "example_aoai_mi" {
-  principal_id         = azurerm_user_assigned_identity.example.principal_id
+  count                = local.deploy_azure_openai && local.deploy_azure_workload_identity ? 1 : 0
+  principal_id         = azurerm_user_assigned_identity.example[0].principal_id
   role_definition_name = "Cognitive Services OpenAI User"
-  scope                = azurerm_cognitive_account.example.id
+  scope                = azurerm_cognitive_account.example[0].id
 }
\ No newline at end of file
diff --git a/infra/terraform/outputs.tf b/infra/terraform/outputs.tf
index ed7a8339..6c25b134 100644
--- a/infra/terraform/outputs.tf
+++ b/infra/terraform/outputs.tf
@@ -11,58 +11,63 @@ output "AZURE_AKS_CLUSTER_NAME" {
 }
 
 output "AZURE_OPENAI_MODEL_NAME" {
-  value = var.openai_model_name
+  value = local.deploy_azure_openai ? var.openai_model_name : ""
 }
 
 output "AZURE_OPENAI_ENDPOINT" {
-  value = azurerm_cognitive_account.example.endpoint
+  value = local.deploy_azure_openai ? azurerm_cognitive_account.example[0].endpoint : ""
+}
+
+output "AZURE_OPENAI_KEY" {
+  value     = local.deploy_azure_openai ? azurerm_key_vault_secret.openai_key[0].name : ""
+  sensitive = true
 }
 
 output "AZURE_IDENTITY_CLIENT_ID" {
-  value = azurerm_user_assigned_identity.example.client_id
+  value = local.deploy_azure_workload_identity ? azurerm_user_assigned_identity.example[0].client_id : ""
 }
 
 output "AZURE_SERVICE_BUS_HOST" {
-  value = "${azurerm_servicebus_namespace.example.name}.servicebus.windows.net"
+  value = local.deploy_azure_servicebus ? "${azurerm_servicebus_namespace.example[0].name}.servicebus.windows.net" : ""
 }
 
 output "AZURE_SERVICE_BUS_URI" {
-  value     = "amqps://${azurerm_servicebus_namespace.example.name}.servicebus.windows.net"
+  value     = local.deploy_azure_servicebus ? "amqps://${azurerm_servicebus_namespace.example[0].name}.servicebus.windows.net" : ""
   sensitive = true
 }
 
 output "AZURE_SERVICE_BUS_LISTENER_NAME" {
-  value = azurerm_servicebus_namespace_authorization_rule.example.name
+  value = local.deploy_azure_servicebus ? azurerm_servicebus_namespace_authorization_rule.example[0].name : ""
 }
 
 output "AZURE_SERVICE_BUS_LISTENER_KEY" {
-  value     = azurerm_key_vault_secret.listener_key.name
+  value     = local.deploy_azure_servicebus ? azurerm_key_vault_secret.listener_key[0].name : ""
   sensitive = true
 }
 
 output "AZURE_SERVICE_BUS_SENDER_NAME" {
-  value = azurerm_servicebus_queue_authorization_rule.example.name
+  value = local.deploy_azure_servicebus ? azurerm_servicebus_queue_authorization_rule.example[0].name : ""
 }
 
 output "AZURE_SERVICE_BUS_SENDER_KEY" {
-  value     = azurerm_key_vault_secret.sender_key.name
+  value     = local.deploy_azure_servicebus ? azurerm_key_vault_secret.sender_key[0].name : ""
   sensitive = true
 }
 
 output "AZURE_COSMOS_DATABASE_NAME" {
-  value = azurerm_cosmosdb_account.example.name
+  value = local.deploy_azure_cosmosdb ? azurerm_cosmosdb_account.example[0].name : ""
 }
 
 output "AZURE_COSMOS_DATABASE_URI" {
-  value = local.cosmosdb_account_kind == "MongoDB" ? "mongodb://${azurerm_cosmosdb_account.example.name}.mongo.cosmos.azure.com:10255/?retryWrites=false" : "https://${azurerm_cosmosdb_account.example.name}.documents.azure.com:443/"
+  value = local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "MongoDB" ? "mongodb://${azurerm_cosmosdb_account.example[0].name}.mongo.cosmos.azure.com:10255/?retryWrites=false" : local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "GlobalDocumentDB" ? "https://${azurerm_cosmosdb_account.example[0].name}.documents.azure.com:443/" : ""
 }
 
 output "AZURE_DATABASE_API" {
-  value = local.cosmosdb_account_kind == "MongoDB" ? "mongodb" : "cosmosdbsql"
+  value = local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "MongoDB" ? "mongodb" : local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "GlobalDocumentDB" ? "cosmosdbsql" : ""
 }
 
 output "AZURE_COSMOS_DATABASE_KEY" {
-  value     = azurerm_key_vault_secret.cosmosdb_key.name
+  value     = local.deploy_azure_cosmosdb ? azurerm_key_vault_secret.cosmosdb_key[0].name : ""
   sensitive = true
 }
 
@@ -75,9 +80,9 @@ output "AZURE_KEY_VAULT_NAME" {
 }
 
 output "AZURE_REGISTRY_NAME" {
-  value = local.deploy_acr ? azurerm_container_registry.example[0].name : ""
+  value = local.deploy_azure_container_registry ? azurerm_container_registry.example[0].name : ""
 }
 
 output "AZURE_REGISTRY_URI" {
-  value = local.deploy_acr ? azurerm_container_registry.example[0].login_server : "ghcr.io/azure-samples"
+  value = local.deploy_azure_container_registry ? azurerm_container_registry.example[0].login_server : "ghcr.io/azure-samples"
 }
diff --git a/infra/terraform/servicebus.tf b/infra/terraform/servicebus.tf
index fcead1ef..3692e995 100644
--- a/infra/terraform/servicebus.tf
+++ b/infra/terraform/servicebus.tf
@@ -1,4 +1,5 @@
 resource "azurerm_servicebus_namespace" "example" {
+  count               = local.deploy_azure_servicebus ? 1 : 0
   name                = "sb-${local.name}"
   location            = azurerm_resource_group.example.location
   resource_group_name = azurerm_resource_group.example.name
@@ -6,8 +7,9 @@ resource "azurerm_servicebus_namespace" "example" {
 }
 
 resource "azurerm_servicebus_namespace_authorization_rule" "example" {
+  count        = local.deploy_azure_servicebus ? 1 : 0
   name         = "listener"
-  namespace_id = azurerm_servicebus_namespace.example.id
+  namespace_id = azurerm_servicebus_namespace.example[0].id
 
   listen = true
   send   = false
@@ -15,13 +17,15 @@ resource "azurerm_servicebus_namespace_authorization_rule" "example" {
 }
 
 resource "azurerm_servicebus_queue" "example" {
+  count        = local.deploy_azure_servicebus ? 1 : 0
   name         = "orders"
-  namespace_id = azurerm_servicebus_namespace.example.id
+  namespace_id = azurerm_servicebus_namespace.example[0].id
 }
 
 resource "azurerm_servicebus_queue_authorization_rule" "example" {
+  count    = local.deploy_azure_servicebus ? 1 : 0
   name     = "sender"
-  queue_id = azurerm_servicebus_queue.example.id
+  queue_id = azurerm_servicebus_queue.example[0].id
 
   listen = false
   send   = true
diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf
index f27d8672..20051537 100644
--- a/infra/terraform/variables.tf
+++ b/infra/terraform/variables.tf
@@ -31,7 +31,7 @@ variable "k8s_namespace" {
 }
 
 variable "cosmosdb_account_kind" {
-  description = "value of cosmosdb account kind. this string value will be used to set the local variable"
+  description = "value of azure cosmosdb account kind. this string value defaults to MongoDB and will be used to set the local variable"
   type        = string
   default     = "MongoDB"
 
@@ -41,8 +41,38 @@ variable "cosmosdb_account_kind" {
   # }
 }
 
-variable "deploy_acr" {
-  description = "value of deploy acr. this string value will be used to set the local variable"
+variable "deploy_azure_container_registry" {
+  description = "value of setting to deploy azure container registry. this string value will be used to set the local boolean variable"
   type        = string
   default     = "false"
 }
+
+variable "deploy_azure_workload_identity" {
+  description = "value of setting to deploy azure workload identity for service authentication. this string value will be used to set the local boolean variable"
+  type        = string
+  default     = "false"
+}
+
+variable "deploy_azure_openai" {
+  description = "value of setting to deploy azure openai. this string value will be used to set the local boolean variable"
+  type        = string
+  default     = "false"
+}
+
+variable "deploy_azure_servicebus" {
+  description = "value of setting to deploy azure service bus. this string value will be used to set the local boolean variable"
+  type        = string
+  default     = "false"
+}
+
+variable "deploy_azure_cosmosdb" {
+  description = "value of setting to deploy azure cosmosdb. this string value will be used to set the local boolean variable"
+  type        = string
+  default     = "false"
+}
+
+variable "deploy_observability_tools" {
+  description = "value of setting to deploy observability stack which includes prometheus, grafana, and container insights. this string value will be used to set the local boolean variable"
+  type        = string
+  default     = "false"
+}
\ No newline at end of file

From d0a33399c49c48a76700bea8a1f6db7f0ea4c860 Mon Sep 17 00:00:00 2001
From: Paul Yu <paul.d.yu@gmail.com>
Date: Wed, 14 Feb 2024 15:53:20 -0800
Subject: [PATCH 2/6] feat: adding AzureMonitorMetricsControlPlanePreview
 feature

---
 azd-hooks/preprovision.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/azd-hooks/preprovision.sh b/azd-hooks/preprovision.sh
index bdc09024..8f09617c 100755
--- a/azd-hooks/preprovision.sh
+++ b/azd-hooks/preprovision.sh
@@ -7,5 +7,6 @@ az feature register --namespace Microsoft.ContainerService --name AKS-Prometheus
 az feature register --namespace Microsoft.ContainerService --name EnableWorkloadIdentityPreview
 az feature register --namespace Microsoft.ContainerService --name NetworkObservabilityPreview
 az feature register --namespace "Microsoft.ContainerService" --name "NodeOsUpgradeChannelPreview"
+az feature register --namespace "Microsoft.ContainerService" --name "AzureMonitorMetricsControlPlanePreview" 
 az extension add --upgrade --name aks-preview
 az extension add --upgrade --name amg
\ No newline at end of file

From 2a50337a366db84e42cb4f1b8b584a385ef09e7f Mon Sep 17 00:00:00 2001
From: Paul Yu <paul.d.yu@gmail.com>
Date: Wed, 14 Feb 2024 16:11:07 -0800
Subject: [PATCH 3/6] feat: purge log analytics workspace

---
 infra/terraform/main.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/infra/terraform/main.tf b/infra/terraform/main.tf
index 8224ed11..cbdf87c4 100644
--- a/infra/terraform/main.tf
+++ b/infra/terraform/main.tf
@@ -30,6 +30,10 @@ provider "azurerm" {
     key_vault {
       purge_soft_delete_on_destroy = true
     }
+
+    log_analytics_workspace {
+      permanently_delete_on_destroy = true
+    }
   }
 }
 

From c3f11788c0a9a0fb60abcfb10927b5d273fd51e8 Mon Sep 17 00:00:00 2001
From: Paul Yu <paul.d.yu@gmail.com>
Date: Wed, 14 Feb 2024 19:05:25 -0800
Subject: [PATCH 4/6] feat: add namespace to helm deployment

---
 azd-hooks/predeploy.sh       | 2 ++
 infra/terraform/variables.tf | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/azd-hooks/predeploy.sh b/azd-hooks/predeploy.sh
index b2423607..9a86db28 100755
--- a/azd-hooks/predeploy.sh
+++ b/azd-hooks/predeploy.sh
@@ -6,6 +6,8 @@ az aks get-credentials --resource-group ${AZURE_RESOURCEGROUP_NAME} --name ${AZU
 echo "Deploy Helm chart"
 cmd="helm upgrade aks-store-demo ./charts/aks-store-demo \
   --install \
+  --namespace ${AZURE_AKS_NAMESPACE} \
+  --create-namespace \
   --set aiService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/ai-service \
   --set orderService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/order-service \
   --set makelineService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/makeline-service \
diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf
index 20051537..c015770d 100644
--- a/infra/terraform/variables.tf
+++ b/infra/terraform/variables.tf
@@ -27,7 +27,7 @@ variable "openai_model_capacity" {
 variable "k8s_namespace" {
   description = "value of kubernetes namespace"
   type        = string
-  default     = "default"
+  default     = "pets"
 }
 
 variable "cosmosdb_account_kind" {

From d48ec1b6bc1494567a3215a0173f938a612aeaf6 Mon Sep 17 00:00:00 2001
From: Paul Yu <paul.d.yu@gmail.com>
Date: Fri, 16 Feb 2024 13:11:49 -0800
Subject: [PATCH 5/6] feat: adding additional aks output params

---
 infra/terraform/outputs.tf | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/infra/terraform/outputs.tf b/infra/terraform/outputs.tf
index 6c25b134..a9052e30 100644
--- a/infra/terraform/outputs.tf
+++ b/infra/terraform/outputs.tf
@@ -10,6 +10,22 @@ output "AZURE_AKS_CLUSTER_NAME" {
   value = azurerm_kubernetes_cluster.example.name
 }
 
+output "AZURE_AKS_NAMESPACE" {
+  value = var.k8s_namespace
+}
+
+output "AZURE_AKS_CLUSTER_ID" {
+  value = azurerm_kubernetes_cluster.example.id
+}
+
+output "AZURE_AKS_CLUSTER_NODE_RESOURCEGROUP_NAME" {
+  value = azurerm_kubernetes_cluster.example.node_resource_group
+}
+
+output "AZURE_AKS_OIDC_ISSUER_URL" {
+  value = azurerm_kubernetes_cluster.example.oidc_issuer_url
+}
+
 output "AZURE_OPENAI_MODEL_NAME" {
   value = local.deploy_azure_openai ? var.openai_model_name : ""
 }
@@ -24,7 +40,7 @@ output "AZURE_OPENAI_KEY" {
 }
 
 output "AZURE_IDENTITY_CLIENT_ID" {
-  value = local.deploy_azure_workload_identity ? azurerm_user_assigned_identity.example[0].client_id : ""
+  value = local.deploy_azure_openai && local.deploy_azure_workload_identity ? azurerm_user_assigned_identity.example[0].client_id : ""
 }
 
 output "AZURE_SERVICE_BUS_HOST" {
@@ -71,10 +87,6 @@ output "AZURE_COSMOS_DATABASE_KEY" {
   sensitive = true
 }
 
-output "AZURE_AKS_NAMESPACE" {
-  value = var.k8s_namespace
-}
-
 output "AZURE_KEY_VAULT_NAME" {
   value = azurerm_key_vault.example.name
 }
@@ -86,3 +98,7 @@ output "AZURE_REGISTRY_NAME" {
 output "AZURE_REGISTRY_URI" {
   value = local.deploy_azure_container_registry ? azurerm_container_registry.example[0].login_server : "ghcr.io/azure-samples"
 }
+
+output "AZURE_TENANT_ID" {
+  value = data.azurerm_client_config.current.tenant_id
+}

From 08a6279e7863667f8f528c2da0b04a33f866ff5c Mon Sep 17 00:00:00 2001
From: Paul Yu <paul.d.yu@gmail.com>
Date: Fri, 16 Feb 2024 15:10:24 -0800
Subject: [PATCH 6/6] feat: adding var to optionally set kv access mode to
 azure rbac

---
 infra/terraform/keyvault.tf  | 81 +++++++++++++++++++++++++++++++++++-
 infra/terraform/variables.tf |  6 +++
 2 files changed, 86 insertions(+), 1 deletion(-)

diff --git a/infra/terraform/keyvault.tf b/infra/terraform/keyvault.tf
index b85bb66c..0592b59c 100644
--- a/infra/terraform/keyvault.tf
+++ b/infra/terraform/keyvault.tf
@@ -5,10 +5,89 @@ resource "azurerm_key_vault" "example" {
   enabled_for_disk_encryption = true
   tenant_id                   = data.azurerm_client_config.current.tenant_id
   sku_name                    = "standard"
-  enable_rbac_authorization   = true
+  enable_rbac_authorization   = var.kv_rbac_enabled
+
+  dynamic "access_policy" {
+    for_each = var.kv_rbac_enabled ? [] : [1]
+    content {
+      tenant_id = data.azurerm_client_config.current.tenant_id
+      object_id = data.azurerm_client_config.current.object_id
+
+      certificate_permissions = [
+        "Backup",
+        "Create",
+        "Delete",
+        "DeleteIssuers",
+        "Get",
+        "GetIssuers",
+        "Import",
+        "List",
+        "ListIssuers",
+        "ManageContacts",
+        "ManageIssuers",
+        "Purge",
+        "Recover",
+        "Restore",
+        "SetIssuers",
+        "Update"
+      ]
+
+      key_permissions = [
+        "Backup",
+        "Create",
+        "Decrypt",
+        "Delete",
+        "Encrypt",
+        "Get",
+        "Import",
+        "List",
+        "Purge",
+        "Recover",
+        "Restore",
+        "Sign",
+        "UnwrapKey",
+        "Update",
+        "Verify",
+        "WrapKey",
+        "Release",
+        "Rotate",
+        "GetRotationPolicy",
+        "SetRotationPolicy"
+      ]
+
+      secret_permissions = [
+        "Backup",
+        "Delete",
+        "Get",
+        "List",
+        "Purge",
+        "Recover",
+        "Restore",
+        "Set"
+      ]
+
+      storage_permissions = [
+        "Backup",
+        "Delete",
+        "DeleteSAS",
+        "Get",
+        "GetSAS",
+        "List",
+        "ListSAS",
+        "Purge",
+        "Recover",
+        "RegenerateKey",
+        "Restore",
+        "Set",
+        "SetSAS",
+        "Update"
+      ]
+    }
+  }
 }
 
 resource "azurerm_role_assignment" "example_akv_rbac" {
+  count                = var.kv_rbac_enabled ? 1 : 0
   principal_id         = data.azurerm_client_config.current.object_id
   role_definition_name = "Key Vault Administrator"
   scope                = azurerm_key_vault.example.id
diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf
index c015770d..e81cbd22 100644
--- a/infra/terraform/variables.tf
+++ b/infra/terraform/variables.tf
@@ -2,6 +2,12 @@ variable "location" {
   type = string
 }
 
+variable "kv_rbac_enabled" {
+  description = "value of keyvault rbac enabled. when set to true, key vault will use azure role-based access control"
+  type        = bool
+  default     = false
+}
+
 variable "ai_location" {
   description = "value of azure region for deploying azure ai service"
   type        = string