Skip to content

Commit

Permalink
Merge pull request #113 from pauldotyu/modular_terraform
Browse files Browse the repository at this point in the history
feat: modularize tf deployment
  • Loading branch information
pauldotyu authored Feb 20, 2024
2 parents 702c6ad + 08a6279 commit 4f6ba49
Show file tree
Hide file tree
Showing 15 changed files with 377 additions and 132 deletions.
64 changes: 44 additions & 20 deletions azd-hooks/predeploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,55 @@ echo "Retrieving cluster credentials"
az aks get-credentials --resource-group ${AZURE_RESOURCEGROUP_NAME} --name ${AZURE_AKS_CLUSTER_NAME} --overwrite-existing

echo "Deploy Helm chart"
helm upgrade aks-store-demo ./charts/aks-store-demo \
cmd="helm upgrade aks-store-demo ./charts/aks-store-demo \
--install \
--set aiService.create=true \
--set aiService.modelDeploymentName=${AZURE_OPENAI_MODEL_NAME} \
--set aiService.openAiEndpoint=${AZURE_OPENAI_ENDPOINT} \
--set aiService.managedIdentityClientId=${AZURE_IDENTITY_CLIENT_ID} \
--namespace ${AZURE_AKS_NAMESPACE} \
--create-namespace \
--set aiService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/ai-service \
--set orderService.useAzureServiceBus=true \
--set orderService.queueHost=${AZURE_SERVICE_BUS_HOST} \
--set orderService.queuePort=5671 \
--set orderService.queueUsername=${AZURE_SERVICE_BUS_SENDER_NAME} \
--set orderService.queuePassword=$(az keyvault secret show --name ${AZURE_SERVICE_BUS_SENDER_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv) \
--set orderService.queueTransport=tls \
--set orderService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/order-service \
--set makelineService.useAzureCosmosDB=true \
--set makelineService.orderQueueUri=${AZURE_SERVICE_BUS_URI} \
--set makelineService.orderQueueUsername=${AZURE_SERVICE_BUS_LISTENER_NAME} \
--set makelineService.orderQueuePassword=$(az keyvault secret show --name ${AZURE_SERVICE_BUS_LISTENER_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv) \
--set makelineService.orderDBUri=${AZURE_COSMOS_DATABASE_URI} \
--set makelineService.orderDBUsername=${AZURE_COSMOS_DATABASE_NAME} \
--set makelineService.orderDBPassword=$(az keyvault secret show --name ${AZURE_COSMOS_DATABASE_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv) \
--set makelineService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/makeline-service \
--set productService.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/product-service \
--set storeAdmin.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/store-admin \
--set storeFront.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/store-front \
--set virtualCustomer.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/virtual-customer \
--set virtualWorker.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/virtual-worker \
$(if [ "${AZURE_DATABASE_API}" == "cosmosdbsql" ]; then echo "--set makelineService.useSqlApi=true"; fi)
--set virtualWorker.image.repository=${AZURE_REGISTRY_URI}/aks-store-demo/virtual-worker"

if [ -n "${AZURE_OPENAI_ENDPOINT}" ]; then
cmd+=" --set aiService.create=true \
--set aiService.openAiEndpoint=${AZURE_OPENAI_ENDPOINT} \
--set aiService.modelDeploymentName=${AZURE_OPENAI_MODEL_NAME} \
--set aiService.useAzureOpenAi=true"

if [ -n "${AZURE_IDENTITY_CLIENT_ID}" ]; then
cmd+=" --set aiService.managedIdentityClientId=${AZURE_IDENTITY_CLIENT_ID} \
--set aiService.useAzureAd=true"
else
cmd+=" --set aiService.openAiKey=$(az keyvault secret show --name ${AZURE_OPENAI_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv) \
--set aiService.useAzureAd=false"
fi
fi

if [ -n "${AZURE_SERVICE_BUS_HOST}" ]; then
cmd+=" --set orderService.useAzureServiceBus=true \
--set orderService.queueHost=${AZURE_SERVICE_BUS_HOST} \
--set orderService.queuePort=5671 \
--set orderService.queueUsername=${AZURE_SERVICE_BUS_SENDER_NAME} \
--set orderService.queuePassword=$(az keyvault secret show --name ${AZURE_SERVICE_BUS_SENDER_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv) \
--set orderService.queueTransport=tls \
--set makelineService.orderQueueUri=${AZURE_SERVICE_BUS_URI} \
--set makelineService.orderQueueUsername=${AZURE_SERVICE_BUS_LISTENER_NAME} \
--set makelineService.orderQueuePassword=$(az keyvault secret show --name ${AZURE_SERVICE_BUS_LISTENER_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv)"
fi

if [ -n "${AZURE_COSMOS_DATABASE_URI}" ]; then
cmd+=" --set makelineService.useAzureCosmosDB=true \
--set makelineService.orderDBUri=${AZURE_COSMOS_DATABASE_URI} \
--set makelineService.orderDBUsername=${AZURE_COSMOS_DATABASE_NAME} \
--set makelineService.orderDBPassword=$(az keyvault secret show --name ${AZURE_COSMOS_DATABASE_KEY} --vault-name ${AZURE_KEY_VAULT_NAME} --query value -o tsv)"

if [ "${AZURE_DATABASE_API}" == "cosmosdbsql" ]; then
cmd+=" --set makelineService.useSqlApi=true"
fi
fi

eval $cmd
2 changes: 2 additions & 0 deletions azd-hooks/preprovision.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,7 @@ az feature register --namespace Microsoft.ContainerService --name AKS-KedaPrevie
az feature register --namespace Microsoft.ContainerService --name AKS-PrometheusAddonPreview
az feature register --namespace Microsoft.ContainerService --name EnableWorkloadIdentityPreview
az feature register --namespace Microsoft.ContainerService --name NetworkObservabilityPreview
az feature register --namespace "Microsoft.ContainerService" --name "NodeOsUpgradeChannelPreview"
az feature register --namespace "Microsoft.ContainerService" --name "AzureMonitorMetricsControlPlanePreview"
az extension add --upgrade --name aks-preview
az extension add --upgrade --name amg
46 changes: 36 additions & 10 deletions charts/aks-store-demo/templates/ai-service.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,41 @@ metadata:
data:
AZURE_OPENAI_DEPLOYMENT_NAME: "{{ .Values.aiService.modelDeploymentName }}"
AZURE_OPENAI_ENDPOINT: "{{ .Values.aiService.openAiEndpoint }}"
{{- if .Values.aiService.useAzureOpenAi }}
{{- /*
Use Azure OpenAI or OpenAI
*/}}
{{- if eq .Values.aiService.useAzureOpenAi true }}
USE_AZURE_OPENAI: "True"
{{- else }}
USE_AZURE_OPENAI: "False"
OPENAI_ORG_ID: "{{ .Values.aiService.openAiOrgId }}"
{{- end }}
{{- if .Values.aiService.useAzureAd }}

{{- /*
Use Azure AD or OpenAI API Key
*/}}
{{- if eq .Values.aiService.useAzureAd true }}
USE_AZURE_AD: "True"
{{- else }}
USE_AZURE_AD: "False"
{{- end }}
---
{{- if eq .Values.aiService.useAzureAd false }}

{{- /*
If Azure AD is used, create a service account with managed identity
*/}}
{{- if eq .Values.aiService.useAzureAd true }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: ai-service-account
annotations:
azure.workload.identity/client-id: "{{ .Values.aiService.managedIdentityClientId }}"
---
{{- /*
If Azure AD is not used, create a secret with OpenAI API Key
*/}}
{{- else }}
apiVersion: v1
kind: Secret
metadata:
Expand All @@ -27,13 +49,7 @@ data:
OPENAI_API_KEY: "{{ .Values.aiService.openAiKey | b64enc }}"
---
{{- end }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: ai-service-account
annotations:
azure.workload.identity/client-id: "{{ .Values.aiService.managedIdentityClientId }}"
---

apiVersion: apps/v1
kind: Deployment
metadata:
Expand All @@ -47,9 +63,19 @@ spec:
metadata:
labels:
app: ai-service
{{- /*
If Azure AD is used, set the label to use workload identity
*/}}
{{- if eq .Values.aiService.useAzureAd true }}
azure.workload.identity/use: "true"
{{- end }}
spec:
{{- /*
If Azure AD is used, use the service account
*/}}
{{- if eq .Values.aiService.useAzureAd true }}
serviceAccount: ai-service-account
{{- end }}
nodeSelector:
"kubernetes.io/os": linux
containers:
Expand Down
4 changes: 2 additions & 2 deletions charts/aks-store-demo/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ aiService:
openAiKey: ""
openAiOrgId: ""
managedIdentityClientId: ""
useAzureOpenAi: true
useAzureAd: true
useAzureOpenAi: false
useAzureAd: false
image:
repository: "ghcr.io/azure-samples/aks-store-demo/ai-service"
tag: "latest"
Expand Down
31 changes: 18 additions & 13 deletions infra/terraform/cosmosdb.tf
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
resource "azurerm_cosmosdb_account" "example" {
count = local.deploy_azure_cosmosdb ? 1 : 0
name = "db-${local.name}"
location = azurerm_resource_group.example.location
resource_group_name = azurerm_resource_group.example.name
Expand Down Expand Up @@ -32,39 +33,43 @@ resource "azurerm_cosmosdb_account" "example" {
}

resource "azurerm_cosmosdb_mongo_database" "example" {
count = local.cosmosdb_account_kind == "MongoDB" ? 1 : 0
count = local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "MongoDB" ? 1 : 0
name = "orderdb"
resource_group_name = azurerm_cosmosdb_account.example.resource_group_name
account_name = azurerm_cosmosdb_account.example.name
resource_group_name = azurerm_cosmosdb_account.example[0].resource_group_name
account_name = azurerm_cosmosdb_account.example[0].name
throughput = 400
}

resource "azurerm_cosmosdb_mongo_collection" "example" {
count = local.cosmosdb_account_kind == "MongoDB" ? 1 : 0
count = local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "MongoDB" ? 1 : 0
name = "orders"
resource_group_name = azurerm_cosmosdb_account.example.resource_group_name
account_name = azurerm_cosmosdb_account.example.name
resource_group_name = azurerm_cosmosdb_account.example[0].resource_group_name
account_name = azurerm_cosmosdb_account.example[0].name
database_name = azurerm_cosmosdb_mongo_database.example[0].name
throughput = 400

index {
keys = ["_id"]
keys = ["_id"]
}

lifecycle {
ignore_changes = [index]
}
}

resource "azurerm_cosmosdb_sql_database" "example" {
count = local.cosmosdb_account_kind == "GlobalDocumentDB" ? 1 : 0
count = local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "GlobalDocumentDB" ? 1 : 0
name = "orderdb"
resource_group_name = azurerm_cosmosdb_account.example.resource_group_name
account_name = azurerm_cosmosdb_account.example.name
resource_group_name = azurerm_cosmosdb_account.example[0].resource_group_name
account_name = azurerm_cosmosdb_account.example[0].name
throughput = 400
}

resource "azurerm_cosmosdb_sql_container" "example" {
count = local.cosmosdb_account_kind == "GlobalDocumentDB" ? 1 : 0
count = local.deploy_azure_cosmosdb && local.cosmosdb_account_kind == "GlobalDocumentDB" ? 1 : 0
name = "orders"
resource_group_name = azurerm_cosmosdb_account.example.resource_group_name
account_name = azurerm_cosmosdb_account.example.name
resource_group_name = azurerm_cosmosdb_account.example[0].resource_group_name
account_name = azurerm_cosmosdb_account.example[0].name
database_name = azurerm_cosmosdb_sql_database.example[0].name
partition_key_path = "/storeId"
partition_key_version = 1
Expand Down
135 changes: 109 additions & 26 deletions infra/terraform/keyvault.tf
Original file line number Diff line number Diff line change
@@ -1,43 +1,126 @@
resource "azurerm_key_vault" "example" {
name = "akv-${local.name}"
location = azurerm_resource_group.example.location
resource_group_name = azurerm_resource_group.example.name
enabled_for_disk_encryption = true
tenant_id = data.azurerm_client_config.current.tenant_id
sku_name = "standard"
enable_rbac_authorization = true
name = "akv-${local.name}"
location = azurerm_resource_group.example.location
resource_group_name = azurerm_resource_group.example.name
enabled_for_disk_encryption = true
tenant_id = data.azurerm_client_config.current.tenant_id
sku_name = "standard"
enable_rbac_authorization = var.kv_rbac_enabled

dynamic "access_policy" {
for_each = var.kv_rbac_enabled ? [] : [1]
content {
tenant_id = data.azurerm_client_config.current.tenant_id
object_id = data.azurerm_client_config.current.object_id

certificate_permissions = [
"Backup",
"Create",
"Delete",
"DeleteIssuers",
"Get",
"GetIssuers",
"Import",
"List",
"ListIssuers",
"ManageContacts",
"ManageIssuers",
"Purge",
"Recover",
"Restore",
"SetIssuers",
"Update"
]

key_permissions = [
"Backup",
"Create",
"Decrypt",
"Delete",
"Encrypt",
"Get",
"Import",
"List",
"Purge",
"Recover",
"Restore",
"Sign",
"UnwrapKey",
"Update",
"Verify",
"WrapKey",
"Release",
"Rotate",
"GetRotationPolicy",
"SetRotationPolicy"
]

secret_permissions = [
"Backup",
"Delete",
"Get",
"List",
"Purge",
"Recover",
"Restore",
"Set"
]

storage_permissions = [
"Backup",
"Delete",
"DeleteSAS",
"Get",
"GetSAS",
"List",
"ListSAS",
"Purge",
"Recover",
"RegenerateKey",
"Restore",
"Set",
"SetSAS",
"Update"
]
}
}
}

resource "azurerm_role_assignment" "example_akv_rbac" {
principal_id = data.azurerm_client_config.current.object_id
role_definition_name = "Key Vault Administrator"
scope = azurerm_key_vault.example.id
count = var.kv_rbac_enabled ? 1 : 0
principal_id = data.azurerm_client_config.current.object_id
role_definition_name = "Key Vault Administrator"
scope = azurerm_key_vault.example.id
}

resource "azurerm_key_vault_secret" "openai_key" {
name = "AZURE-OPENAI-KEY"
value = azurerm_cognitive_account.example.primary_access_key
key_vault_id = azurerm_key_vault.example.id
depends_on = [ azurerm_role_assignment.example_akv_rbac ]
count = local.deploy_azure_openai ? 1 : 0
name = "AZURE-OPENAI-KEY"
value = azurerm_cognitive_account.example[0].primary_access_key
key_vault_id = azurerm_key_vault.example.id
depends_on = [azurerm_role_assignment.example_akv_rbac]
}

resource "azurerm_key_vault_secret" "cosmosdb_key" {
name = "AZURE-COSMOS-KEY"
value = azurerm_cosmosdb_account.example.primary_key
key_vault_id = azurerm_key_vault.example.id
depends_on = [ azurerm_role_assignment.example_akv_rbac ]
count = local.deploy_azure_cosmosdb ? 1 : 0
name = "AZURE-COSMOS-KEY"
value = azurerm_cosmosdb_account.example[0].primary_key
key_vault_id = azurerm_key_vault.example.id
depends_on = [azurerm_role_assignment.example_akv_rbac]
}

resource "azurerm_key_vault_secret" "listener_key" {
name = "AZURE-SERVICE-BUS-LISTENER-KEY"
value = azurerm_servicebus_namespace_authorization_rule.example.primary_key
key_vault_id = azurerm_key_vault.example.id
depends_on = [ azurerm_role_assignment.example_akv_rbac ]
count = local.deploy_azure_servicebus ? 1 : 0
name = "AZURE-SERVICE-BUS-LISTENER-KEY"
value = azurerm_servicebus_namespace_authorization_rule.example[0].primary_key
key_vault_id = azurerm_key_vault.example.id
depends_on = [azurerm_role_assignment.example_akv_rbac]
}

resource "azurerm_key_vault_secret" "sender_key" {
name = "AZURE-SERVICE-BUS-SENDER-KEY"
value = azurerm_servicebus_queue_authorization_rule.example.primary_key
key_vault_id = azurerm_key_vault.example.id
depends_on = [ azurerm_role_assignment.example_akv_rbac ]
count = local.deploy_azure_servicebus ? 1 : 0
name = "AZURE-SERVICE-BUS-SENDER-KEY"
value = azurerm_servicebus_queue_authorization_rule.example[0].primary_key
key_vault_id = azurerm_key_vault.example.id
depends_on = [azurerm_role_assignment.example_akv_rbac]
}
Loading

0 comments on commit 4f6ba49

Please sign in to comment.