From 6a2594a4c258f8b96277f21af456029d771b96fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Pei=C3=9Fker?= <30857628+JPEasier@users.noreply.github.com> Date: Fri, 12 Jan 2024 16:50:02 +0100 Subject: [PATCH] feat: New Module avm-res-container-service-managed-cluster (#665) ## Description New Module for AKS migrated from CARML | Pipeline | |----------| | [![avm.res.container-service.managed-cluster](https://github.com/JPEasier/bicep-registry-modules/actions/workflows/avm.res.container-service.managed-cluster.yml/badge.svg?branch=users%2Fjpeasier%2Favm-aks)](https://github.com/JPEasier/bicep-registry-modules/actions/workflows/avm.res.container-service.managed-cluster.yml) | --------- Co-authored-by: Alexander Sehr --- .github/CODEOWNERS | 2 +- ....res.container-service.managed-cluster.yml | 83 + .../managed-cluster/README.md | 3177 +++++++++++++++++ .../managed-cluster/agent-pool/README.md | 404 +++ .../managed-cluster/agent-pool/main.bicep | 205 ++ .../managed-cluster/agent-pool/main.json | 380 ++ .../managed-cluster/main.bicep | 1035 ++++++ .../managed-cluster/main.json | 2756 ++++++++++++++ .../tests/e2e/azure/dependencies.bicep | 185 + .../tests/e2e/azure/main.test.bicep | 281 ++ .../tests/e2e/defaults/main.test.bicep | 53 + .../tests/e2e/kubenet/dependencies.bicep | 27 + .../tests/e2e/kubenet/main.test.bicep | 179 + .../tests/e2e/priv/dependencies.bicep | 87 + .../tests/e2e/priv/main.test.bicep | 137 + .../tests/e2e/waf-aligned/dependencies.bicep | 87 + .../tests/e2e/waf-aligned/main.test.bicep | 194 + .../managed-cluster/version.json | 7 + 18 files changed, 9278 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/avm.res.container-service.managed-cluster.yml create mode 100644 avm/res/container-service/managed-cluster/README.md create mode 100644 avm/res/container-service/managed-cluster/agent-pool/README.md create mode 100644 avm/res/container-service/managed-cluster/agent-pool/main.bicep create mode 100644 avm/res/container-service/managed-cluster/agent-pool/main.json create mode 100644 avm/res/container-service/managed-cluster/main.bicep create mode 100644 avm/res/container-service/managed-cluster/main.json create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/azure/main.test.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/defaults/main.test.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/kubenet/dependencies.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/kubenet/main.test.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/priv/dependencies.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/priv/main.test.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/dependencies.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/main.test.bicep create mode 100644 avm/res/container-service/managed-cluster/version.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index baf9bd2b2c..26d7a976d6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -33,7 +33,7 @@ #/avm/res/consumption/budget/ @Azure/avm-res-consumption-budget-module-owners-bicep @Azure/avm-core-team-technical-bicep #/avm/res/container-instance/container-group/ @Azure/avm-res-containerinstance-containergroup-module-owners-bicep @Azure/avm-core-team-technical-bicep #/avm/res/container-registry/registry/ @Azure/avm-res-containerregistry-registry-module-owners-bicep @Azure/avm-core-team-technical-bicep -#/avm/res/container-service/managed-cluster/ @Azure/avm-res-containerservice-managedcluster-module-owners-bicep @Azure/avm-core-team-technical-bicep +/avm/res/container-service/managed-cluster/ @Azure/avm-res-containerservice-managedcluster-module-owners-bicep @Azure/avm-core-team-technical-bicep #/avm/res/databricks/access-connector/ @Azure/avm-res-databricks-accessconnector-module-owners-bicep @Azure/avm-core-team-technical-bicep #/avm/res/databricks/workspace/ @Azure/avm-res-databricks-workspace-module-owners-bicep @Azure/avm-core-team-technical-bicep /avm/res/data-factory/factory/ @Azure/avm-res-datafactory-factory-module-owners-bicep @Azure/avm-core-team-technical-bicep diff --git a/.github/workflows/avm.res.container-service.managed-cluster.yml b/.github/workflows/avm.res.container-service.managed-cluster.yml new file mode 100644 index 0000000000..a2eb2dd467 --- /dev/null +++ b/.github/workflows/avm.res.container-service.managed-cluster.yml @@ -0,0 +1,83 @@ +name: "avm.res.container-service.managed-cluster" + +on: + schedule: + - cron: "0 12 1/15 * *" # Bi-Weekly Test (on 1st & 15th of month) + workflow_dispatch: + inputs: + staticValidation: + type: boolean + description: "Execute static validation" + required: false + default: true + deploymentValidation: + type: boolean + description: "Execute deployment validation" + required: false + default: true + removeDeployment: + type: boolean + description: "Remove deployed module" + required: false + default: true + + push: + branches: + - main + paths: + - ".github/actions/templates/avm-**" + - ".github/workflows/avm.template.module.yml" + - ".github/workflows/avm.res.container-service.managed-cluster.yml" + - "avm/res/container-service/managed-cluster/**" + - "avm/utilities/pipelines/**" + - "!*/**/README.md" + +env: + modulePath: "avm/res/container-service/managed-cluster" + workflowPath: ".github/workflows/avm.res.container-service.managed-cluster.yml" + +concurrency: + group: ${{ github.workflow }} + +jobs: + ########################### + # Initialize pipeline # + ########################### + job_initialize_pipeline: + runs-on: ubuntu-20.04 + name: "Initialize pipeline" + steps: + - name: "Checkout" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "Set input parameters to output variables" + id: get-workflow-param + uses: ./.github/actions/templates/avm-getWorkflowInput + with: + workflowPath: "${{ env.workflowPath}}" + - name: "Get parameter file paths" + id: get-module-test-file-paths + uses: ./.github/actions/templates/avm-getModuleTestFiles + with: + modulePath: "${{ env.modulePath }}" + outputs: + workflowInput: ${{ steps.get-workflow-param.outputs.workflowInput }} + moduleTestFilePaths: ${{ steps.get-module-test-file-paths.outputs.moduleTestFilePaths }} + psRuleModuleTestFilePaths: ${{ steps.get-module-test-file-paths.outputs.psRuleModuleTestFilePaths }} + modulePath: "${{ env.modulePath }}" + + ############################## + # Call reusable workflow # + ############################## + call-workflow-passing-data: + name: "Module" + needs: + - job_initialize_pipeline + uses: ./.github/workflows/avm.template.module.yml + with: + workflowInput: "${{ needs.job_initialize_pipeline.outputs.workflowInput }}" + moduleTestFilePaths: "${{ needs.job_initialize_pipeline.outputs.moduleTestFilePaths }}" + psRuleModuleTestFilePaths: "${{ needs.job_initialize_pipeline.outputs.psRuleModuleTestFilePaths }}" + modulePath: "${{ needs.job_initialize_pipeline.outputs.modulePath}}" + secrets: inherit diff --git a/avm/res/container-service/managed-cluster/README.md b/avm/res/container-service/managed-cluster/README.md new file mode 100644 index 0000000000..800c225de5 --- /dev/null +++ b/avm/res/container-service/managed-cluster/README.md @@ -0,0 +1,3177 @@ +# Azure Kubernetes Service (AKS) Managed Clusters `[Microsoft.ContainerService/managedClusters]` + +This module deploys an Azure Kubernetes Service (AKS) Managed Cluster. + +## Navigation + +- [Resource Types](#Resource-Types) +- [Usage examples](#Usage-examples) +- [Parameters](#Parameters) +- [Outputs](#Outputs) +- [Cross-referenced modules](#Cross-referenced-modules) +- [Data Collection](#Data-Collection) + +## Resource Types + +| Resource Type | API Version | +| :-- | :-- | +| `Microsoft.Authorization/locks` | [2020-05-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Authorization/2020-05-01/locks) | +| `Microsoft.Authorization/roleAssignments` | [2022-04-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Authorization/2022-04-01/roleAssignments) | +| `Microsoft.ContainerService/managedClusters` | [2023-07-02-preview](https://learn.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2023-07-02-preview/managedClusters) | +| `Microsoft.ContainerService/managedClusters/agentPools` | [2023-07-02-preview](https://learn.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2023-07-02-preview/managedClusters/agentPools) | +| `Microsoft.Insights/diagnosticSettings` | [2021-05-01-preview](https://learn.microsoft.com/en-us/azure/templates/Microsoft.Insights/2021-05-01-preview/diagnosticSettings) | +| `Microsoft.KubernetesConfiguration/extensions` | [2022-03-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/2022-03-01/extensions) | +| `Microsoft.KubernetesConfiguration/fluxConfigurations` | [2022-03-01](https://learn.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/2022-03-01/fluxConfigurations) | + +## Usage examples + +The following section provides usage examples for the module, which were used to validate and deploy the module successfully. For a full reference, please review the module's test folder in its repository. + +>**Note**: Each example lists all the required parameters first, followed by the rest - each in alphabetical order. + +>**Note**: To reference the module, please use the following syntax `br/public:avm/res/container-service/managed-cluster:`. + +- [Using Azure CNI Network Plugin.](#example-1-using-azure-cni-network-plugin) +- [Using only defaults](#example-2-using-only-defaults) +- [Using Kubenet Network Plugin.](#example-3-using-kubenet-network-plugin) +- [Using Private Cluster.](#example-4-using-private-cluster) +- [WAF-aligned](#example-5-waf-aligned) + +### Example 1: _Using Azure CNI Network Plugin._ + +This instance deploys the module with Azure CNI network plugin . + + +
+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: '${uniqueString(deployment().name, location)}-test-csmaz' + params: { + // Required parameters + name: 'csmaz001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + ] + // Non-required parameters + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + proximityPlacementGroupResourceId: '' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + ] + autoUpgradeProfileUpgradeChannel: 'stable' + customerManagedKey: { + keyName: '' + keyVaultNetworkAccess: 'Public' + keyVaultResourceId: '' + } + diagnosticSettings: [ + { + eventHubAuthorizationRuleResourceId: '' + eventHubName: '' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + name: 'customSetting' + storageAccountResourceId: '' + workspaceResourceId: '' + } + ] + diskEncryptionSetResourceId: '' + enableAzureDefender: true + enableAzureMonitorProfileMetrics: true + enableKeyvaultSecretsProvider: true + enableOidcIssuerProfile: true + enablePodSecurityPolicy: false + enableStorageProfileBlobCSIDriver: true + enableStorageProfileDiskCSIDriver: true + enableStorageProfileFileCSIDriver: true + enableStorageProfileSnapshotController: true + enableWorkloadIdentity: true + fluxExtension: { + configurations: [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + namespace: 'flux-system' + } + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + kustomizations: { + apps: { + dependsOn: [ + 'infra' + ] + path: './apps/staging' + prune: true + retryIntervalInSeconds: 120 + syncIntervalInSeconds: 600 + timeoutInSeconds: 600 + } + infra: { + dependsOn: [] + path: './infrastructure' + prune: true + syncIntervalInSeconds: 600 + timeoutInSeconds: 600 + validation: 'none' + } + } + namespace: 'flux-system-helm' + } + ] + configurationSettings: { + 'helm-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'source-controller.enabled': 'true' + } + } + identityProfile: { + kubeletidentity: { + resourceId: '' + } + } + location: '' + lock: { + kind: 'CanNotDelete' + name: 'myCustomLockName' + } + managedIdentities: { + userAssignedResourcesIds: [ + '' + ] + } + monitoringWorkspaceId: '' + networkDataplane: 'azure' + networkPlugin: 'azure' + networkPluginMode: 'overlay' + omsAgentEnabled: true + openServiceMeshEnabled: true + roleAssignments: [ + { + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'Owner' + } + { + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } + { + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: '' + } + ] + tags: { + Environment: 'Non-Prod' + 'hidden-title': 'This is visible in the resource name' + Role: 'DeploymentValidation' + } + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmaz001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 1, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "mode": "System", + "name": "systempool", + "osDiskSizeGB": 0, + "osType": "Linux", + "serviceCidr": "", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + } + ] + }, + // Non-required parameters + "agentPools": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool1", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 128, + "osType": "Linux", + "proximityPlacementGroupResourceId": "", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + }, + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool2", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + } + ] + }, + "autoUpgradeProfileUpgradeChannel": { + "value": "stable" + }, + "customerManagedKey": { + "value": { + "keyName": "", + "keyVaultNetworkAccess": "Public", + "keyVaultResourceId": "" + } + }, + "diagnosticSettings": { + "value": [ + { + "eventHubAuthorizationRuleResourceId": "", + "eventHubName": "", + "metricCategories": [ + { + "category": "AllMetrics" + } + ], + "name": "customSetting", + "storageAccountResourceId": "", + "workspaceResourceId": "" + } + ] + }, + "diskEncryptionSetResourceId": { + "value": "" + }, + "enableAzureDefender": { + "value": true + }, + "enableAzureMonitorProfileMetrics": { + "value": true + }, + "enableKeyvaultSecretsProvider": { + "value": true + }, + "enableOidcIssuerProfile": { + "value": true + }, + "enablePodSecurityPolicy": { + "value": false + }, + "enableStorageProfileBlobCSIDriver": { + "value": true + }, + "enableStorageProfileDiskCSIDriver": { + "value": true + }, + "enableStorageProfileFileCSIDriver": { + "value": true + }, + "enableStorageProfileSnapshotController": { + "value": true + }, + "enableWorkloadIdentity": { + "value": true + }, + "fluxExtension": { + "value": { + "configurations": [ + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + }, + "namespace": "flux-system" + }, + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/Azure/gitops-flux2-kustomize-helm-mt" + }, + "kustomizations": { + "apps": { + "dependsOn": [ + "infra" + ], + "path": "./apps/staging", + "prune": true, + "retryIntervalInSeconds": 120, + "syncIntervalInSeconds": 600, + "timeoutInSeconds": 600 + }, + "infra": { + "dependsOn": [], + "path": "./infrastructure", + "prune": true, + "syncIntervalInSeconds": 600, + "timeoutInSeconds": 600, + "validation": "none" + } + }, + "namespace": "flux-system-helm" + } + ], + "configurationSettings": { + "helm-controller.enabled": "true", + "image-automation-controller.enabled": "false", + "image-reflector-controller.enabled": "false", + "kustomize-controller.enabled": "true", + "notification-controller.enabled": "true", + "source-controller.enabled": "true" + } + } + }, + "identityProfile": { + "value": { + "kubeletidentity": { + "resourceId": "" + } + } + }, + "location": { + "value": "" + }, + "lock": { + "value": { + "kind": "CanNotDelete", + "name": "myCustomLockName" + } + }, + "managedIdentities": { + "value": { + "userAssignedResourcesIds": [ + "" + ] + } + }, + "monitoringWorkspaceId": { + "value": "" + }, + "networkDataplane": { + "value": "azure" + }, + "networkPlugin": { + "value": "azure" + }, + "networkPluginMode": { + "value": "overlay" + }, + "omsAgentEnabled": { + "value": true + }, + "openServiceMeshEnabled": { + "value": true + }, + "roleAssignments": { + "value": [ + { + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "Owner" + }, + { + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "b24988ac-6180-42a0-ab88-20f7382dd24c" + }, + { + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "" + } + ] + }, + "tags": { + "value": { + "Environment": "Non-Prod", + "hidden-title": "This is visible in the resource name", + "Role": "DeploymentValidation" + } + } + } +} +``` + +
+

+ +### Example 2: _Using only defaults_ + +This instance deploys the module with the minimum set of required parameters. + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: '${uniqueString(deployment().name, location)}-test-csmin' + params: { + // Required parameters + name: 'csmin001' + primaryAgentPoolProfile: [ + { + count: 1 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + location: '' + managedIdentities: { + systemAssigned: true + } + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmin001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "count": 1, + "mode": "System", + "name": "systempool", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "location": { + "value": "" + }, + "managedIdentities": { + "value": { + "systemAssigned": true + } + } + } +} +``` + +
+

+ +### Example 3: _Using Kubenet Network Plugin._ + +This instance deploys the module with Kubenet network plugin . + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: '${uniqueString(deployment().name, location)}-test-csmkube' + params: { + // Required parameters + name: 'csmkube001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + diagnosticSettings: [ + { + eventHubAuthorizationRuleResourceId: '' + eventHubName: '' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + name: 'customSetting' + storageAccountResourceId: '' + workspaceResourceId: '' + } + ] + location: '' + managedIdentities: { + userAssignedResourcesIds: [ + '' + ] + } + networkPlugin: 'kubenet' + roleAssignments: [ + { + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'Owner' + } + { + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } + { + principalId: '' + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: '' + } + ] + tags: { + Environment: 'Non-Prod' + 'hidden-title': 'This is visible in the resource name' + Role: 'DeploymentValidation' + } + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmkube001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 1, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "mode": "System", + "name": "systempool", + "osDiskSizeGB": 0, + "osType": "Linux", + "serviceCidr": "", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "agentPools": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool1", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + }, + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool2", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + } + ] + }, + "diagnosticSettings": { + "value": [ + { + "eventHubAuthorizationRuleResourceId": "", + "eventHubName": "", + "metricCategories": [ + { + "category": "AllMetrics" + } + ], + "name": "customSetting", + "storageAccountResourceId": "", + "workspaceResourceId": "" + } + ] + }, + "location": { + "value": "" + }, + "managedIdentities": { + "value": { + "userAssignedResourcesIds": [ + "" + ] + } + }, + "networkPlugin": { + "value": "kubenet" + }, + "roleAssignments": { + "value": [ + { + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "Owner" + }, + { + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "b24988ac-6180-42a0-ab88-20f7382dd24c" + }, + { + "principalId": "", + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "" + } + ] + }, + "tags": { + "value": { + "Environment": "Non-Prod", + "hidden-title": "This is visible in the resource name", + "Role": "DeploymentValidation" + } + } + } +} +``` + +
+

+ +### Example 4: _Using Private Cluster._ + +This instance deploys the module with a private cluster instance. + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: '${uniqueString(deployment().name, location)}-test-csmpriv' + params: { + // Required parameters + name: 'csmpriv001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + ] + // Non-required parameters + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + dnsServiceIP: '10.10.200.10' + enablePrivateCluster: true + location: '' + managedIdentities: { + userAssignedResourcesIds: [ + '' + ] + } + networkPlugin: 'azure' + privateDNSZone: '' + serviceCidr: '10.10.200.0/24' + skuTier: 'Standard' + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmpriv001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 1, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "mode": "System", + "name": "systempool", + "osDiskSizeGB": 0, + "osType": "Linux", + "serviceCidr": "", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + } + ] + }, + // Non-required parameters + "agentPools": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool1", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + }, + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool2", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + } + ] + }, + "dnsServiceIP": { + "value": "10.10.200.10" + }, + "enablePrivateCluster": { + "value": true + }, + "location": { + "value": "" + }, + "managedIdentities": { + "value": { + "userAssignedResourcesIds": [ + "" + ] + } + }, + "networkPlugin": { + "value": "azure" + }, + "privateDNSZone": { + "value": "" + }, + "serviceCidr": { + "value": "10.10.200.0/24" + }, + "skuTier": { + "value": "Standard" + } + } +} +``` + +
+

+ +### Example 5: _WAF-aligned_ + +This instance deploys the module in alignment with the best-practices of the Well-Architected Framework. + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: '${uniqueString(deployment().name, location)}-test-cswaf' + params: { + // Required parameters + name: 'cswaf001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + ] + // Non-required parameters + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 60 + osDiskType: 'Ephemeral' + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 60 + osDiskType: 'Ephemeral' + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + autoUpgradeProfileUpgradeChannel: 'stable' + diagnosticSettings: [ + { + eventHubAuthorizationRuleResourceId: '' + eventHubName: '' + logCategoriesAndGroups: [ + { + category: 'kube-apiserver' + } + { + category: 'kube-controller-manager' + } + { + category: 'kube-scheduler' + } + { + category: 'cluster-autoscaler' + } + ] + metricCategories: [ + { + category: 'AllMetrics' + } + ] + name: 'customSetting' + storageAccountResourceId: '' + workspaceResourceId: '' + } + ] + disableLocalAccounts: true + dnsServiceIP: '10.10.200.10' + enableAzureDefender: true + enablePrivateCluster: true + location: '' + managedIdentities: { + userAssignedResourcesIds: [ + '' + ] + } + monitoringWorkspaceId: '' + networkPlugin: 'azure' + networkPolicy: 'azure' + omsAgentEnabled: true + privateDNSZone: '' + serviceCidr: '10.10.200.0/24' + skuTier: 'Standard' + tags: { + Environment: 'Non-Prod' + 'hidden-title': 'This is visible in the resource name' + Role: 'DeploymentValidation' + } + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "cswaf001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 3, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 50, + "minCount": 1, + "mode": "System", + "name": "systempool", + "osDiskSizeGB": 0, + "osType": "Linux", + "serviceCidr": "", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + } + ] + }, + // Non-required parameters + "agentPools": { + "value": [ + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 50, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool1", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 60, + "osDiskType": "Ephemeral", + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + }, + { + "availabilityZones": [ + "3" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 50, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool2", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 60, + "osDiskType": "Ephemeral", + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + } + ] + }, + "autoUpgradeProfileUpgradeChannel": { + "value": "stable" + }, + "diagnosticSettings": { + "value": [ + { + "eventHubAuthorizationRuleResourceId": "", + "eventHubName": "", + "logCategoriesAndGroups": [ + { + "category": "kube-apiserver" + }, + { + "category": "kube-controller-manager" + }, + { + "category": "kube-scheduler" + }, + { + "category": "cluster-autoscaler" + } + ], + "metricCategories": [ + { + "category": "AllMetrics" + } + ], + "name": "customSetting", + "storageAccountResourceId": "", + "workspaceResourceId": "" + } + ] + }, + "disableLocalAccounts": { + "value": true + }, + "dnsServiceIP": { + "value": "10.10.200.10" + }, + "enableAzureDefender": { + "value": true + }, + "enablePrivateCluster": { + "value": true + }, + "location": { + "value": "" + }, + "managedIdentities": { + "value": { + "userAssignedResourcesIds": [ + "" + ] + } + }, + "monitoringWorkspaceId": { + "value": "" + }, + "networkPlugin": { + "value": "azure" + }, + "networkPolicy": { + "value": "azure" + }, + "omsAgentEnabled": { + "value": true + }, + "privateDNSZone": { + "value": "" + }, + "serviceCidr": { + "value": "10.10.200.0/24" + }, + "skuTier": { + "value": "Standard" + }, + "tags": { + "value": { + "Environment": "Non-Prod", + "hidden-title": "This is visible in the resource name", + "Role": "DeploymentValidation" + } + } + } +} +``` + +
+

+ + +## Parameters + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`name`](#parameter-name) | string | Specifies the name of the AKS cluster. | +| [`primaryAgentPoolProfile`](#parameter-primaryagentpoolprofile) | array | Properties of the primary agent pool. | + +**Conditional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`aksServicePrincipalProfile`](#parameter-aksserviceprincipalprofile) | object | Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster. | +| [`appGatewayResourceId`](#parameter-appgatewayresourceid) | string | Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`aadProfileAdminGroupObjectIDs`](#parameter-aadprofileadmingroupobjectids) | array | Specifies the AAD group object IDs that will have admin role of the cluster. | +| [`aadProfileClientAppID`](#parameter-aadprofileclientappid) | string | The client AAD application ID. | +| [`aadProfileEnableAzureRBAC`](#parameter-aadprofileenableazurerbac) | bool | Specifies whether to enable Azure RBAC for Kubernetes authorization. | +| [`aadProfileManaged`](#parameter-aadprofilemanaged) | bool | Specifies whether to enable managed AAD integration. | +| [`aadProfileServerAppID`](#parameter-aadprofileserverappid) | string | The server AAD application ID. | +| [`aadProfileServerAppSecret`](#parameter-aadprofileserverappsecret) | string | The server AAD application secret. | +| [`aadProfileTenantId`](#parameter-aadprofiletenantid) | string | Specifies the tenant ID of the Azure Active Directory used by the AKS cluster for authentication. | +| [`aciConnectorLinuxEnabled`](#parameter-aciconnectorlinuxenabled) | bool | Specifies whether the aciConnectorLinux add-on is enabled or not. | +| [`adminUsername`](#parameter-adminusername) | string | Specifies the administrator username of Linux virtual machines. | +| [`agentPools`](#parameter-agentpools) | array | Define one or more secondary/additional agent pools. | +| [`authorizedIPRanges`](#parameter-authorizedipranges) | array | IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer. | +| [`autoScalerProfileBalanceSimilarNodeGroups`](#parameter-autoscalerprofilebalancesimilarnodegroups) | string | Specifies the balance of similar node groups for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileExpander`](#parameter-autoscalerprofileexpander) | string | Specifies the expand strategy for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileMaxEmptyBulkDelete`](#parameter-autoscalerprofilemaxemptybulkdelete) | string | Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileMaxGracefulTerminationSec`](#parameter-autoscalerprofilemaxgracefulterminationsec) | string | Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileMaxNodeProvisionTime`](#parameter-autoscalerprofilemaxnodeprovisiontime) | string | Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an "m". No unit of time other than minutes (m) is supported. | +| [`autoScalerProfileMaxTotalUnreadyPercentage`](#parameter-autoscalerprofilemaxtotalunreadypercentage) | string | Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0. | +| [`autoScalerProfileNewPodScaleUpDelay`](#parameter-autoscalerprofilenewpodscaleupdelay) | string | For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit ("s" for seconds, "m" for minutes, "h" for hours, etc). | +| [`autoScalerProfileOkTotalUnreadyCount`](#parameter-autoscalerprofileoktotalunreadycount) | string | Specifies the OK total unready count for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScaleDownDelayAfterAdd`](#parameter-autoscalerprofilescaledowndelayafteradd) | string | Specifies the scale down delay after add of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScaleDownDelayAfterDelete`](#parameter-autoscalerprofilescaledowndelayafterdelete) | string | Specifies the scale down delay after delete of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScaleDownDelayAfterFailure`](#parameter-autoscalerprofilescaledowndelayafterfailure) | string | Specifies scale down delay after failure of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScaleDownUnneededTime`](#parameter-autoscalerprofilescaledownunneededtime) | string | Specifies the scale down unneeded time of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScaleDownUnreadyTime`](#parameter-autoscalerprofilescaledownunreadytime) | string | Specifies the scale down unready time of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileScanInterval`](#parameter-autoscalerprofilescaninterval) | string | Specifies the scan interval of the auto-scaler of the AKS cluster. | +| [`autoScalerProfileSkipNodesWithLocalStorage`](#parameter-autoscalerprofileskipnodeswithlocalstorage) | string | Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileSkipNodesWithSystemPods`](#parameter-autoscalerprofileskipnodeswithsystempods) | string | Specifies if nodes with system pods should be skipped for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileUtilizationThreshold`](#parameter-autoscalerprofileutilizationthreshold) | string | Specifies the utilization threshold of the auto-scaler of the AKS cluster. | +| [`autoUpgradeProfileUpgradeChannel`](#parameter-autoupgradeprofileupgradechannel) | string | Auto-upgrade channel on the AKS cluster. | +| [`azurePolicyEnabled`](#parameter-azurepolicyenabled) | bool | Specifies whether the azurepolicy add-on is enabled or not. For security reasons, this setting should be enabled. | +| [`azurePolicyVersion`](#parameter-azurepolicyversion) | string | Specifies the azure policy version to use. | +| [`customerManagedKey`](#parameter-customermanagedkey) | object | The customer managed key definition. | +| [`diagnosticSettings`](#parameter-diagnosticsettings) | array | The diagnostic settings of the service. | +| [`disableLocalAccounts`](#parameter-disablelocalaccounts) | bool | If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled. | +| [`disableRunCommand`](#parameter-disableruncommand) | bool | Whether to disable run command for the cluster or not. | +| [`diskEncryptionSetResourceId`](#parameter-diskencryptionsetresourceid) | string | The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided. | +| [`dnsPrefix`](#parameter-dnsprefix) | string | Specifies the DNS prefix specified when creating the managed cluster. | +| [`dnsServiceIP`](#parameter-dnsserviceip) | string | Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. | +| [`dnsZoneResourceId`](#parameter-dnszoneresourceid) | string | Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`. | +| [`enableAppMonitoring`](#parameter-enableappmonitoring) | bool | Indicates if Application Monitoring of the kubenetes cluster is enabled. | +| [`enableAppMonitoringOpenTelemetryMetrics`](#parameter-enableappmonitoringopentelemetrymetrics) | bool | Indicates if Application Monitoring Open Telemetry Metrics is enabled. | +| [`enableAzureDefender`](#parameter-enableazuredefender) | bool | Whether to enable Azure Defender. | +| [`enableAzureMonitorProfileLogs`](#parameter-enableazuremonitorprofilelogs) | bool | Whether the Logs profile for the Azure Monitor Infrastructure and Application Logs is enabled. | +| [`enableAzureMonitorProfileMetrics`](#parameter-enableazuremonitorprofilemetrics) | bool | Whether the metric state of the kubenetes cluster is enabled. | +| [`enableContainerInsights`](#parameter-enablecontainerinsights) | bool | Indicates if Azure Monitor Container Insights Logs Addon is enabled. | +| [`enableDnsZoneContributorRoleAssignment`](#parameter-enablednszonecontributorroleassignment) | bool | Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided. | +| [`enableKeyvaultSecretsProvider`](#parameter-enablekeyvaultsecretsprovider) | bool | Specifies whether the KeyvaultSecretsProvider add-on is enabled or not. | +| [`enableOidcIssuerProfile`](#parameter-enableoidcissuerprofile) | bool | Whether the The OIDC issuer profile of the Managed Cluster is enabled. | +| [`enablePodSecurityPolicy`](#parameter-enablepodsecuritypolicy) | bool | Whether to enable Kubernetes pod security policy. Requires enabling the pod security policy feature flag on the subscription. | +| [`enablePrivateCluster`](#parameter-enableprivatecluster) | bool | Specifies whether to create the cluster as a private cluster or not. | +| [`enablePrivateClusterPublicFQDN`](#parameter-enableprivateclusterpublicfqdn) | bool | Whether to create additional public FQDN for private cluster or not. | +| [`enableRBAC`](#parameter-enablerbac) | bool | Whether to enable Kubernetes Role-Based Access Control. | +| [`enableSecretRotation`](#parameter-enablesecretrotation) | string | Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation. | +| [`enableStorageProfileBlobCSIDriver`](#parameter-enablestorageprofileblobcsidriver) | bool | Whether the AzureBlob CSI Driver for the storage profile is enabled. | +| [`enableStorageProfileDiskCSIDriver`](#parameter-enablestorageprofilediskcsidriver) | bool | Whether the AzureDisk CSI Driver for the storage profile is enabled. | +| [`enableStorageProfileFileCSIDriver`](#parameter-enablestorageprofilefilecsidriver) | bool | Whether the AzureFile CSI Driver for the storage profile is enabled. | +| [`enableStorageProfileSnapshotController`](#parameter-enablestorageprofilesnapshotcontroller) | bool | Whether the snapshot controller for the storage profile is enabled. | +| [`enableTelemetry`](#parameter-enabletelemetry) | bool | Enable/Disable usage telemetry for module. | +| [`enableWindowsHostLogs`](#parameter-enablewindowshostlogs) | bool | Whether the Windows Log Collection for Azure Monitor Container Insights Logs Addon is enabled. | +| [`enableWorkloadIdentity`](#parameter-enableworkloadidentity) | bool | Whether to enable Workload Identity. Requires OIDC issuer profile to be enabled. | +| [`fluxExtension`](#parameter-fluxextension) | object | Settings and configurations for the flux extension. | +| [`httpApplicationRoutingEnabled`](#parameter-httpapplicationroutingenabled) | bool | Specifies whether the httpApplicationRouting add-on is enabled or not. | +| [`httpProxyConfig`](#parameter-httpproxyconfig) | object | Configurations for provisioning the cluster with HTTP proxy servers. | +| [`identityProfile`](#parameter-identityprofile) | object | Identities associated with the cluster. | +| [`ingressApplicationGatewayEnabled`](#parameter-ingressapplicationgatewayenabled) | bool | Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. | +| [`kubeDashboardEnabled`](#parameter-kubedashboardenabled) | bool | Specifies whether the kubeDashboard add-on is enabled or not. | +| [`kubernetesVersion`](#parameter-kubernetesversion) | string | Version of Kubernetes specified when creating the managed cluster. | +| [`loadBalancerSku`](#parameter-loadbalancersku) | string | Specifies the sku of the load balancer used by the virtual machine scale sets used by nodepools. | +| [`location`](#parameter-location) | string | Specifies the location of AKS cluster. It picks up Resource Group's location by default. | +| [`lock`](#parameter-lock) | object | The lock settings of the service. | +| [`managedIdentities`](#parameter-managedidentities) | object | The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both. | +| [`managedOutboundIPCount`](#parameter-managedoutboundipcount) | int | Outbound IP Count for the Load balancer. | +| [`metricAnnotationsAllowList`](#parameter-metricannotationsallowlist) | string | A comma-separated list of Kubernetes cluster metrics annotations. | +| [`metricLabelsAllowlist`](#parameter-metriclabelsallowlist) | string | A comma-separated list of kubernetes cluster metrics labels. | +| [`monitoringWorkspaceId`](#parameter-monitoringworkspaceid) | string | Resource ID of the monitoring log analytics workspace. | +| [`networkDataplane`](#parameter-networkdataplane) | string | Network dataplane used in the Kubernetes cluster. Not compatible with kubenet network plugin. | +| [`networkPlugin`](#parameter-networkplugin) | string | Specifies the network plugin used for building Kubernetes network. | +| [`networkPluginMode`](#parameter-networkpluginmode) | string | Network plugin mode used for building the Kubernetes network. Not compatible with kubenet network plugin. | +| [`networkPolicy`](#parameter-networkpolicy) | string | Specifies the network policy used for building Kubernetes network. - calico or azure. | +| [`nodeResourceGroup`](#parameter-noderesourcegroup) | string | Name of the resource group containing agent pool nodes. | +| [`omsAgentEnabled`](#parameter-omsagentenabled) | bool | Specifies whether the OMS agent is enabled. | +| [`openServiceMeshEnabled`](#parameter-openservicemeshenabled) | bool | Specifies whether the openServiceMesh add-on is enabled or not. | +| [`outboundType`](#parameter-outboundtype) | string | Specifies outbound (egress) routing method. - loadBalancer or userDefinedRouting. | +| [`podCidr`](#parameter-podcidr) | string | Specifies the CIDR notation IP range from which to assign pod IPs when kubenet is used. | +| [`podIdentityProfileAllowNetworkPluginKubenet`](#parameter-podidentityprofileallownetworkpluginkubenet) | bool | Running in Kubenet is disabled by default due to the security related nature of AAD Pod Identity and the risks of IP spoofing. | +| [`podIdentityProfileEnable`](#parameter-podidentityprofileenable) | bool | Whether the pod identity addon is enabled. | +| [`podIdentityProfileUserAssignedIdentities`](#parameter-podidentityprofileuserassignedidentities) | array | The pod identities to use in the cluster. | +| [`podIdentityProfileUserAssignedIdentityExceptions`](#parameter-podidentityprofileuserassignedidentityexceptions) | array | The pod identity exceptions to allow. | +| [`privateDNSZone`](#parameter-privatednszone) | string | Private DNS Zone configuration. Set to 'system' and AKS will create a private DNS zone in the node resource group. Set to '' to disable private DNS Zone creation and use public DNS. Supply the resource ID here of an existing Private DNS zone to use an existing zone. | +| [`roleAssignments`](#parameter-roleassignments) | array | Array of role assignments to create. | +| [`serviceCidr`](#parameter-servicecidr) | string | A CIDR notation IP range from which to assign service cluster IPs. It must not overlap with any Subnet IP ranges. | +| [`skuTier`](#parameter-skutier) | string | Tier of a managed cluster SKU. | +| [`sshPublicKey`](#parameter-sshpublickey) | string | Specifies the SSH RSA public key string for the Linux nodes. | +| [`supportPlan`](#parameter-supportplan) | string | The support plan for the Managed Cluster. | +| [`tags`](#parameter-tags) | object | Tags of the resource. | +| [`webApplicationRoutingEnabled`](#parameter-webapplicationroutingenabled) | bool | Specifies whether the webApplicationRoutingEnabled add-on is enabled or not. | + +### Parameter: `name` + +Specifies the name of the AKS cluster. + +- Required: Yes +- Type: string + +### Parameter: `primaryAgentPoolProfile` + +Properties of the primary agent pool. + +- Required: Yes +- Type: array + +### Parameter: `aksServicePrincipalProfile` + +Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster. + +- Required: No +- Type: object + +### Parameter: `appGatewayResourceId` + +Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`. + +- Required: No +- Type: string + +### Parameter: `aadProfileAdminGroupObjectIDs` + +Specifies the AAD group object IDs that will have admin role of the cluster. + +- Required: No +- Type: array + +### Parameter: `aadProfileClientAppID` + +The client AAD application ID. + +- Required: No +- Type: string + +### Parameter: `aadProfileEnableAzureRBAC` + +Specifies whether to enable Azure RBAC for Kubernetes authorization. + +- Required: No +- Type: bool +- Default: `[parameters('enableRBAC')]` + +### Parameter: `aadProfileManaged` + +Specifies whether to enable managed AAD integration. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `aadProfileServerAppID` + +The server AAD application ID. + +- Required: No +- Type: string + +### Parameter: `aadProfileServerAppSecret` + +The server AAD application secret. + +- Required: No +- Type: string + +### Parameter: `aadProfileTenantId` + +Specifies the tenant ID of the Azure Active Directory used by the AKS cluster for authentication. + +- Required: No +- Type: string +- Default: `[subscription().tenantId]` + +### Parameter: `aciConnectorLinuxEnabled` + +Specifies whether the aciConnectorLinux add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `adminUsername` + +Specifies the administrator username of Linux virtual machines. + +- Required: No +- Type: string +- Default: `'azureuser'` + +### Parameter: `agentPools` + +Define one or more secondary/additional agent pools. + +- Required: No +- Type: array + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`name`](#parameter-agentpoolsname) | string | The name of the agent pool. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`availabilityZones`](#parameter-agentpoolsavailabilityzones) | array | The availability zones of the agent pool. | +| [`count`](#parameter-agentpoolscount) | int | The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). | +| [`enableAutoScaling`](#parameter-agentpoolsenableautoscaling) | bool | Whether to enable auto-scaling for the agent pool. | +| [`enableDefaultTelemetry`](#parameter-agentpoolsenabledefaulttelemetry) | bool | The enable default telemetry of the agent pool. | +| [`enableEncryptionAtHost`](#parameter-agentpoolsenableencryptionathost) | bool | Whether to enable encryption at host for the agent pool. | +| [`enableFIPS`](#parameter-agentpoolsenablefips) | bool | Whether to enable FIPS for the agent pool. | +| [`enableNodePublicIP`](#parameter-agentpoolsenablenodepublicip) | bool | Whether to enable node public IP for the agent pool. | +| [`enableUltraSSD`](#parameter-agentpoolsenableultrassd) | bool | Whether to enable Ultra SSD for the agent pool. | +| [`gpuInstanceProfile`](#parameter-agentpoolsgpuinstanceprofile) | string | The GPU instance profile of the agent pool. | +| [`kubeletDiskType`](#parameter-agentpoolskubeletdisktype) | string | The kubelet disk type of the agent pool. | +| [`maxCount`](#parameter-agentpoolsmaxcount) | int | The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). | +| [`maxPods`](#parameter-agentpoolsmaxpods) | int | The maximum number of pods that can run on a node. | +| [`maxSurge`](#parameter-agentpoolsmaxsurge) | string | The maximum number of nodes that can be created during an upgrade. | +| [`minCount`](#parameter-agentpoolsmincount) | int | The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). | +| [`minPods`](#parameter-agentpoolsminpods) | int | The minimum number of pods that can run on a node. | +| [`mode`](#parameter-agentpoolsmode) | string | The mode of the agent pool. | +| [`nodeLabels`](#parameter-agentpoolsnodelabels) | object | The node labels of the agent pool. | +| [`nodePublicIpPrefixId`](#parameter-agentpoolsnodepublicipprefixid) | string | The node public IP prefix ID of the agent pool. | +| [`nodeTaints`](#parameter-agentpoolsnodetaints) | array | The node taints of the agent pool. | +| [`orchestratorVersion`](#parameter-agentpoolsorchestratorversion) | string | The Kubernetes version of the agent pool. | +| [`osDiskSizeGB`](#parameter-agentpoolsosdisksizegb) | int | The OS disk size in GB of the agent pool. | +| [`osDiskType`](#parameter-agentpoolsosdisktype) | string | The OS disk type of the agent pool. | +| [`osSku`](#parameter-agentpoolsossku) | string | The OS SKU of the agent pool. | +| [`osType`](#parameter-agentpoolsostype) | string | The OS type of the agent pool. | +| [`podSubnetId`](#parameter-agentpoolspodsubnetid) | string | The pod subnet ID of the agent pool. | +| [`proximityPlacementGroupResourceId`](#parameter-agentpoolsproximityplacementgroupresourceid) | string | The proximity placement group resource ID of the agent pool. | +| [`scaleDownMode`](#parameter-agentpoolsscaledownmode) | string | The scale down mode of the agent pool. | +| [`scaleSetEvictionPolicy`](#parameter-agentpoolsscalesetevictionpolicy) | string | The scale set eviction policy of the agent pool. | +| [`scaleSetPriority`](#parameter-agentpoolsscalesetpriority) | string | The scale set priority of the agent pool. | +| [`sourceResourceId`](#parameter-agentpoolssourceresourceid) | string | The source resource ID to create the agent pool from. | +| [`spotMaxPrice`](#parameter-agentpoolsspotmaxprice) | int | The spot max price of the agent pool. | +| [`tags`](#parameter-agentpoolstags) | object | The tags of the agent pool. | +| [`type`](#parameter-agentpoolstype) | string | The type of the agent pool. | +| [`vmSize`](#parameter-agentpoolsvmsize) | string | The VM size of the agent pool. | +| [`vnetSubnetID`](#parameter-agentpoolsvnetsubnetid) | string | The VNet subnet ID of the agent pool. | +| [`workloadRuntime`](#parameter-agentpoolsworkloadruntime) | string | The workload runtime of the agent pool. | + +### Parameter: `agentPools.name` + +The name of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.availabilityZones` + +The availability zones of the agent pool. + +- Required: No +- Type: array + +### Parameter: `agentPools.count` + +The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). + +- Required: No +- Type: int + +### Parameter: `agentPools.enableAutoScaling` + +Whether to enable auto-scaling for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.enableDefaultTelemetry` + +The enable default telemetry of the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.enableEncryptionAtHost` + +Whether to enable encryption at host for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.enableFIPS` + +Whether to enable FIPS for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.enableNodePublicIP` + +Whether to enable node public IP for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.enableUltraSSD` + +Whether to enable Ultra SSD for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `agentPools.gpuInstanceProfile` + +The GPU instance profile of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'MIG1g' + 'MIG2g' + 'MIG3g' + 'MIG4g' + 'MIG7g' + ] + ``` + +### Parameter: `agentPools.kubeletDiskType` + +The kubelet disk type of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.maxCount` + +The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). + +- Required: No +- Type: int + +### Parameter: `agentPools.maxPods` + +The maximum number of pods that can run on a node. + +- Required: No +- Type: int + +### Parameter: `agentPools.maxSurge` + +The maximum number of nodes that can be created during an upgrade. + +- Required: No +- Type: string + +### Parameter: `agentPools.minCount` + +The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). + +- Required: No +- Type: int + +### Parameter: `agentPools.minPods` + +The minimum number of pods that can run on a node. + +- Required: No +- Type: int + +### Parameter: `agentPools.mode` + +The mode of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'System' + 'User' + ] + ``` + +### Parameter: `agentPools.nodeLabels` + +The node labels of the agent pool. + +- Required: No +- Type: object + +### Parameter: `agentPools.nodePublicIpPrefixId` + +The node public IP prefix ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.nodeTaints` + +The node taints of the agent pool. + +- Required: No +- Type: array + +### Parameter: `agentPools.orchestratorVersion` + +The Kubernetes version of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.osDiskSizeGB` + +The OS disk size in GB of the agent pool. + +- Required: No +- Type: int + +### Parameter: `agentPools.osDiskType` + +The OS disk type of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.osSku` + +The OS SKU of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.osType` + +The OS type of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Linux' + 'Windows' + ] + ``` + +### Parameter: `agentPools.podSubnetId` + +The pod subnet ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.proximityPlacementGroupResourceId` + +The proximity placement group resource ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.scaleDownMode` + +The scale down mode of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Delete' + 'DeleteRequeue' + 'Pause' + 'Requeue' + ] + ``` + +### Parameter: `agentPools.scaleSetEvictionPolicy` + +The scale set eviction policy of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Deallocate' + 'Delete' + ] + ``` + +### Parameter: `agentPools.scaleSetPriority` + +The scale set priority of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Low' + 'Regular' + 'Spot' + ] + ``` + +### Parameter: `agentPools.sourceResourceId` + +The source resource ID to create the agent pool from. + +- Required: No +- Type: string + +### Parameter: `agentPools.spotMaxPrice` + +The spot max price of the agent pool. + +- Required: No +- Type: int + +### Parameter: `agentPools.tags` + +The tags of the agent pool. + +- Required: No +- Type: object + +### Parameter: `agentPools.type` + +The type of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'AvailabilitySet' + 'VirtualMachineScaleSets' + ] + ``` + +### Parameter: `agentPools.vmSize` + +The VM size of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.vnetSubnetID` + +The VNet subnet ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `agentPools.workloadRuntime` + +The workload runtime of the agent pool. + +- Required: No +- Type: string + +### Parameter: `authorizedIPRanges` + +IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer. + +- Required: No +- Type: array + +### Parameter: `autoScalerProfileBalanceSimilarNodeGroups` + +Specifies the balance of similar node groups for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'false'` +- Allowed: + ```Bicep + [ + 'false' + 'true' + ] + ``` + +### Parameter: `autoScalerProfileExpander` + +Specifies the expand strategy for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'random'` +- Allowed: + ```Bicep + [ + 'least-waste' + 'most-pods' + 'priority' + 'random' + ] + ``` + +### Parameter: `autoScalerProfileMaxEmptyBulkDelete` + +Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'10'` + +### Parameter: `autoScalerProfileMaxGracefulTerminationSec` + +Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'600'` + +### Parameter: `autoScalerProfileMaxNodeProvisionTime` + +Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an "m". No unit of time other than minutes (m) is supported. + +- Required: No +- Type: string +- Default: `'15m'` + +### Parameter: `autoScalerProfileMaxTotalUnreadyPercentage` + +Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0. + +- Required: No +- Type: string +- Default: `'45'` + +### Parameter: `autoScalerProfileNewPodScaleUpDelay` + +For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit ("s" for seconds, "m" for minutes, "h" for hours, etc). + +- Required: No +- Type: string +- Default: `'0s'` + +### Parameter: `autoScalerProfileOkTotalUnreadyCount` + +Specifies the OK total unready count for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'3'` + +### Parameter: `autoScalerProfileScaleDownDelayAfterAdd` + +Specifies the scale down delay after add of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'10m'` + +### Parameter: `autoScalerProfileScaleDownDelayAfterDelete` + +Specifies the scale down delay after delete of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'20s'` + +### Parameter: `autoScalerProfileScaleDownDelayAfterFailure` + +Specifies scale down delay after failure of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'3m'` + +### Parameter: `autoScalerProfileScaleDownUnneededTime` + +Specifies the scale down unneeded time of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'10m'` + +### Parameter: `autoScalerProfileScaleDownUnreadyTime` + +Specifies the scale down unready time of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'20m'` + +### Parameter: `autoScalerProfileScanInterval` + +Specifies the scan interval of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'10s'` + +### Parameter: `autoScalerProfileSkipNodesWithLocalStorage` + +Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'true'` +- Allowed: + ```Bicep + [ + 'false' + 'true' + ] + ``` + +### Parameter: `autoScalerProfileSkipNodesWithSystemPods` + +Specifies if nodes with system pods should be skipped for the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'true'` +- Allowed: + ```Bicep + [ + 'false' + 'true' + ] + ``` + +### Parameter: `autoScalerProfileUtilizationThreshold` + +Specifies the utilization threshold of the auto-scaler of the AKS cluster. + +- Required: No +- Type: string +- Default: `'0.5'` + +### Parameter: `autoUpgradeProfileUpgradeChannel` + +Auto-upgrade channel on the AKS cluster. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'node-image' + 'none' + 'patch' + 'rapid' + 'stable' + ] + ``` + +### Parameter: `azurePolicyEnabled` + +Specifies whether the azurepolicy add-on is enabled or not. For security reasons, this setting should be enabled. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `azurePolicyVersion` + +Specifies the azure policy version to use. + +- Required: No +- Type: string +- Default: `'v2'` + +### Parameter: `customerManagedKey` + +The customer managed key definition. + +- Required: No +- Type: object + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`keyName`](#parameter-customermanagedkeykeyname) | string | The name of the customer managed key to use for encryption. | +| [`keyVaultNetworkAccess`](#parameter-customermanagedkeykeyvaultnetworkaccess) | string | Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public. | +| [`keyVaultResourceId`](#parameter-customermanagedkeykeyvaultresourceid) | string | The resource ID of a key vault to reference a customer managed key for encryption from. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`keyVersion`](#parameter-customermanagedkeykeyversion) | string | The version of the customer managed key to reference for encryption. If not provided, using 'latest'. | + +### Parameter: `customerManagedKey.keyName` + +The name of the customer managed key to use for encryption. + +- Required: Yes +- Type: string + +### Parameter: `customerManagedKey.keyVaultNetworkAccess` + +Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public. + +- Required: Yes +- Type: string +- Allowed: + ```Bicep + [ + 'Private' + 'Public' + ] + ``` + +### Parameter: `customerManagedKey.keyVaultResourceId` + +The resource ID of a key vault to reference a customer managed key for encryption from. + +- Required: Yes +- Type: string + +### Parameter: `customerManagedKey.keyVersion` + +The version of the customer managed key to reference for encryption. If not provided, using 'latest'. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings` + +The diagnostic settings of the service. + +- Required: No +- Type: array + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`eventHubAuthorizationRuleResourceId`](#parameter-diagnosticsettingseventhubauthorizationruleresourceid) | string | Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to. | +| [`eventHubName`](#parameter-diagnosticsettingseventhubname) | string | Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. | +| [`logAnalyticsDestinationType`](#parameter-diagnosticsettingsloganalyticsdestinationtype) | string | A string indicating whether the export to Log Analytics should use the default destination type, i.e. AzureDiagnostics, or use a destination type. | +| [`logCategoriesAndGroups`](#parameter-diagnosticsettingslogcategoriesandgroups) | array | The name of logs that will be streamed. "allLogs" includes all possible logs for the resource. Set to '' to disable log collection. | +| [`marketplacePartnerResourceId`](#parameter-diagnosticsettingsmarketplacepartnerresourceid) | string | The full ARM resource ID of the Marketplace resource to which you would like to send Diagnostic Logs. | +| [`metricCategories`](#parameter-diagnosticsettingsmetriccategories) | array | The name of metrics that will be streamed. "allMetrics" includes all possible metrics for the resource. Set to '' to disable metric collection. | +| [`name`](#parameter-diagnosticsettingsname) | string | The name of diagnostic setting. | +| [`storageAccountResourceId`](#parameter-diagnosticsettingsstorageaccountresourceid) | string | Resource ID of the diagnostic storage account. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. | +| [`workspaceResourceId`](#parameter-diagnosticsettingsworkspaceresourceid) | string | Resource ID of the diagnostic log analytics workspace. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. | + +### Parameter: `diagnosticSettings.eventHubAuthorizationRuleResourceId` + +Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.eventHubName` + +Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.logAnalyticsDestinationType` + +A string indicating whether the export to Log Analytics should use the default destination type, i.e. AzureDiagnostics, or use a destination type. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'AzureDiagnostics' + 'Dedicated' + ] + ``` + +### Parameter: `diagnosticSettings.logCategoriesAndGroups` + +The name of logs that will be streamed. "allLogs" includes all possible logs for the resource. Set to '' to disable log collection. + +- Required: No +- Type: array + +### Parameter: `diagnosticSettings.marketplacePartnerResourceId` + +The full ARM resource ID of the Marketplace resource to which you would like to send Diagnostic Logs. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.metricCategories` + +The name of metrics that will be streamed. "allMetrics" includes all possible metrics for the resource. Set to '' to disable metric collection. + +- Required: No +- Type: array + +### Parameter: `diagnosticSettings.name` + +The name of diagnostic setting. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.storageAccountResourceId` + +Resource ID of the diagnostic storage account. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. + +- Required: No +- Type: string + +### Parameter: `diagnosticSettings.workspaceResourceId` + +Resource ID of the diagnostic log analytics workspace. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub. + +- Required: No +- Type: string + +### Parameter: `disableLocalAccounts` + +If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `disableRunCommand` + +Whether to disable run command for the cluster or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `diskEncryptionSetResourceId` + +The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided. + +- Required: No +- Type: string + +### Parameter: `dnsPrefix` + +Specifies the DNS prefix specified when creating the managed cluster. + +- Required: No +- Type: string +- Default: `[parameters('name')]` + +### Parameter: `dnsServiceIP` + +Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. + +- Required: No +- Type: string + +### Parameter: `dnsZoneResourceId` + +Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`. + +- Required: No +- Type: string + +### Parameter: `enableAppMonitoring` + +Indicates if Application Monitoring of the kubenetes cluster is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableAppMonitoringOpenTelemetryMetrics` + +Indicates if Application Monitoring Open Telemetry Metrics is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableAzureDefender` + +Whether to enable Azure Defender. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableAzureMonitorProfileLogs` + +Whether the Logs profile for the Azure Monitor Infrastructure and Application Logs is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableAzureMonitorProfileMetrics` + +Whether the metric state of the kubenetes cluster is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableContainerInsights` + +Indicates if Azure Monitor Container Insights Logs Addon is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableDnsZoneContributorRoleAssignment` + +Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `enableKeyvaultSecretsProvider` + +Specifies whether the KeyvaultSecretsProvider add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableOidcIssuerProfile` + +Whether the The OIDC issuer profile of the Managed Cluster is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enablePodSecurityPolicy` + +Whether to enable Kubernetes pod security policy. Requires enabling the pod security policy feature flag on the subscription. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enablePrivateCluster` + +Specifies whether to create the cluster as a private cluster or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enablePrivateClusterPublicFQDN` + +Whether to create additional public FQDN for private cluster or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableRBAC` + +Whether to enable Kubernetes Role-Based Access Control. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `enableSecretRotation` + +Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation. + +- Required: No +- Type: string +- Default: `'false'` +- Allowed: + ```Bicep + [ + 'false' + 'true' + ] + ``` + +### Parameter: `enableStorageProfileBlobCSIDriver` + +Whether the AzureBlob CSI Driver for the storage profile is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableStorageProfileDiskCSIDriver` + +Whether the AzureDisk CSI Driver for the storage profile is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableStorageProfileFileCSIDriver` + +Whether the AzureFile CSI Driver for the storage profile is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableStorageProfileSnapshotController` + +Whether the snapshot controller for the storage profile is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableTelemetry` + +Enable/Disable usage telemetry for module. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `enableWindowsHostLogs` + +Whether the Windows Log Collection for Azure Monitor Container Insights Logs Addon is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableWorkloadIdentity` + +Whether to enable Workload Identity. Requires OIDC issuer profile to be enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `fluxExtension` + +Settings and configurations for the flux extension. + +- Required: No +- Type: object + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`name`](#parameter-fluxextensionname) | string | The name of the extension. | +| [`releaseTrain`](#parameter-fluxextensionreleasetrain) | string | The release train of the extension. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`configurationProtectedSettings`](#parameter-fluxextensionconfigurationprotectedsettings) | object | The configuration protected settings of the extension. | +| [`configurations`](#parameter-fluxextensionconfigurations) | array | The flux configurations of the extension. | +| [`configurationSettings`](#parameter-fluxextensionconfigurationsettings) | object | The configuration settings of the extension. | +| [`releaseNamespace`](#parameter-fluxextensionreleasenamespace) | string | Namespace where the extension Release must be placed. | +| [`targetNamespace`](#parameter-fluxextensiontargetnamespace) | string | Namespace where the extension will be created for an Namespace scoped extension. | +| [`version`](#parameter-fluxextensionversion) | string | The version of the extension. | + +### Parameter: `fluxExtension.name` + +The name of the extension. + +- Required: No +- Type: string + +### Parameter: `fluxExtension.releaseTrain` + +The release train of the extension. + +- Required: No +- Type: string + +### Parameter: `fluxExtension.configurationProtectedSettings` + +The configuration protected settings of the extension. + +- Required: No +- Type: object + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`sshPrivateKey`](#parameter-fluxextensionconfigurationprotectedsettingssshprivatekey) | string | The SSH private key to use for Git authentication. | + +### Parameter: `fluxExtension.configurationProtectedSettings.sshPrivateKey` + +The SSH private key to use for Git authentication. + +- Required: No +- Type: string + +### Parameter: `fluxExtension.configurations` + +The flux configurations of the extension. + +- Required: No +- Type: array + +### Parameter: `fluxExtension.configurationSettings` + +The configuration settings of the extension. + +- Required: No +- Type: object + +### Parameter: `fluxExtension.releaseNamespace` + +Namespace where the extension Release must be placed. + +- Required: No +- Type: string + +### Parameter: `fluxExtension.targetNamespace` + +Namespace where the extension will be created for an Namespace scoped extension. + +- Required: No +- Type: string + +### Parameter: `fluxExtension.version` + +The version of the extension. + +- Required: No +- Type: string + +### Parameter: `httpApplicationRoutingEnabled` + +Specifies whether the httpApplicationRouting add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `httpProxyConfig` + +Configurations for provisioning the cluster with HTTP proxy servers. + +- Required: No +- Type: object + +### Parameter: `identityProfile` + +Identities associated with the cluster. + +- Required: No +- Type: object + +### Parameter: `ingressApplicationGatewayEnabled` + +Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `kubeDashboardEnabled` + +Specifies whether the kubeDashboard add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `kubernetesVersion` + +Version of Kubernetes specified when creating the managed cluster. + +- Required: No +- Type: string + +### Parameter: `loadBalancerSku` + +Specifies the sku of the load balancer used by the virtual machine scale sets used by nodepools. + +- Required: No +- Type: string +- Default: `'standard'` +- Allowed: + ```Bicep + [ + 'basic' + 'standard' + ] + ``` + +### Parameter: `location` + +Specifies the location of AKS cluster. It picks up Resource Group's location by default. + +- Required: No +- Type: string +- Default: `[resourceGroup().location]` + +### Parameter: `lock` + +The lock settings of the service. + +- Required: No +- Type: object + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`kind`](#parameter-lockkind) | string | Specify the type of lock. | +| [`name`](#parameter-lockname) | string | Specify the name of lock. | + +### Parameter: `lock.kind` + +Specify the type of lock. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'CanNotDelete' + 'None' + 'ReadOnly' + ] + ``` + +### Parameter: `lock.name` + +Specify the name of lock. + +- Required: No +- Type: string + +### Parameter: `managedIdentities` + +The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both. + +- Required: No +- Type: object + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`systemAssigned`](#parameter-managedidentitiessystemassigned) | bool | Enables system assigned managed identity on the resource. | +| [`userAssignedResourcesIds`](#parameter-managedidentitiesuserassignedresourcesids) | array | The resource ID(s) to assign to the resource. | + +### Parameter: `managedIdentities.systemAssigned` + +Enables system assigned managed identity on the resource. + +- Required: No +- Type: bool + +### Parameter: `managedIdentities.userAssignedResourcesIds` + +The resource ID(s) to assign to the resource. + +- Required: No +- Type: array + +### Parameter: `managedOutboundIPCount` + +Outbound IP Count for the Load balancer. + +- Required: No +- Type: int +- Default: `0` + +### Parameter: `metricAnnotationsAllowList` + +A comma-separated list of Kubernetes cluster metrics annotations. + +- Required: No +- Type: string +- Default: `''` + +### Parameter: `metricLabelsAllowlist` + +A comma-separated list of kubernetes cluster metrics labels. + +- Required: No +- Type: string +- Default: `''` + +### Parameter: `monitoringWorkspaceId` + +Resource ID of the monitoring log analytics workspace. + +- Required: No +- Type: string + +### Parameter: `networkDataplane` + +Network dataplane used in the Kubernetes cluster. Not compatible with kubenet network plugin. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'azure' + 'cilium' + ] + ``` + +### Parameter: `networkPlugin` + +Specifies the network plugin used for building Kubernetes network. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'azure' + 'kubenet' + ] + ``` + +### Parameter: `networkPluginMode` + +Network plugin mode used for building the Kubernetes network. Not compatible with kubenet network plugin. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'overlay' + ] + ``` + +### Parameter: `networkPolicy` + +Specifies the network policy used for building Kubernetes network. - calico or azure. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'azure' + 'calico' + ] + ``` + +### Parameter: `nodeResourceGroup` + +Name of the resource group containing agent pool nodes. + +- Required: No +- Type: string +- Default: `[format('{0}_aks_{1}_nodes', resourceGroup().name, parameters('name'))]` + +### Parameter: `omsAgentEnabled` + +Specifies whether the OMS agent is enabled. + +- Required: No +- Type: bool +- Default: `True` + +### Parameter: `openServiceMeshEnabled` + +Specifies whether the openServiceMesh add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `outboundType` + +Specifies outbound (egress) routing method. - loadBalancer or userDefinedRouting. + +- Required: No +- Type: string +- Default: `'loadBalancer'` +- Allowed: + ```Bicep + [ + 'loadBalancer' + 'userDefinedRouting' + ] + ``` + +### Parameter: `podCidr` + +Specifies the CIDR notation IP range from which to assign pod IPs when kubenet is used. + +- Required: No +- Type: string + +### Parameter: `podIdentityProfileAllowNetworkPluginKubenet` + +Running in Kubenet is disabled by default due to the security related nature of AAD Pod Identity and the risks of IP spoofing. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `podIdentityProfileEnable` + +Whether the pod identity addon is enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `podIdentityProfileUserAssignedIdentities` + +The pod identities to use in the cluster. + +- Required: No +- Type: array + +### Parameter: `podIdentityProfileUserAssignedIdentityExceptions` + +The pod identity exceptions to allow. + +- Required: No +- Type: array + +### Parameter: `privateDNSZone` + +Private DNS Zone configuration. Set to 'system' and AKS will create a private DNS zone in the node resource group. Set to '' to disable private DNS Zone creation and use public DNS. Supply the resource ID here of an existing Private DNS zone to use an existing zone. + +- Required: No +- Type: string + +### Parameter: `roleAssignments` + +Array of role assignments to create. + +- Required: No +- Type: array + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`principalId`](#parameter-roleassignmentsprincipalid) | string | The principal ID of the principal (user/group/identity) to assign the role to. | +| [`roleDefinitionIdOrName`](#parameter-roleassignmentsroledefinitionidorname) | string | The role to assign. You can provide either the display name of the role definition, the role definition GUID, or its fully qualified ID in the following format: '/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11'. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`condition`](#parameter-roleassignmentscondition) | string | The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase "foo_storage_container". | +| [`conditionVersion`](#parameter-roleassignmentsconditionversion) | string | Version of the condition. | +| [`delegatedManagedIdentityResourceId`](#parameter-roleassignmentsdelegatedmanagedidentityresourceid) | string | The Resource Id of the delegated managed identity resource. | +| [`description`](#parameter-roleassignmentsdescription) | string | The description of the role assignment. | +| [`principalType`](#parameter-roleassignmentsprincipaltype) | string | The principal type of the assigned principal ID. | + +### Parameter: `roleAssignments.principalId` + +The principal ID of the principal (user/group/identity) to assign the role to. + +- Required: Yes +- Type: string + +### Parameter: `roleAssignments.roleDefinitionIdOrName` + +The role to assign. You can provide either the display name of the role definition, the role definition GUID, or its fully qualified ID in the following format: '/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11'. + +- Required: Yes +- Type: string + +### Parameter: `roleAssignments.condition` + +The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase "foo_storage_container". + +- Required: No +- Type: string + +### Parameter: `roleAssignments.conditionVersion` + +Version of the condition. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + '2.0' + ] + ``` + +### Parameter: `roleAssignments.delegatedManagedIdentityResourceId` + +The Resource Id of the delegated managed identity resource. + +- Required: No +- Type: string + +### Parameter: `roleAssignments.description` + +The description of the role assignment. + +- Required: No +- Type: string + +### Parameter: `roleAssignments.principalType` + +The principal type of the assigned principal ID. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Device' + 'ForeignGroup' + 'Group' + 'ServicePrincipal' + 'User' + ] + ``` + +### Parameter: `serviceCidr` + +A CIDR notation IP range from which to assign service cluster IPs. It must not overlap with any Subnet IP ranges. + +- Required: No +- Type: string + +### Parameter: `skuTier` + +Tier of a managed cluster SKU. + +- Required: No +- Type: string +- Default: `'Standard'` +- Allowed: + ```Bicep + [ + 'Free' + 'Premium' + 'Standard' + ] + ``` + +### Parameter: `sshPublicKey` + +Specifies the SSH RSA public key string for the Linux nodes. + +- Required: No +- Type: string + +### Parameter: `supportPlan` + +The support plan for the Managed Cluster. + +- Required: No +- Type: string +- Default: `'KubernetesOfficial'` +- Allowed: + ```Bicep + [ + 'AKSLongTermSupport' + 'KubernetesOfficial' + ] + ``` + +### Parameter: `tags` + +Tags of the resource. + +- Required: No +- Type: object + +### Parameter: `webApplicationRoutingEnabled` + +Specifies whether the webApplicationRoutingEnabled add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + + +## Outputs + +| Output | Type | Description | +| :-- | :-- | :-- | +| `addonProfiles` | object | The addonProfiles of the Kubernetes cluster. | +| `controlPlaneFQDN` | string | The control plane FQDN of the managed cluster. | +| `ingressApplicationGatewayIdentityObjectId` | string | The Object ID of Application Gateway Ingress Controller (AGIC) identity. | +| `keyvaultIdentityClientId` | string | The Client ID of the Key Vault Secrets Provider identity. | +| `keyvaultIdentityObjectId` | string | The Object ID of the Key Vault Secrets Provider identity. | +| `kubeletidentityObjectId` | string | The Object ID of the AKS identity. | +| `location` | string | The location the resource was deployed into. | +| `name` | string | The name of the managed cluster. | +| `oidcIssuerUrl` | string | The OIDC token issuer URL. | +| `omsagentIdentityObjectId` | string | The Object ID of the OMS agent identity. | +| `resourceGroupName` | string | The resource group the managed cluster was deployed into. | +| `resourceId` | string | The resource ID of the managed cluster. | +| `systemAssignedMIPrincipalId` | string | The principal ID of the system assigned identity. | +| `webAppRoutingIdentityObjectId` | string | The Object ID of Web Application Routing. | + +## Cross-referenced modules + +This section gives you an overview of all local-referenced module files (i.e., other CARML modules that are referenced in this module) and all remote-referenced files (i.e., Bicep modules that are referenced from a Bicep Registry or Template Specs). + +| Reference | Type | +| :-- | :-- | +| `br/public:avm/res/kubernetes-configuration/extension:0.2.0` | Remote reference | + +## Data Collection + +The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the [repository](https://aka.ms/avm/telemetry). There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft’s privacy statement. Our privacy statement is located at . You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. diff --git a/avm/res/container-service/managed-cluster/agent-pool/README.md b/avm/res/container-service/managed-cluster/agent-pool/README.md new file mode 100644 index 0000000000..b86400ae45 --- /dev/null +++ b/avm/res/container-service/managed-cluster/agent-pool/README.md @@ -0,0 +1,404 @@ +# Azure Kubernetes Service (AKS) Managed Cluster Agent Pools `[Microsoft.ContainerService/managedClusters/agentPools]` + +This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool. + +## Navigation + +- [Resource Types](#Resource-Types) +- [Parameters](#Parameters) +- [Outputs](#Outputs) +- [Cross-referenced modules](#Cross-referenced-modules) +- [Data Collection](#Data-Collection) + +## Resource Types + +| Resource Type | API Version | +| :-- | :-- | +| `Microsoft.ContainerService/managedClusters/agentPools` | [2023-07-02-preview](https://learn.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2023-07-02-preview/managedClusters/agentPools) | + +## Parameters + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`name`](#parameter-name) | string | Name of the agent pool. | + +**Conditional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`managedClusterName`](#parameter-managedclustername) | string | The name of the parent managed cluster. Required if the template is used in a standalone deployment. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`availabilityZones`](#parameter-availabilityzones) | array | The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is "VirtualMachineScaleSets". | +| [`count`](#parameter-count) | int | Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1. | +| [`enableAutoScaling`](#parameter-enableautoscaling) | bool | Whether to enable auto-scaler. | +| [`enableEncryptionAtHost`](#parameter-enableencryptionathost) | bool | This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption. For security reasons, this setting should be enabled. | +| [`enableFIPS`](#parameter-enablefips) | bool | See Add a FIPS-enabled node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details. | +| [`enableNodePublicIP`](#parameter-enablenodepublicip) | bool | Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools). | +| [`enableUltraSSD`](#parameter-enableultrassd) | bool | Whether to enable UltraSSD. | +| [`gpuInstanceProfile`](#parameter-gpuinstanceprofile) | string | GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU. | +| [`kubeletDiskType`](#parameter-kubeletdisktype) | string | Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage. | +| [`maxCount`](#parameter-maxcount) | int | The maximum number of nodes for auto-scaling. | +| [`maxPods`](#parameter-maxpods) | int | The maximum number of pods that can run on a node. | +| [`maxSurge`](#parameter-maxsurge) | string | This can either be set to an integer (e.g. "5") or a percentage (e.g. "50%"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade. | +| [`minCount`](#parameter-mincount) | int | The minimum number of nodes for auto-scaling. | +| [`mode`](#parameter-mode) | string | A cluster must have at least one "System" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools. | +| [`nodeLabels`](#parameter-nodelabels) | object | The node labels to be persisted across all nodes in agent pool. | +| [`nodePublicIpPrefixId`](#parameter-nodepublicipprefixid) | string | ResourceId of the node PublicIPPrefix. | +| [`nodeTaints`](#parameter-nodetaints) | array | The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule. | +| [`orchestratorVersion`](#parameter-orchestratorversion) | string | As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool). | +| [`osDiskSizeGB`](#parameter-osdisksizegb) | int | OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified. | +| [`osDiskType`](#parameter-osdisktype) | string | The default is "Ephemeral" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to "Managed". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os). | +| [`osSku`](#parameter-ossku) | string | Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows. | +| [`osType`](#parameter-ostype) | string | The operating system type. The default is Linux. | +| [`podSubnetId`](#parameter-podsubnetid) | string | Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. | +| [`proximityPlacementGroupResourceId`](#parameter-proximityplacementgroupresourceid) | string | The ID for the Proximity Placement Group. | +| [`scaleDownMode`](#parameter-scaledownmode) | string | Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing). | +| [`scaleSetEvictionPolicy`](#parameter-scalesetevictionpolicy) | string | The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs. | +| [`scaleSetPriority`](#parameter-scalesetpriority) | string | The Virtual Machine Scale Set priority. | +| [`sourceResourceId`](#parameter-sourceresourceid) | string | This is the ARM ID of the source object to be used to create the target object. | +| [`spotMaxPrice`](#parameter-spotmaxprice) | int | Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://learn.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing). | +| [`tags`](#parameter-tags) | object | Tags of the resource. | +| [`type`](#parameter-type) | string | The type of Agent Pool. | +| [`vmSize`](#parameter-vmsize) | string | VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions. | +| [`vnetSubnetId`](#parameter-vnetsubnetid) | string | Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. | +| [`workloadRuntime`](#parameter-workloadruntime) | string | Determines the type of workload a node can run. | + +### Parameter: `name` + +Name of the agent pool. + +- Required: Yes +- Type: string + +### Parameter: `managedClusterName` + +The name of the parent managed cluster. Required if the template is used in a standalone deployment. + +- Required: Yes +- Type: string + +### Parameter: `availabilityZones` + +The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is "VirtualMachineScaleSets". + +- Required: No +- Type: array + +### Parameter: `count` + +Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1. + +- Required: No +- Type: int +- Default: `1` + +### Parameter: `enableAutoScaling` + +Whether to enable auto-scaler. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableEncryptionAtHost` + +This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption. For security reasons, this setting should be enabled. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableFIPS` + +See Add a FIPS-enabled node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableNodePublicIP` + +Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools). + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `enableUltraSSD` + +Whether to enable UltraSSD. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `gpuInstanceProfile` + +GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'MIG1g' + 'MIG2g' + 'MIG3g' + 'MIG4g' + 'MIG7g' + ] + ``` + +### Parameter: `kubeletDiskType` + +Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage. + +- Required: No +- Type: string + +### Parameter: `maxCount` + +The maximum number of nodes for auto-scaling. + +- Required: No +- Type: int + +### Parameter: `maxPods` + +The maximum number of pods that can run on a node. + +- Required: No +- Type: int + +### Parameter: `maxSurge` + +This can either be set to an integer (e.g. "5") or a percentage (e.g. "50%"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade. + +- Required: No +- Type: string + +### Parameter: `minCount` + +The minimum number of nodes for auto-scaling. + +- Required: No +- Type: int + +### Parameter: `mode` + +A cluster must have at least one "System" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools. + +- Required: No +- Type: string + +### Parameter: `nodeLabels` + +The node labels to be persisted across all nodes in agent pool. + +- Required: No +- Type: object + +### Parameter: `nodePublicIpPrefixId` + +ResourceId of the node PublicIPPrefix. + +- Required: No +- Type: string + +### Parameter: `nodeTaints` + +The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule. + +- Required: No +- Type: array + +### Parameter: `orchestratorVersion` + +As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool). + +- Required: No +- Type: string + +### Parameter: `osDiskSizeGB` + +OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified. + +- Required: No +- Type: int + +### Parameter: `osDiskType` + +The default is "Ephemeral" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to "Managed". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os). + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Ephemeral' + 'Managed' + ] + ``` + +### Parameter: `osSku` + +Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'AzureLinux' + 'CBLMariner' + 'Ubuntu' + 'Windows2019' + 'Windows2022' + ] + ``` + +### Parameter: `osType` + +The operating system type. The default is Linux. + +- Required: No +- Type: string +- Default: `'Linux'` +- Allowed: + ```Bicep + [ + 'Linux' + 'Windows' + ] + ``` + +### Parameter: `podSubnetId` + +Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. + +- Required: No +- Type: string + +### Parameter: `proximityPlacementGroupResourceId` + +The ID for the Proximity Placement Group. + +- Required: No +- Type: string + +### Parameter: `scaleDownMode` + +Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing). + +- Required: No +- Type: string +- Default: `'Delete'` +- Allowed: + ```Bicep + [ + 'Deallocate' + 'Delete' + ] + ``` + +### Parameter: `scaleSetEvictionPolicy` + +The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs. + +- Required: No +- Type: string +- Default: `'Delete'` +- Allowed: + ```Bicep + [ + 'Deallocate' + 'Delete' + ] + ``` + +### Parameter: `scaleSetPriority` + +The Virtual Machine Scale Set priority. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Regular' + 'Spot' + ] + ``` + +### Parameter: `sourceResourceId` + +This is the ARM ID of the source object to be used to create the target object. + +- Required: No +- Type: string + +### Parameter: `spotMaxPrice` + +Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://learn.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing). + +- Required: No +- Type: int + +### Parameter: `tags` + +Tags of the resource. + +- Required: No +- Type: object + +### Parameter: `type` + +The type of Agent Pool. + +- Required: No +- Type: string + +### Parameter: `vmSize` + +VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions. + +- Required: No +- Type: string +- Default: `'Standard_D2s_v3'` + +### Parameter: `vnetSubnetId` + +Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. + +- Required: No +- Type: string + +### Parameter: `workloadRuntime` + +Determines the type of workload a node can run. + +- Required: No +- Type: string + + +## Outputs + +| Output | Type | Description | +| :-- | :-- | :-- | +| `name` | string | The name of the agent pool. | +| `resourceGroupName` | string | The resource group the agent pool was deployed into. | +| `resourceId` | string | The resource ID of the agent pool. | + +## Cross-referenced modules + +_None_ + +## Data Collection + +The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the [repository](https://aka.ms/avm/telemetry). There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft’s privacy statement. Our privacy statement is located at . You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices. diff --git a/avm/res/container-service/managed-cluster/agent-pool/main.bicep b/avm/res/container-service/managed-cluster/agent-pool/main.bicep new file mode 100644 index 0000000000..f34cd88385 --- /dev/null +++ b/avm/res/container-service/managed-cluster/agent-pool/main.bicep @@ -0,0 +1,205 @@ +metadata name = 'Azure Kubernetes Service (AKS) Managed Cluster Agent Pools' +metadata description = 'This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.' +metadata owner = 'Azure/module-maintainers' + +@description('Conditional. The name of the parent managed cluster. Required if the template is used in a standalone deployment.') +param managedClusterName string + +@description('Required. Name of the agent pool.') +param name string + +@description('Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is "VirtualMachineScaleSets".') +param availabilityZones array? + +@description('Optional. Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1.') +@minValue(0) +@maxValue(1000) +param count int = 1 + +@description('Optional. This is the ARM ID of the source object to be used to create the target object.') +param sourceResourceId string? + +@description('Optional. Whether to enable auto-scaler.') +param enableAutoScaling bool = false + +@description('Optional. This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption. For security reasons, this setting should be enabled.') +param enableEncryptionAtHost bool = false + +@description('Optional. See Add a FIPS-enabled node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details.') +param enableFIPS bool = false + +@description('Optional. Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools).') +param enableNodePublicIP bool = false + +@description('Optional. Whether to enable UltraSSD.') +param enableUltraSSD bool = false + +@description('Optional. GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU.') +@allowed([ + 'MIG1g' + 'MIG2g' + 'MIG3g' + 'MIG4g' + 'MIG7g' +]) +param gpuInstanceProfile string? + +@description('Optional. Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage.') +param kubeletDiskType string? + +@description('Optional. The maximum number of nodes for auto-scaling.') +param maxCount int? + +@description('Optional. The maximum number of pods that can run on a node.') +param maxPods int? + +@description('Optional. The minimum number of nodes for auto-scaling.') +param minCount int? + +@description('Optional. A cluster must have at least one "System" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools.') +param mode string? + +@description('Optional. The node labels to be persisted across all nodes in agent pool.') +param nodeLabels object? + +@description('Optional. ResourceId of the node PublicIPPrefix.') +param nodePublicIpPrefixId string? + +@description('Optional. The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule.') +param nodeTaints array? + +@description('Optional. As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool).') +param orchestratorVersion string? + +@description('Optional. OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified.') +param osDiskSizeGB int? + +@description('Optional. The default is "Ephemeral" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to "Managed". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os).') +@allowed([ + 'Ephemeral' + 'Managed' +]) +param osDiskType string? + +@description('Optional. Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows.') +@allowed([ + 'AzureLinux' + 'CBLMariner' + 'Ubuntu' + 'Windows2019' + 'Windows2022' +]) +param osSku string? + +@description('Optional. The operating system type. The default is Linux.') +@allowed([ + 'Linux' + 'Windows' +]) +param osType string = 'Linux' + +@description('Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}.') +param podSubnetId string? + +@description('Optional. The ID for the Proximity Placement Group.') +param proximityPlacementGroupResourceId string? + +@description('Optional. Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing).') +@allowed([ + 'Deallocate' + 'Delete' +]) +param scaleDownMode string = 'Delete' + +@description('Optional. The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs.') +@allowed([ + 'Deallocate' + 'Delete' +]) +param scaleSetEvictionPolicy string = 'Delete' + +@description('Optional. The Virtual Machine Scale Set priority.') +@allowed([ + 'Regular' + 'Spot' +]) +param scaleSetPriority string? + +@description('Optional. Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://learn.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing).') +param spotMaxPrice int? + +@description('Optional. Tags of the resource.') +param tags object? + +@description('Optional. The type of Agent Pool.') +param type string? + +@description('Optional. This can either be set to an integer (e.g. "5") or a percentage (e.g. "50%"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade.') +param maxSurge string? + +@description('Optional. VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions.') +param vmSize string = 'Standard_D2s_v3' + +@description('Optional. Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}.') +param vnetSubnetId string? + +@description('Optional. Determines the type of workload a node can run.') +param workloadRuntime string? + +resource managedCluster 'Microsoft.ContainerService/managedClusters@2023-07-02-preview' existing = { + name: managedClusterName +} + +resource agentPool 'Microsoft.ContainerService/managedClusters/agentPools@2023-07-02-preview' = { + name: name + parent: managedCluster + properties: { + availabilityZones: availabilityZones + count: count + creationData: !empty(sourceResourceId) ? { + sourceResourceId: sourceResourceId + } : null + enableAutoScaling: enableAutoScaling + enableEncryptionAtHost: enableEncryptionAtHost + enableFIPS: enableFIPS + enableNodePublicIP: enableNodePublicIP + enableUltraSSD: enableUltraSSD + gpuInstanceProfile: gpuInstanceProfile + kubeletDiskType: kubeletDiskType + maxCount: maxCount + maxPods: maxPods + minCount: minCount + mode: mode + nodeLabels: nodeLabels + nodePublicIPPrefixID: nodePublicIpPrefixId + nodeTaints: nodeTaints + orchestratorVersion: orchestratorVersion + osDiskSizeGB: osDiskSizeGB + osDiskType: osDiskType + osSKU: osSku + osType: osType + podSubnetID: podSubnetId + proximityPlacementGroupID: proximityPlacementGroupResourceId + scaleDownMode: scaleDownMode + scaleSetEvictionPolicy: scaleSetEvictionPolicy + scaleSetPriority: scaleSetPriority + spotMaxPrice: spotMaxPrice + tags: tags + type: type + upgradeSettings: { + maxSurge: maxSurge + } + vmSize: vmSize + vnetSubnetID: vnetSubnetId + workloadRuntime: workloadRuntime + } +} + +@description('The name of the agent pool.') +output name string = agentPool.name + +@description('The resource ID of the agent pool.') +output resourceId string = agentPool.id + +@description('The resource group the agent pool was deployed into.') +output resourceGroupName string = resourceGroup().name diff --git a/avm/res/container-service/managed-cluster/agent-pool/main.json b/avm/res/container-service/managed-cluster/agent-pool/main.json new file mode 100644 index 0000000000..ccc5e14fe0 --- /dev/null +++ b/avm/res/container-service/managed-cluster/agent-pool/main.json @@ -0,0 +1,380 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.24.24.22086", + "templateHash": "59537816730486645" + }, + "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", + "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "managedClusterName": { + "type": "string", + "metadata": { + "description": "Conditional. The name of the parent managed cluster. Required if the template is used in a standalone deployment." + } + }, + "name": { + "type": "string", + "metadata": { + "description": "Required. Name of the agent pool." + } + }, + "availabilityZones": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is \"VirtualMachineScaleSets\"." + } + }, + "count": { + "type": "int", + "defaultValue": 1, + "minValue": 0, + "maxValue": 1000, + "metadata": { + "description": "Optional. Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1." + } + }, + "sourceResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. This is the ARM ID of the source object to be used to create the target object." + } + }, + "enableAutoScaling": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable auto-scaler." + } + }, + "enableEncryptionAtHost": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption. For security reasons, this setting should be enabled." + } + }, + "enableFIPS": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. See Add a FIPS-enabled node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details." + } + }, + "enableNodePublicIP": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools)." + } + }, + "enableUltraSSD": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable UltraSSD." + } + }, + "gpuInstanceProfile": { + "type": "string", + "nullable": true, + "allowedValues": [ + "MIG1g", + "MIG2g", + "MIG3g", + "MIG4g", + "MIG7g" + ], + "metadata": { + "description": "Optional. GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU." + } + }, + "kubeletDiskType": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage." + } + }, + "maxCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of nodes for auto-scaling." + } + }, + "maxPods": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of pods that can run on a node." + } + }, + "minCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The minimum number of nodes for auto-scaling." + } + }, + "mode": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. A cluster must have at least one \"System\" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools." + } + }, + "nodeLabels": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The node labels to be persisted across all nodes in agent pool." + } + }, + "nodePublicIpPrefixId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. ResourceId of the node PublicIPPrefix." + } + }, + "nodeTaints": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule." + } + }, + "orchestratorVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool)." + } + }, + "osDiskSizeGB": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified." + } + }, + "osDiskType": { + "type": "string", + "nullable": true, + "allowedValues": [ + "Ephemeral", + "Managed" + ], + "metadata": { + "description": "Optional. The default is \"Ephemeral\" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to \"Managed\". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os)." + } + }, + "osSku": { + "type": "string", + "nullable": true, + "allowedValues": [ + "AzureLinux", + "CBLMariner", + "Ubuntu", + "Windows2019", + "Windows2022" + ], + "metadata": { + "description": "Optional. Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows." + } + }, + "osType": { + "type": "string", + "defaultValue": "Linux", + "allowedValues": [ + "Linux", + "Windows" + ], + "metadata": { + "description": "Optional. The operating system type. The default is Linux." + } + }, + "podSubnetId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." + } + }, + "proximityPlacementGroupResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The ID for the Proximity Placement Group." + } + }, + "scaleDownMode": { + "type": "string", + "defaultValue": "Delete", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "metadata": { + "description": "Optional. Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing)." + } + }, + "scaleSetEvictionPolicy": { + "type": "string", + "defaultValue": "Delete", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "metadata": { + "description": "Optional. The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs." + } + }, + "scaleSetPriority": { + "type": "string", + "nullable": true, + "allowedValues": [ + "Regular", + "Spot" + ], + "metadata": { + "description": "Optional. The Virtual Machine Scale Set priority." + } + }, + "spotMaxPrice": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://learn.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing)." + } + }, + "tags": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Tags of the resource." + } + }, + "type": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The type of Agent Pool." + } + }, + "maxSurge": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. This can either be set to an integer (e.g. \"5\") or a percentage (e.g. \"50%\"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade." + } + }, + "vmSize": { + "type": "string", + "defaultValue": "Standard_D2s_v3", + "metadata": { + "description": "Optional. VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions." + } + }, + "vnetSubnetId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." + } + }, + "workloadRuntime": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Determines the type of workload a node can run." + } + } + }, + "resources": { + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2023-07-02-preview", + "name": "[parameters('managedClusterName')]" + }, + "agentPool": { + "type": "Microsoft.ContainerService/managedClusters/agentPools", + "apiVersion": "2023-07-02-preview", + "name": "[format('{0}/{1}', parameters('managedClusterName'), parameters('name'))]", + "properties": { + "availabilityZones": "[parameters('availabilityZones')]", + "count": "[parameters('count')]", + "creationData": "[if(not(empty(parameters('sourceResourceId'))), createObject('sourceResourceId', parameters('sourceResourceId')), null())]", + "enableAutoScaling": "[parameters('enableAutoScaling')]", + "enableEncryptionAtHost": "[parameters('enableEncryptionAtHost')]", + "enableFIPS": "[parameters('enableFIPS')]", + "enableNodePublicIP": "[parameters('enableNodePublicIP')]", + "enableUltraSSD": "[parameters('enableUltraSSD')]", + "gpuInstanceProfile": "[parameters('gpuInstanceProfile')]", + "kubeletDiskType": "[parameters('kubeletDiskType')]", + "maxCount": "[parameters('maxCount')]", + "maxPods": "[parameters('maxPods')]", + "minCount": "[parameters('minCount')]", + "mode": "[parameters('mode')]", + "nodeLabels": "[parameters('nodeLabels')]", + "nodePublicIPPrefixID": "[parameters('nodePublicIpPrefixId')]", + "nodeTaints": "[parameters('nodeTaints')]", + "orchestratorVersion": "[parameters('orchestratorVersion')]", + "osDiskSizeGB": "[parameters('osDiskSizeGB')]", + "osDiskType": "[parameters('osDiskType')]", + "osSKU": "[parameters('osSku')]", + "osType": "[parameters('osType')]", + "podSubnetID": "[parameters('podSubnetId')]", + "proximityPlacementGroupID": "[parameters('proximityPlacementGroupResourceId')]", + "scaleDownMode": "[parameters('scaleDownMode')]", + "scaleSetEvictionPolicy": "[parameters('scaleSetEvictionPolicy')]", + "scaleSetPriority": "[parameters('scaleSetPriority')]", + "spotMaxPrice": "[parameters('spotMaxPrice')]", + "tags": "[parameters('tags')]", + "type": "[parameters('type')]", + "upgradeSettings": { + "maxSurge": "[parameters('maxSurge')]" + }, + "vmSize": "[parameters('vmSize')]", + "vnetSubnetID": "[parameters('vnetSubnetId')]", + "workloadRuntime": "[parameters('workloadRuntime')]" + }, + "dependsOn": [ + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the agent pool." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the agent pool." + }, + "value": "[resourceId('Microsoft.ContainerService/managedClusters/agentPools', parameters('managedClusterName'), parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The resource group the agent pool was deployed into." + }, + "value": "[resourceGroup().name]" + } + } +} \ No newline at end of file diff --git a/avm/res/container-service/managed-cluster/main.bicep b/avm/res/container-service/managed-cluster/main.bicep new file mode 100644 index 0000000000..9dd7c344bd --- /dev/null +++ b/avm/res/container-service/managed-cluster/main.bicep @@ -0,0 +1,1035 @@ +metadata name = 'Azure Kubernetes Service (AKS) Managed Clusters' +metadata description = 'This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.' +metadata owner = 'Azure/module-maintainers' + +@description('Required. Specifies the name of the AKS cluster.') +param name string + +@description('Optional. Specifies the location of AKS cluster. It picks up Resource Group\'s location by default.') +param location string = resourceGroup().location + +@description('Optional. Specifies the DNS prefix specified when creating the managed cluster.') +param dnsPrefix string = name + +@description('Optional. The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both.') +param managedIdentities managedIdentitiesType + +@description('Optional. Network dataplane used in the Kubernetes cluster. Not compatible with kubenet network plugin.') +@allowed([ + 'azure' + 'cilium' +]) +param networkDataplane string? + +@description('Optional. Specifies the network plugin used for building Kubernetes network.') +@allowed([ + 'azure' + 'kubenet' +]) +param networkPlugin string? + +@description('Optional. Network plugin mode used for building the Kubernetes network. Not compatible with kubenet network plugin.') +@allowed([ + 'overlay' +]) +param networkPluginMode string? + +@description('Optional. Specifies the network policy used for building Kubernetes network. - calico or azure.') +@allowed([ + 'azure' + 'calico' +]) +param networkPolicy string? + +@description('Optional. Specifies the CIDR notation IP range from which to assign pod IPs when kubenet is used.') +param podCidr string? + +@description('Optional. A CIDR notation IP range from which to assign service cluster IPs. It must not overlap with any Subnet IP ranges.') +param serviceCidr string? + +@description('Optional. Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr.') +param dnsServiceIP string? + +@description('Optional. Specifies the sku of the load balancer used by the virtual machine scale sets used by nodepools.') +@allowed([ + 'basic' + 'standard' +]) +param loadBalancerSku string = 'standard' + +@description('Optional. Outbound IP Count for the Load balancer.') +param managedOutboundIPCount int = 0 + +@description('Optional. Specifies outbound (egress) routing method. - loadBalancer or userDefinedRouting.') +@allowed([ + 'loadBalancer' + 'userDefinedRouting' +]) +param outboundType string = 'loadBalancer' + +@description('Optional. Tier of a managed cluster SKU.') +@allowed([ + 'Free' + 'Premium' + 'Standard' +]) +param skuTier string = 'Standard' + +@description('Optional. Version of Kubernetes specified when creating the managed cluster.') +param kubernetesVersion string? + +@description('Optional. Specifies the administrator username of Linux virtual machines.') +param adminUsername string = 'azureuser' + +@description('Optional. Specifies the SSH RSA public key string for the Linux nodes.') +param sshPublicKey string? + +@description('Conditional. Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster.') +param aksServicePrincipalProfile object? + +@description('Optional. The client AAD application ID.') +param aadProfileClientAppID string? + +@description('Optional. The server AAD application ID.') +param aadProfileServerAppID string? + +@description('Optional. The server AAD application secret.') +#disable-next-line secure-secrets-in-params // Not a secret +param aadProfileServerAppSecret string? + +@description('Optional. Specifies the tenant ID of the Azure Active Directory used by the AKS cluster for authentication.') +param aadProfileTenantId string = subscription().tenantId + +@description('Optional. Specifies the AAD group object IDs that will have admin role of the cluster.') +param aadProfileAdminGroupObjectIDs array? + +@description('Optional. Specifies whether to enable managed AAD integration.') +param aadProfileManaged bool = true + +@description('Optional. Whether to enable Kubernetes Role-Based Access Control.') +param enableRBAC bool = true + +@description('Optional. Specifies whether to enable Azure RBAC for Kubernetes authorization.') +param aadProfileEnableAzureRBAC bool = enableRBAC + +@description('Optional. If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled.') +param disableLocalAccounts bool = false + +@description('Optional. Name of the resource group containing agent pool nodes.') +param nodeResourceGroup string = '${resourceGroup().name}_aks_${name}_nodes' + +@description('Optional. IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer.') +param authorizedIPRanges array? + +@description('Optional. Whether to disable run command for the cluster or not.') +param disableRunCommand bool = false + +@description('Optional. Specifies whether to create the cluster as a private cluster or not.') +param enablePrivateCluster bool = false + +@description('Optional. Whether to create additional public FQDN for private cluster or not.') +param enablePrivateClusterPublicFQDN bool = false + +@description('Optional. Private DNS Zone configuration. Set to \'system\' and AKS will create a private DNS zone in the node resource group. Set to \'\' to disable private DNS Zone creation and use public DNS. Supply the resource ID here of an existing Private DNS zone to use an existing zone.') +param privateDNSZone string? + +@description('Required. Properties of the primary agent pool.') +param primaryAgentPoolProfile array + +@description('Optional. Define one or more secondary/additional agent pools.') +param agentPools agentPoolType + +@description('Optional. Specifies whether the httpApplicationRouting add-on is enabled or not.') +param httpApplicationRoutingEnabled bool = false + +@description('Optional. Specifies whether the webApplicationRoutingEnabled add-on is enabled or not.') +param webApplicationRoutingEnabled bool = false + +@description('Optional. Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`.') +param dnsZoneResourceId string? + +@description('Optional. Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided.') +param enableDnsZoneContributorRoleAssignment bool = true + +@description('Optional. Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not.') +param ingressApplicationGatewayEnabled bool = false + +@description('Conditional. Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`.') +param appGatewayResourceId string? + +@description('Optional. Specifies whether the aciConnectorLinux add-on is enabled or not.') +param aciConnectorLinuxEnabled bool = false + +@description('Optional. Specifies whether the azurepolicy add-on is enabled or not. For security reasons, this setting should be enabled.') +param azurePolicyEnabled bool = true + +@description('Optional. Specifies whether the openServiceMesh add-on is enabled or not.') +param openServiceMeshEnabled bool = false + +@description('Optional. Specifies the azure policy version to use.') +param azurePolicyVersion string = 'v2' + +@description('Optional. Specifies whether the kubeDashboard add-on is enabled or not.') +param kubeDashboardEnabled bool = false + +@description('Optional. Specifies whether the KeyvaultSecretsProvider add-on is enabled or not.') +#disable-next-line secure-secrets-in-params // Not a secret +param enableKeyvaultSecretsProvider bool = false + +@allowed([ + 'false' + 'true' +]) +@description('Optional. Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation.') +#disable-next-line secure-secrets-in-params // Not a secret +param enableSecretRotation string = 'false' + +@description('Optional. Specifies the scan interval of the auto-scaler of the AKS cluster.') +param autoScalerProfileScanInterval string = '10s' + +@description('Optional. Specifies the scale down delay after add of the auto-scaler of the AKS cluster.') +param autoScalerProfileScaleDownDelayAfterAdd string = '10m' + +@description('Optional. Specifies the scale down delay after delete of the auto-scaler of the AKS cluster.') +param autoScalerProfileScaleDownDelayAfterDelete string = '20s' + +@description('Optional. Specifies scale down delay after failure of the auto-scaler of the AKS cluster.') +param autoScalerProfileScaleDownDelayAfterFailure string = '3m' + +@description('Optional. Specifies the scale down unneeded time of the auto-scaler of the AKS cluster.') +param autoScalerProfileScaleDownUnneededTime string = '10m' + +@description('Optional. Specifies the scale down unready time of the auto-scaler of the AKS cluster.') +param autoScalerProfileScaleDownUnreadyTime string = '20m' + +@description('Optional. Specifies the utilization threshold of the auto-scaler of the AKS cluster.') +param autoScalerProfileUtilizationThreshold string = '0.5' + +@description('Optional. Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster.') +param autoScalerProfileMaxGracefulTerminationSec string = '600' + +@allowed([ + 'false' + 'true' +]) +@description('Optional. Specifies the balance of similar node groups for the auto-scaler of the AKS cluster.') +param autoScalerProfileBalanceSimilarNodeGroups string = 'false' + +@allowed([ + 'least-waste' + 'most-pods' + 'priority' + 'random' +]) +@description('Optional. Specifies the expand strategy for the auto-scaler of the AKS cluster.') +param autoScalerProfileExpander string = 'random' + +@description('Optional. Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster.') +param autoScalerProfileMaxEmptyBulkDelete string = '10' + +@description('Optional. Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an "m". No unit of time other than minutes (m) is supported.') +param autoScalerProfileMaxNodeProvisionTime string = '15m' + +@description('Optional. Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0.') +param autoScalerProfileMaxTotalUnreadyPercentage string = '45' + +@description('Optional. For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit ("s" for seconds, "m" for minutes, "h" for hours, etc).') +param autoScalerProfileNewPodScaleUpDelay string = '0s' + +@description('Optional. Specifies the OK total unready count for the auto-scaler of the AKS cluster.') +param autoScalerProfileOkTotalUnreadyCount string = '3' + +@allowed([ + 'false' + 'true' +]) +@description('Optional. Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster.') +param autoScalerProfileSkipNodesWithLocalStorage string = 'true' + +@allowed([ + 'false' + 'true' +]) +@description('Optional. Specifies if nodes with system pods should be skipped for the auto-scaler of the AKS cluster.') +param autoScalerProfileSkipNodesWithSystemPods string = 'true' + +@allowed([ + 'node-image' + 'none' + 'patch' + 'rapid' + 'stable' +]) +@description('Optional. Auto-upgrade channel on the AKS cluster.') +param autoUpgradeProfileUpgradeChannel string? + +@description('Optional. Running in Kubenet is disabled by default due to the security related nature of AAD Pod Identity and the risks of IP spoofing.') +param podIdentityProfileAllowNetworkPluginKubenet bool = false + +@description('Optional. Whether the pod identity addon is enabled.') +param podIdentityProfileEnable bool = false + +@description('Optional. The pod identities to use in the cluster.') +param podIdentityProfileUserAssignedIdentities array? + +@description('Optional. The pod identity exceptions to allow.') +param podIdentityProfileUserAssignedIdentityExceptions array? + +@description('Optional. Whether the The OIDC issuer profile of the Managed Cluster is enabled.') +param enableOidcIssuerProfile bool = false + +@description('Optional. Whether to enable Workload Identity. Requires OIDC issuer profile to be enabled.') +param enableWorkloadIdentity bool = false + +@description('Optional. Whether to enable Azure Defender.') +param enableAzureDefender bool = false + +@description('Optional. Whether to enable Kubernetes pod security policy. Requires enabling the pod security policy feature flag on the subscription.') +param enablePodSecurityPolicy bool = false + +@description('Optional. Whether the AzureBlob CSI Driver for the storage profile is enabled.') +param enableStorageProfileBlobCSIDriver bool = false + +@description('Optional. Whether the AzureDisk CSI Driver for the storage profile is enabled.') +param enableStorageProfileDiskCSIDriver bool = false + +@description('Optional. Whether the AzureFile CSI Driver for the storage profile is enabled.') +param enableStorageProfileFileCSIDriver bool = false + +@description('Optional. Whether the snapshot controller for the storage profile is enabled.') +param enableStorageProfileSnapshotController bool = false + +@allowed([ + 'AKSLongTermSupport' + 'KubernetesOfficial' +]) +@description('Optional. The support plan for the Managed Cluster.') +param supportPlan string = 'KubernetesOfficial' + +@description('Optional. The diagnostic settings of the service.') +param diagnosticSettings diagnosticSettingType + +@description('Optional. Specifies whether the OMS agent is enabled.') +param omsAgentEnabled bool = true + +@description('Optional. Resource ID of the monitoring log analytics workspace.') +param monitoringWorkspaceId string? + +@description('Optional. Enable/Disable usage telemetry for module.') +param enableTelemetry bool = true + +@description('Optional. Array of role assignments to create.') +param roleAssignments roleAssignmentType + +@description('Optional. The lock settings of the service.') +param lock lockType + +@description('Optional. Tags of the resource.') +param tags object? + +@description('Optional. The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided.') +param diskEncryptionSetResourceId string? + +@description('Optional. Settings and configurations for the flux extension.') +param fluxExtension extensionType + +@description('Optional. Configurations for provisioning the cluster with HTTP proxy servers.') +param httpProxyConfig object? + +@description('Optional. Identities associated with the cluster.') +param identityProfile object? + +@description('Optional. The customer managed key definition.') +param customerManagedKey customerManagedKeyType + +@description('Optional. Whether the metric state of the kubenetes cluster is enabled.') +param enableAzureMonitorProfileMetrics bool = false + +@description('Optional. Whether the Logs profile for the Azure Monitor Infrastructure and Application Logs is enabled.') +param enableAzureMonitorProfileLogs bool = false + +@description('Optional. Indicates if Application Monitoring of the kubenetes cluster is enabled.') +param enableAppMonitoring bool = false + +@description('Optional. Whether the Windows Log Collection for Azure Monitor Container Insights Logs Addon is enabled.') +param enableWindowsHostLogs bool = false + +@description('Optional. Indicates if Azure Monitor Container Insights Logs Addon is enabled.') +param enableContainerInsights bool = false + +@description('Optional. Indicates if Application Monitoring Open Telemetry Metrics is enabled.') +param enableAppMonitoringOpenTelemetryMetrics bool = false + +@description('Optional. A comma-separated list of kubernetes cluster metrics labels.') +param metricLabelsAllowlist string = '' + +@description('Optional. A comma-separated list of Kubernetes cluster metrics annotations.') +param metricAnnotationsAllowList string = '' + +// =========== // +// Variables // +// =========== // + +var formattedUserAssignedIdentities = reduce(map((managedIdentities.?userAssignedResourcesIds ?? []), (id) => { '${id}': {} }), {}, (cur, next) => union(cur, next)) // Converts the flat array to an object like { '${id1}': {}, '${id2}': {} } + +var identity = !empty(managedIdentities) ? { + type: (managedIdentities.?systemAssigned ?? false) ? 'SystemAssigned' : (!empty(managedIdentities.?userAssignedResourcesIds ?? {}) ? 'UserAssigned' : null) + userAssignedIdentities: !empty(formattedUserAssignedIdentities) ? formattedUserAssignedIdentities : null +} : null + +var builtInRoleNames = { + 'Azure Kubernetes Fleet Manager Contributor Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '63bb64ad-9799-4770-b5c3-24ed299a07bf') + 'Azure Kubernetes Fleet Manager RBAC Admin': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '434fb43a-c01c-447e-9f67-c3ad923cfaba') + 'Azure Kubernetes Fleet Manager RBAC Cluster Admin': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '18ab4d3d-a1bf-4477-8ad9-8359bc988f69') + 'Azure Kubernetes Fleet Manager RBAC Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '30b27cfc-9c84-438e-b0ce-70e35255df80') + 'Azure Kubernetes Fleet Manager RBAC Writer': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '5af6afb3-c06c-4fa4-8848-71a8aee05683') + 'Azure Kubernetes Service Cluster Admin Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '0ab0b1a8-8aac-4efd-b8c2-3ee1fb270be8') + 'Azure Kubernetes Service Cluster Monitoring User': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '1afdec4b-e479-420e-99e7-f82237c7c5e6') + 'Azure Kubernetes Service Cluster User Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4abbcc35-e782-43d8-92c5-2d3f1bd2253f') + 'Azure Kubernetes Service Contributor Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ed7f3fbd-7b88-4dd4-9017-9adb7ce333f8') + 'Azure Kubernetes Service RBAC Admin': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3498e952-d568-435e-9b2c-8d77e338d7f7') + 'Azure Kubernetes Service RBAC Cluster Admin': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b') + 'Azure Kubernetes Service RBAC Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f6c6a51-bcf8-42ba-9220-52d62157d7db') + 'Azure Kubernetes Service RBAC Writer': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a7ffa36f-339b-4b5c-8bdf-e2c188b2c0eb') + Contributor: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') + 'Kubernetes Agentless Operator': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'd5a2ae44-610b-4500-93be-660a0c5f5ca6') + Owner: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') + Reader: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7') + 'Role Based Access Control Administrator (Preview)': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f58310d9-a9f6-439a-9e8d-f62e7b41a168') + 'User Access Administrator': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '18d7d88d-d35e-4fb5-a5c3-7773c20a72d9') +} + +// ============ // +// Dependencies // +// ============ // + +resource avmTelemetry 'Microsoft.Resources/deployments@2023-07-01' = if (enableTelemetry) { + name: '46d3xbcp.res.containerservice-managedcluster.${replace('-..--..-', '.', '-')}.${substring(uniqueString(deployment().name, location), 0, 4)}' + properties: { + mode: 'Incremental' + template: { + '$schema': 'https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#' + contentVersion: '1.0.0.0' + resources: [] + outputs: { + telemetry: { + type: 'String' + value: 'For more information, see https://aka.ms/avm/TelemetryInfo' + } + } + } + } +} + +resource cMKKeyVault 'Microsoft.KeyVault/vaults@2023-02-01' existing = if (!empty(customerManagedKey.?keyVaultResourceId)) { + name: last(split((customerManagedKey.?keyVaultResourceId ?? 'dummyVault'), '/')) + scope: resourceGroup(split((customerManagedKey.?keyVaultResourceId ?? '//'), '/')[2], split((customerManagedKey.?keyVaultResourceId ?? '////'), '/')[4]) + + resource cMKKey 'keys@2023-02-01' existing = if (!empty(customerManagedKey.?keyVaultResourceId) && !empty(customerManagedKey.?keyName)) { + name: customerManagedKey.?keyName ?? 'dummyKey' + } +} + +// ============== // +// Main Resources // +// ============== // + +resource managedCluster 'Microsoft.ContainerService/managedClusters@2023-07-02-preview' = { + name: name + location: location + tags: tags + identity: identity + sku: { + name: 'Base' + tier: skuTier + } + properties: { + httpProxyConfig: httpProxyConfig + identityProfile: identityProfile + diskEncryptionSetID: diskEncryptionSetResourceId + kubernetesVersion: kubernetesVersion + dnsPrefix: dnsPrefix + agentPoolProfiles: primaryAgentPoolProfile + linuxProfile: !empty(sshPublicKey) ? { + adminUsername: adminUsername + ssh: { + publicKeys: [ + { + keyData: sshPublicKey + } + ] + } + } : null + servicePrincipalProfile: aksServicePrincipalProfile + ingressProfile: { + webAppRouting: { + enabled: webApplicationRoutingEnabled + dnsZoneResourceIds: !empty(dnsZoneResourceId) ? [ + any(dnsZoneResourceId) + ] : null + } + } + addonProfiles: { + httpApplicationRouting: { + enabled: httpApplicationRoutingEnabled + } + ingressApplicationGateway: { + enabled: ingressApplicationGatewayEnabled && !empty(appGatewayResourceId) + config: ingressApplicationGatewayEnabled && !empty(appGatewayResourceId) ? { + applicationGatewayId: appGatewayResourceId + effectiveApplicationGatewayId: appGatewayResourceId + } : null + } + omsagent: { + enabled: omsAgentEnabled && !empty(monitoringWorkspaceId) + config: omsAgentEnabled && !empty(monitoringWorkspaceId) ? { + logAnalyticsWorkspaceResourceID: monitoringWorkspaceId + } : null + } + aciConnectorLinux: { + enabled: aciConnectorLinuxEnabled + } + azurepolicy: { + enabled: azurePolicyEnabled + config: azurePolicyEnabled ? { + version: azurePolicyVersion + } : null + } + openServiceMesh: { + enabled: openServiceMeshEnabled + config: openServiceMeshEnabled ? {} : null + } + kubeDashboard: { + enabled: kubeDashboardEnabled + } + azureKeyvaultSecretsProvider: { + enabled: enableKeyvaultSecretsProvider + config: enableKeyvaultSecretsProvider ? { + enableSecretRotation: enableSecretRotation + } : null + } + } + oidcIssuerProfile: enableOidcIssuerProfile ? { + enabled: enableOidcIssuerProfile + } : null + enableRBAC: enableRBAC + disableLocalAccounts: disableLocalAccounts + nodeResourceGroup: nodeResourceGroup + enablePodSecurityPolicy: enablePodSecurityPolicy + networkProfile: { + networkDataplane: networkDataplane + networkPlugin: networkPlugin + networkPluginMode: networkPluginMode + networkPolicy: networkPolicy + podCidr: podCidr + serviceCidr: serviceCidr + dnsServiceIP: dnsServiceIP + outboundType: outboundType + loadBalancerSku: loadBalancerSku + loadBalancerProfile: managedOutboundIPCount != 0 ? { + managedOutboundIPs: { + count: managedOutboundIPCount + } + effectiveOutboundIPs: [] + } : null + } + aadProfile: { + clientAppID: aadProfileClientAppID + serverAppID: aadProfileServerAppID + serverAppSecret: aadProfileServerAppSecret + managed: aadProfileManaged + enableAzureRBAC: aadProfileEnableAzureRBAC + adminGroupObjectIDs: aadProfileAdminGroupObjectIDs + tenantID: aadProfileTenantId + } + autoScalerProfile: { + 'balance-similar-node-groups': autoScalerProfileBalanceSimilarNodeGroups + expander: autoScalerProfileExpander + 'max-empty-bulk-delete': autoScalerProfileMaxEmptyBulkDelete + 'max-graceful-termination-sec': autoScalerProfileMaxGracefulTerminationSec + 'max-node-provision-time': autoScalerProfileMaxNodeProvisionTime + 'max-total-unready-percentage': autoScalerProfileMaxTotalUnreadyPercentage + 'new-pod-scale-up-delay': autoScalerProfileNewPodScaleUpDelay + 'ok-total-unready-count': autoScalerProfileOkTotalUnreadyCount + 'scale-down-delay-after-add': autoScalerProfileScaleDownDelayAfterAdd + 'scale-down-delay-after-delete': autoScalerProfileScaleDownDelayAfterDelete + 'scale-down-delay-after-failure': autoScalerProfileScaleDownDelayAfterFailure + 'scale-down-unneeded-time': autoScalerProfileScaleDownUnneededTime + 'scale-down-unready-time': autoScalerProfileScaleDownUnreadyTime + 'scale-down-utilization-threshold': autoScalerProfileUtilizationThreshold + 'scan-interval': autoScalerProfileScanInterval + 'skip-nodes-with-local-storage': autoScalerProfileSkipNodesWithLocalStorage + 'skip-nodes-with-system-pods': autoScalerProfileSkipNodesWithSystemPods + } + autoUpgradeProfile: { + upgradeChannel: autoUpgradeProfileUpgradeChannel + } + apiServerAccessProfile: { + authorizedIPRanges: authorizedIPRanges + disableRunCommand: disableRunCommand + enablePrivateCluster: enablePrivateCluster + enablePrivateClusterPublicFQDN: enablePrivateClusterPublicFQDN + privateDNSZone: privateDNSZone + } + azureMonitorProfile: { + logs: enableAzureMonitorProfileLogs ? { + enabled: enableAzureMonitorProfileLogs + appMonitoring: { + enabled: enableAppMonitoring + } + containerInsights: { + enabled: enableContainerInsights + logAnalyticsWorkspaceResourceId: !empty(monitoringWorkspaceId) ? monitoringWorkspaceId : null + windowsHostLogs: { + enabled: enableWindowsHostLogs + } + } + } : null + metrics: enableAzureMonitorProfileMetrics ? { + enabled: enableAzureMonitorProfileMetrics + appMonitoringOpenTelemetryMetrics: { + enabled: enableAppMonitoringOpenTelemetryMetrics + } + kubeStateMetrics: { + metricLabelsAllowlist: metricLabelsAllowlist + metricAnnotationsAllowList: metricAnnotationsAllowList + } + } : null + } + podIdentityProfile: { + allowNetworkPluginKubenet: podIdentityProfileAllowNetworkPluginKubenet + enabled: podIdentityProfileEnable + userAssignedIdentities: podIdentityProfileUserAssignedIdentities + userAssignedIdentityExceptions: podIdentityProfileUserAssignedIdentityExceptions + } + securityProfile: { + defender: enableAzureDefender ? { + securityMonitoring: { + enabled: enableAzureDefender + } + logAnalyticsWorkspaceResourceId: monitoringWorkspaceId + } : null + workloadIdentity: enableWorkloadIdentity ? { + enabled: enableWorkloadIdentity + } : null + } + storageProfile: { + blobCSIDriver: { + enabled: enableStorageProfileBlobCSIDriver + } + diskCSIDriver: { + enabled: enableStorageProfileDiskCSIDriver + } + fileCSIDriver: { + enabled: enableStorageProfileFileCSIDriver + } + snapshotController: { + enabled: enableStorageProfileSnapshotController + } + } + supportPlan: supportPlan + } +} + +module managedCluster_agentPools 'agent-pool/main.bicep' = [for (agentPool, index) in (agentPools ?? []): { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-AgentPool-${index}' + params: { + managedClusterName: managedCluster.?name + name: agentPool.name + availabilityZones: agentPool.?availabilityZones + count: agentPool.?count + sourceResourceId: agentPool.?sourceResourceId + enableAutoScaling: agentPool.?enableAutoScaling + enableEncryptionAtHost: agentPool.?enableEncryptionAtHost + enableFIPS: agentPool.?enableFIPS + enableNodePublicIP: agentPool.?enableNodePublicIP + enableUltraSSD: agentPool.?enableUltraSSD + gpuInstanceProfile: agentPool.?gpuInstanceProfile + kubeletDiskType: agentPool.?kubeletDiskType + maxCount: agentPool.?maxCount + maxPods: agentPool.?maxPods + minCount: agentPool.?minCount + mode: agentPool.?mode + nodeLabels: agentPool.?nodeLabels + nodePublicIpPrefixId: agentPool.?nodePublicIpPrefixId + nodeTaints: agentPool.?nodeTaints + orchestratorVersion: agentPool.?orchestratorVersion ?? kubernetesVersion + osDiskSizeGB: agentPool.?osDiskSizeGB + osDiskType: agentPool.?osDiskType + osSku: agentPool.?osSku + osType: agentPool.?osType + podSubnetId: agentPool.?podSubnetId + proximityPlacementGroupResourceId: agentPool.?proximityPlacementGroupResourceId + scaleDownMode: agentPool.?scaleDownMode + scaleSetEvictionPolicy: agentPool.?scaleSetEvictionPolicy + scaleSetPriority: agentPool.?scaleSetPriority + spotMaxPrice: agentPool.?spotMaxPrice + tags: agentPool.?tags ?? tags + type: agentPool.?type + maxSurge: agentPool.?maxSurge + vmSize: agentPool.?vmSize + vnetSubnetId: agentPool.?vnetSubnetId + workloadRuntime: agentPool.?workloadRuntime + } +}] + +module managedCluster_extension 'br/public:avm/res/kubernetes-configuration/extension:0.2.0' = if (!empty(fluxExtension)) { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxExtension' + params: { + clusterName: managedCluster.name + configurationProtectedSettings: fluxExtension.?configurationProtectedSettings + configurationSettings: fluxExtension.?configurationSettings + enableTelemetry: enableTelemetry + extensionType: 'microsoft.flux' + fluxConfigurations: fluxExtension.?configurations + location: location + name: 'flux' + releaseNamespace: fluxExtension.?releaseNamespace ?? 'flux-system' + releaseTrain: fluxExtension.?releaseTrain ?? 'Stable' + version: fluxExtension.?version + } +} + +resource managedCluster_lock 'Microsoft.Authorization/locks@2020-05-01' = if (!empty(lock ?? {}) && lock.?kind != 'None') { + name: lock.?name ?? 'lock-${name}' + properties: { + level: lock.?kind ?? '' + notes: lock.?kind == 'CanNotDelete' ? 'Cannot delete resource or child resources.' : 'Cannot delete or modify the resource or child resources.' + } + scope: managedCluster +} + +resource managedCluster_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = [for (diagnosticSetting, index) in (diagnosticSettings ?? []): { + name: diagnosticSetting.?name ?? '${name}-diagnosticSettings' + properties: { + storageAccountId: diagnosticSetting.?storageAccountResourceId + workspaceId: diagnosticSetting.?workspaceResourceId + eventHubAuthorizationRuleId: diagnosticSetting.?eventHubAuthorizationRuleResourceId + eventHubName: diagnosticSetting.?eventHubName + metrics: diagnosticSetting.?metricCategories ?? [ + { + category: 'AllMetrics' + timeGrain: null + enabled: true + } + ] + logs: diagnosticSetting.?logCategoriesAndGroups ?? [ + { + categoryGroup: 'AllLogs' + enabled: true + } + ] + marketplacePartnerId: diagnosticSetting.?marketplacePartnerResourceId + logAnalyticsDestinationType: diagnosticSetting.?logAnalyticsDestinationType + } + scope: managedCluster +}] + +resource managedCluster_roleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for (roleAssignment, index) in (roleAssignments ?? []): { + name: guid(managedCluster.id, roleAssignment.principalId, roleAssignment.roleDefinitionIdOrName) + properties: { + roleDefinitionId: contains(builtInRoleNames, roleAssignment.roleDefinitionIdOrName) ? builtInRoleNames[roleAssignment.roleDefinitionIdOrName] : contains(roleAssignment.roleDefinitionIdOrName, '/providers/Microsoft.Authorization/roleDefinitions/') ? roleAssignment.roleDefinitionIdOrName : subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleAssignment.roleDefinitionIdOrName) + principalId: roleAssignment.principalId + description: roleAssignment.?description + principalType: roleAssignment.?principalType + condition: roleAssignment.?condition + conditionVersion: !empty(roleAssignment.?condition) ? (roleAssignment.?conditionVersion ?? '2.0') : null // Must only be set if condtion is set + delegatedManagedIdentityResourceId: roleAssignment.?delegatedManagedIdentityResourceId + } + scope: managedCluster +}] + +resource dnsZone 'Microsoft.Network/dnsZones@2018-05-01' existing = if (enableDnsZoneContributorRoleAssignment == true && dnsZoneResourceId != null && webApplicationRoutingEnabled) { + name: last(split((!empty(dnsZoneResourceId) ? any(dnsZoneResourceId) : '/dummmyZone'), '/'))! +} + +resource dnsZone_roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (enableDnsZoneContributorRoleAssignment == true && dnsZoneResourceId != null && webApplicationRoutingEnabled) { + name: guid(dnsZone.id, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314'), 'DNS Zone Contributor') + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314') // 'DNS Zone Contributor' + principalId: managedCluster.properties.ingressProfile.webAppRouting.identity.objectId + principalType: 'ServicePrincipal' + } + scope: dnsZone +} + +@description('The resource ID of the managed cluster.') +output resourceId string = managedCluster.id + +@description('The resource group the managed cluster was deployed into.') +output resourceGroupName string = resourceGroup().name + +@description('The name of the managed cluster.') +output name string = managedCluster.name + +@description('The control plane FQDN of the managed cluster.') +output controlPlaneFQDN string = enablePrivateCluster ? managedCluster.properties.privateFQDN : managedCluster.properties.fqdn + +@description('The principal ID of the system assigned identity.') +output systemAssignedMIPrincipalId string = managedCluster.?identity.?principalId ?? '' + +@description('The Object ID of the AKS identity.') +output kubeletidentityObjectId string = managedCluster.properties.?identityProfile.?kubeletidentity.?objectId ?? '' + +@description('The Object ID of the OMS agent identity.') +output omsagentIdentityObjectId string = managedCluster.properties.?addonProfiles.?omsagent.?identity.?objectId ?? '' + +@description('The Object ID of the Key Vault Secrets Provider identity.') +output keyvaultIdentityObjectId string = managedCluster.properties.?addonProfiles.?azureKeyvaultSecretsProvider.?identity.?objectId ?? '' + +@description('The Client ID of the Key Vault Secrets Provider identity.') +output keyvaultIdentityClientId string = managedCluster.properties.?addonProfiles.?azureKeyvaultSecretsProvider.?identity.?clientId ?? '' + +@description('The Object ID of Application Gateway Ingress Controller (AGIC) identity.') +output ingressApplicationGatewayIdentityObjectId string = managedCluster.properties.?addonProfiles.?ingressApplicationGateway.?identity.?objectId ?? '' + +@description('The location the resource was deployed into.') +output location string = managedCluster.location + +@description('The OIDC token issuer URL.') +output oidcIssuerUrl string = managedCluster.properties.?oidcIssuerProfile.?issuerURL ?? '' + +@description('The addonProfiles of the Kubernetes cluster.') +output addonProfiles object = managedCluster.properties.?addonProfiles ?? {} + +@description('The Object ID of Web Application Routing.') +output webAppRoutingIdentityObjectId string = managedCluster.properties.?ingressProfile.?webAppRouting.?identity.?objectId ?? '' + +// =============== // +// Definitions // +// =============== // + +type agentPoolType = { + @description('Required. The name of the agent pool.') + name: string? + + @description('Optional. The availability zones of the agent pool.') + availabilityZones: string[]? + + @description('Optional. The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive).') + count: int? + + @description('Optional. The source resource ID to create the agent pool from.') + sourceResourceId: string? + + @description('Optional. Whether to enable auto-scaling for the agent pool.') + enableAutoScaling: bool? + + @description('Optional. Whether to enable encryption at host for the agent pool.') + enableEncryptionAtHost: bool? + + @description('Optional. Whether to enable FIPS for the agent pool.') + enableFIPS: bool? + + @description('Optional. Whether to enable node public IP for the agent pool.') + enableNodePublicIP: bool? + + @description('Optional. Whether to enable Ultra SSD for the agent pool.') + enableUltraSSD: bool? + + @description('Optional. The GPU instance profile of the agent pool.') + gpuInstanceProfile: ('MIG1g' | 'MIG2g' | 'MIG3g' | 'MIG4g' | 'MIG7g')? + + @description('Optional. The kubelet disk type of the agent pool.') + kubeletDiskType: string? + + @description('Optional. The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive).') + maxCount: int? + + @description('Optional. The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive).') + minCount: int? + + @description('Optional. The maximum number of pods that can run on a node.') + maxPods: int? + + @description('Optional. The minimum number of pods that can run on a node.') + minPods: int? + + @description('Optional. The mode of the agent pool.') + mode: ('System' | 'User')? + + @description('Optional. The node labels of the agent pool.') + nodeLabels: object? + + @description('Optional. The node public IP prefix ID of the agent pool.') + nodePublicIpPrefixId: string? + + @description('Optional. The node taints of the agent pool.') + nodeTaints: string[]? + + @description('Optional. The Kubernetes version of the agent pool.') + orchestratorVersion: string? + + @description('Optional. The OS disk size in GB of the agent pool.') + osDiskSizeGB: int? + + @description('Optional. The OS disk type of the agent pool.') + osDiskType: string? + + @description('Optional. The OS SKU of the agent pool.') + osSku: string? + + @description('Optional. The OS type of the agent pool.') + osType: ('Linux' | 'Windows')? + + @description('Optional. The pod subnet ID of the agent pool.') + podSubnetId: string? + + @description('Optional. The proximity placement group resource ID of the agent pool.') + proximityPlacementGroupResourceId: string? + + @description('Optional. The scale down mode of the agent pool.') + scaleDownMode: ('Delete' | 'Pause' | 'Requeue' | 'DeleteRequeue')? + + @description('Optional. The scale set eviction policy of the agent pool.') + scaleSetEvictionPolicy: ('Delete' | 'Deallocate')? + + @description('Optional. The scale set priority of the agent pool.') + scaleSetPriority: ('Low' | 'Regular' | 'Spot')? + + @description('Optional. The spot max price of the agent pool.') + spotMaxPrice: int? + + @description('Optional. The tags of the agent pool.') + tags: object? + + @description('Optional. The type of the agent pool.') + type: ('AvailabilitySet' | 'VirtualMachineScaleSets')? + + @description('Optional. The maximum number of nodes that can be created during an upgrade.') + maxSurge: string? + + @description('Optional. The VM size of the agent pool.') + vmSize: string? + + @description('Optional. The VNet subnet ID of the agent pool.') + vnetSubnetID: string? + + @description('Optional. The workload runtime of the agent pool.') + workloadRuntime: string? + + @description('Optional. The enable default telemetry of the agent pool.') + enableDefaultTelemetry: bool? +}[]? + +type managedIdentitiesType = { + @description('Optional. Enables system assigned managed identity on the resource.') + systemAssigned: bool? + + @description('Optional. The resource ID(s) to assign to the resource.') + userAssignedResourcesIds: string[]? +}? + +type lockType = { + @description('Optional. Specify the name of lock.') + name: string? + + @description('Optional. Specify the type of lock.') + kind: ('CanNotDelete' | 'ReadOnly' | 'None')? +}? + +type roleAssignmentType = { + @description('Required. The role to assign. You can provide either the display name of the role definition, the role definition GUID, or its fully qualified ID in the following format: \'/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11\'.') + roleDefinitionIdOrName: string + + @description('Required. The principal ID of the principal (user/group/identity) to assign the role to.') + principalId: string + + @description('Optional. The principal type of the assigned principal ID.') + principalType: ('ServicePrincipal' | 'Group' | 'User' | 'ForeignGroup' | 'Device')? + + @description('Optional. The description of the role assignment.') + description: string? + + @description('Optional. The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase "foo_storage_container".') + condition: string? + + @description('Optional. Version of the condition.') + conditionVersion: '2.0'? + + @description('Optional. The Resource Id of the delegated managed identity resource.') + delegatedManagedIdentityResourceId: string? +}[]? + +type diagnosticSettingType = { + @description('Optional. The name of diagnostic setting.') + name: string? + + @description('Optional. The name of logs that will be streamed. "allLogs" includes all possible logs for the resource. Set to \'\' to disable log collection.') + logCategoriesAndGroups: { + @description('Optional. Name of a Diagnostic Log category for a resource type this setting is applied to. Set the specific logs to collect here.') + category: string? + + @description('Optional. Name of a Diagnostic Log category group for a resource type this setting is applied to. Set to \'AllLogs\' to collect all logs.') + categoryGroup: string? + }[]? + + @description('Optional. The name of metrics that will be streamed. "allMetrics" includes all possible metrics for the resource. Set to \'\' to disable metric collection.') + metricCategories: { + @description('Required. Name of a Diagnostic Metric category for a resource type this setting is applied to. Set to \'AllMetrics\' to collect all metrics.') + category: string + }[]? + + @description('Optional. A string indicating whether the export to Log Analytics should use the default destination type, i.e. AzureDiagnostics, or use a destination type.') + logAnalyticsDestinationType: ('Dedicated' | 'AzureDiagnostics' | null)? + + @description('Optional. Resource ID of the diagnostic log analytics workspace. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub.') + workspaceResourceId: string? + + @description('Optional. Resource ID of the diagnostic storage account. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub.') + storageAccountResourceId: string? + + @description('Optional. Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to.') + eventHubAuthorizationRuleResourceId: string? + + @description('Optional. Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub.') + eventHubName: string? + + @description('Optional. The full ARM resource ID of the Marketplace resource to which you would like to send Diagnostic Logs.') + marketplacePartnerResourceId: string? +}[]? + +type fluxConfigurationProtectedSettingsType = { + @description('Optional. The SSH private key to use for Git authentication.') + sshPrivateKey: string? +}? + +type extensionType = { + @description('Required. The name of the extension.') + name: string? + + @description('Optional. Namespace where the extension Release must be placed.') + releaseNamespace: string? + + @description('Optional. Namespace where the extension will be created for an Namespace scoped extension.') + targetNamespace: string? + + @description('Required. The release train of the extension.') + releaseTrain: string? + + @description('Optional. The configuration protected settings of the extension.') + configurationProtectedSettings: fluxConfigurationProtectedSettingsType? + + @description('Optional. The configuration settings of the extension.') + configurationSettings: object? + + @description('Optional. The version of the extension.') + version: string? + + @description('Optional. The flux configurations of the extension.') + configurations: array? +}? + +type customerManagedKeyType = { + @description('Required. The resource ID of a key vault to reference a customer managed key for encryption from.') + keyVaultResourceId: string + + @description('Required. The name of the customer managed key to use for encryption.') + keyName: string + + @description('Optional. The version of the customer managed key to reference for encryption. If not provided, using \'latest\'.') + keyVersion: string? + + @description('Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public.') + keyVaultNetworkAccess: ('Private' | 'Public') +}? diff --git a/avm/res/container-service/managed-cluster/main.json b/avm/res/container-service/managed-cluster/main.json new file mode 100644 index 0000000000..1caedb332b --- /dev/null +++ b/avm/res/container-service/managed-cluster/main.json @@ -0,0 +1,2756 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.24.24.22086", + "templateHash": "4735311217553069306" + }, + "name": "Azure Kubernetes Service (AKS) Managed Clusters", + "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.", + "owner": "Azure/module-maintainers" + }, + "definitions": { + "agentPoolType": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Required. The name of the agent pool." + } + }, + "availabilityZones": { + "type": "array", + "items": { + "type": "string" + }, + "nullable": true, + "metadata": { + "description": "Optional. The availability zones of the agent pool." + } + }, + "count": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." + } + }, + "sourceResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The source resource ID to create the agent pool from." + } + }, + "enableAutoScaling": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable auto-scaling for the agent pool." + } + }, + "enableEncryptionAtHost": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable encryption at host for the agent pool." + } + }, + "enableFIPS": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable FIPS for the agent pool." + } + }, + "enableNodePublicIP": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable node public IP for the agent pool." + } + }, + "enableUltraSSD": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable Ultra SSD for the agent pool." + } + }, + "gpuInstanceProfile": { + "type": "string", + "allowedValues": [ + "MIG1g", + "MIG2g", + "MIG3g", + "MIG4g", + "MIG7g" + ], + "nullable": true, + "metadata": { + "description": "Optional. The GPU instance profile of the agent pool." + } + }, + "kubeletDiskType": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The kubelet disk type of the agent pool." + } + }, + "maxCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." + } + }, + "minCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." + } + }, + "maxPods": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of pods that can run on a node." + } + }, + "minPods": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The minimum number of pods that can run on a node." + } + }, + "mode": { + "type": "string", + "allowedValues": [ + "System", + "User" + ], + "nullable": true, + "metadata": { + "description": "Optional. The mode of the agent pool." + } + }, + "nodeLabels": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The node labels of the agent pool." + } + }, + "nodePublicIpPrefixId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The node public IP prefix ID of the agent pool." + } + }, + "nodeTaints": { + "type": "array", + "items": { + "type": "string" + }, + "nullable": true, + "metadata": { + "description": "Optional. The node taints of the agent pool." + } + }, + "orchestratorVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The Kubernetes version of the agent pool." + } + }, + "osDiskSizeGB": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The OS disk size in GB of the agent pool." + } + }, + "osDiskType": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The OS disk type of the agent pool." + } + }, + "osSku": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The OS SKU of the agent pool." + } + }, + "osType": { + "type": "string", + "allowedValues": [ + "Linux", + "Windows" + ], + "nullable": true, + "metadata": { + "description": "Optional. The OS type of the agent pool." + } + }, + "podSubnetId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The pod subnet ID of the agent pool." + } + }, + "proximityPlacementGroupResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The proximity placement group resource ID of the agent pool." + } + }, + "scaleDownMode": { + "type": "string", + "allowedValues": [ + "Delete", + "DeleteRequeue", + "Pause", + "Requeue" + ], + "nullable": true, + "metadata": { + "description": "Optional. The scale down mode of the agent pool." + } + }, + "scaleSetEvictionPolicy": { + "type": "string", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "nullable": true, + "metadata": { + "description": "Optional. The scale set eviction policy of the agent pool." + } + }, + "scaleSetPriority": { + "type": "string", + "allowedValues": [ + "Low", + "Regular", + "Spot" + ], + "nullable": true, + "metadata": { + "description": "Optional. The scale set priority of the agent pool." + } + }, + "spotMaxPrice": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The spot max price of the agent pool." + } + }, + "tags": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The tags of the agent pool." + } + }, + "type": { + "type": "string", + "allowedValues": [ + "AvailabilitySet", + "VirtualMachineScaleSets" + ], + "nullable": true, + "metadata": { + "description": "Optional. The type of the agent pool." + } + }, + "maxSurge": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of nodes that can be created during an upgrade." + } + }, + "vmSize": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The VM size of the agent pool." + } + }, + "vnetSubnetID": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The VNet subnet ID of the agent pool." + } + }, + "workloadRuntime": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The workload runtime of the agent pool." + } + }, + "enableDefaultTelemetry": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. The enable default telemetry of the agent pool." + } + } + } + }, + "nullable": true + }, + "managedIdentitiesType": { + "type": "object", + "properties": { + "systemAssigned": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Enables system assigned managed identity on the resource." + } + }, + "userAssignedResourcesIds": { + "type": "array", + "items": { + "type": "string" + }, + "nullable": true, + "metadata": { + "description": "Optional. The resource ID(s) to assign to the resource." + } + } + }, + "nullable": true + }, + "lockType": { + "type": "object", + "properties": { + "name": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Specify the name of lock." + } + }, + "kind": { + "type": "string", + "allowedValues": [ + "CanNotDelete", + "None", + "ReadOnly" + ], + "nullable": true, + "metadata": { + "description": "Optional. Specify the type of lock." + } + } + }, + "nullable": true + }, + "roleAssignmentType": { + "type": "array", + "items": { + "type": "object", + "properties": { + "roleDefinitionIdOrName": { + "type": "string", + "metadata": { + "description": "Required. The role to assign. You can provide either the display name of the role definition, the role definition GUID, or its fully qualified ID in the following format: '/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11'." + } + }, + "principalId": { + "type": "string", + "metadata": { + "description": "Required. The principal ID of the principal (user/group/identity) to assign the role to." + } + }, + "principalType": { + "type": "string", + "allowedValues": [ + "Device", + "ForeignGroup", + "Group", + "ServicePrincipal", + "User" + ], + "nullable": true, + "metadata": { + "description": "Optional. The principal type of the assigned principal ID." + } + }, + "description": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The description of the role assignment." + } + }, + "condition": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase \"foo_storage_container\"." + } + }, + "conditionVersion": { + "type": "string", + "allowedValues": [ + "2.0" + ], + "nullable": true, + "metadata": { + "description": "Optional. Version of the condition." + } + }, + "delegatedManagedIdentityResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The Resource Id of the delegated managed identity resource." + } + } + } + }, + "nullable": true + }, + "diagnosticSettingType": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The name of diagnostic setting." + } + }, + "logCategoriesAndGroups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "category": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Name of a Diagnostic Log category for a resource type this setting is applied to. Set the specific logs to collect here." + } + }, + "categoryGroup": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Name of a Diagnostic Log category group for a resource type this setting is applied to. Set to 'AllLogs' to collect all logs." + } + } + } + }, + "nullable": true, + "metadata": { + "description": "Optional. The name of logs that will be streamed. \"allLogs\" includes all possible logs for the resource. Set to '' to disable log collection." + } + }, + "metricCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "category": { + "type": "string", + "metadata": { + "description": "Required. Name of a Diagnostic Metric category for a resource type this setting is applied to. Set to 'AllMetrics' to collect all metrics." + } + } + } + }, + "nullable": true, + "metadata": { + "description": "Optional. The name of metrics that will be streamed. \"allMetrics\" includes all possible metrics for the resource. Set to '' to disable metric collection." + } + }, + "logAnalyticsDestinationType": { + "type": "string", + "allowedValues": [ + "AzureDiagnostics", + "Dedicated" + ], + "nullable": true, + "metadata": { + "description": "Optional. A string indicating whether the export to Log Analytics should use the default destination type, i.e. AzureDiagnostics, or use a destination type." + } + }, + "workspaceResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Resource ID of the diagnostic log analytics workspace. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub." + } + }, + "storageAccountResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Resource ID of the diagnostic storage account. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub." + } + }, + "eventHubAuthorizationRuleResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to." + } + }, + "eventHubName": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. For security reasons, it is recommended to set diagnostic settings to send data to either storage account, log analytics workspace or event hub." + } + }, + "marketplacePartnerResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The full ARM resource ID of the Marketplace resource to which you would like to send Diagnostic Logs." + } + } + } + }, + "nullable": true + }, + "fluxConfigurationProtectedSettingsType": { + "type": "object", + "properties": { + "sshPrivateKey": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The SSH private key to use for Git authentication." + } + } + }, + "nullable": true + }, + "extensionType": { + "type": "object", + "properties": { + "name": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Required. The name of the extension." + } + }, + "releaseNamespace": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Namespace where the extension Release must be placed." + } + }, + "targetNamespace": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Namespace where the extension will be created for an Namespace scoped extension." + } + }, + "releaseTrain": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Required. The release train of the extension." + } + }, + "configurationProtectedSettings": { + "$ref": "#/definitions/fluxConfigurationProtectedSettingsType", + "nullable": true, + "metadata": { + "description": "Optional. The configuration protected settings of the extension." + } + }, + "configurationSettings": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The configuration settings of the extension." + } + }, + "version": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The version of the extension." + } + }, + "configurations": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The flux configurations of the extension." + } + } + }, + "nullable": true + }, + "customerManagedKeyType": { + "type": "object", + "properties": { + "keyVaultResourceId": { + "type": "string", + "metadata": { + "description": "Required. The resource ID of a key vault to reference a customer managed key for encryption from." + } + }, + "keyName": { + "type": "string", + "metadata": { + "description": "Required. The name of the customer managed key to use for encryption." + } + }, + "keyVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The version of the customer managed key to reference for encryption. If not provided, using 'latest'." + } + }, + "keyVaultNetworkAccess": { + "type": "string", + "allowedValues": [ + "Private", + "Public" + ], + "metadata": { + "description": "Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public." + } + } + }, + "nullable": true + } + }, + "parameters": { + "name": { + "type": "string", + "metadata": { + "description": "Required. Specifies the name of the AKS cluster." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Specifies the location of AKS cluster. It picks up Resource Group's location by default." + } + }, + "dnsPrefix": { + "type": "string", + "defaultValue": "[parameters('name')]", + "metadata": { + "description": "Optional. Specifies the DNS prefix specified when creating the managed cluster." + } + }, + "managedIdentities": { + "$ref": "#/definitions/managedIdentitiesType", + "metadata": { + "description": "Optional. The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both." + } + }, + "networkDataplane": { + "type": "string", + "nullable": true, + "allowedValues": [ + "azure", + "cilium" + ], + "metadata": { + "description": "Optional. Network dataplane used in the Kubernetes cluster. Not compatible with kubenet network plugin." + } + }, + "networkPlugin": { + "type": "string", + "nullable": true, + "allowedValues": [ + "azure", + "kubenet" + ], + "metadata": { + "description": "Optional. Specifies the network plugin used for building Kubernetes network." + } + }, + "networkPluginMode": { + "type": "string", + "nullable": true, + "allowedValues": [ + "overlay" + ], + "metadata": { + "description": "Optional. Network plugin mode used for building the Kubernetes network. Not compatible with kubenet network plugin." + } + }, + "networkPolicy": { + "type": "string", + "nullable": true, + "allowedValues": [ + "azure", + "calico" + ], + "metadata": { + "description": "Optional. Specifies the network policy used for building Kubernetes network. - calico or azure." + } + }, + "podCidr": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Specifies the CIDR notation IP range from which to assign pod IPs when kubenet is used." + } + }, + "serviceCidr": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. A CIDR notation IP range from which to assign service cluster IPs. It must not overlap with any Subnet IP ranges." + } + }, + "dnsServiceIP": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr." + } + }, + "loadBalancerSku": { + "type": "string", + "defaultValue": "standard", + "allowedValues": [ + "basic", + "standard" + ], + "metadata": { + "description": "Optional. Specifies the sku of the load balancer used by the virtual machine scale sets used by nodepools." + } + }, + "managedOutboundIPCount": { + "type": "int", + "defaultValue": 0, + "metadata": { + "description": "Optional. Outbound IP Count for the Load balancer." + } + }, + "outboundType": { + "type": "string", + "defaultValue": "loadBalancer", + "allowedValues": [ + "loadBalancer", + "userDefinedRouting" + ], + "metadata": { + "description": "Optional. Specifies outbound (egress) routing method. - loadBalancer or userDefinedRouting." + } + }, + "skuTier": { + "type": "string", + "defaultValue": "Standard", + "allowedValues": [ + "Free", + "Premium", + "Standard" + ], + "metadata": { + "description": "Optional. Tier of a managed cluster SKU." + } + }, + "kubernetesVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Version of Kubernetes specified when creating the managed cluster." + } + }, + "adminUsername": { + "type": "string", + "defaultValue": "azureuser", + "metadata": { + "description": "Optional. Specifies the administrator username of Linux virtual machines." + } + }, + "sshPublicKey": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Specifies the SSH RSA public key string for the Linux nodes." + } + }, + "aksServicePrincipalProfile": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Conditional. Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster." + } + }, + "aadProfileClientAppID": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The client AAD application ID." + } + }, + "aadProfileServerAppID": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The server AAD application ID." + } + }, + "aadProfileServerAppSecret": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The server AAD application secret." + } + }, + "aadProfileTenantId": { + "type": "string", + "defaultValue": "[subscription().tenantId]", + "metadata": { + "description": "Optional. Specifies the tenant ID of the Azure Active Directory used by the AKS cluster for authentication." + } + }, + "aadProfileAdminGroupObjectIDs": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. Specifies the AAD group object IDs that will have admin role of the cluster." + } + }, + "aadProfileManaged": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Specifies whether to enable managed AAD integration." + } + }, + "enableRBAC": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Whether to enable Kubernetes Role-Based Access Control." + } + }, + "aadProfileEnableAzureRBAC": { + "type": "bool", + "defaultValue": "[parameters('enableRBAC')]", + "metadata": { + "description": "Optional. Specifies whether to enable Azure RBAC for Kubernetes authorization." + } + }, + "disableLocalAccounts": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled." + } + }, + "nodeResourceGroup": { + "type": "string", + "defaultValue": "[format('{0}_aks_{1}_nodes', resourceGroup().name, parameters('name'))]", + "metadata": { + "description": "Optional. Name of the resource group containing agent pool nodes." + } + }, + "authorizedIPRanges": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer." + } + }, + "disableRunCommand": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to disable run command for the cluster or not." + } + }, + "enablePrivateCluster": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether to create the cluster as a private cluster or not." + } + }, + "enablePrivateClusterPublicFQDN": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to create additional public FQDN for private cluster or not." + } + }, + "privateDNSZone": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Private DNS Zone configuration. Set to 'system' and AKS will create a private DNS zone in the node resource group. Set to '' to disable private DNS Zone creation and use public DNS. Supply the resource ID here of an existing Private DNS zone to use an existing zone." + } + }, + "primaryAgentPoolProfile": { + "type": "array", + "metadata": { + "description": "Required. Properties of the primary agent pool." + } + }, + "agentPools": { + "$ref": "#/definitions/agentPoolType", + "metadata": { + "description": "Optional. Define one or more secondary/additional agent pools." + } + }, + "httpApplicationRoutingEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the httpApplicationRouting add-on is enabled or not." + } + }, + "webApplicationRoutingEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the webApplicationRoutingEnabled add-on is enabled or not." + } + }, + "dnsZoneResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`." + } + }, + "enableDnsZoneContributorRoleAssignment": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided." + } + }, + "ingressApplicationGatewayEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not." + } + }, + "appGatewayResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Conditional. Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`." + } + }, + "aciConnectorLinuxEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the aciConnectorLinux add-on is enabled or not." + } + }, + "azurePolicyEnabled": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Specifies whether the azurepolicy add-on is enabled or not. For security reasons, this setting should be enabled." + } + }, + "openServiceMeshEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the openServiceMesh add-on is enabled or not." + } + }, + "azurePolicyVersion": { + "type": "string", + "defaultValue": "v2", + "metadata": { + "description": "Optional. Specifies the azure policy version to use." + } + }, + "kubeDashboardEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the kubeDashboard add-on is enabled or not." + } + }, + "enableKeyvaultSecretsProvider": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the KeyvaultSecretsProvider add-on is enabled or not." + } + }, + "enableSecretRotation": { + "type": "string", + "defaultValue": "false", + "allowedValues": [ + "false", + "true" + ], + "metadata": { + "description": "Optional. Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation." + } + }, + "autoScalerProfileScanInterval": { + "type": "string", + "defaultValue": "10s", + "metadata": { + "description": "Optional. Specifies the scan interval of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileScaleDownDelayAfterAdd": { + "type": "string", + "defaultValue": "10m", + "metadata": { + "description": "Optional. Specifies the scale down delay after add of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileScaleDownDelayAfterDelete": { + "type": "string", + "defaultValue": "20s", + "metadata": { + "description": "Optional. Specifies the scale down delay after delete of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileScaleDownDelayAfterFailure": { + "type": "string", + "defaultValue": "3m", + "metadata": { + "description": "Optional. Specifies scale down delay after failure of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileScaleDownUnneededTime": { + "type": "string", + "defaultValue": "10m", + "metadata": { + "description": "Optional. Specifies the scale down unneeded time of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileScaleDownUnreadyTime": { + "type": "string", + "defaultValue": "20m", + "metadata": { + "description": "Optional. Specifies the scale down unready time of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileUtilizationThreshold": { + "type": "string", + "defaultValue": "0.5", + "metadata": { + "description": "Optional. Specifies the utilization threshold of the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileMaxGracefulTerminationSec": { + "type": "string", + "defaultValue": "600", + "metadata": { + "description": "Optional. Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileBalanceSimilarNodeGroups": { + "type": "string", + "defaultValue": "false", + "allowedValues": [ + "false", + "true" + ], + "metadata": { + "description": "Optional. Specifies the balance of similar node groups for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileExpander": { + "type": "string", + "defaultValue": "random", + "allowedValues": [ + "least-waste", + "most-pods", + "priority", + "random" + ], + "metadata": { + "description": "Optional. Specifies the expand strategy for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileMaxEmptyBulkDelete": { + "type": "string", + "defaultValue": "10", + "metadata": { + "description": "Optional. Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileMaxNodeProvisionTime": { + "type": "string", + "defaultValue": "15m", + "metadata": { + "description": "Optional. Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an \"m\". No unit of time other than minutes (m) is supported." + } + }, + "autoScalerProfileMaxTotalUnreadyPercentage": { + "type": "string", + "defaultValue": "45", + "metadata": { + "description": "Optional. Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0." + } + }, + "autoScalerProfileNewPodScaleUpDelay": { + "type": "string", + "defaultValue": "0s", + "metadata": { + "description": "Optional. For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit (\"s\" for seconds, \"m\" for minutes, \"h\" for hours, etc)." + } + }, + "autoScalerProfileOkTotalUnreadyCount": { + "type": "string", + "defaultValue": "3", + "metadata": { + "description": "Optional. Specifies the OK total unready count for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileSkipNodesWithLocalStorage": { + "type": "string", + "defaultValue": "true", + "allowedValues": [ + "false", + "true" + ], + "metadata": { + "description": "Optional. Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster." + } + }, + "autoScalerProfileSkipNodesWithSystemPods": { + "type": "string", + "defaultValue": "true", + "allowedValues": [ + "false", + "true" + ], + "metadata": { + "description": "Optional. Specifies if nodes with system pods should be skipped for the auto-scaler of the AKS cluster." + } + }, + "autoUpgradeProfileUpgradeChannel": { + "type": "string", + "nullable": true, + "allowedValues": [ + "node-image", + "none", + "patch", + "rapid", + "stable" + ], + "metadata": { + "description": "Optional. Auto-upgrade channel on the AKS cluster." + } + }, + "podIdentityProfileAllowNetworkPluginKubenet": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Running in Kubenet is disabled by default due to the security related nature of AAD Pod Identity and the risks of IP spoofing." + } + }, + "podIdentityProfileEnable": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the pod identity addon is enabled." + } + }, + "podIdentityProfileUserAssignedIdentities": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The pod identities to use in the cluster." + } + }, + "podIdentityProfileUserAssignedIdentityExceptions": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The pod identity exceptions to allow." + } + }, + "enableOidcIssuerProfile": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the The OIDC issuer profile of the Managed Cluster is enabled." + } + }, + "enableWorkloadIdentity": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable Workload Identity. Requires OIDC issuer profile to be enabled." + } + }, + "enableAzureDefender": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable Azure Defender." + } + }, + "enablePodSecurityPolicy": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable Kubernetes pod security policy. Requires enabling the pod security policy feature flag on the subscription." + } + }, + "enableStorageProfileBlobCSIDriver": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the AzureBlob CSI Driver for the storage profile is enabled." + } + }, + "enableStorageProfileDiskCSIDriver": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the AzureDisk CSI Driver for the storage profile is enabled." + } + }, + "enableStorageProfileFileCSIDriver": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the AzureFile CSI Driver for the storage profile is enabled." + } + }, + "enableStorageProfileSnapshotController": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the snapshot controller for the storage profile is enabled." + } + }, + "supportPlan": { + "type": "string", + "defaultValue": "KubernetesOfficial", + "allowedValues": [ + "AKSLongTermSupport", + "KubernetesOfficial" + ], + "metadata": { + "description": "Optional. The support plan for the Managed Cluster." + } + }, + "diagnosticSettings": { + "$ref": "#/definitions/diagnosticSettingType", + "metadata": { + "description": "Optional. The diagnostic settings of the service." + } + }, + "omsAgentEnabled": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Specifies whether the OMS agent is enabled." + } + }, + "monitoringWorkspaceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Resource ID of the monitoring log analytics workspace." + } + }, + "enableTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Enable/Disable usage telemetry for module." + } + }, + "roleAssignments": { + "$ref": "#/definitions/roleAssignmentType", + "metadata": { + "description": "Optional. Array of role assignments to create." + } + }, + "lock": { + "$ref": "#/definitions/lockType", + "metadata": { + "description": "Optional. The lock settings of the service." + } + }, + "tags": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Tags of the resource." + } + }, + "diskEncryptionSetResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided." + } + }, + "fluxExtension": { + "$ref": "#/definitions/extensionType", + "metadata": { + "description": "Optional. Settings and configurations for the flux extension." + } + }, + "httpProxyConfig": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Configurations for provisioning the cluster with HTTP proxy servers." + } + }, + "identityProfile": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Identities associated with the cluster." + } + }, + "customerManagedKey": { + "$ref": "#/definitions/customerManagedKeyType", + "metadata": { + "description": "Optional. The customer managed key definition." + } + }, + "enableAzureMonitorProfileMetrics": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the metric state of the kubenetes cluster is enabled." + } + }, + "enableAzureMonitorProfileLogs": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the Logs profile for the Azure Monitor Infrastructure and Application Logs is enabled." + } + }, + "enableAppMonitoring": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Indicates if Application Monitoring of the kubenetes cluster is enabled." + } + }, + "enableWindowsHostLogs": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether the Windows Log Collection for Azure Monitor Container Insights Logs Addon is enabled." + } + }, + "enableContainerInsights": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Indicates if Azure Monitor Container Insights Logs Addon is enabled." + } + }, + "enableAppMonitoringOpenTelemetryMetrics": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Indicates if Application Monitoring Open Telemetry Metrics is enabled." + } + }, + "metricLabelsAllowlist": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Optional. A comma-separated list of kubernetes cluster metrics labels." + } + }, + "metricAnnotationsAllowList": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Optional. A comma-separated list of Kubernetes cluster metrics annotations." + } + } + }, + "variables": { + "formattedUserAssignedIdentities": "[reduce(map(coalesce(tryGet(parameters('managedIdentities'), 'userAssignedResourcesIds'), createArray()), lambda('id', createObject(format('{0}', lambdaVariables('id')), createObject()))), createObject(), lambda('cur', 'next', union(lambdaVariables('cur'), lambdaVariables('next'))))]", + "identity": "[if(not(empty(parameters('managedIdentities'))), createObject('type', if(coalesce(tryGet(parameters('managedIdentities'), 'systemAssigned'), false()), 'SystemAssigned', if(not(empty(coalesce(tryGet(parameters('managedIdentities'), 'userAssignedResourcesIds'), createObject()))), 'UserAssigned', null())), 'userAssignedIdentities', if(not(empty(variables('formattedUserAssignedIdentities'))), variables('formattedUserAssignedIdentities'), null())), null())]", + "builtInRoleNames": { + "Azure Kubernetes Fleet Manager Contributor Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '63bb64ad-9799-4770-b5c3-24ed299a07bf')]", + "Azure Kubernetes Fleet Manager RBAC Admin": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '434fb43a-c01c-447e-9f67-c3ad923cfaba')]", + "Azure Kubernetes Fleet Manager RBAC Cluster Admin": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '18ab4d3d-a1bf-4477-8ad9-8359bc988f69')]", + "Azure Kubernetes Fleet Manager RBAC Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '30b27cfc-9c84-438e-b0ce-70e35255df80')]", + "Azure Kubernetes Fleet Manager RBAC Writer": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '5af6afb3-c06c-4fa4-8848-71a8aee05683')]", + "Azure Kubernetes Service Cluster Admin Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '0ab0b1a8-8aac-4efd-b8c2-3ee1fb270be8')]", + "Azure Kubernetes Service Cluster Monitoring User": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '1afdec4b-e479-420e-99e7-f82237c7c5e6')]", + "Azure Kubernetes Service Cluster User Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4abbcc35-e782-43d8-92c5-2d3f1bd2253f')]", + "Azure Kubernetes Service Contributor Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ed7f3fbd-7b88-4dd4-9017-9adb7ce333f8')]", + "Azure Kubernetes Service RBAC Admin": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3498e952-d568-435e-9b2c-8d77e338d7f7')]", + "Azure Kubernetes Service RBAC Cluster Admin": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b')]", + "Azure Kubernetes Service RBAC Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f6c6a51-bcf8-42ba-9220-52d62157d7db')]", + "Azure Kubernetes Service RBAC Writer": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a7ffa36f-339b-4b5c-8bdf-e2c188b2c0eb')]", + "Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", + "Kubernetes Agentless Operator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'd5a2ae44-610b-4500-93be-660a0c5f5ca6')]", + "Owner": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", + "Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]", + "Role Based Access Control Administrator (Preview)": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f58310d9-a9f6-439a-9e8d-f62e7b41a168')]", + "User Access Administrator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '18d7d88d-d35e-4fb5-a5c3-7773c20a72d9')]" + } + }, + "resources": { + "cMKKeyVault::cMKKey": { + "condition": "[and(not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'))), and(not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'))), not(empty(tryGet(parameters('customerManagedKey'), 'keyName')))))]", + "existing": true, + "type": "Microsoft.KeyVault/vaults/keys", + "apiVersion": "2023-02-01", + "subscriptionId": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '//'), '/')[2]]", + "resourceGroup": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '////'), '/')[4]]", + "name": "[format('{0}/{1}', last(split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), 'dummyVault'), '/')), coalesce(tryGet(parameters('customerManagedKey'), 'keyName'), 'dummyKey'))]", + "dependsOn": [ + "cMKKeyVault" + ] + }, + "avmTelemetry": { + "condition": "[parameters('enableTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2023-07-01", + "name": "[format('46d3xbcp.res.containerservice-managedcluster.{0}.{1}', replace('-..--..-', '.', '-'), substring(uniqueString(deployment().name, parameters('location')), 0, 4))]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [], + "outputs": { + "telemetry": { + "type": "String", + "value": "For more information, see https://aka.ms/avm/TelemetryInfo" + } + } + } + } + }, + "cMKKeyVault": { + "condition": "[not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId')))]", + "existing": true, + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2023-02-01", + "subscriptionId": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '//'), '/')[2]]", + "resourceGroup": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '////'), '/')[4]]", + "name": "[last(split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), 'dummyVault'), '/'))]" + }, + "managedCluster": { + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2023-07-02-preview", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "identity": "[variables('identity')]", + "sku": { + "name": "Base", + "tier": "[parameters('skuTier')]" + }, + "properties": { + "httpProxyConfig": "[parameters('httpProxyConfig')]", + "identityProfile": "[parameters('identityProfile')]", + "diskEncryptionSetID": "[parameters('diskEncryptionSetResourceId')]", + "kubernetesVersion": "[parameters('kubernetesVersion')]", + "dnsPrefix": "[parameters('dnsPrefix')]", + "agentPoolProfiles": "[parameters('primaryAgentPoolProfile')]", + "linuxProfile": "[if(not(empty(parameters('sshPublicKey'))), createObject('adminUsername', parameters('adminUsername'), 'ssh', createObject('publicKeys', createArray(createObject('keyData', parameters('sshPublicKey'))))), null())]", + "servicePrincipalProfile": "[parameters('aksServicePrincipalProfile')]", + "ingressProfile": { + "webAppRouting": { + "enabled": "[parameters('webApplicationRoutingEnabled')]", + "dnsZoneResourceIds": "[if(not(empty(parameters('dnsZoneResourceId'))), createArray(parameters('dnsZoneResourceId')), null())]" + } + }, + "addonProfiles": { + "httpApplicationRouting": { + "enabled": "[parameters('httpApplicationRoutingEnabled')]" + }, + "ingressApplicationGateway": { + "enabled": "[and(parameters('ingressApplicationGatewayEnabled'), not(empty(parameters('appGatewayResourceId'))))]", + "config": "[if(and(parameters('ingressApplicationGatewayEnabled'), not(empty(parameters('appGatewayResourceId')))), createObject('applicationGatewayId', parameters('appGatewayResourceId'), 'effectiveApplicationGatewayId', parameters('appGatewayResourceId')), null())]" + }, + "omsagent": { + "enabled": "[and(parameters('omsAgentEnabled'), not(empty(parameters('monitoringWorkspaceId'))))]", + "config": "[if(and(parameters('omsAgentEnabled'), not(empty(parameters('monitoringWorkspaceId')))), createObject('logAnalyticsWorkspaceResourceID', parameters('monitoringWorkspaceId')), null())]" + }, + "aciConnectorLinux": { + "enabled": "[parameters('aciConnectorLinuxEnabled')]" + }, + "azurepolicy": { + "enabled": "[parameters('azurePolicyEnabled')]", + "config": "[if(parameters('azurePolicyEnabled'), createObject('version', parameters('azurePolicyVersion')), null())]" + }, + "openServiceMesh": { + "enabled": "[parameters('openServiceMeshEnabled')]", + "config": "[if(parameters('openServiceMeshEnabled'), createObject(), null())]" + }, + "kubeDashboard": { + "enabled": "[parameters('kubeDashboardEnabled')]" + }, + "azureKeyvaultSecretsProvider": { + "enabled": "[parameters('enableKeyvaultSecretsProvider')]", + "config": "[if(parameters('enableKeyvaultSecretsProvider'), createObject('enableSecretRotation', parameters('enableSecretRotation')), null())]" + } + }, + "oidcIssuerProfile": "[if(parameters('enableOidcIssuerProfile'), createObject('enabled', parameters('enableOidcIssuerProfile')), null())]", + "enableRBAC": "[parameters('enableRBAC')]", + "disableLocalAccounts": "[parameters('disableLocalAccounts')]", + "nodeResourceGroup": "[parameters('nodeResourceGroup')]", + "enablePodSecurityPolicy": "[parameters('enablePodSecurityPolicy')]", + "networkProfile": { + "networkDataplane": "[parameters('networkDataplane')]", + "networkPlugin": "[parameters('networkPlugin')]", + "networkPluginMode": "[parameters('networkPluginMode')]", + "networkPolicy": "[parameters('networkPolicy')]", + "podCidr": "[parameters('podCidr')]", + "serviceCidr": "[parameters('serviceCidr')]", + "dnsServiceIP": "[parameters('dnsServiceIP')]", + "outboundType": "[parameters('outboundType')]", + "loadBalancerSku": "[parameters('loadBalancerSku')]", + "loadBalancerProfile": "[if(not(equals(parameters('managedOutboundIPCount'), 0)), createObject('managedOutboundIPs', createObject('count', parameters('managedOutboundIPCount')), 'effectiveOutboundIPs', createArray()), null())]" + }, + "aadProfile": { + "clientAppID": "[parameters('aadProfileClientAppID')]", + "serverAppID": "[parameters('aadProfileServerAppID')]", + "serverAppSecret": "[parameters('aadProfileServerAppSecret')]", + "managed": "[parameters('aadProfileManaged')]", + "enableAzureRBAC": "[parameters('aadProfileEnableAzureRBAC')]", + "adminGroupObjectIDs": "[parameters('aadProfileAdminGroupObjectIDs')]", + "tenantID": "[parameters('aadProfileTenantId')]" + }, + "autoScalerProfile": { + "balance-similar-node-groups": "[parameters('autoScalerProfileBalanceSimilarNodeGroups')]", + "expander": "[parameters('autoScalerProfileExpander')]", + "max-empty-bulk-delete": "[parameters('autoScalerProfileMaxEmptyBulkDelete')]", + "max-graceful-termination-sec": "[parameters('autoScalerProfileMaxGracefulTerminationSec')]", + "max-node-provision-time": "[parameters('autoScalerProfileMaxNodeProvisionTime')]", + "max-total-unready-percentage": "[parameters('autoScalerProfileMaxTotalUnreadyPercentage')]", + "new-pod-scale-up-delay": "[parameters('autoScalerProfileNewPodScaleUpDelay')]", + "ok-total-unready-count": "[parameters('autoScalerProfileOkTotalUnreadyCount')]", + "scale-down-delay-after-add": "[parameters('autoScalerProfileScaleDownDelayAfterAdd')]", + "scale-down-delay-after-delete": "[parameters('autoScalerProfileScaleDownDelayAfterDelete')]", + "scale-down-delay-after-failure": "[parameters('autoScalerProfileScaleDownDelayAfterFailure')]", + "scale-down-unneeded-time": "[parameters('autoScalerProfileScaleDownUnneededTime')]", + "scale-down-unready-time": "[parameters('autoScalerProfileScaleDownUnreadyTime')]", + "scale-down-utilization-threshold": "[parameters('autoScalerProfileUtilizationThreshold')]", + "scan-interval": "[parameters('autoScalerProfileScanInterval')]", + "skip-nodes-with-local-storage": "[parameters('autoScalerProfileSkipNodesWithLocalStorage')]", + "skip-nodes-with-system-pods": "[parameters('autoScalerProfileSkipNodesWithSystemPods')]" + }, + "autoUpgradeProfile": { + "upgradeChannel": "[parameters('autoUpgradeProfileUpgradeChannel')]" + }, + "apiServerAccessProfile": { + "authorizedIPRanges": "[parameters('authorizedIPRanges')]", + "disableRunCommand": "[parameters('disableRunCommand')]", + "enablePrivateCluster": "[parameters('enablePrivateCluster')]", + "enablePrivateClusterPublicFQDN": "[parameters('enablePrivateClusterPublicFQDN')]", + "privateDNSZone": "[parameters('privateDNSZone')]" + }, + "azureMonitorProfile": { + "logs": "[if(parameters('enableAzureMonitorProfileLogs'), createObject('enabled', parameters('enableAzureMonitorProfileLogs'), 'appMonitoring', createObject('enabled', parameters('enableAppMonitoring')), 'containerInsights', createObject('enabled', parameters('enableContainerInsights'), 'logAnalyticsWorkspaceResourceId', if(not(empty(parameters('monitoringWorkspaceId'))), parameters('monitoringWorkspaceId'), null()), 'windowsHostLogs', createObject('enabled', parameters('enableWindowsHostLogs')))), null())]", + "metrics": "[if(parameters('enableAzureMonitorProfileMetrics'), createObject('enabled', parameters('enableAzureMonitorProfileMetrics'), 'appMonitoringOpenTelemetryMetrics', createObject('enabled', parameters('enableAppMonitoringOpenTelemetryMetrics')), 'kubeStateMetrics', createObject('metricLabelsAllowlist', parameters('metricLabelsAllowlist'), 'metricAnnotationsAllowList', parameters('metricAnnotationsAllowList'))), null())]" + }, + "podIdentityProfile": { + "allowNetworkPluginKubenet": "[parameters('podIdentityProfileAllowNetworkPluginKubenet')]", + "enabled": "[parameters('podIdentityProfileEnable')]", + "userAssignedIdentities": "[parameters('podIdentityProfileUserAssignedIdentities')]", + "userAssignedIdentityExceptions": "[parameters('podIdentityProfileUserAssignedIdentityExceptions')]" + }, + "securityProfile": { + "defender": "[if(parameters('enableAzureDefender'), createObject('securityMonitoring', createObject('enabled', parameters('enableAzureDefender')), 'logAnalyticsWorkspaceResourceId', parameters('monitoringWorkspaceId')), null())]", + "workloadIdentity": "[if(parameters('enableWorkloadIdentity'), createObject('enabled', parameters('enableWorkloadIdentity')), null())]" + }, + "storageProfile": { + "blobCSIDriver": { + "enabled": "[parameters('enableStorageProfileBlobCSIDriver')]" + }, + "diskCSIDriver": { + "enabled": "[parameters('enableStorageProfileDiskCSIDriver')]" + }, + "fileCSIDriver": { + "enabled": "[parameters('enableStorageProfileFileCSIDriver')]" + }, + "snapshotController": { + "enabled": "[parameters('enableStorageProfileSnapshotController')]" + } + }, + "supportPlan": "[parameters('supportPlan')]" + } + }, + "managedCluster_lock": { + "condition": "[and(not(empty(coalesce(parameters('lock'), createObject()))), not(equals(tryGet(parameters('lock'), 'kind'), 'None')))]", + "type": "Microsoft.Authorization/locks", + "apiVersion": "2020-05-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('name'))]", + "name": "[coalesce(tryGet(parameters('lock'), 'name'), format('lock-{0}', parameters('name')))]", + "properties": { + "level": "[coalesce(tryGet(parameters('lock'), 'kind'), '')]", + "notes": "[if(equals(tryGet(parameters('lock'), 'kind'), 'CanNotDelete'), 'Cannot delete resource or child resources.', 'Cannot delete or modify the resource or child resources.')]" + }, + "dependsOn": [ + "managedCluster" + ] + }, + "managedCluster_diagnosticSettings": { + "copy": { + "name": "managedCluster_diagnosticSettings", + "count": "[length(coalesce(parameters('diagnosticSettings'), createArray()))]" + }, + "type": "Microsoft.Insights/diagnosticSettings", + "apiVersion": "2021-05-01-preview", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('name'))]", + "name": "[coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'name'), format('{0}-diagnosticSettings', parameters('name')))]", + "properties": { + "storageAccountId": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'storageAccountResourceId')]", + "workspaceId": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'workspaceResourceId')]", + "eventHubAuthorizationRuleId": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'eventHubAuthorizationRuleResourceId')]", + "eventHubName": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'eventHubName')]", + "metrics": "[coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'metricCategories'), createArray(createObject('category', 'AllMetrics', 'timeGrain', null(), 'enabled', true())))]", + "logs": "[coalesce(tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'logCategoriesAndGroups'), createArray(createObject('categoryGroup', 'AllLogs', 'enabled', true())))]", + "marketplacePartnerId": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'marketplacePartnerResourceId')]", + "logAnalyticsDestinationType": "[tryGet(coalesce(parameters('diagnosticSettings'), createArray())[copyIndex()], 'logAnalyticsDestinationType')]" + }, + "dependsOn": [ + "managedCluster" + ] + }, + "managedCluster_roleAssignments": { + "copy": { + "name": "managedCluster_roleAssignments", + "count": "[length(coalesce(parameters('roleAssignments'), createArray()))]" + }, + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('name'))]", + "name": "[guid(resourceId('Microsoft.ContainerService/managedClusters', parameters('name')), coalesce(parameters('roleAssignments'), createArray())[copyIndex()].principalId, coalesce(parameters('roleAssignments'), createArray())[copyIndex()].roleDefinitionIdOrName)]", + "properties": { + "roleDefinitionId": "[if(contains(variables('builtInRoleNames'), coalesce(parameters('roleAssignments'), createArray())[copyIndex()].roleDefinitionIdOrName), variables('builtInRoleNames')[coalesce(parameters('roleAssignments'), createArray())[copyIndex()].roleDefinitionIdOrName], if(contains(coalesce(parameters('roleAssignments'), createArray())[copyIndex()].roleDefinitionIdOrName, '/providers/Microsoft.Authorization/roleDefinitions/'), coalesce(parameters('roleAssignments'), createArray())[copyIndex()].roleDefinitionIdOrName, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', coalesce(parameters('roleAssignments'), createArray())[copyIndex()].roleDefinitionIdOrName)))]", + "principalId": "[coalesce(parameters('roleAssignments'), createArray())[copyIndex()].principalId]", + "description": "[tryGet(coalesce(parameters('roleAssignments'), createArray())[copyIndex()], 'description')]", + "principalType": "[tryGet(coalesce(parameters('roleAssignments'), createArray())[copyIndex()], 'principalType')]", + "condition": "[tryGet(coalesce(parameters('roleAssignments'), createArray())[copyIndex()], 'condition')]", + "conditionVersion": "[if(not(empty(tryGet(coalesce(parameters('roleAssignments'), createArray())[copyIndex()], 'condition'))), coalesce(tryGet(coalesce(parameters('roleAssignments'), createArray())[copyIndex()], 'conditionVersion'), '2.0'), null())]", + "delegatedManagedIdentityResourceId": "[tryGet(coalesce(parameters('roleAssignments'), createArray())[copyIndex()], 'delegatedManagedIdentityResourceId')]" + }, + "dependsOn": [ + "managedCluster" + ] + }, + "dnsZone": { + "condition": "[and(and(equals(parameters('enableDnsZoneContributorRoleAssignment'), true()), not(equals(parameters('dnsZoneResourceId'), null()))), parameters('webApplicationRoutingEnabled'))]", + "existing": true, + "type": "Microsoft.Network/dnsZones", + "apiVersion": "2018-05-01", + "name": "[last(split(if(not(empty(parameters('dnsZoneResourceId'))), parameters('dnsZoneResourceId'), '/dummmyZone'), '/'))]" + }, + "dnsZone_roleAssignment": { + "condition": "[and(and(equals(parameters('enableDnsZoneContributorRoleAssignment'), true()), not(equals(parameters('dnsZoneResourceId'), null()))), parameters('webApplicationRoutingEnabled'))]", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "scope": "[format('Microsoft.Network/dnsZones/{0}', last(split(if(not(empty(parameters('dnsZoneResourceId'))), parameters('dnsZoneResourceId'), '/dummmyZone'), '/')))]", + "name": "[guid(resourceId('Microsoft.Network/dnsZones', last(split(if(not(empty(parameters('dnsZoneResourceId'))), parameters('dnsZoneResourceId'), '/dummmyZone'), '/'))), subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314'), 'DNS Zone Contributor')]", + "properties": { + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314')]", + "principalId": "[reference('managedCluster').ingressProfile.webAppRouting.identity.objectId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "dnsZone", + "managedCluster" + ] + }, + "managedCluster_agentPools": { + "copy": { + "name": "managedCluster_agentPools", + "count": "[length(coalesce(parameters('agentPools'), createArray()))]" + }, + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-ManagedCluster-AgentPool-{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "managedClusterName": { + "value": "[parameters('name')]" + }, + "name": { + "value": "[coalesce(parameters('agentPools'), createArray())[copyIndex()].name]" + }, + "availabilityZones": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'availabilityZones')]" + }, + "count": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'count')]" + }, + "sourceResourceId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'sourceResourceId')]" + }, + "enableAutoScaling": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'enableAutoScaling')]" + }, + "enableEncryptionAtHost": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'enableEncryptionAtHost')]" + }, + "enableFIPS": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'enableFIPS')]" + }, + "enableNodePublicIP": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'enableNodePublicIP')]" + }, + "enableUltraSSD": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'enableUltraSSD')]" + }, + "gpuInstanceProfile": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'gpuInstanceProfile')]" + }, + "kubeletDiskType": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'kubeletDiskType')]" + }, + "maxCount": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'maxCount')]" + }, + "maxPods": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'maxPods')]" + }, + "minCount": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'minCount')]" + }, + "mode": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'mode')]" + }, + "nodeLabels": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'nodeLabels')]" + }, + "nodePublicIpPrefixId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'nodePublicIpPrefixId')]" + }, + "nodeTaints": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'nodeTaints')]" + }, + "orchestratorVersion": { + "value": "[coalesce(tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'orchestratorVersion'), parameters('kubernetesVersion'))]" + }, + "osDiskSizeGB": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'osDiskSizeGB')]" + }, + "osDiskType": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'osDiskType')]" + }, + "osSku": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'osSku')]" + }, + "osType": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'osType')]" + }, + "podSubnetId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'podSubnetId')]" + }, + "proximityPlacementGroupResourceId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'proximityPlacementGroupResourceId')]" + }, + "scaleDownMode": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'scaleDownMode')]" + }, + "scaleSetEvictionPolicy": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'scaleSetEvictionPolicy')]" + }, + "scaleSetPriority": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'scaleSetPriority')]" + }, + "spotMaxPrice": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'spotMaxPrice')]" + }, + "tags": { + "value": "[coalesce(tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'tags'), parameters('tags'))]" + }, + "type": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'type')]" + }, + "maxSurge": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'maxSurge')]" + }, + "vmSize": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'vmSize')]" + }, + "vnetSubnetId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'vnetSubnetId')]" + }, + "workloadRuntime": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'workloadRuntime')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.24.24.22086", + "templateHash": "59537816730486645" + }, + "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", + "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "managedClusterName": { + "type": "string", + "metadata": { + "description": "Conditional. The name of the parent managed cluster. Required if the template is used in a standalone deployment." + } + }, + "name": { + "type": "string", + "metadata": { + "description": "Required. Name of the agent pool." + } + }, + "availabilityZones": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is \"VirtualMachineScaleSets\"." + } + }, + "count": { + "type": "int", + "defaultValue": 1, + "minValue": 0, + "maxValue": 1000, + "metadata": { + "description": "Optional. Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1." + } + }, + "sourceResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. This is the ARM ID of the source object to be used to create the target object." + } + }, + "enableAutoScaling": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable auto-scaler." + } + }, + "enableEncryptionAtHost": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption. For security reasons, this setting should be enabled." + } + }, + "enableFIPS": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. See Add a FIPS-enabled node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details." + } + }, + "enableNodePublicIP": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools)." + } + }, + "enableUltraSSD": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable UltraSSD." + } + }, + "gpuInstanceProfile": { + "type": "string", + "nullable": true, + "allowedValues": [ + "MIG1g", + "MIG2g", + "MIG3g", + "MIG4g", + "MIG7g" + ], + "metadata": { + "description": "Optional. GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU." + } + }, + "kubeletDiskType": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage." + } + }, + "maxCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of nodes for auto-scaling." + } + }, + "maxPods": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of pods that can run on a node." + } + }, + "minCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The minimum number of nodes for auto-scaling." + } + }, + "mode": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. A cluster must have at least one \"System\" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools." + } + }, + "nodeLabels": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The node labels to be persisted across all nodes in agent pool." + } + }, + "nodePublicIpPrefixId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. ResourceId of the node PublicIPPrefix." + } + }, + "nodeTaints": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule." + } + }, + "orchestratorVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool)." + } + }, + "osDiskSizeGB": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified." + } + }, + "osDiskType": { + "type": "string", + "nullable": true, + "allowedValues": [ + "Ephemeral", + "Managed" + ], + "metadata": { + "description": "Optional. The default is \"Ephemeral\" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to \"Managed\". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os)." + } + }, + "osSku": { + "type": "string", + "nullable": true, + "allowedValues": [ + "AzureLinux", + "CBLMariner", + "Ubuntu", + "Windows2019", + "Windows2022" + ], + "metadata": { + "description": "Optional. Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows." + } + }, + "osType": { + "type": "string", + "defaultValue": "Linux", + "allowedValues": [ + "Linux", + "Windows" + ], + "metadata": { + "description": "Optional. The operating system type. The default is Linux." + } + }, + "podSubnetId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." + } + }, + "proximityPlacementGroupResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The ID for the Proximity Placement Group." + } + }, + "scaleDownMode": { + "type": "string", + "defaultValue": "Delete", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "metadata": { + "description": "Optional. Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing)." + } + }, + "scaleSetEvictionPolicy": { + "type": "string", + "defaultValue": "Delete", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "metadata": { + "description": "Optional. The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs." + } + }, + "scaleSetPriority": { + "type": "string", + "nullable": true, + "allowedValues": [ + "Regular", + "Spot" + ], + "metadata": { + "description": "Optional. The Virtual Machine Scale Set priority." + } + }, + "spotMaxPrice": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://learn.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing)." + } + }, + "tags": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Tags of the resource." + } + }, + "type": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The type of Agent Pool." + } + }, + "maxSurge": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. This can either be set to an integer (e.g. \"5\") or a percentage (e.g. \"50%\"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade." + } + }, + "vmSize": { + "type": "string", + "defaultValue": "Standard_D2s_v3", + "metadata": { + "description": "Optional. VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions." + } + }, + "vnetSubnetId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." + } + }, + "workloadRuntime": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Determines the type of workload a node can run." + } + } + }, + "resources": { + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2023-07-02-preview", + "name": "[parameters('managedClusterName')]" + }, + "agentPool": { + "type": "Microsoft.ContainerService/managedClusters/agentPools", + "apiVersion": "2023-07-02-preview", + "name": "[format('{0}/{1}', parameters('managedClusterName'), parameters('name'))]", + "properties": { + "availabilityZones": "[parameters('availabilityZones')]", + "count": "[parameters('count')]", + "creationData": "[if(not(empty(parameters('sourceResourceId'))), createObject('sourceResourceId', parameters('sourceResourceId')), null())]", + "enableAutoScaling": "[parameters('enableAutoScaling')]", + "enableEncryptionAtHost": "[parameters('enableEncryptionAtHost')]", + "enableFIPS": "[parameters('enableFIPS')]", + "enableNodePublicIP": "[parameters('enableNodePublicIP')]", + "enableUltraSSD": "[parameters('enableUltraSSD')]", + "gpuInstanceProfile": "[parameters('gpuInstanceProfile')]", + "kubeletDiskType": "[parameters('kubeletDiskType')]", + "maxCount": "[parameters('maxCount')]", + "maxPods": "[parameters('maxPods')]", + "minCount": "[parameters('minCount')]", + "mode": "[parameters('mode')]", + "nodeLabels": "[parameters('nodeLabels')]", + "nodePublicIPPrefixID": "[parameters('nodePublicIpPrefixId')]", + "nodeTaints": "[parameters('nodeTaints')]", + "orchestratorVersion": "[parameters('orchestratorVersion')]", + "osDiskSizeGB": "[parameters('osDiskSizeGB')]", + "osDiskType": "[parameters('osDiskType')]", + "osSKU": "[parameters('osSku')]", + "osType": "[parameters('osType')]", + "podSubnetID": "[parameters('podSubnetId')]", + "proximityPlacementGroupID": "[parameters('proximityPlacementGroupResourceId')]", + "scaleDownMode": "[parameters('scaleDownMode')]", + "scaleSetEvictionPolicy": "[parameters('scaleSetEvictionPolicy')]", + "scaleSetPriority": "[parameters('scaleSetPriority')]", + "spotMaxPrice": "[parameters('spotMaxPrice')]", + "tags": "[parameters('tags')]", + "type": "[parameters('type')]", + "upgradeSettings": { + "maxSurge": "[parameters('maxSurge')]" + }, + "vmSize": "[parameters('vmSize')]", + "vnetSubnetID": "[parameters('vnetSubnetId')]", + "workloadRuntime": "[parameters('workloadRuntime')]" + }, + "dependsOn": [ + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the agent pool." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the agent pool." + }, + "value": "[resourceId('Microsoft.ContainerService/managedClusters/agentPools', parameters('managedClusterName'), parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The resource group the agent pool was deployed into." + }, + "value": "[resourceGroup().name]" + } + } + } + }, + "dependsOn": [ + "managedCluster" + ] + }, + "managedCluster_extension": { + "condition": "[not(empty(parameters('fluxExtension')))]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-ManagedCluster-FluxExtension', uniqueString(deployment().name, parameters('location')))]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "clusterName": { + "value": "[parameters('name')]" + }, + "configurationProtectedSettings": { + "value": "[tryGet(parameters('fluxExtension'), 'configurationProtectedSettings')]" + }, + "configurationSettings": { + "value": "[tryGet(parameters('fluxExtension'), 'configurationSettings')]" + }, + "enableTelemetry": { + "value": "[parameters('enableTelemetry')]" + }, + "extensionType": { + "value": "microsoft.flux" + }, + "fluxConfigurations": { + "value": "[tryGet(parameters('fluxExtension'), 'configurations')]" + }, + "location": { + "value": "[parameters('location')]" + }, + "name": { + "value": "flux" + }, + "releaseNamespace": { + "value": "[coalesce(tryGet(parameters('fluxExtension'), 'releaseNamespace'), 'flux-system')]" + }, + "releaseTrain": { + "value": "[coalesce(tryGet(parameters('fluxExtension'), 'releaseTrain'), 'Stable')]" + }, + "version": { + "value": "[tryGet(parameters('fluxExtension'), 'version')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.23.1.45101", + "templateHash": "12293754418506359991" + }, + "name": "Kubernetes Configuration Extensions", + "description": "This module deploys a Kubernetes Configuration Extension.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "name": { + "type": "string", + "metadata": { + "description": "Required. The name of the Flux Configuration." + } + }, + "enableTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Enable/Disable usage telemetry for module." + } + }, + "clusterName": { + "type": "string", + "metadata": { + "description": "Required. The name of the AKS cluster that should be configured." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Location for all resources." + } + }, + "configurationProtectedSettings": { + "type": "secureObject", + "nullable": true, + "metadata": { + "description": "Optional. Configuration settings that are sensitive, as name-value pairs for configuring this extension." + } + }, + "configurationSettings": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Configuration settings, as name-value pairs for configuring this extension." + } + }, + "extensionType": { + "type": "string", + "metadata": { + "description": "Required. Type of the extension, of which this resource is an instance of. It must be one of the Extension Types registered with Microsoft.KubernetesConfiguration by the extension publisher." + } + }, + "releaseTrain": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is \"true\"." + } + }, + "releaseNamespace": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Namespace where the extension Release must be placed, for a Cluster scoped extension. If this namespace does not exist, it will be created." + } + }, + "targetNamespace": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Namespace where the extension will be created for an Namespace scoped extension. If this namespace does not exist, it will be created." + } + }, + "version": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. Version of the extension for this extension, if it is \"pinned\" to a specific version." + } + }, + "fluxConfigurations": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. A list of flux configuraitons." + } + } + }, + "resources": { + "avmTelemetry": { + "condition": "[parameters('enableTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2023-07-01", + "name": "[format('46d3xbcp.res.kubernetesconfiguration-fluxconfig.{0}.{1}', replace('0.2.0', '.', '-'), substring(uniqueString(deployment().name, parameters('location')), 0, 4))]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [], + "outputs": { + "telemetry": { + "type": "String", + "value": "For more information, see https://aka.ms/avm/TelemetryInfo" + } + } + } + } + }, + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2022-07-01", + "name": "[parameters('clusterName')]" + }, + "extension": { + "type": "Microsoft.KubernetesConfiguration/extensions", + "apiVersion": "2022-03-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('clusterName'))]", + "name": "[parameters('name')]", + "properties": { + "autoUpgradeMinorVersion": "[if(not(empty(parameters('version'))), false(), true())]", + "configurationProtectedSettings": "[parameters('configurationProtectedSettings')]", + "configurationSettings": "[parameters('configurationSettings')]", + "extensionType": "[parameters('extensionType')]", + "releaseTrain": "[parameters('releaseTrain')]", + "scope": { + "cluster": "[if(not(empty(coalesce(parameters('releaseNamespace'), ''))), createObject('releaseNamespace', parameters('releaseNamespace')), null())]", + "namespace": "[if(not(empty(coalesce(parameters('targetNamespace'), ''))), createObject('targetNamespace', parameters('targetNamespace')), null())]" + }, + "version": "[parameters('version')]" + }, + "dependsOn": [ + "managedCluster" + ] + }, + "fluxConfiguration": { + "copy": { + "name": "fluxConfiguration", + "count": "[length(coalesce(parameters('fluxConfigurations'), createArray()))]" + }, + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-ManagedCluster-FluxConfiguration{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "enableTelemetry": { + "value": "[parameters('enableTelemetry')]" + }, + "clusterName": { + "value": "[parameters('clusterName')]" + }, + "scope": { + "value": "[coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()].scope]" + }, + "namespace": { + "value": "[coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()].namespace]" + }, + "sourceKind": "[if(contains(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'gitRepository'), createObject('value', 'GitRepository'), createObject('value', 'Bucket'))]", + "name": { + "value": "[coalesce(tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'name'), toLower(format('{0}-fluxconfiguration{1}', parameters('clusterName'), copyIndex())))]" + }, + "bucket": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'bucket')]" + }, + "configurationProtectedSettings": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'configurationProtectedSettings')]" + }, + "gitRepository": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'gitRepository')]" + }, + "kustomizations": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'kustomizations')]" + }, + "suspend": { + "value": "[tryGet(coalesce(parameters('fluxConfigurations'), createArray())[copyIndex()], 'suspend')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.23.1.45101", + "templateHash": "13420454476526931427" + }, + "name": "Kubernetes Configuration Flux Configurations", + "description": "This module deploys a Kubernetes Configuration Flux Configuration.", + "owner": "Azure/module-maintainers" + }, + "parameters": { + "name": { + "type": "string", + "metadata": { + "description": "Required. The name of the Flux Configuration." + } + }, + "enableTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Enable/Disable usage telemetry for module." + } + }, + "clusterName": { + "type": "string", + "metadata": { + "description": "Required. The name of the AKS cluster that should be configured." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Location for all resources." + } + }, + "bucket": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Conditional. Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `Bucket`." + } + }, + "configurationProtectedSettings": { + "type": "secureObject", + "nullable": true, + "metadata": { + "description": "Optional. Key-value pairs of protected configuration settings for the configuration." + } + }, + "gitRepository": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Conditional. Parameters to reconcile to the GitRepository source kind type. Required if `sourceKind` is `GitRepository`." + } + }, + "kustomizations": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. Array of kustomizations used to reconcile the artifact pulled by the source type on the cluster." + } + }, + "namespace": { + "type": "string", + "metadata": { + "description": "Required. The namespace to which this configuration is installed to. Maximum of 253 lower case alphanumeric characters, hyphen and period only." + } + }, + "scope": { + "type": "string", + "allowedValues": [ + "cluster", + "namespace" + ], + "metadata": { + "description": "Required. Scope at which the configuration will be installed." + } + }, + "sourceKind": { + "type": "string", + "allowedValues": [ + "Bucket", + "GitRepository" + ], + "metadata": { + "description": "Required. Source Kind to pull the configuration data from." + } + }, + "suspend": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether this configuration should suspend its reconciliation of its kustomizations and sources." + } + } + }, + "resources": { + "avmTelemetry": { + "condition": "[parameters('enableTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2023-07-01", + "name": "[format('46d3xbcp.res.kubernetesconfiguration-extension.{0}.{1}', replace('0.2.0', '.', '-'), substring(uniqueString(deployment().name, parameters('location')), 0, 4))]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [], + "outputs": { + "telemetry": { + "type": "String", + "value": "For more information, see https://aka.ms/avm/TelemetryInfo" + } + } + } + } + }, + "managedCluster": { + "existing": true, + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2022-07-01", + "name": "[parameters('clusterName')]" + }, + "fluxConfiguration": { + "type": "Microsoft.KubernetesConfiguration/fluxConfigurations", + "apiVersion": "2022-03-01", + "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('clusterName'))]", + "name": "[parameters('name')]", + "properties": { + "bucket": "[parameters('bucket')]", + "configurationProtectedSettings": "[parameters('configurationProtectedSettings')]", + "gitRepository": "[parameters('gitRepository')]", + "kustomizations": "[parameters('kustomizations')]", + "namespace": "[parameters('namespace')]", + "scope": "[parameters('scope')]", + "sourceKind": "[parameters('sourceKind')]", + "suspend": "[parameters('suspend')]" + }, + "dependsOn": [ + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the flux configuration." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the flux configuration." + }, + "value": "[extensionResourceId(resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName')), 'Microsoft.KubernetesConfiguration/fluxConfigurations', parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The name of the resource group the flux configuration was deployed into." + }, + "value": "[resourceGroup().name]" + } + } + } + }, + "dependsOn": [ + "extension", + "managedCluster" + ] + } + }, + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the extension." + }, + "value": "[parameters('name')]" + }, + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the extension." + }, + "value": "[extensionResourceId(resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName')), 'Microsoft.KubernetesConfiguration/extensions', parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The name of the resource group the extension was deployed into." + }, + "value": "[resourceGroup().name]" + } + } + } + }, + "dependsOn": [ + "managedCluster" + ] + } + }, + "outputs": { + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the managed cluster." + }, + "value": "[resourceId('Microsoft.ContainerService/managedClusters', parameters('name'))]" + }, + "resourceGroupName": { + "type": "string", + "metadata": { + "description": "The resource group the managed cluster was deployed into." + }, + "value": "[resourceGroup().name]" + }, + "name": { + "type": "string", + "metadata": { + "description": "The name of the managed cluster." + }, + "value": "[parameters('name')]" + }, + "controlPlaneFQDN": { + "type": "string", + "metadata": { + "description": "The control plane FQDN of the managed cluster." + }, + "value": "[if(parameters('enablePrivateCluster'), reference('managedCluster').privateFQDN, reference('managedCluster').fqdn)]" + }, + "systemAssignedMIPrincipalId": { + "type": "string", + "metadata": { + "description": "The principal ID of the system assigned identity." + }, + "value": "[coalesce(tryGet(tryGet(reference('managedCluster', '2023-07-02-preview', 'full'), 'identity'), 'principalId'), '')]" + }, + "kubeletidentityObjectId": { + "type": "string", + "metadata": { + "description": "The Object ID of the AKS identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(reference('managedCluster'), 'identityProfile'), 'kubeletidentity'), 'objectId'), '')]" + }, + "omsagentIdentityObjectId": { + "type": "string", + "metadata": { + "description": "The Object ID of the OMS agent identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(tryGet(reference('managedCluster'), 'addonProfiles'), 'omsagent'), 'identity'), 'objectId'), '')]" + }, + "keyvaultIdentityObjectId": { + "type": "string", + "metadata": { + "description": "The Object ID of the Key Vault Secrets Provider identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(tryGet(reference('managedCluster'), 'addonProfiles'), 'azureKeyvaultSecretsProvider'), 'identity'), 'objectId'), '')]" + }, + "keyvaultIdentityClientId": { + "type": "string", + "metadata": { + "description": "The Client ID of the Key Vault Secrets Provider identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(tryGet(reference('managedCluster'), 'addonProfiles'), 'azureKeyvaultSecretsProvider'), 'identity'), 'clientId'), '')]" + }, + "ingressApplicationGatewayIdentityObjectId": { + "type": "string", + "metadata": { + "description": "The Object ID of Application Gateway Ingress Controller (AGIC) identity." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(tryGet(reference('managedCluster'), 'addonProfiles'), 'ingressApplicationGateway'), 'identity'), 'objectId'), '')]" + }, + "location": { + "type": "string", + "metadata": { + "description": "The location the resource was deployed into." + }, + "value": "[reference('managedCluster', '2023-07-02-preview', 'full').location]" + }, + "oidcIssuerUrl": { + "type": "string", + "metadata": { + "description": "The OIDC token issuer URL." + }, + "value": "[coalesce(tryGet(tryGet(reference('managedCluster'), 'oidcIssuerProfile'), 'issuerURL'), '')]" + }, + "addonProfiles": { + "type": "object", + "metadata": { + "description": "The addonProfiles of the Kubernetes cluster." + }, + "value": "[coalesce(tryGet(reference('managedCluster'), 'addonProfiles'), createObject())]" + }, + "webAppRoutingIdentityObjectId": { + "type": "string", + "metadata": { + "description": "The Object ID of Web Application Routing." + }, + "value": "[coalesce(tryGet(tryGet(tryGet(tryGet(reference('managedCluster'), 'ingressProfile'), 'webAppRouting'), 'identity'), 'objectId'), '')]" + } + } +} \ No newline at end of file diff --git a/avm/res/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep b/avm/res/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep new file mode 100644 index 0000000000..39364978b3 --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep @@ -0,0 +1,185 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the Virtual Network to create.') +param virtualNetworkName string + +@description('Required. The name of the Managed Identity to create.') +param managedIdentityName string + +@description('Required. The name of the Kubelet Identity Managed Identity to create.') +param managedIdentityKubeletIdentityName string + +@description('Required. The name of the Disk Encryption Set to create.') +param diskEncryptionSetName string + +@description('Required. The name of the Key Vault to create.') +param keyVaultName string + +@description('Required. The name of the Proximity Placement Group to create.') +param proximityPlacementGroupName string + +@description('Required. The name of the DNS Zone to create.') +param dnsZoneName string + +@description('Required. The name of the log analytics workspace to create.') +param logAnalyticsWorkspaceName string + +var addressPrefix = '10.1.0.0/22' + +resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = { + name: virtualNetworkName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + addressPrefix + ] + } + subnets: map(range(0, 3), i => { + name: 'subnet-${i}' + properties: { + addressPrefix: cidrSubnet(addressPrefix, 24, i) + } + }) + } +} + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { + name: managedIdentityName + location: location +} + +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { + name: logAnalyticsWorkspaceName + location: location +} + +resource keyVault 'Microsoft.KeyVault/vaults@2022-11-01' = { + name: keyVaultName + location: location + properties: { + sku: { + family: 'A' + name: 'standard' + } + tenantId: tenant().tenantId + enablePurgeProtection: true // Required by nodepool vmss + softDeleteRetentionInDays: 7 + enabledForTemplateDeployment: true + enabledForDiskEncryption: true + enabledForDeployment: true + enableRbacAuthorization: true + accessPolicies: [] + } + + resource key 'keys@2022-07-01' = { + name: 'encryptionKey' + properties: { + kty: 'RSA' + } + } + + resource kmskey 'keys@2022-07-01' = { + name: 'kmsEncryptionKey' + properties: { + kty: 'RSA' + } + } +} + +resource diskEncryptionSet 'Microsoft.Compute/diskEncryptionSets@2022-07-02' = { + name: diskEncryptionSetName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + activeKey: { + sourceVault: { + id: keyVault.id + } + keyUrl: keyVault::key.properties.keyUriWithVersion + } + encryptionType: 'EncryptionAtRestWithCustomerKey' + } +} + +resource keyPermissionsKeyVaultCryptoUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-KeyVault-Crypto-User-RoleAssignment') + scope: keyVault + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') // KeyVault-Crypto-User + principalType: 'ServicePrincipal' + } +} + +resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-KeyVault-Key-Read-RoleAssignment') + scope: keyVault + properties: { + principalId: diskEncryptionSet.identity.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') // Key Vault Crypto User + principalType: 'ServicePrincipal' + } +} + +resource proximityPlacementGroup 'Microsoft.Compute/proximityPlacementGroups@2022-03-01' = { + name: proximityPlacementGroupName + location: location +} + +@description('The resource ID of the created Virtual Network Subnet.') +output subnetResourceIds array = [ + virtualNetwork.properties.subnets[0].id + virtualNetwork.properties.subnets[1].id + virtualNetwork.properties.subnets[2].id +] + +resource dnsZone 'Microsoft.Network/dnsZones@2018-05-01' = { + name: dnsZoneName + location: 'global' +} + +resource managedIdentityKubeletIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { + name: managedIdentityKubeletIdentityName + location: location +} + +resource roleAssignmentKubeletIdentity 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid('msi-${location}-${managedIdentityKubeletIdentity.id}-ManagedIdentityOperator-RoleAssignment') + scope: managedIdentityKubeletIdentity + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f1a07417-d97a-45cb-824c-7a7467783830') // Managed Identity Operator Role used for Kubelet identity. + principalType: 'ServicePrincipal' + } +} + +@description('The principal ID of the created Managed Identity.') +output managedIdentityPrincipalId string = managedIdentity.properties.principalId + +@description('The resource ID of the created Managed Identity.') +output managedIdentityResourceId string = managedIdentity.id + +@description('The resource ID of the created Kubelet Identity Managed Identity.') +output managedIdentityKubeletIdentityResourceId string = managedIdentityKubeletIdentity.id + +@description('The resource ID of the created Disk Encryption Set.') +output diskEncryptionSetResourceId string = diskEncryptionSet.id + +@description('The resource ID of the created Key Vault.') +output keyVaultResourceId string = keyVault.id + +@description('The name of the Key Vault Encryption Key.') +output keyVaultEncryptionKeyName string = keyVault::key.name + +@description('The resource ID of the created Proximity Placement Group.') +output proximityPlacementGroupResourceId string = proximityPlacementGroup.id + +@description('The resource ID of the created DNS Zone.') +output dnsZoneResourceId string = dnsZone.id + +@description('The resource ID of the created Log Analytics Workspace.') +output logAnalyticsWorkspaceResourceId string = logAnalyticsWorkspace.id diff --git a/avm/res/container-service/managed-cluster/tests/e2e/azure/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/azure/main.test.bicep new file mode 100644 index 0000000000..6d44c21a54 --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/azure/main.test.bicep @@ -0,0 +1,281 @@ +targetScope = 'subscription' +metadata name = 'Using Azure CNI Network Plugin.' +metadata description = 'This instance deploys the module with Azure CNI network plugin .' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmaz' + +@description('Generated. Used as a basis for unique resource names.') +param baseTime string = utcNow('u') + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: location +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-nestedDependencies' + params: { + virtualNetworkName: 'dep-${namePrefix}-vnet-${serviceShort}' + managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' + managedIdentityKubeletIdentityName: 'dep-${namePrefix}-msiki-${serviceShort}' + diskEncryptionSetName: 'dep-${namePrefix}-des-${serviceShort}' + proximityPlacementGroupName: 'dep-${namePrefix}-ppg-${serviceShort}' + // Adding base time to make the name unique as purge protection must be enabled (but may not be longer than 24 characters total) + keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}' + dnsZoneName: 'dep-${namePrefix}-dns-${serviceShort}.com' + logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' + location: location + } +} + +// Diagnostics +// =========== +module diagnosticDependencies '../../../../../../utilities/e2e-template-assets/templates/diagnostic.dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-diagnosticDependencies' + params: { + storageAccountName: 'dep${namePrefix}diasa${serviceShort}01' + logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' + eventHubNamespaceEventHubName: 'dep-${namePrefix}-evh-${serviceShort}' + eventHubNamespaceName: 'dep-${namePrefix}-evhns-${serviceShort}' + location: location + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [for iteration in [ 'init', 'idem' ]: { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-test-${serviceShort}-${iteration}' + params: { + location: location + name: '${namePrefix}${serviceShort}001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[0] + } + ] + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[1] + proximityPlacementGroupResourceId: nestedDependencies.outputs.proximityPlacementGroupResourceId + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[2] + } + ] + autoUpgradeProfileUpgradeChannel: 'stable' + enableWorkloadIdentity: true + enableOidcIssuerProfile: true + networkPlugin: 'azure' + networkDataplane: 'azure' + networkPluginMode: 'overlay' + diagnosticSettings: [ + { + name: 'customSetting' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + eventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName + eventHubAuthorizationRuleResourceId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId + storageAccountResourceId: diagnosticDependencies.outputs.storageAccountResourceId + workspaceResourceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + } + ] + diskEncryptionSetResourceId: nestedDependencies.outputs.diskEncryptionSetResourceId + openServiceMeshEnabled: true + enableStorageProfileBlobCSIDriver: true + enableStorageProfileDiskCSIDriver: true + enableStorageProfileFileCSIDriver: true + enableStorageProfileSnapshotController: true + managedIdentities: { + userAssignedResourcesIds: [ + nestedDependencies.outputs.managedIdentityResourceId + ] + } + identityProfile: { + kubeletidentity: { + resourceId: nestedDependencies.outputs.managedIdentityKubeletIdentityResourceId + } + } + omsAgentEnabled: true + monitoringWorkspaceId: nestedDependencies.outputs.logAnalyticsWorkspaceResourceId + enableAzureDefender: true + enableKeyvaultSecretsProvider: true + enablePodSecurityPolicy: false + enableAzureMonitorProfileMetrics: true + customerManagedKey: { + keyName: nestedDependencies.outputs.keyVaultEncryptionKeyName + keyVaultNetworkAccess: 'Public' + keyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId + } + lock: { + kind: 'CanNotDelete' + name: 'myCustomLockName' + } + roleAssignments: [ + { + roleDefinitionIdOrName: 'Owner' + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + { + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + { + roleDefinitionIdOrName: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7') + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + ] + tags: { + 'hidden-title': 'This is visible in the resource name' + Environment: 'Non-Prod' + Role: 'DeploymentValidation' + } + fluxExtension: { + configurationSettings: { + 'helm-controller.enabled': 'true' + 'source-controller.enabled': 'true' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + } + configurations: [ + { + namespace: 'flux-system' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + } + { + namespace: 'flux-system-helm' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + kustomizations: { + infra: { + path: './infrastructure' + dependsOn: [] + timeoutInSeconds: 600 + syncIntervalInSeconds: 600 + validation: 'none' + prune: true + } + apps: { + path: './apps/staging' + dependsOn: [ + 'infra' + ] + timeoutInSeconds: 600 + syncIntervalInSeconds: 600 + retryIntervalInSeconds: 120 + prune: true + } + } + } + ] + } + } +}] diff --git a/avm/res/container-service/managed-cluster/tests/e2e/defaults/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/defaults/main.test.bicep new file mode 100644 index 0000000000..be98dd9270 --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/defaults/main.test.bicep @@ -0,0 +1,53 @@ +targetScope = 'subscription' + +metadata name = 'Using only defaults' +metadata description = 'This instance deploys the module with the minimum set of required parameters.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmin' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: location +} + +@batchSize(1) +module testDeployment '../../../main.bicep' = [for iteration in [ 'init', 'idem' ]: { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: location + managedIdentities: { + systemAssigned: true + } + primaryAgentPoolProfile: [ + { + name: 'systempool' + count: 1 + vmSize: 'Standard_DS2_v2' + mode: 'System' + } + ] + } +}] diff --git a/avm/res/container-service/managed-cluster/tests/e2e/kubenet/dependencies.bicep b/avm/res/container-service/managed-cluster/tests/e2e/kubenet/dependencies.bicep new file mode 100644 index 0000000000..bcd58414ee --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/kubenet/dependencies.bicep @@ -0,0 +1,27 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the Managed Identity to create.') +param managedIdentityName string + +@description('Required. The name of the DNS Zone to create.') +param dnsZoneName string + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: managedIdentityName + location: location +} + +resource dnsZone 'Microsoft.Network/dnsZones@2018-05-01' = { + name: dnsZoneName + location: 'global' +} + +@description('The principal ID of the created Managed Identity.') +output managedIdentityPrincipalId string = managedIdentity.properties.principalId + +@description('The resource ID of the created Managed Identity.') +output managedIdentityResourceId string = managedIdentity.id + +@description('The resource ID of the created DNS Zone.') +output dnsZoneResourceId string = dnsZone.id diff --git a/avm/res/container-service/managed-cluster/tests/e2e/kubenet/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/kubenet/main.test.bicep new file mode 100644 index 0000000000..b5ff8718cd --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/kubenet/main.test.bicep @@ -0,0 +1,179 @@ +targetScope = 'subscription' + +metadata name = 'Using Kubenet Network Plugin.' +metadata description = 'This instance deploys the module with Kubenet network plugin .' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmkube' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: location +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-nestedDependencies' + params: { + managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' + dnsZoneName: 'dep-${namePrefix}-dns-${serviceShort}.com' + location: location + } +} + +// Diagnostics +// =========== +module diagnosticDependencies '../../../../../../utilities/e2e-template-assets/templates/diagnostic.dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-diagnosticDependencies' + params: { + storageAccountName: 'dep${namePrefix}diasa${serviceShort}01' + logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' + eventHubNamespaceEventHubName: 'dep-${namePrefix}-evh-${serviceShort}' + eventHubNamespaceName: 'dep-${namePrefix}-evhns-${serviceShort}' + location: location + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [for iteration in [ 'init', 'idem' ]: { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: location + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + networkPlugin: 'kubenet' + diagnosticSettings: [ + { + name: 'customSetting' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + eventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName + eventHubAuthorizationRuleResourceId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId + storageAccountResourceId: diagnosticDependencies.outputs.storageAccountResourceId + workspaceResourceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + } + ] + roleAssignments: [ + { + roleDefinitionIdOrName: 'Owner' + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + { + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + { + roleDefinitionIdOrName: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7') + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + principalType: 'ServicePrincipal' + } + ] + managedIdentities: { + userAssignedResourcesIds: [ + nestedDependencies.outputs.managedIdentityResourceId + ] + } + tags: { + 'hidden-title': 'This is visible in the resource name' + Environment: 'Non-Prod' + Role: 'DeploymentValidation' + } + } +}] diff --git a/avm/res/container-service/managed-cluster/tests/e2e/priv/dependencies.bicep b/avm/res/container-service/managed-cluster/tests/e2e/priv/dependencies.bicep new file mode 100644 index 0000000000..b74bb113ac --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/priv/dependencies.bicep @@ -0,0 +1,87 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the Managed Identity to create.') +param managedIdentityName string + +@description('Required. The Private DNS Zone Name to create for Private AKS Cluster.') +param privateDnsZoneName string + +@description('Required. The Name of the Virtual Network to create.') +param virtualNetworkName string + +var addressPrefix = '10.0.0.0/16' + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: managedIdentityName + location: location +} + +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: privateDnsZoneName + location: 'global' +} + +resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = { + name: virtualNetworkName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + addressPrefix + ] + } + subnets: [ + { + name: 'defaultSubnet' + properties: { + addressPrefix: cidrSubnet(addressPrefix, 16, 0) + } + } + ] + } +} + +resource privateDNSZoneVNetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + name: 'pDnsLink-${virtualNetworkName}-${privateDnsZoneName}' + location: 'global' + parent: privateDnsZone + properties: { + registrationEnabled: true + virtualNetwork: { + id: virtualNetwork.id + } + } +} + +resource msiVnetRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, 'NetworkContributor', managedIdentity.id) + scope: virtualNetwork + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7') // Network Contributor + principalType: 'ServicePrincipal' + } +} + +resource msiPrivDnsZoneRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, 'PrivateDNSZoneContributor', managedIdentity.id) + scope: privateDnsZone + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b12aa53e-6015-4669-85d0-8515ebb3ae7f') // Private DNS Zone Contributor + principalType: 'ServicePrincipal' + } +} + +@description('The principal ID of the created Managed Identity.') +output managedIdentityPrincipalId string = managedIdentity.properties.principalId + +@description('The resource ID of the created Managed Identity.') +output managedIdentityResourceId string = managedIdentity.id + +@description('The resource ID of the private DNS Zone created.') +output privateDnsZoneResourceId string = privateDnsZone.id + +@description('The resource ID of the VirtualNetwork created.') +output vNetResourceId string = virtualNetwork.id diff --git a/avm/res/container-service/managed-cluster/tests/e2e/priv/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/priv/main.test.bicep new file mode 100644 index 0000000000..b33d9febfc --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/priv/main.test.bicep @@ -0,0 +1,137 @@ +targetScope = 'subscription' + +metadata name = 'Using Private Cluster.' +metadata description = 'This instance deploys the module with a private cluster instance.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmpriv' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: location +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-nestedDependencies' + params: { + managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' + privateDnsZoneName: 'privatelink.${location}.azmk8s.io' + virtualNetworkName: 'dep-${namePrefix}-vnet-${serviceShort}' + location: location + } +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [for iteration in [ 'init', 'idem' ]: { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: location + enablePrivateCluster: true + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + } + ] + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + networkPlugin: 'azure' + skuTier: 'Standard' + dnsServiceIP: '10.10.200.10' + serviceCidr: '10.10.200.0/24' + privateDNSZone: nestedDependencies.outputs.privateDnsZoneResourceId + managedIdentities: { + userAssignedResourcesIds: [ + nestedDependencies.outputs.managedIdentityResourceId + ] + } + } +}] diff --git a/avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/dependencies.bicep b/avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/dependencies.bicep new file mode 100644 index 0000000000..b74bb113ac --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/dependencies.bicep @@ -0,0 +1,87 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the Managed Identity to create.') +param managedIdentityName string + +@description('Required. The Private DNS Zone Name to create for Private AKS Cluster.') +param privateDnsZoneName string + +@description('Required. The Name of the Virtual Network to create.') +param virtualNetworkName string + +var addressPrefix = '10.0.0.0/16' + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: managedIdentityName + location: location +} + +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: privateDnsZoneName + location: 'global' +} + +resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = { + name: virtualNetworkName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + addressPrefix + ] + } + subnets: [ + { + name: 'defaultSubnet' + properties: { + addressPrefix: cidrSubnet(addressPrefix, 16, 0) + } + } + ] + } +} + +resource privateDNSZoneVNetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + name: 'pDnsLink-${virtualNetworkName}-${privateDnsZoneName}' + location: 'global' + parent: privateDnsZone + properties: { + registrationEnabled: true + virtualNetwork: { + id: virtualNetwork.id + } + } +} + +resource msiVnetRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, 'NetworkContributor', managedIdentity.id) + scope: virtualNetwork + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7') // Network Contributor + principalType: 'ServicePrincipal' + } +} + +resource msiPrivDnsZoneRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, 'PrivateDNSZoneContributor', managedIdentity.id) + scope: privateDnsZone + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b12aa53e-6015-4669-85d0-8515ebb3ae7f') // Private DNS Zone Contributor + principalType: 'ServicePrincipal' + } +} + +@description('The principal ID of the created Managed Identity.') +output managedIdentityPrincipalId string = managedIdentity.properties.principalId + +@description('The resource ID of the created Managed Identity.') +output managedIdentityResourceId string = managedIdentity.id + +@description('The resource ID of the private DNS Zone created.') +output privateDnsZoneResourceId string = privateDnsZone.id + +@description('The resource ID of the VirtualNetwork created.') +output vNetResourceId string = virtualNetwork.id diff --git a/avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/main.test.bicep new file mode 100644 index 0000000000..25ed4da8c6 --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/main.test.bicep @@ -0,0 +1,194 @@ +targetScope = 'subscription' + +metadata name = 'WAF-aligned' +metadata description = 'This instance deploys the module in alignment with the best-practices of the Well-Architected Framework.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +// e.g., for a module 'network/private-endpoint' you could use 'dep-dev-network.privateendpoints-${serviceShort}-rg' +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +// e.g., for a module 'network/private-endpoint' you could use 'npe' as a prefix and then 'waf' as a suffix for the waf-aligned test +param serviceShort string = 'cswaf' + +@description('Optional. A token to inject into the name of each resource. This value can be automatically injected by the CI.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-nestedDependencies' + params: { + managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' + privateDnsZoneName: 'privatelink.${location}.azmk8s.io' + virtualNetworkName: 'dep-${namePrefix}-vnet-${serviceShort}' + location: location + } +} + +// Diagnostics +// =========== +module diagnosticDependencies '../../../../../../utilities/e2e-template-assets/templates/diagnostic.dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-diagnosticDependencies' + params: { + storageAccountName: 'dep${namePrefix}diasa${serviceShort}01' + logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' + eventHubNamespaceEventHubName: 'dep-${namePrefix}-evh-${serviceShort}' + eventHubNamespaceName: 'dep-${namePrefix}-evhns-${serviceShort}' + location: location + } +} + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: location +} + +// ============== // +// Test Execution // +// ============== // + +@batchSize(1) +module testDeployment '../../../main.bicep' = [for iteration in [ 'init', 'idem' ]: { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: location + enablePrivateCluster: true + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 3 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + } + ] + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskType: 'Ephemeral' + osDiskSizeGB: 60 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 50 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskType: 'Ephemeral' + osDiskSizeGB: 60 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + autoUpgradeProfileUpgradeChannel: 'stable' + networkPlugin: 'azure' + networkPolicy: 'azure' + skuTier: 'Standard' + dnsServiceIP: '10.10.200.10' + serviceCidr: '10.10.200.0/24' + omsAgentEnabled: true + monitoringWorkspaceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + disableLocalAccounts: true + enableAzureDefender: true + diagnosticSettings: [ + { + name: 'customSetting' + logCategoriesAndGroups: [ + { + category: 'kube-apiserver' + } + { + category: 'kube-controller-manager' + } + { + category: 'kube-scheduler' + } + { + category: 'cluster-autoscaler' + } + ] + metricCategories: [ + { + category: 'AllMetrics' + } + ] + eventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName + eventHubAuthorizationRuleResourceId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId + storageAccountResourceId: diagnosticDependencies.outputs.storageAccountResourceId + workspaceResourceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + } + ] + privateDNSZone: nestedDependencies.outputs.privateDnsZoneResourceId + managedIdentities: { + userAssignedResourcesIds: [ + nestedDependencies.outputs.managedIdentityResourceId + ] + } + tags: { + 'hidden-title': 'This is visible in the resource name' + Environment: 'Non-Prod' + Role: 'DeploymentValidation' + } + } +}] diff --git a/avm/res/container-service/managed-cluster/version.json b/avm/res/container-service/managed-cluster/version.json new file mode 100644 index 0000000000..8def869ede --- /dev/null +++ b/avm/res/container-service/managed-cluster/version.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://aka.ms/bicep-registry-module-version-file-schema#", + "version": "0.1", + "pathFilters": [ + "./main.json" + ] +}