diff --git a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/main.bicep b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/main.bicep index 793699a0..10a0bcd2 100644 --- a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/main.bicep +++ b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/main.bicep @@ -6,8 +6,6 @@ param vnetHubName string param hubVNETaddPrefixes array param azfwName string param rtVMSubnetName string -param fwapplicationRuleCollections array -param fwnetworkRuleCollections array param fwnatRuleCollections array param location string = deployment().location param availabilityZones array @@ -22,6 +20,10 @@ param azureBastionSubnetAddressPrefix string param vmsubnetSubnetName string param vmsubnetSubnetAddressPrefix string +@description('The prefix for the spoke subnet AKS') +param spokeSubnetAKSPrefix string = '10.1.1.0/24' + + module rg 'br/public:avm/res/resources/resource-group:0.2.3' = { name: rgName params: { @@ -164,6 +166,202 @@ module telemetry './telemetry.bicep' = { params: { enableTelemetry: enableTelemetry telemetryId: telemetryId - location: location } } + + +// Defining firewal config here so that the spokeSubnetAKSPrefix can be passed dynamically +// this way, if customer isnt using the default subnetprefix for AKS, they can configure the +// firewall settings to make it allow AKS deploy successfully by specifying their own subnet prefix +@description('Returns the firewall application rule collections') +param fwapplicationRuleCollections array = [ + { + name: 'Helper-tools' + properties: { + priority: 101 + action: { + type: 'Allow' + } + rules: [ + { + name: 'Allow-ifconfig' + protocols: [ + { + port: 80 + protocolType: 'Http' + } + { + port: 443 + protocolType: 'Https' + } + ] + targetFqdns: [ + 'ifconfig.co' + 'api.snapcraft.io' + 'jsonip.com' + 'kubernaut.io' + 'motd.ubuntu.com' + ] + sourceAddresses: [ + spokeSubnetAKSPrefix + ] + } + ] + } + } + { + name: 'AKS-egress-application' + properties: { + priority: 102 + action: { + type: 'Allow' + } + rules: [ + { + name: 'Egress' + protocols: [ + { + port: 443 + protocolType: 'Https' + } + ] + targetFqdns: [ + '*.azmk8s.io' + 'aksrepos.azurecr.io' + '*.blob.core.windows.net' + '*.cdn.mscr.io' + '*.opinsights.azure.com' + '*.monitoring.azure.com' + ] + sourceAddresses: [ + '10.1.1.0/24' + ] + } + { + name: 'Registries' + protocols: [ + { + port: 443 + protocolType: 'Https' + } + ] + targetFqdns: [ + '*.azurecr.io' + '*.gcr.io' + '*.docker.io' + 'quay.io' + '*.quay.io' + '*.cloudfront.net' + 'production.cloudflare.docker.com' + ] + sourceAddresses: [ + spokeSubnetAKSPrefix + ] + } + { + name: 'Additional-Usefull-Address' + protocols: [ + { + port: 443 + protocolType: 'Https' + } + ] + targetFqdns: [ + 'grafana.net' + 'grafana.com' + 'stats.grafana.org' + 'github.com' + 'charts.bitnami.com' + 'raw.githubusercontent.com' + '*.letsencrypt.org' + 'usage.projectcalico.org' + 'vortex.data.microsoft.com' + ] + sourceAddresses: [ + spokeSubnetAKSPrefix + ] + } + { + name: 'AKS-FQDN-TAG' + protocols: [ + { + port: 80 + protocolType: 'Http' + } + { + port: 443 + protocolType: 'Https' + } + ] + targetFqdns: [] + fqdnTags: [ + 'AzureKubernetesService' + ] + sourceAddresses: [ + spokeSubnetAKSPrefix + ] + } + ] + } + } +] + +@description('Returns the firewall network rule collections') +param fwnetworkRuleCollections array = [ + { + name: 'AKS-egress' + properties: { + priority: 200 + action: { + type: 'Allow' + } + rules: [ + { + name: 'NTP' + protocols: [ + 'UDP' + ] + sourceAddresses: [ + spokeSubnetAKSPrefix + ] + destinationAddresses: [ + '*' + ] + destinationPorts: [ + '123' + ] + } + { + name: 'APITCP' + protocols: [ + 'TCP' + ] + sourceAddresses: [ + spokeSubnetAKSPrefix + ] + destinationAddresses: [ + '*' + ] + destinationPorts: [ + '9000' + ] + } + { + name: 'APIUDP' + protocols: [ + 'UDP' + ] + sourceAddresses: [ + spokeSubnetAKSPrefix + ] + destinationAddresses: [ + '*' + ] + destinationPorts: [ + '1194' + ] + } + ] + } + } +] diff --git a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/parameters-main.json b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/parameters-main.json index 8253f9c3..6f56bab6 100644 --- a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/parameters-main.json +++ b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/parameters-main.json @@ -23,199 +23,6 @@ "rtVMSubnetName":{ "value": "vm-subnet-rt" }, - "fwapplicationRuleCollections": { - "value": [ - { - "name": "Helper-tools", - "properties": { - "priority": 101, - "action": { - "type": "Allow" - }, - "rules": [ - { - "name": "Allow-ifconfig", - "protocols": [ - { - "port": 80, - "protocolType": "Http" - }, - { - "port": 443, - "protocolType": "Https" - } - ], - "targetFqdns": [ - "ifconfig.co", - "api.snapcraft.io", - "jsonip.com", - "kubernaut.io", - "motd.ubuntu.com" - ], - "sourceAddresses": [ - "10.1.1.0/24" - ] - } - ] - } - }, - { - "name": "AKS-egress-application", - "properties": { - "priority": 102, - "action": { - "type": "Allow" - }, - "rules": [ - { - "name": "Egress", - "protocols": [ - { - "port": 443, - "protocolType": "Https" - } - ], - "targetFqdns": [ - "*.azmk8s.io", - "aksrepos.azurecr.io", - "*.blob.core.windows.net", - "*.cdn.mscr.io", - "*.opinsights.azure.com", - "*.monitoring.azure.com" - ], - "sourceAddresses": [ - "10.1.1.0/24" - ] - }, - { - "name": "Registries", - "protocols": [ - { - "port": 443, - "protocolType": "Https" - } - ], - "targetFqdns": [ - "*.azurecr.io", - "*.gcr.io", - "*.docker.io", - "quay.io", - "*.quay.io", - "*.cloudfront.net", - "production.cloudflare.docker.com" - ], - "sourceAddresses": [ - "10.1.1.0/24" - ] - }, - { - "name": "Additional-Usefull-Address", - "protocols": [ - { - "port": 443, - "protocolType": "Https" - } - ], - "targetFqdns": [ - "grafana.net", - "grafana.com", - "stats.grafana.org", - "github.com", - "charts.bitnami.com", - "raw.githubusercontent.com", - "*.letsencrypt.org", - "usage.projectcalico.org", - "vortex.data.microsoft.com" - ], - "sourceAddresses": [ - "10.1.1.0/24" - ] - }, - { - "name": "AKS-FQDN-TAG", - "protocols": [ - { - "port": 80, - "protocolType": "Http" - }, - { - "port": 443, - "protocolType": "Https" - } - ], - "targetFqdns": [], - "fqdnTags": [ - "AzureKubernetesService" - ], - "sourceAddresses": [ - "10.1.1.0/24" - ] - } - ] - } - } - ] - }, - "fwnetworkRuleCollections": { - "value": [ - { - "name": "AKS-egress", - "properties": { - "priority": 200, - "action": { - "type": "Allow" - }, - "rules": [ - { - "name": "NTP", - "protocols": [ - "UDP" - ], - "sourceAddresses": [ - "10.1.1.0/24" - ], - "destinationAddresses": [ - "*" - ], - "destinationPorts": [ - "123" - ] - }, - { - "name": "APITCP", - "protocols": [ - "TCP" - ], - "sourceAddresses": [ - "10.1.1.0/24" - ], - "destinationAddresses": [ - "*" - ], - "destinationPorts": [ - "9000" - ] - }, - { - "name": "APIUDP", - "protocols": [ - "UDP" - ], - "sourceAddresses": [ - "10.1.1.0/24" - ], - "destinationAddresses": [ - "*" - ], - "destinationPorts": [ - "1194" - ] - } - ] - } - } - ] - }, "fwnatRuleCollections": { "value": [] }, diff --git a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/telemetry.bicep b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/telemetry.bicep index 93d99a4e..2b27ecf2 100644 --- a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/telemetry.bicep +++ b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/03-Network-Hub/telemetry.bicep @@ -1,9 +1,7 @@ param enableTelemetry bool param telemetryId string -param location string resource telemetrydeployment 'Microsoft.Resources/deployments@2021-04-01' = if (enableTelemetry) { name: telemetryId - location: location properties: { mode: 'Incremental' template: { diff --git a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/04-Network-LZ/main.bicep b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/04-Network-LZ/main.bicep index 15a970c3..967ef1c6 100644 --- a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/04-Network-LZ/main.bicep +++ b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/04-Network-LZ/main.bicep @@ -11,11 +11,18 @@ param vnetHUBRGName string param nsgAKSName string param nsgAppGWName string param rtAppGWSubnetName string +param enablePrivateCluster bool = true // param dnsServers array param location string = deployment().location param availabilityZones array param appGwyAutoScale object param securityRules array = [] +param spokeSubnetDefaultPrefix string = '10.1.0.0/24' +param spokeSubnetAKSPrefix string = '10.1.1.0/24' +param spokeSubnetAppGWPrefix string = '10.1.2.0/27' +param spokeSubnetVMPrefix string = '10.1.3.0/24' +param spokeSubnetPLinkervicePrefix string = '10.1.4.0/24' +param remotePeeringName string = 'spoke-hub-peering' var privateDNSZoneAKSSuffixes = { AzureCloud: '.azmk8s.io' @@ -48,24 +55,24 @@ module vnetspoke 'br/public:avm/res/network/virtual-network:0.1.1' = { subnets: [ { name: 'default' - addressPrefix: '10.1.0.0/24' + addressPrefix: spokeSubnetDefaultPrefix } { name: 'AKS' - addressPrefix: '10.1.1.0/24' + addressPrefix: spokeSubnetAKSPrefix routeTableResourceId: routeTable.outputs.resourceId } { name: 'AppGWSubnet' - addressPrefix: '10.1.2.0/27' + addressPrefix: spokeSubnetAppGWPrefix } { name: 'vmsubnet' - addressPrefix: '10.1.3.0/24' + addressPrefix: spokeSubnetVMPrefix } { name: 'servicespe' - addressPrefix: '10.1.4.0/24' + addressPrefix: spokeSubnetPLinkervicePrefix } ] enableTelemetry: true @@ -78,7 +85,7 @@ module vnetspoke 'br/public:avm/res/network/virtual-network:0.1.1' = { remotePeeringAllowForwardedTraffic: true remotePeeringAllowVirtualNetworkAccess: true remotePeeringEnabled: true - remotePeeringName: 'spoke-hub-peering' + remotePeeringName: remotePeeringName remoteVirtualNetworkId: vnethub.id useRemoteGateways: false } @@ -256,7 +263,7 @@ module privateDnsZoneSA 'br/public:avm/res/network/private-dns-zone:0.2.4' = { } } -module privateDnsZoneAKS 'br/public:avm/res/network/private-dns-zone:0.2.4' = { +module privateDnsZoneAKS 'br/public:avm/res/network/private-dns-zone:0.2.4' = if (enablePrivateCluster) { scope: resourceGroup(rg.name) name: 'privatednsAKSZone' params: { diff --git a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/06-AKS-cluster/main.bicep b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/06-AKS-cluster/main.bicep index 98bb5c00..85574c57 100644 --- a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/06-AKS-cluster/main.bicep +++ b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/06-AKS-cluster/main.bicep @@ -14,6 +14,7 @@ param keyvaultName string @description('The name of the Container registry you deployed in the previous step (check Azure portal if you need to).') param acrName string param aksClusterName string +param enablePrivateCluster bool = true @allowed([ @@ -36,7 +37,7 @@ resource aksIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-3 name: aksIdentityName } -resource pvtdnsAKSZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { +resource pvtdnsAKSZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (enablePrivateCluster) { name: privateDNSZoneAKSName scope: resourceGroup(rg.name) } @@ -134,8 +135,8 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:0.1.2 serviceCidr: '192.168.100.0/24' networkPolicy: 'calico' podCidr: networkPlugin == 'kubenet' ? '172.17.0.0/16' : null - enablePrivateCluster: true - privateDNSZone: pvtdnsAKSZone.id + enablePrivateCluster: enablePrivateCluster + privateDNSZone: enablePrivateCluster ? pvtdnsAKSZone.id : null enablePrivateClusterPublicFQDN: false enableRBAC: true aadProfileAdminGroupObjectIDs: [ diff --git a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/README.md b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/README.md index 0824e80d..8e1d97dd 100644 --- a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/README.md +++ b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/README.md @@ -59,7 +59,7 @@ LOCATION= ``` ```bash -az deployment sub create --location $LOCATION --template-file main.bicep --parameters @main.json --name aksLZAAllInOne --parameters aksadminaccessprincipalId +az deployment sub create --location $LOCATION --template-file main.bicep --parameters @main.json --name aksLZAAllInOne --parameters aksadminaccessprincipalId= ``` ### Step 3 - Deploy the application to AKS. diff --git a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.bicep b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.bicep index 7fe8a1f4..344d64fa 100644 --- a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.bicep +++ b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.bicep @@ -13,6 +13,12 @@ targetScope = 'subscription' ///////////////// // 03-network-Hub ///////////////// +@description('Set this to true if you want to deploy the hub network and its resources') +param deployHub bool = true + +@description('Set this to true if you want your aks cluster to be private') +param enablePrivateCluster bool = true + param rgHubName string = 'AKS-LZA-HUB' param vnetHubName string = 'VNet-HUB' param hubVNETaddPrefixes array = ['10.0.0.0/16'] @@ -218,6 +224,12 @@ param rgSpokeName string = 'AKS-LZA-SPOKE' param vnetSpokeName string = 'VNet-SPOKE' //param availabilityZones array = ['1', '2', '3'] param spokeVNETaddPrefixes array = ['10.1.0.0/16'] +param spokeSubnetDefaultPrefix string = '10.1.0.0/24' +param spokeSubnetAKSPrefix string = '10.1.1.0/24' +param spokeSubnetAppGWPrefix string = '10.1.2.0/27' +param spokeSubnetVMPrefix string = '10.1.3.0/24' +param spokeSubnetPLinkervicePrefix string = '10.1.4.0/24' +param remotePeeringName string = 'spoke-hub-peering' param rtAKSSubnetName string = 'AKS-RT' param firewallIP string = '10.0.1.4' //param vnetHubName string = 'VNet-HUB' @@ -299,11 +311,12 @@ param aksClusterName string = 'aksCluster' // 03-network-Hub ///////////////// -module networkHub '../03-Network-Hub/main.bicep' = { +module networkHub '../03-Network-Hub/main.bicep' = if (deployHub) { name: 'hubDeploy' params: { rgName: rgHubName availabilityZones: availabilityZones + spokeSubnetAKSPrefix: spokeSubnetAKSPrefix vnetHubName: vnetHubName azfwName: azfwName rtVMSubnetName: rtVMSubnetName @@ -332,6 +345,7 @@ module networkSpoke '../04-Network-LZ/main.bicep' = { name: 'lzSpokeDeploy' params: { rgName: rgSpokeName + enablePrivateCluster: enablePrivateCluster vnetSpokeName: vnetSpokeName availabilityZones: availabilityZones spokeVNETaddPrefixes: spokeVNETaddPrefixes @@ -346,8 +360,15 @@ module networkSpoke '../04-Network-LZ/main.bicep' = { //dnsServers: dnsServers appGwyAutoScale: appGwyAutoScale securityRules: securityRules + spokeSubnetDefaultPrefix: spokeSubnetDefaultPrefix + spokeSubnetAKSPrefix: spokeSubnetAKSPrefix + spokeSubnetAppGWPrefix: spokeSubnetAppGWPrefix + spokeSubnetVMPrefix:spokeSubnetVMPrefix + spokeSubnetPLinkervicePrefix: spokeSubnetPLinkervicePrefix + remotePeeringName: remotePeeringName + } - dependsOn: [networkHub] + dependsOn: deployHub ? [networkHub] : [] } ///////////////// @@ -377,6 +398,7 @@ module aksCluster '../06-AKS-Cluster/main.bicep' = { name: 'aksCluster' params: { rgName: rgSpokeName + enablePrivateCluster: enablePrivateCluster vnetName: vnetSpokeName subnetName: aksSubnetName aksIdentityName: aksIdentityName diff --git a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.json b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.json index 9290f534..7e9aeb66 100644 --- a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.json +++ b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.json @@ -5,10 +5,24 @@ "_generator": { "name": "bicep", "version": "0.28.1.47646", - "templateHash": "7106384231315853249" + "templateHash": "5950662107880979924" } }, "parameters": { + "deployHub": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Set this to true if you want to deploy the hub network and its resources" + } + }, + "enablePrivateCluster": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Set this to true if you want your aks cluster to be private" + } + }, "rgHubName": { "type": "string", "defaultValue": "AKS-LZA-HUB" @@ -252,6 +266,30 @@ "10.1.0.0/16" ] }, + "spokeSubnetDefaultPrefix": { + "type": "string", + "defaultValue": "10.1.0.0/24" + }, + "spokeSubnetAKSPrefix": { + "type": "string", + "defaultValue": "10.1.1.0/24" + }, + "spokeSubnetAppGWPrefix": { + "type": "string", + "defaultValue": "10.1.2.0/27" + }, + "spokeSubnetVMPrefix": { + "type": "string", + "defaultValue": "10.1.3.0/24" + }, + "spokeSubnetPLinkervicePrefix": { + "type": "string", + "defaultValue": "10.1.4.0/24" + }, + "remotePeeringName": { + "type": "string", + "defaultValue": "spoke-hub-peering" + }, "rtAKSSubnetName": { "type": "string", "defaultValue": "AKS-RT" @@ -386,8 +424,7 @@ } }, "aksadminaccessprincipalId": { - "type": "string", - "defaultValue": "" + "type": "string" }, "kubernetesVersion": { "type": "string", @@ -404,6 +441,7 @@ }, "resources": [ { + "condition": "[parameters('deployHub')]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", "name": "hubDeploy", @@ -420,6 +458,9 @@ "availabilityZones": { "value": "[parameters('availabilityZones')]" }, + "spokeSubnetAKSPrefix": { + "value": "[parameters('spokeSubnetAKSPrefix')]" + }, "vnetHubName": { "value": "[parameters('vnetHubName')]" }, @@ -479,7 +520,7 @@ "_generator": { "name": "bicep", "version": "0.28.1.47646", - "templateHash": "842494715723055146" + "templateHash": "14622037900232296803" } }, "parameters": { @@ -498,12 +539,6 @@ "rtVMSubnetName": { "type": "string" }, - "fwapplicationRuleCollections": { - "type": "array" - }, - "fwnetworkRuleCollections": { - "type": "array" - }, "fwnatRuleCollections": { "type": "array" }, @@ -543,8 +578,226 @@ }, "vmsubnetSubnetAddressPrefix": { "type": "string" + }, + "spokeSubnetAKSPrefix": { + "type": "string", + "defaultValue": "10.1.1.0/24", + "metadata": { + "description": "The prefix for the spoke subnet AKS" + } + }, + "enableTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Enable usage and telemetry feedback to Microsoft." + } + }, + "fwapplicationRuleCollections": { + "type": "array", + "defaultValue": [ + { + "name": "Helper-tools", + "properties": { + "priority": 101, + "action": { + "type": "Allow" + }, + "rules": [ + { + "name": "Allow-ifconfig", + "protocols": [ + { + "port": 80, + "protocolType": "Http" + }, + { + "port": 443, + "protocolType": "Https" + } + ], + "targetFqdns": [ + "ifconfig.co", + "api.snapcraft.io", + "jsonip.com", + "kubernaut.io", + "motd.ubuntu.com" + ], + "sourceAddresses": [ + "[parameters('spokeSubnetAKSPrefix')]" + ] + } + ] + } + }, + { + "name": "AKS-egress-application", + "properties": { + "priority": 102, + "action": { + "type": "Allow" + }, + "rules": [ + { + "name": "Egress", + "protocols": [ + { + "port": 443, + "protocolType": "Https" + } + ], + "targetFqdns": [ + "*.azmk8s.io", + "aksrepos.azurecr.io", + "*.blob.core.windows.net", + "*.cdn.mscr.io", + "*.opinsights.azure.com", + "*.monitoring.azure.com" + ], + "sourceAddresses": [ + "10.1.1.0/24" + ] + }, + { + "name": "Registries", + "protocols": [ + { + "port": 443, + "protocolType": "Https" + } + ], + "targetFqdns": [ + "*.azurecr.io", + "*.gcr.io", + "*.docker.io", + "quay.io", + "*.quay.io", + "*.cloudfront.net", + "production.cloudflare.docker.com" + ], + "sourceAddresses": [ + "[parameters('spokeSubnetAKSPrefix')]" + ] + }, + { + "name": "Additional-Usefull-Address", + "protocols": [ + { + "port": 443, + "protocolType": "Https" + } + ], + "targetFqdns": [ + "grafana.net", + "grafana.com", + "stats.grafana.org", + "github.com", + "charts.bitnami.com", + "raw.githubusercontent.com", + "*.letsencrypt.org", + "usage.projectcalico.org", + "vortex.data.microsoft.com" + ], + "sourceAddresses": [ + "[parameters('spokeSubnetAKSPrefix')]" + ] + }, + { + "name": "AKS-FQDN-TAG", + "protocols": [ + { + "port": 80, + "protocolType": "Http" + }, + { + "port": 443, + "protocolType": "Https" + } + ], + "targetFqdns": [], + "fqdnTags": [ + "AzureKubernetesService" + ], + "sourceAddresses": [ + "[parameters('spokeSubnetAKSPrefix')]" + ] + } + ] + } + } + ], + "metadata": { + "description": "Returns the firewall application rule collections" + } + }, + "fwnetworkRuleCollections": { + "type": "array", + "defaultValue": [ + { + "name": "AKS-egress", + "properties": { + "priority": 200, + "action": { + "type": "Allow" + }, + "rules": [ + { + "name": "NTP", + "protocols": [ + "UDP" + ], + "sourceAddresses": [ + "[parameters('spokeSubnetAKSPrefix')]" + ], + "destinationAddresses": [ + "*" + ], + "destinationPorts": [ + "123" + ] + }, + { + "name": "APITCP", + "protocols": [ + "TCP" + ], + "sourceAddresses": [ + "[parameters('spokeSubnetAKSPrefix')]" + ], + "destinationAddresses": [ + "*" + ], + "destinationPorts": [ + "9000" + ] + }, + { + "name": "APIUDP", + "protocols": [ + "UDP" + ], + "sourceAddresses": [ + "[parameters('spokeSubnetAKSPrefix')]" + ], + "destinationAddresses": [ + "*" + ], + "destinationPorts": [ + "1194" + ] + } + ] + } + } + ], + "metadata": { + "description": "Returns the firewall network rule collections" + } } }, + "variables": { + "telemetryId": "[format('0d807b2d-f7c3-4710-9a65-e88257df1ea0-{0}', parameters('location'))]" + }, "resources": [ { "type": "Microsoft.Resources/deployments", @@ -7356,6 +7609,64 @@ "[subscriptionResourceId('Microsoft.Resources/deployments', parameters('rgName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('rgName')), 'Microsoft.Resources/deployments', parameters('vnetHubName'))]" ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "telemetry", + "resourceGroup": "[parameters('rgName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "enableTelemetry": { + "value": "[parameters('enableTelemetry')]" + }, + "telemetryId": { + "value": "[variables('telemetryId')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.28.1.47646", + "templateHash": "2475204222112158833" + } + }, + "parameters": { + "enableTelemetry": { + "type": "bool" + }, + "telemetryId": { + "type": "string" + } + }, + "resources": [ + { + "condition": "[parameters('enableTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2021-04-01", + "name": "[parameters('telemetryId')]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "resources": {} + } + } + } + ] + } + }, + "dependsOn": [ + "[subscriptionResourceId('Microsoft.Resources/deployments', parameters('rgName'))]" + ] } ] } @@ -7375,6 +7686,9 @@ "rgName": { "value": "[parameters('rgSpokeName')]" }, + "enablePrivateCluster": { + "value": "[parameters('enablePrivateCluster')]" + }, "vnetSpokeName": { "value": "[parameters('vnetSpokeName')]" }, @@ -7413,6 +7727,24 @@ }, "securityRules": { "value": "[parameters('securityRules')]" + }, + "spokeSubnetDefaultPrefix": { + "value": "[parameters('spokeSubnetDefaultPrefix')]" + }, + "spokeSubnetAKSPrefix": { + "value": "[parameters('spokeSubnetAKSPrefix')]" + }, + "spokeSubnetAppGWPrefix": { + "value": "[parameters('spokeSubnetAppGWPrefix')]" + }, + "spokeSubnetVMPrefix": { + "value": "[parameters('spokeSubnetVMPrefix')]" + }, + "spokeSubnetPLinkervicePrefix": { + "value": "[parameters('spokeSubnetPLinkervicePrefix')]" + }, + "remotePeeringName": { + "value": "[parameters('remotePeeringName')]" } }, "template": { @@ -7422,7 +7754,7 @@ "_generator": { "name": "bicep", "version": "0.28.1.47646", - "templateHash": "12902392534524655418" + "templateHash": "1962386068872565768" } }, "parameters": { @@ -7459,6 +7791,10 @@ "rtAppGWSubnetName": { "type": "string" }, + "enablePrivateCluster": { + "type": "bool", + "defaultValue": true + }, "location": { "type": "string", "defaultValue": "[deployment().location]" @@ -7472,6 +7808,30 @@ "securityRules": { "type": "array", "defaultValue": [] + }, + "spokeSubnetDefaultPrefix": { + "type": "string", + "defaultValue": "10.1.0.0/24" + }, + "spokeSubnetAKSPrefix": { + "type": "string", + "defaultValue": "10.1.1.0/24" + }, + "spokeSubnetAppGWPrefix": { + "type": "string", + "defaultValue": "10.1.2.0/27" + }, + "spokeSubnetVMPrefix": { + "type": "string", + "defaultValue": "10.1.3.0/24" + }, + "spokeSubnetPLinkervicePrefix": { + "type": "string", + "defaultValue": "10.1.4.0/24" + }, + "remotePeeringName": { + "type": "string", + "defaultValue": "spoke-hub-peering" } }, "variables": { @@ -7965,24 +8325,24 @@ "value": [ { "name": "default", - "addressPrefix": "10.1.0.0/24" + "addressPrefix": "[parameters('spokeSubnetDefaultPrefix')]" }, { "name": "AKS", - "addressPrefix": "10.1.1.0/24", + "addressPrefix": "[parameters('spokeSubnetAKSPrefix')]", "routeTableResourceId": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('rgName')), 'Microsoft.Resources/deployments', parameters('rtAKSSubnetName')), '2022-09-01').outputs.resourceId.value]" }, { "name": "AppGWSubnet", - "addressPrefix": "10.1.2.0/27" + "addressPrefix": "[parameters('spokeSubnetAppGWPrefix')]" }, { "name": "vmsubnet", - "addressPrefix": "10.1.3.0/24" + "addressPrefix": "[parameters('spokeSubnetVMPrefix')]" }, { "name": "servicespe", - "addressPrefix": "10.1.4.0/24" + "addressPrefix": "[parameters('spokeSubnetPLinkervicePrefix')]" } ] }, @@ -7998,7 +8358,7 @@ "remotePeeringAllowForwardedTraffic": true, "remotePeeringAllowVirtualNetworkAccess": true, "remotePeeringEnabled": true, - "remotePeeringName": "spoke-hub-peering", + "remotePeeringName": "[parameters('remotePeeringName')]", "remoteVirtualNetworkId": "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('vnetHUBRGName')), 'Microsoft.Network/virtualNetworks', parameters('vnetHubName'))]", "useRemoteGateways": false } @@ -18574,6 +18934,7 @@ ] }, { + "condition": "[parameters('enablePrivateCluster')]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", "name": "privatednsAKSZone", @@ -37534,6 +37895,9 @@ "rgName": { "value": "[parameters('rgSpokeName')]" }, + "enablePrivateCluster": { + "value": "[parameters('enablePrivateCluster')]" + }, "vnetName": { "value": "[parameters('vnetSpokeName')]" }, @@ -37575,7 +37939,7 @@ "_generator": { "name": "bicep", "version": "0.28.1.47646", - "templateHash": "14695252208240220213" + "templateHash": "1662336602186095921" } }, "parameters": { @@ -37622,6 +37986,10 @@ "aksClusterName": { "type": "string" }, + "enablePrivateCluster": { + "type": "bool", + "defaultValue": true + }, "networkPlugin": { "type": "string", "allowedValues": [ @@ -40153,11 +40521,9 @@ }, "podCidr": "[if(equals(parameters('networkPlugin'), 'kubenet'), createObject('value', '172.17.0.0/16'), createObject('value', null()))]", "enablePrivateCluster": { - "value": true - }, - "privateDNSZone": { - "value": "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('rgName')), 'Microsoft.Network/privateDnsZones', variables('privateDNSZoneAKSName'))]" + "value": "[parameters('enablePrivateCluster')]" }, + "privateDNSZone": "[if(parameters('enablePrivateCluster'), createObject('value', extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('rgName')), 'Microsoft.Network/privateDnsZones', variables('privateDNSZoneAKSName'))), createObject('value', null()))]", "enablePrivateClusterPublicFQDN": { "value": false }, diff --git a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.portal.ui.json b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.portal.ui.json index 019260d7..a1669f98 100644 --- a/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.portal.ui.json +++ b/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/All-in-One-Bicep/main.portal.ui.json @@ -114,6 +114,22 @@ "name": "hubnetwork", "label": "Hub network settings", "elements": [ + { + "name": "deployHub", + "type": "Microsoft.Common.CheckBox", + "label": "Deploy Hub", + "subLabel": "", + "defaultValue": true, + "toolTip": "Enable this if you don't already have a hub network. Best to already have a hub network using Azure Landing Zones. You will still have to provide the information about your existing hub network in this page and ensure the virtual network is preconfigued with the subnets in this page, a Bastion host and an Azure firewall.", + "constraints": { + "required": true, + "regex": "", + "validationMessage": "", + "validations": [] + }, + "infoMessages": [], + "visible": true + }, { "name": "vnetAddressPrefixes", "type": "Microsoft.Common.TextBox", @@ -393,6 +409,110 @@ }, "infoMessages": [], "visible": true + }, + + { + "name": "spokeSubnetDefaultPrefix", + "type": "Microsoft.Common.TextBox", + "label": "Spoke network default subnet address space", + "subLabel": "", + "defaultValue": "10.1.0.0/24", + "placeholder": "10.1.0.0/24", + "toolTip": "Address space for the default subnet within the spoke vnet.", + "constraints": { + "required": true, + "regex": "", + "validationMessage": "", + "validations": [] + }, + "infoMessages": [], + "visible": true + }, + { + "name": "spokeSubnetAKSPrefix", + "type": "Microsoft.Common.TextBox", + "label": "Spoke network AKS subnet address space", + "subLabel": "", + "defaultValue": "10.1.1.0/24", + "placeholder": "10.1.1.0/24", + "toolTip": "Address space for the subnet AKS will be deployed to within the spoke vnet.", + "constraints": { + "required": true, + "regex": "", + "validationMessage": "", + "validations": [] + }, + "infoMessages": [], + "visible": true + }, + { + "name": "spokeSubnetAppGWPrefix", + "type": "Microsoft.Common.TextBox", + "label": "Spoke network AGW subnet address space", + "subLabel": "", + "defaultValue": "10.1.2.0/27", + "placeholder": "10.1.2.0/27", + "toolTip": "Address space for the subnet App gateway will be deployed to within the spoke vnet.", + "constraints": { + "required": true, + "regex": "", + "validationMessage": "", + "validations": [] + }, + "infoMessages": [], + "visible": true + }, + { + "name": "spokeSubnetVMPrefix", + "type": "Microsoft.Common.TextBox", + "label": "Spoke network Jumpbox subnet address space", + "subLabel": "", + "defaultValue": "10.1.3.0/24", + "placeholder": "10.1.3.0/24", + "toolTip": "Address space for the subnet Jumpbox will be deployed to within the spoke vnet.", + "constraints": { + "required": true, + "regex": "", + "validationMessage": "", + "validations": [] + }, + "infoMessages": [], + "visible": true + }, + + { + "name": "spokeSubnetPLinkervicePrefix", + "type": "Microsoft.Common.TextBox", + "label": "Spoke network PLink subnet address space", + "subLabel": "", + "defaultValue": "10.1.4.0/16", + "placeholder": "10.1.4.0/16", + "toolTip": "Address space for the private link services that will be deployed to within the spoke vnet.", + "constraints": { + "required": true, + "regex": "", + "validationMessage": "", + "validations": [] + }, + "infoMessages": [], + "visible": true + }, + { + "name": "remotePeeringName", + "type": "Microsoft.Common.TextBox", + "label": "Hub to spoke network peering name", + "subLabel": "", + "defaultValue": "spoke-hub-peering", + "placeholder": "spoke-hub-peering", + "toolTip": "Name that will be used for the network peering between the hub and spoke network.", + "constraints": { + "required": false, + "regex": "", + "validationMessage": "", + "validations": [] + }, + "infoMessages": [], + "visible": true } ] }, @@ -431,7 +551,7 @@ "text": "The new cluster will be created with Azure Entra (AAD) integration and members of the given security group will have the ClusterAdmin role inside Kubernetes.", "link": { "label": "Create the required security group before running this deployment.", - "uri": "https://github.com/Azure/AKS-Landing-Zone-Accelerator/blob/main/Scenarios/AKS-Secure-Baseline-Private-AVM/Bicep/02-eid.md" + "uri": "https://github.com/Azure/AKS-Landing-Zone-Accelerator/blob/main/Scenarios/AKS-Secure-Baseline-PrivateCluster/Bicep/02-eid.md" } } } @@ -469,6 +589,22 @@ } ] }, + { + "name": "enablePrivateCluster", + "type": "Microsoft.Common.CheckBox", + "label": "Make cluster private", + "subLabel": "", + "defaultValue": true, + "toolTip": "Check this box to make the cluster a private cluster.", + "constraints": { + "required": true, + "regex": "", + "validationMessage": "", + "validations": [] + }, + "infoMessages": [], + "visible": true + }, { "name": "aksSubnetName", "type": "Microsoft.Common.TextBox",