From 1991374acd45c35eb11b6108eb9136537799e7a1 Mon Sep 17 00:00:00 2001 From: Mariano Grande Date: Thu, 15 Sep 2022 13:01:20 -0300 Subject: [PATCH] AKS Baseline regulated - Bicep migration - Networking files (#63) Co-authored-by: Chad Kittel Co-authored-by: Fernando Antivero --- docs/deploy/04-subscription.md | 5 +- docs/deploy/05-networking-hub.md | 4 +- docs/deploy/06-aks-jumpboximage.md | 8 +- docs/deploy/08-cluster-networking.md | 6 +- modules/subscriptionPolicyAssignment.bicep | 4 +- networking/README.md | 10 +- networking/hub-region.v0.bicep | 547 +++++++ networking/hub-region.v0.json | 673 --------- networking/hub-region.v1.bicep | 787 ++++++++++ networking/hub-region.v1.json | 897 ------------ networking/hub-region.v2.bicep | 1024 +++++++++++++ networking/hub-region.v2.json | 1130 --------------- ...sterVNetShouldNotHaveNICwithpublicIP.bicep | 36 + networking/modules/flowlogsDeployment.bicep | 52 + .../modules/hubsSpokesPeeringDeployment.bicep | 39 + networking/spoke-BU0001A0005-00.bicep | 317 ++++ networking/spoke-BU0001A0005-00.json | 384 ----- networking/spoke-BU0001A0005-01.bicep | 1023 +++++++++++++ networking/spoke-BU0001A0005-01.json | 1284 ----------------- 19 files changed, 3843 insertions(+), 4387 deletions(-) create mode 100644 networking/hub-region.v0.bicep delete mode 100644 networking/hub-region.v0.json create mode 100644 networking/hub-region.v1.bicep delete mode 100644 networking/hub-region.v1.json create mode 100644 networking/hub-region.v2.bicep delete mode 100644 networking/hub-region.v2.json create mode 100644 networking/modules/ClusterVNetShouldNotHaveNICwithpublicIP.bicep create mode 100644 networking/modules/flowlogsDeployment.bicep create mode 100644 networking/modules/hubsSpokesPeeringDeployment.bicep create mode 100644 networking/spoke-BU0001A0005-00.bicep delete mode 100644 networking/spoke-BU0001A0005-00.json create mode 100644 networking/spoke-BU0001A0005-01.bicep delete mode 100644 networking/spoke-BU0001A0005-01.json diff --git a/docs/deploy/04-subscription.md b/docs/deploy/04-subscription.md index d0363afe..b21f5e0d 100644 --- a/docs/deploy/04-subscription.md +++ b/docs/deploy/04-subscription.md @@ -118,9 +118,8 @@ It is recommended that your Azure _subscription_ have the **Azure Security Bench 1. Open the [**Regulatory Compliance** screen in Microsoft Defender for Cloud](https://portal.azure.com/#blade/Microsoft_Azure_Security/SecurityMenuBlade/22) 1. Click on **Manage compliance policies** -1. Click on your subscription -1. Ensure the **Azure Security Benchmark** is applied as the **Microsoft Defender for Cloud default policy**. -1. You'll want to ensure all relevant standards (e.g. **PCI DSS 3.2.1**) are **Enabled** under **Industry & regulatory standards** +1. Expand the tree, and click on your subscription +1. Navigate to **Security policy** blade since you'll want to ensure the **Azure Security Benchmark** and all relevant standards (e.g. **PCI DSS 3.2.1**) are **Enabled** under **Industry & regulatory standards** 1. The **Regulatory Compliance** dashboard in Microsoft Defender for Cloud might take an hour or two to reflect any modifications you've made. **None of the above is required for this walkthrough**, but we want to ensure you're aware of these subscription-level policies and how you can enable them for your final implementation. All subscriptions containing PCI workloads should have the **PCI DSS 3.2.1** Industry & regulatory standards reports enabled, which _requires_ that the **Azure Security Benchmark** is applied as the default policy. diff --git a/docs/deploy/05-networking-hub.md b/docs/deploy/05-networking-hub.md index 1a5fe31d..8ceb4ec5 100644 --- a/docs/deploy/05-networking-hub.md +++ b/docs/deploy/05-networking-hub.md @@ -12,7 +12,7 @@ Egressing your spoke traffic through a hub network (following the hub-spoke mode After executing these steps you'll have the `rg-enterprise-networking-hubs` resource group populated with a regional virtual network (vnet), Azure Firewall, Azure Bastion, and Azure Monitor & Flow Log storage for network observability. No spokes will have been created yet, so the default firewall rules are maximally restrictive, as there is no expected outflow of traffic so none is allowed. We'll open up access on an as-needed bases throughout this walk through. -Specifically, you'll see networking/hub-region.v​_n_.json referenced a couple times in this process. Think of this as an evolution of a _single_ ARM template as the number and needs of the connected spokes change over time. You can diff the v​_n_ and v​_n+1_ ARM templates to see this progression over time. Typically your network team would have encapsulated this hub in a file named something like `hub-eastus2.json` and updated it in their source control as dependencies/requirements dictate. It likely would have not taken as many parameters as either, as those would be constants that could be easily defined directly in the template as the file would be specific to the region's spokes. To keep this reference implementation more flexible on what region you select, you'll be asked to provide deployment parameters and the filename can remain the generic name of hub-​_region_. +Specifically, you'll see networking/hub-region.v​_n_.bicep referenced a couple times in this process. Think of this as an evolution of a _single_ ARM template as the number and needs of the connected spokes change over time. You can diff the v​_n_ and v​_n+1_ ARM templates to see this progression over time. Typically your network team would have encapsulated this hub in a file named something like `hub-eastus2.bicep` and updated it in their source control as dependencies/requirements dictate. It likely would have not taken as many parameters as either, as those would be constants that could be easily defined directly in the template as the file would be specific to the region's spokes. To keep this reference implementation more flexible on what region you select, you'll be asked to provide deployment parameters and the filename can remain the generic name of hub-​_region_. The examples that follow use `eastus2` as the primary region. You're welcome to change this in the ARM template parameters throughout this walkthrough. Clusters are regional resources; and the expectation is that your regional hub, regional spoke, and regional workload are all sharing the same region. So if you make a change to the region, be sure you change it in _all_ places along the way. For a reference architecture of a general-purpose, multi-region cluster, see [AKS Baseline for Multi-Region Topology](https://github.com/mspnp/aks-baseline-multi-region). @@ -29,7 +29,7 @@ Since this walkthrough is expected to be deployed isolated from existing infrast ```bash # [This takes about eight minutes to run.] - az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v0.json -p location=eastus2 + az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v0.bicep -p location=eastus2 ``` The hub deployment will output the following: diff --git a/docs/deploy/06-aks-jumpboximage.md b/docs/deploy/06-aks-jumpboximage.md index e6f02cdb..8aa268d1 100644 --- a/docs/deploy/06-aks-jumpboximage.md +++ b/docs/deploy/06-aks-jumpboximage.md @@ -40,20 +40,20 @@ You are going to be using Azure Image Builder to generate a Kubernetes-specific RESOURCEID_VNET_HUB=$(az deployment group show -g rg-enterprise-networking-hubs -n hub-region.v0 --query properties.outputs.hubVnetId.value -o tsv) # [This takes about one minute to run.] - az deployment group create -g rg-enterprise-networking-spokes -f networking/spoke-BU0001A0005-00.json -p location=eastus2 hubVnetResourceId="${RESOURCEID_VNET_HUB}" + az deployment group create -g rg-enterprise-networking-spokes -f networking/spoke-BU0001A0005-00.bicep -p location=eastus2 hubVnetResourceId="${RESOURCEID_VNET_HUB}" ``` 1. Update the regional hub deployment to account for the requirements of the spoke. Now that the first spoke network is created, the hub network's firewall needs to be updated to support the Azure Image Builder process that will execute in there. The hub firewall does NOT have any default permissive egress rules, and as such, each needed egress endpoint needs to be specifically allowed. This deployment builds on the prior with the added allowances in the firewall. - > :eyes: If you're curious to see what changed in the regional hub, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v0.json&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v1.json). + > :eyes: If you're curious to see what changed in the regional hub, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v0.bicep&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v1.bicep). ```bash RESOURCEID_SUBNET_AIB=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-00 --query properties.outputs.imageBuilderSubnetResourceId.value -o tsv) # [This takes about five minutes to run.] - az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v1.json -p location=eastus2 aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}" + az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v1.bicep -p location=eastus2 aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}" ``` ### Build and deploy the jump box image @@ -62,7 +62,7 @@ Now that we have our image building network created, egressing through our hub, 1. Download the ARM templates from the AKS Jump Box Image Builder repository. - Ideally core templates like this would be part of your private Bicep registry. For this walk through, we are simply downloading the remote Bicep templates locally for execution. + Ideally core templates like this would be part of your private Bicep registry. For this walk through, we are simply downloading the remote ARM templates locally for execution. ```bash wget -B https://raw.githubusercontent.com/mspnp/aks-jumpbox-imagebuilder/main/ -x -nH --cut-dirs=3 -i jumpbox/jumpbox-bicep.txt -P jumpbox diff --git a/docs/deploy/08-cluster-networking.md b/docs/deploy/08-cluster-networking.md index 7355f1da..8de40929 100644 --- a/docs/deploy/08-cluster-networking.md +++ b/docs/deploy/08-cluster-networking.md @@ -23,14 +23,14 @@ Your `rg-enterprise-networking-spokes` will be populated with the dedicated regi RESOURCEID_VNET_HUB=$(az deployment group show -g rg-enterprise-networking-hubs -n hub-region.v0 --query properties.outputs.hubVnetId.value -o tsv) # [This takes about five minutes to run.] - az deployment group create -g rg-enterprise-networking-spokes -f networking/spoke-BU0001A0005-01.json -p location=eastus2 hubVnetResourceId="${RESOURCEID_VNET_HUB}" + az deployment group create -g rg-enterprise-networking-spokes -f networking/spoke-BU0001A0005-01.bicep -p location=eastus2 hubVnetResourceId="${RESOURCEID_VNET_HUB}" ``` 1. Update the regional hub deployment to account for the runtime requirements of the virtual network. This is an evolution of same hub template you used before, but now updated with Azure Firewall rules specific to this AKS cluster infrastructure. - > :eyes: If you're curious to see what changed in the regional hub, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v1.json&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v2.json). + > :eyes: If you're curious to see what changed in the regional hub, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v1.bicep&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v2.bicep). ```bash RESOURCEID_SUBNET_AIB=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-00 --query properties.outputs.imageBuilderSubnetResourceId.value -o tsv) @@ -38,7 +38,7 @@ Your `rg-enterprise-networking-spokes` will be populated with the dedicated regi RESOURCEID_SUBNET_JUMPBOX=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-01 --query properties.outputs.jumpboxSubnetResourceId.value -o tsv) # [This takes about seven minutes to run.] - az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v2.json -p location=eastus2 aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}" nodepoolSubnetResourceIds="${RESOURCEID_SUBNET_NODEPOOLS}" aksJumpboxSubnetResourceId="${RESOURCEID_SUBNET_JUMPBOX}" + az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v2.bicep -p location=eastus2 aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}" nodepoolSubnetResourceIds="${RESOURCEID_SUBNET_NODEPOOLS}" aksJumpboxSubnetResourceId="${RESOURCEID_SUBNET_JUMPBOX}" ``` ### Next step diff --git a/modules/subscriptionPolicyAssignment.bicep b/modules/subscriptionPolicyAssignment.bicep index 75c12330..3dd5adf1 100644 --- a/modules/subscriptionPolicyAssignment.bicep +++ b/modules/subscriptionPolicyAssignment.bicep @@ -13,7 +13,7 @@ param enforcementMode string = 'Default' @minLength(4) param location string -@description('The name of the policy set to assign.') +@description('The name of the policy or policy set to assign.') @minLength(36) @maxLength(36) param policyDefinitionSetName string @@ -33,7 +33,7 @@ var builtIntPolicyDefinitionSetId = subscriptionResourceId('Microsoft.Authorizat @description('Assignment of policy') resource policyAssignment 'Microsoft.Authorization/policyAssignments@2021-06-01' = { - name: guid(builtIntPolicyDefinitionSetId) + name: guid(builtIntPolicyDefinitionSetId, subscription().id) identity: { type: 'SystemAssigned' } diff --git a/networking/README.md b/networking/README.md index f2ad4146..45e2d480 100644 --- a/networking/README.md +++ b/networking/README.md @@ -4,11 +4,11 @@ This is part of the Azure Kubernetes Service (AKS) Baseline Cluster for Regulate ## Files -* [`hub-region.v0.json`](./hub-region.v0.json) is a file that defines a generic regional hub. All regional hubs can generally be considered a fork of this base template. -* [`hub-region.v1.json`](./hub-region.v1.json) is an updated version that defines a specific region's hub (for example, it might be named `hub-eastus2.json`). This is the long-lived template that defines this specific region's hub. This version has support for our image builder process. -* [`hub-region.v2.json`](./hub-region.v2.json) is an even more updated version that defines a specific region's hub (it would still be the same named `hub-eastus2.json` file). This version has support for our image builder process plus our AKS cluster's needs. -* [`spoke-BU0001A0005-00.json`](./spoke-BU0001A0005-00.json) is a file that defines a specific spoke in the topology. A spoke is created for each workload in a business unit, hence the naming pattern in the file name. This spoke contains the networking resources for the image builder process. -* [`spoke-BU0001A0005-01.json`](./spoke-BU0001A0005-01.json) is a file that defines a specific spoke in the topology. This spoke contains the networking resources for the AKS cluster. +* [`hub-region.v0.bicep`](./hub-region.v0.bicep) is a file that defines a generic regional hub. All regional hubs can generally be considered a fork of this base template. +* [`hub-region.v1.bicep`](./hub-region.v1.bicep) is an updated version that defines a specific region's hub (for example, it might be named `hub-eastus2.bicep`). This is the long-lived template that defines this specific region's hub. This version has support for our image builder process. +* [`hub-region.v2.bicep`](./hub-region.v2.bicep) is an even more updated version that defines a specific region's hub (it would still be the same named `hub-eastus2.bicep` file). This version has support for our image builder process plus our AKS cluster's needs. +* [`spoke-BU0001A0005-00.bicep`](./spoke-BU0001A0005-00.bicep) is a file that defines a specific spoke in the topology. A spoke is created for each workload in a business unit, hence the naming pattern in the file name. This spoke contains the networking resources for the image builder process. +* [`spoke-BU0001A0005-01.bicep`](./spoke-BU0001A0005-01.bicep) is a file that defines a specific spoke in the topology. This spoke contains the networking resources for the AKS cluster. Your organization will likely have its own standards for their hub-spoke or vwan implementation. Be sure to follow your organizational guidelines. diff --git a/networking/hub-region.v0.bicep b/networking/hub-region.v0.bicep new file mode 100644 index 00000000..6b192854 --- /dev/null +++ b/networking/hub-region.v0.bicep @@ -0,0 +1,547 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@allowed([ + 'australiaeast' + 'canadacentral' + 'centralus' + 'eastus' + 'eastus2' + 'westus2' + 'francecentral' + 'germanywestcentral' + 'northeurope' + 'southafricanorth' + 'southcentralus' + 'uksouth' + 'westeurope' + 'japaneast' + 'southeastasia' +]) +@description('The hub\'s regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support.') +@minLength(4) +param location string + +@description('A /24 to contain the regional firewall, management, and gateway subnet') +@minLength(10) +@maxLength(18) +param hubVnetAddressSpace string = '10.200.0.0/24' + +@description('A /26 under the VNet Address Space for the regional Azure Firewall') +@minLength(10) +@maxLength(18) +param azureFirewallSubnetAddressSpace string = '10.200.0.0/26' + +@description('A /27 under the VNet Address Space for our regional On-Prem Gateway') +@minLength(10) +@maxLength(18) +param azureGatewaySubnetAddressSpace string = '10.200.0.64/27' + +@description('A /27 under the VNet Address Space for regional Azure Bastion') +@minLength(10) +@maxLength(18) +param azureBastionSubnetAddressSpace string = '10.200.0.96/27' + +@description('Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing "false" to this parameter.') +param deployFlowLogResources bool = true + +/*** EXISTING RESOURCES ***/ + +resource networkWatcherResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = if (deployFlowLogResources) { + scope: subscription() + name: 'networkWatcherRG' +} + +/*** RESOURCES ***/ + +@description('This Log Analytics workspace stores logs from the regional hub network, its spokes, and bastion. Log analytics is a regional resource, as such there will be one workspace per hub (region)') +resource laHub 'Microsoft.OperationalInsights/workspaces@2021-06-01' = { + name: 'la-hub-${location}-${uniqueString(resourceGroup().id, 'vnet-${location}-hub')}' + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 90 + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + } +} + +@description('Enables Azure Sentinal integration.') +var laHubSolutionName = 'SecurityInsights(${laHub.name})' +resource laHubSolution 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: laHubSolutionName + location: location + plan: { + name: laHubSolutionName + promotionCode: '' + product: 'OMSGallery/SecurityInsights' + publisher: 'Microsoft' + } + properties: { + workspaceResourceId: laHub.id + } +} + +@description('Wraps the AzureBastion subnet in this regional hub. Source: https://docs.microsoft.com/azure/bastion/bastion-nsg') +resource nsgBastionSubnet 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-${location}-bastion' + location: location + properties: { + securityRules: [ + { + name: 'AllowWebExperienceInbound' + properties: { + description: 'Allow our users in. Update this to be as restrictive as possible.' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '443' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 100 + direction: 'Inbound' + } + } + { + name: 'AllowControlPlaneInbound' + properties: { + description: 'Service Requirement. Allow control plane access. Regional Tag not yet supported.' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '443' + sourceAddressPrefix: 'GatewayManager' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 110 + direction: 'Inbound' + } + } + { + name: 'AllowHealthProbesInbound' + properties: { + description: 'Service Requirement. Allow Health Probes.' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '443' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 120 + direction: 'Inbound' + } + } + { + name: 'AllowBastionHostToHostInbound' + properties: { + description: 'Service Requirement. Allow Required Host to Host Communication.' + protocol: '*' + sourcePortRange: '*' + destinationPortRanges: [ + '8080' + '5701' + ] + sourceAddressPrefix: 'VirtualNetwork' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 130 + direction: 'Inbound' + } + } + { + name: 'DenyAllInbound' + properties: { + description: 'No further inbound traffic allowed.' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Inbound' + } + } + { + name: 'AllowSshToVnetOutbound' + properties: { + description: 'Allow SSH out to the virtual network' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '22' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 100 + direction: 'Outbound' + } + } + { + name: 'AllowRdpToVnetOutbound' + properties: { + description: 'Allow RDP out to the virtual network' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '3389' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 110 + direction: 'Outbound' + } + } + { + name: 'AllowControlPlaneOutbound' + properties: { + description: 'Required for control plane outbound. Regional prefix not yet supported' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '443' + destinationAddressPrefix: 'AzureCloud' + access: 'Allow' + priority: 120 + direction: 'Outbound' + } + } + { + name: 'AllowBastionHostToHostOutbound' + properties: { + description: 'Service Requirement. Allow Required Host to Host Communication.' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRanges: [ + '8080' + '5701' + ] + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 130 + direction: 'Outbound' + } + } + { + name: 'AllowBastionCertificateValidationOutbound' + properties: { + description: 'Service Requirement. Allow Required Session and Certificate Validation.' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '80' + destinationAddressPrefix: 'Internet' + access: 'Allow' + priority: 140 + direction: 'Outbound' + } + } + { + name: 'DenyAllOutbound' + properties: { + description: 'No further outbound traffic allowed.' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Outbound' + } + } + ] + } +} + +resource nsgBastionSubnet_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: nsgBastionSubnet + properties: { + workspaceId: laHub.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('The regional hub network') +resource vnetHub 'Microsoft.Network/virtualNetworks@2021-05-01' = { + name: 'vnet-${location}-hub' + location: location + properties: { + addressSpace: { + addressPrefixes: [ + hubVnetAddressSpace + ] + } + subnets: [ + { + name: 'AzureFirewallSubnet' + properties: { + addressPrefix: azureFirewallSubnetAddressSpace + } + } + { + name: 'GatewaySubnet' + properties: { + addressPrefix: azureGatewaySubnetAddressSpace + } + } + { + name: 'AzureBastionSubnet' + properties: { + addressPrefix: azureBastionSubnetAddressSpace + networkSecurityGroup: { + id: nsgBastionSubnet.id + } + } + } + ] + } + + resource azureFirewallSubnet 'subnets' existing = { + name: 'AzureFirewallSubnet' + } + + resource azureBastionSubnet 'subnets' existing = { + name: 'AzureBastionSubnet' + } +} + +resource vnetHub_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: vnetHub + properties: { + workspaceId: laHub.id + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('Allocate three public IP addresses to the firewall') +var numFirewallIpAddressesToAssign = 3 +resource pipsAzureFirewall 'Microsoft.Network/publicIPAddresses@2021-05-01' = [for i in range(0, numFirewallIpAddressesToAssign): { + name: 'pip-fw-${location}-${padLeft(i, 2, '0')}' + location: location + sku: { + name: 'Standard' + } + zones: [ + '1' + '2' + '3' + ] + properties: { + publicIPAllocationMethod: 'Static' + idleTimeoutInMinutes: 4 + publicIPAddressVersion: 'IPv4' + } +}] + +@description('The public IP for the regional hub\'s Azure Bastion service.') +resource pipAzureBastion 'Microsoft.Network/publicIPAddresses@2021-05-01' = { + name: 'pip-ab-${location}' + location: location + sku: { + name: 'Standard' + } + zones: [ + '1' + '2' + '3' + ] + properties: { + publicIPAllocationMethod: 'Static' + idleTimeoutInMinutes: 4 + publicIPAddressVersion: 'IPv4' + } +} + +@description('This regional hub\'s Azure Bastion service.') +resource azureBastion 'Microsoft.Network/bastionHosts@2021-05-01' = { + name: 'ab-${location}' + location: location + properties: { + ipConfigurations: [ + { + name: 'hub-subnet' + properties: { + privateIPAllocationMethod: 'Dynamic' + subnet: { + id: vnetHub::azureBastionSubnet.id + } + publicIPAddress: { + id: pipAzureBastion.id + } + } + } + ] + } +} + +resource azureBastion_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: azureBastion + properties: { + workspaceId: laHub.id + logs: [ + { + category: 'BastionAuditLogs' + enabled: true + } + ] + } +} + +@description('Storage account to store the flow logs') +resource flowlogs_storageAccount 'Microsoft.Storage/storageAccounts@2021-08-01' = { + name: substring('stnfl${location}${uniqueString(resourceGroup().id)}', 0, 24) + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + accessTier: 'Hot' + minimumTlsVersion: 'TLS1_2' + supportsHttpsTrafficOnly: true + allowBlobPublicAccess: false + allowSharedKeyAccess: false + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + ipRules: [] + } + } + + resource blobStorage 'blobServices' existing = { + name: 'default' + } +} + +resource flowlogs_storageAccount_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: flowlogs_storageAccount + properties: { + workspaceId: laHub.id + logs: [] + metrics: [ + { + category: 'Transaction' + enabled: true + } + ] + } +} + +resource region_flowlog_storageAccount_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2017-05-01-preview' = { + name: 'default' + scope: flowlogs_storageAccount::blobStorage + properties: { + workspaceId: laHub.id + logs: [ + { + category: 'StorageRead' + enabled: true + } + { + category: 'StorageWrite' + enabled: true + } + { + category: 'StorageDelete' + enabled: true + } + ] + metrics: [ + { + category: 'Transaction' + enabled: true + } + ] + } +} + +@description('This is the regional Azure Firewall that all regional spoke networks can egress through.') +resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' = { + name: 'fw-${location}' + location: location + zones: [ + '1' + '2' + '3' + ] + properties: { + additionalProperties: { + 'Network.DNS.EnableProxy': 'true' + } + sku: { + tier: 'Premium' + name: 'AZFW_VNet' + } + threatIntelMode: 'Deny' + ipConfigurations: [for i in range(0, numFirewallIpAddressesToAssign): { + name: pipsAzureFirewall[i].name + properties: { + subnet: (0 == i) ? { + id: vnetHub::azureFirewallSubnet.id + } : null + publicIPAddress: { + id: pipsAzureFirewall[i].id + } + } + }] + natRuleCollections: [] + networkRuleCollections: [] + applicationRuleCollections: [] + } +} + +resource hubFirewall_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: hubFirewall + properties: { + workspaceId: laHub.id + logs: [ + { + categoryGroup: 'allLogs' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('Flow Logs deployment.') +module regionalFlowlogsDeployment 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'connect-hub-regional-flowlogs' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgBastionSubnet.id + laHubId: laHub.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +/*** OUTPUTS ***/ + +output hubVnetId string = vnetHub.id diff --git a/networking/hub-region.v0.json b/networking/hub-region.v0.json deleted file mode 100644 index 8c8c5bc7..00000000 --- a/networking/hub-region.v0.json +++ /dev/null @@ -1,673 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "location": { - "type": "string", - "allowedValues": [ - "australiaeast", - "canadacentral", - "centralus", - "eastus", - "eastus2", - "westus2", - "francecentral", - "germanywestcentral", - "northeurope", - "southafricanorth", - "southcentralus", - "uksouth", - "westeurope", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "The hub's regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support." - } - }, - "hubVnetAddressSpace": { - "defaultValue": "10.200.0.0/24", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /24 to contain the regional firewall, management, and gateway subnet" - } - }, - "azureFirewallSubnetAddressSpace": { - "defaultValue": "10.200.0.0/26", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /26 under the VNet Address Space for the regional Azure Firewall" - } - }, - "azureGatewaySubnetAddressSpace": { - "defaultValue": "10.200.0.64/27", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /27 under the VNet Address Space for our regional On-Prem Gateway" - } - }, - "azureBastionSubnetAddressSpace": { - "defaultValue": "10.200.0.96/27", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /27 under the VNet Address Space for regional Azure Bastion" - } - }, - "deployFlowLogResources": { - "defaultValue": true, - "type": "bool", - "metadata": { - "description": "Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing 'false' to this parameter." - } - } - }, - "variables": { - "baseFwPipName": "[concat('pip-fw-', parameters('location'))]", - "hubFwPipNames": [ - "[concat(variables('baseFwPipName'), '-default')]", - "[concat(variables('baseFwPipName'), '-01')]", - "[concat(variables('baseFwPipName'), '-02')]" - ], - "hubFwName": "[concat('fw-', parameters('location'))]", - "hubVNetName": "[concat('vnet-', parameters('location'), '-hub')]", - "bastionNetworkNsgName": "[concat('nsg-', parameters('location'), '-bastion')]", - "bastionPipName": "[concat('pip-ab-', parameters('location'))]", - "bastionServiceName": "[concat('ab-', parameters('location'))]", - "hubLaName": "[concat('la-hub-', parameters('location'), '-', uniqueString(resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))))]", - "regionFlowLowStorageAccountName": "[take(concat('stnfl', parameters('location'), uniqueString(resourceGroup().id)), 24)]", - "azureSentialLaSolutionName": "[concat('SecurityInsights(', variables('hubLaName'), ')')]" - }, - "resources": [ - { - "type": "Microsoft.OperationalInsights/workspaces", - "apiVersion": "2020-08-01", - "name": "[variables('hubLaName')]", - "location": "[parameters('location')]", - "comments": "This Log Analytics workspace stores logs from the regional hub network, its spokes, and bastion.", - "properties": { - "sku": { - "name": "PerGB2018" - }, - "retentionInDays": 90, - "publicNetworkAccessForIngestion": "Enabled", - "publicNetworkAccessForQuery": "Enabled" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[variables('azureSentialLaSolutionName')]", - "location": "[parameters('location')]", - "comments": "Enables Azure Sentinal integration.", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "plan": { - "name": "[variables('azureSentialLaSolutionName')]", - "promotionCode": "", - "product": "OMSGallery/SecurityInsights", - "publisher": "Microsoft" - }, - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - } - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[variables('bastionNetworkNsgName')]", - "location": "[parameters('location')]", - "comments": "Wraps the AzureBastion subnet in this regional hub. Source: https://docs.microsoft.com/azure/bastion/bastion-nsg", - "properties": { - "securityRules": [ - { - "name": "AllowWebExperienceInBound", - "properties": { - "description": "Allow our users in. Update this to be as restrictive as possible.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "Internet", - "destinationPortRange": "443", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 100, - "direction": "Inbound" - } - }, - { - "name": "AllowControlPlaneInBound", - "properties": { - "description": "Service Requirement. Allow control plane access. Regional Tag not yet supported.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "GatewayManager", - "destinationPortRange": "443", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 110, - "direction": "Inbound" - } - }, - { - "name": "AllowHealthProbesInBound", - "properties": { - "description": "Service Requirement. Allow Health Probes.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "AzureLoadBalancer", - "destinationPortRange": "443", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 120, - "direction": "Inbound" - } - }, - { - "name": "AllowBastionHostToHostInBound", - "properties": { - "description": "Service Requirement. Allow Required Host to Host Communication.", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRanges": [ - "8080", - "5701" - ], - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 130, - "direction": "Inbound" - } - }, - { - "name": "DenyAllInBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Inbound" - } - }, - { - "name": "AllowSshToVnetOutBound", - "properties": { - "description": "Allow SSH out to the VNet", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "22", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 100, - "direction": "Outbound" - } - }, - { - "name": "AllowRdpToVnetOutBound", - "properties": { - "protocol": "Tcp", - "description": "Unused in this RI, but required for ARM validation.", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "3389", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 110, - "direction": "Outbound" - } - }, - { - "name": "AllowControlPlaneOutBound", - "properties": { - "description": "Required for control plane outbound. Regional prefix not yet supported", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "443", - "destinationAddressPrefix": "AzureCloud", - "access": "Allow", - "priority": 120, - "direction": "Outbound" - } - }, - { - "name": "AllowBastionHostToHostOutBound", - "properties": { - "description": "Service Requirement. Allow Required Host to Host Communication.", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRanges": [ - "8080", - "5701" - ], - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 130, - "direction": "Outbound" - } - }, - { - "name": "AllowBastionCertificateValidationOutBound", - "properties": { - "description": "Service Requirement. Allow Required Session and Certificate Validation.", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "80", - "destinationAddressPrefix": "Internet", - "access": "Allow", - "priority": 140, - "direction": "Outbound" - } - }, - { - "name": "DenyAllOutBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Outbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/virtualNetworks", - "apiVersion": "2020-05-01", - "name": "[variables('hubVnetName')]", - "location": "[parameters('location')]", - "comments": "This is this region's hub network.", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]" - ], - "properties": { - "addressSpace": { - "addressPrefixes": [ - "[parameters('hubVnetAddressSpace')]" - ] - }, - "subnets": [ - { - "name": "AzureFirewallSubnet", - "properties": { - "addressPrefix": "[parameters('azureFirewallSubnetAddressSpace')]" - } - }, - { - "name": "GatewaySubnet", - "properties": { - "addressPrefix": "[parameters('azureGatewaySubnetAddressSpace')]" - } - }, - { - "name": "AzureBastionSubnet", - "properties": { - "addressPrefix": "[parameters('azureBastionSubnetAddressSpace')]", - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]" - } - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/publicIpAddresses", - "apiVersion": "2020-05-01", - "name": "[variables('hubFwPipNames')[copyIndex()]]", - "location": "[parameters('location')]", - "comments": "This is a public IP for this regional hub's firewall. You'll want as many as necessary to avoid SNAT port exhaustion. Typical production hubs may require 20 IPs -- we only deploy three here as a reference.", - "sku": { - "name": "Standard" - }, - "properties": { - "publicIPAllocationMethod": "Static", - "idleTimeoutInMinutes": 4, - "publicIPAddressVersion": "IPv4" - }, - "copy": { - "name": "create-fw-pips", - "count": "[length(variables('hubFwPipNames'))]", - "mode": "Parallel" - } - }, - { - "type": "Microsoft.Network/publicIpAddresses", - "apiVersion": "2020-05-01", - "name": "[variables('bastionPipName')]", - "location": "[parameters('location')]", - "comments": "This is the public IP for this regional hub's Azure Bastion service.", - "sku": { - "name": "Standard" - }, - "properties": { - "publicIPAllocationMethod": "Static", - "idleTimeoutInMinutes": 4, - "publicIPAddressVersion": "IPv4" - } - }, - { - "type": "Microsoft.Network/bastionHosts", - "apiVersion": "2020-05-01", - "name": "[variables('bastionServiceName')]", - "location": "[parameters('location')]", - "comments": "This is this regional hub's Azure Bastion service.", - "dependsOn": [ - "[resourceId('Microsoft.Network/publicIpAddresses', variables('bastionPipName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]" - ], - "properties": { - "ipConfigurations": [ - { - "name": "hub-subnet", - "properties": { - "privateIPAllocationMethod": "Dynamic", - "subnet": { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('hubVnetName'), 'AzureBastionSubnet')]" - }, - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('bastionPipName'))]" - } - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.Network/bastionHosts', variables('bastionServiceName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "category": "BastionAuditLogs", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2021-02-01", - "name": "[variables('regionFlowLowStorageAccountName')]", - "location": "[parameters('location')]", - "sku": { - "name": "standard_LRS" - }, - "kind": "StorageV2", - "properties": { - "accessTier": "Hot", - "minimumTlsVersion": "TLS1_2", - "supportsHttpsTrafficOnly": true, - "allowBlobPublicAccess": false, - "allowSharedKeyAccess": false, - "networkAcls": { - "bypass": "AzureServices", - "defaultAction": "Deny", - "ipRules": [] - } - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [], - "metrics": [ - { - "category": "Transaction", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Storage/storageAccounts/blobServices/providers/diagnosticsettings", - "apiVersion": "2017-05-01-preview", - "name": "[concat(variables('regionFlowLowStorageAccountName'), '/default/Microsoft.Insights/default')]", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "category": "StorageRead", - "enabled": true - }, - { - "category": "StorageWrite", - "enabled": true - }, - { - "category": "StorageDelete", - "enabled": true - } - ], - "metrics": [ - { - "category": "Transaction", - "enabled": true - } - ] - } - }, - { - "type": "Microsoft.Network/azureFirewalls", - "apiVersion": "2020-05-01", - "name": "[variables('hubFwName')]", - "location": "[parameters('location')]", - "comments": "This is the regional Azure Firewall that all regional spoke networks can egress through.", - "zones": [ - "1", - "2", - "3" - ], - "dependsOn": [ - "create-fw-pips", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]" - ], - "properties": { - "additionalProperties": { - "Network.DNS.EnableProxy": "true" - }, - "sku": { - "name": "AZFW_VNet", - "tier": "Standard" - }, - "threatIntelMode": "Deny", - "ipConfigurations": [ - { - "name": "[variables('hubFwPipNames')[0]]", - "properties": { - "subnet": { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('hubVnetName'), 'AzureFirewallSubnet')]" - }, - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('hubFwPipNames')[0])]" - } - } - }, - { - "name": "[variables('hubFwPipNames')[1]]", - "properties": { - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('hubFwPipNames')[1])]" - } - } - }, - { - "name": "[variables('hubFwPipNames')[2]]", - "properties": { - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('hubFwPipNames')[2])]" - } - } - } - ], - "natRuleCollections": [], - "networkRuleCollections": [], - "applicationRuleCollections": [] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/azureFirewalls', variables('hubFwName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "categoryGroup": "allLogs", - "enabled": true - } - ], - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "condition": "[parameters('deployFlowLogResources')]", - "name": "connect-hub-regional-flowlogs", - "type": "Microsoft.Resources/deployments", - "resourceGroup": "networkWatcherRG", - "apiVersion": "2020-10-01", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]", - "[resourceId('Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/azureFirewalls', variables('hubFwName'))]" // This doesn't depend on the FW, but because network watchers are auto-deployed, this helps prevents a race condition between the vnet creation triggering a NetworkWatcher auto provisioning, and the referencing of it here. - ], - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "variables": {}, - "resources": [ - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]", - "storageId": "[resourceId(resourceGroup().name, 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[resourceId(resourceGroup().name, 'Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - } - ] - } - } - } - ], - "outputs": { - "hubVnetId": { - "value": "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]", - "type": "string" - } - } -} diff --git a/networking/hub-region.v1.bicep b/networking/hub-region.v1.bicep new file mode 100644 index 00000000..ffd5b791 --- /dev/null +++ b/networking/hub-region.v1.bicep @@ -0,0 +1,787 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('Subnet resource Id for the AKS image builder subnet') +@minLength(79) +param aksImageBuilderSubnetResourceId string + +@allowed([ + 'australiaeast' + 'canadacentral' + 'centralus' + 'eastus' + 'eastus2' + 'westus2' + 'francecentral' + 'germanywestcentral' + 'northeurope' + 'southafricanorth' + 'southcentralus' + 'uksouth' + 'westeurope' + 'japaneast' + 'southeastasia' +]) +@description('The hub\'s regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support.') +@minLength(4) +param location string = 'eastus2' + +@description('A /24 to contain the regional firewall, management, and gateway subnet') +@minLength(10) +@maxLength(18) +param hubVnetAddressSpace string = '10.200.0.0/24' + +@description('A /26 under the VNet Address Space for the regional Azure Firewall') +@minLength(10) +@maxLength(18) +param azureFirewallSubnetAddressSpace string = '10.200.0.0/26' + +@description('A /27 under the VNet Address Space for our regional On-Prem Gateway') +@minLength(10) +@maxLength(18) +param azureGatewaySubnetAddressSpace string = '10.200.0.64/27' + +@description('A /27 under the VNet Address Space for regional Azure Bastion') +@minLength(10) +@maxLength(18) +param azureBastionSubnetAddressSpace string = '10.200.0.96/27' + +@description('Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing "false" to this parameter.') +param deployFlowLogResources bool = true + +/*** EXISTING RESOURCES ***/ + +@description('The resource group name containing virtual network in which Azure Image Builder will drop the compute into to perform the image build.') +resource rgBuilderVirutalNetwork 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { + scope: subscription() + name: split(aksImageBuilderSubnetResourceId, '/')[4] +} + +@description('AKS Spoke Virtual Network') +resource aksSpokeVnet 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { + scope: rgBuilderVirutalNetwork + name: split(aksImageBuilderSubnetResourceId, '/')[8] +} + +@description('AKS ImageBuilder subnet') +resource aksImageBuilderSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-01-01' existing = { + parent: aksSpokeVnet + name: last(split(aksImageBuilderSubnetResourceId, '/')) +} + +resource networkWatcherResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = if (deployFlowLogResources) { + scope: subscription() + name: 'networkWatcherRG' +} + +/*** RESOURCES ***/ + +@description('This Log Analytics workspace stores logs from the regional hub network, its spokes, and bastion. Log analytics is a regional resource, as such there will be one workspace per hub (region)') +resource laHub 'Microsoft.OperationalInsights/workspaces@2021-06-01' = { + name: 'la-hub-${location}-${uniqueString(resourceGroup().id, 'vnet-${location}-hub')}' + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 90 + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + } +} + +@description('Enables Azure Sentinal integration.') +var laHubSolutionName = 'SecurityInsights(${laHub.name})' +resource laHubSolution 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: laHubSolutionName + location: location + plan: { + name: laHubSolutionName + promotionCode: '' + product: 'OMSGallery/SecurityInsights' + publisher: 'Microsoft' + } + properties: { + workspaceResourceId: laHub.id + } +} + +@description('Wraps the AzureBastion subnet in this regional hub. Source: https://docs.microsoft.com/azure/bastion/bastion-nsg') +resource nsgBastionSubnet 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-${location}-bastion' + location: location + properties: { + securityRules: [ + { + name: 'AllowWebExperienceInbound' + properties: { + description: 'Allow our users in. Update this to be as restrictive as possible.' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '443' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 100 + direction: 'Inbound' + } + } + { + name: 'AllowControlPlaneInbound' + properties: { + description: 'Service Requirement. Allow control plane access. Regional Tag not yet supported.' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '443' + sourceAddressPrefix: 'GatewayManager' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 110 + direction: 'Inbound' + } + } + { + name: 'AllowHealthProbesInbound' + properties: { + description: 'Service Requirement. Allow Health Probes.' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '443' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 120 + direction: 'Inbound' + } + } + { + name: 'AllowBastionHostToHostInbound' + properties: { + description: 'Service Requirement. Allow Required Host to Host Communication.' + protocol: '*' + sourcePortRange: '*' + destinationPortRanges: [ + '8080' + '5701' + ] + sourceAddressPrefix: 'VirtualNetwork' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 130 + direction: 'Inbound' + } + } + { + name: 'DenyAllInbound' + properties: { + description: 'No further inbound traffic allowed.' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Inbound' + } + } + { + name: 'AllowSshToVnetOutbound' + properties: { + description: 'Allow SSH out to the virtual network' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '22' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 100 + direction: 'Outbound' + } + } + { + name: 'AllowRdpToVnetOutbound' + properties: { + description: 'Allow RDP out to the virtual network' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '3389' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 110 + direction: 'Outbound' + } + } + { + name: 'AllowControlPlaneOutbound' + properties: { + description: 'Required for control plane outbound. Regional prefix not yet supported' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '443' + destinationAddressPrefix: 'AzureCloud' + access: 'Allow' + priority: 120 + direction: 'Outbound' + } + } + { + name: 'AllowBastionHostToHostOutbound' + properties: { + description: 'Service Requirement. Allow Required Host to Host Communication.' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRanges: [ + '8080' + '5701' + ] + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 130 + direction: 'Outbound' + } + } + { + name: 'AllowBastionCertificateValidationOutbound' + properties: { + description: 'Service Requirement. Allow Required Session and Certificate Validation.' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '80' + destinationAddressPrefix: 'Internet' + access: 'Allow' + priority: 140 + direction: 'Outbound' + } + } + { + name: 'DenyAllOutbound' + properties: { + description: 'No further outbound traffic allowed.' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Outbound' + } + } + ] + } +} + +resource nsgBastionSubnet_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: nsgBastionSubnet + properties: { + workspaceId: laHub.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('The regional hub network') +resource vnetHub 'Microsoft.Network/virtualNetworks@2021-05-01' = { + name: 'vnet-${location}-hub' + location: location + properties: { + addressSpace: { + addressPrefixes: [ + hubVnetAddressSpace + ] + } + subnets: [ + { + name: 'AzureFirewallSubnet' + properties: { + addressPrefix: azureFirewallSubnetAddressSpace + } + } + { + name: 'GatewaySubnet' + properties: { + addressPrefix: azureGatewaySubnetAddressSpace + } + } + { + name: 'AzureBastionSubnet' + properties: { + addressPrefix: azureBastionSubnetAddressSpace + networkSecurityGroup: { + id: nsgBastionSubnet.id + } + } + } + ] + } + + resource azureFirewallSubnet 'subnets' existing = { + name: 'AzureFirewallSubnet' + } + + resource azureBastionSubnet 'subnets' existing = { + name: 'AzureBastionSubnet' + } +} + +resource vnetHub_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: vnetHub + properties: { + workspaceId: laHub.id + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('Allocate three public IP addresses to the firewall') +var numFirewallIpAddressesToAssign = 3 +resource pipsAzureFirewall 'Microsoft.Network/publicIPAddresses@2021-05-01' = [for i in range(0, numFirewallIpAddressesToAssign): { + name: 'pip-fw-${location}-${padLeft(i, 2, '0')}' + location: location + sku: { + name: 'Standard' + } + zones: [ + '1' + '2' + '3' + ] + properties: { + publicIPAllocationMethod: 'Static' + idleTimeoutInMinutes: 4 + publicIPAddressVersion: 'IPv4' + } +}] + +@description('The public IP for the regional hub\'s Azure Bastion service.') +resource pipAzureBastion 'Microsoft.Network/publicIPAddresses@2021-05-01' = { + name: 'pip-ab-${location}' + location: location + sku: { + name: 'Standard' + } + zones: [ + '1' + '2' + '3' + ] + properties: { + publicIPAllocationMethod: 'Static' + idleTimeoutInMinutes: 4 + publicIPAddressVersion: 'IPv4' + } +} + +@description('This regional hub\'s Azure Bastion service.') +resource azureBastion 'Microsoft.Network/bastionHosts@2021-05-01' = { + name: 'ab-${location}' + location: location + properties: { + ipConfigurations: [ + { + name: 'hub-subnet' + properties: { + privateIPAllocationMethod: 'Dynamic' + subnet: { + id: vnetHub::azureBastionSubnet.id + } + publicIPAddress: { + id: pipAzureBastion.id + } + } + } + ] + } +} + +resource azureBastion_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: azureBastion + properties: { + workspaceId: laHub.id + logs: [ + { + category: 'BastionAuditLogs' + enabled: true + } + ] + } +} + +@description('Storage account to store the flow logs') +resource flowlogs_storageAccount 'Microsoft.Storage/storageAccounts@2021-08-01' = { + name: substring('stnfl${location}${uniqueString(resourceGroup().id)}', 0, 24) + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + accessTier: 'Hot' + minimumTlsVersion: 'TLS1_2' + supportsHttpsTrafficOnly: true + allowBlobPublicAccess: false + allowSharedKeyAccess: false + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + ipRules: [] + } + } + + resource blobStorage 'blobServices' existing = { + name: 'default' + } +} + +resource flowlogs_storageAccount_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: flowlogs_storageAccount + properties: { + workspaceId: laHub.id + logs: [] + metrics: [ + { + category: 'Transaction' + enabled: true + } + ] + } +} + +@description('This holds IP addresses of known AKS Jumpbox image building subnets in attached spokes.') +resource imageBuilder_ipgroups 'Microsoft.Network/ipGroups@2021-05-01' = { + name: 'ipg-${location}-AksJumpboxImageBuilders' + location: location + properties: { + ipAddresses: [ + aksImageBuilderSubnet.properties.addressPrefix + ] + } +} + +resource region_flowlog_storageAccount_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2017-05-01-preview' = if (deployFlowLogResources) { + name: 'default' + scope: flowlogs_storageAccount::blobStorage + properties: { + workspaceId: laHub.id + logs: [ + { + category: 'StorageRead' + enabled: true + } + { + category: 'StorageWrite' + enabled: true + } + { + category: 'StorageDelete' + enabled: true + } + ] + metrics: [ + { + category: 'Transaction' + enabled: true + } + ] + } +} + +@description('This is the regional Azure Firewall that all regional spoke networks can egress through.') +resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' = { + name: 'fw-${location}' + location: location + zones: [ + '1' + '2' + '3' + ] + properties: { + additionalProperties: { + 'Network.DNS.EnableProxy': 'true' + } + sku: { + tier: 'Premium' + name: 'AZFW_VNet' + } + threatIntelMode: 'Deny' + ipConfigurations: [for i in range(0, numFirewallIpAddressesToAssign): { + name: pipsAzureFirewall[i].name + properties: { + subnet: (0 == i) ? { + id: vnetHub::azureFirewallSubnet.id + } : null + publicIPAddress: { + id: pipsAzureFirewall[i].id + } + } + }] + natRuleCollections: [] + networkRuleCollections: [ + { + name: 'allow-ntp' + properties: { + action: { + type: 'Allow' + } + priority: 100 + rules: [ + { + name: 'ntp' + description: 'Network Time Protocol (NTP) time synchronization for image builder VMs.' + sourceIpGroups: [ + resourceId('Microsoft.Network/ipGroups', imageBuilder_ipgroups.name) + ] + protocols: [ + 'UDP' + ] + destinationPorts: [ + '123' + ] + destinationFqdns: [ + 'ntp.ubuntu.com' + ] + } + ] + } + } + ] + applicationRuleCollections: [ + { + name: 'AKSImageBuilder-Requirements' + properties: { + action: { + type: 'Allow' + } + priority: 400 + rules: [ + { + name: 'to-azuremanagement' + description: 'This for AIB VMs to communicate with Azure management API.' + sourceIpGroups: [ + resourceId('Microsoft.Network/ipGroups', imageBuilder_ipgroups.name) + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ +#disable-next-line no-hardcoded-env-urls + 'management.azure.com' + ] + } + { + name: 'to-blobstorage' + description: 'This is required as the Proxy VM and Packer VM both read and write from transient storage accounts (no ability to know what storage accounts before the process starts.)' + sourceIpGroups: [ + resourceId('Microsoft.Network/ipGroups', imageBuilder_ipgroups.name) + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ +#disable-next-line no-hardcoded-env-urls + '*.blob.core.windows.net' + ] + } + { + name: 'apt-get' + description: 'This is required as the Packer VM performs a package upgrade. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + resourceId('Microsoft.Network/ipGroups', imageBuilder_ipgroups.name) + ] + protocols: [ + { + protocolType: 'Http' + port: 80 + } + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'azure.archive.ubuntu.com' + 'packages.microsoft.com' + 'archive.ubuntu.com' + 'security.ubuntu.com' + ] + } + { + name: 'install-azcli' + description: 'This is required as the Packer VM needs to install Azure CLI. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + resourceId('Microsoft.Network/ipGroups', imageBuilder_ipgroups.name) + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'aka.ms' +#disable-next-line no-hardcoded-env-urls + 'azurecliextensionsync.blob.core.windows.net' +#disable-next-line no-hardcoded-env-urls + 'azurecliprod.blob.core.windows.net' + ] + } + { + name: 'install-k8scli' + description: 'This is required as the Packer VM needs to install k8s cli tooling. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + resourceId('Microsoft.Network/ipGroups', imageBuilder_ipgroups.name) + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'objects.githubusercontent.com' + 'storage.googleapis.com' + 'api.github.com' + 'github-releases.githubusercontent.com' + 'github.com' + ] + } + { + name: 'install-helm' + description: 'This is required as the Packer VM needs to install helm cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + resourceId('Microsoft.Network/ipGroups', imageBuilder_ipgroups.name) + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'raw.githubusercontent.com' + 'get.helm.sh' + 'github-releases.githubusercontent.com' + ] + } + { + name: 'install-flux' + description: 'This is required as the Packer VMs needs to install flux cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + resourceId('Microsoft.Network/ipGroups', imageBuilder_ipgroups.name) + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'raw.githubusercontent.com' + 'api.github.com' + 'fluxcd.io' + 'github-releases.githubusercontent.com' + ] + } + { + name: 'install-open-service-mesh' + description: 'This is required as the Packer VMs needs to install open service mesh cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + resourceId('Microsoft.Network/ipGroups', imageBuilder_ipgroups.name) + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'github.com' + 'github-releases.githubusercontent.com' + ] + } + { + name: 'install-terraform' + description: 'This is required as the Packer VMs needs to install HashiCorp Terraform cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + resourceId('Microsoft.Network/ipGroups', imageBuilder_ipgroups.name) + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'releases.hashicorp.com' + ] + } + ] + } + } + ] + } +} + +resource hubFirewall_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: hubFirewall + properties: { + workspaceId: laHub.id + logs: [ + { + categoryGroup: 'allLogs' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('Flow Logs deployment') +module regionalFlowlogsDeployment 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'connect-hub-regional-flowlogs' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgBastionSubnet.id + laHubId: laHub.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +/*** OUTPUTS ***/ + +output hubVnetId string = vnetHub.id diff --git a/networking/hub-region.v1.json b/networking/hub-region.v1.json deleted file mode 100644 index c2907431..00000000 --- a/networking/hub-region.v1.json +++ /dev/null @@ -1,897 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.1.0.0", - "parameters": { - "aksImageBuilderSubnetResourceId": { - "type": "string", - "metadata": { - "description": "Subnet resource Id for the AKS image builder subnet" - } - }, - "location": { - "defaultValue": "eastus2", - "type": "string", - "allowedValues": [ - "australiaeast", - "canadacentral", - "centralus", - "eastus", - "eastus2", - "westus2", - "francecentral", - "germanywestcentral", - "northeurope", - "southafricanorth", - "southcentralus", - "uksouth", - "westeurope", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "The hub's regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support." - } - }, - "hubVnetAddressSpace": { - "defaultValue": "10.200.0.0/24", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /24 to contain the regional firewall, management, and gateway subnet" - } - }, - "azureFirewallSubnetAddressSpace": { - "defaultValue": "10.200.0.0/26", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /26 under the VNet Address Space for the regional Azure Firewall" - } - }, - "azureGatewaySubnetAddressSpace": { - "defaultValue": "10.200.0.64/27", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /27 under the VNet Address Space for our regional On-Prem Gateway" - } - }, - "azureBastionSubnetAddressSpace": { - "defaultValue": "10.200.0.96/27", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /27 under the VNet Address Space for regional Azure Bastion" - } - }, - "deployFlowLogResources": { - "defaultValue": true, - "type": "bool", - "metadata": { - "description": "Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing 'false' to this parameter." - } - } - }, - "variables": { - "imageBuilderIpGroupName": "[concat('ipg-', parameters('location'), '-AksJumpboxImageBuilders')]", - "baseFwPipName": "[concat('pip-fw-', parameters('location'))]", - "hubFwPipNames": [ - "[concat(variables('baseFwPipName'), '-default')]", - "[concat(variables('baseFwPipName'), '-01')]", - "[concat(variables('baseFwPipName'), '-02')]" - ], - "hubFwName": "[concat('fw-', parameters('location'))]", - "hubVNetName": "[concat('vnet-', parameters('location'), '-hub')]", - "bastionNetworkNsgName": "[concat('nsg-', parameters('location'), '-bastion')]", - "bastionPipName": "[concat('pip-ab-', parameters('location'))]", - "bastionServiceName": "[concat('ab-', parameters('location'))]", - "hubLaName": "[concat('la-hub-', parameters('location'), '-', uniqueString(resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))))]", - "regionFlowLowStorageAccountName": "[take(concat('stnfl', parameters('location'), uniqueString(resourceGroup().id)), 24)]", - "azureSentialLaSolutionName": "[concat('SecurityInsights(', variables('hubLaName'), ')')]" - }, - "resources": [ - { - "type": "Microsoft.OperationalInsights/workspaces", - "apiVersion": "2020-08-01", - "name": "[variables('hubLaName')]", - "location": "[parameters('location')]", - "comments": "This Log Analytics workspace stores logs from the regional hub network, its spokes, and bastion.", - "properties": { - "sku": { - "name": "PerGB2018" - }, - "retentionInDays": 90, - "publicNetworkAccessForIngestion": "Enabled", - "publicNetworkAccessForQuery": "Enabled" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[variables('azureSentialLaSolutionName')]", - "location": "[parameters('location')]", - "comments": "Enables Azure Sentinal integration.", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "plan": { - "name": "[variables('azureSentialLaSolutionName')]", - "promotionCode": "", - "product": "OMSGallery/SecurityInsights", - "publisher": "Microsoft" - }, - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - } - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[variables('bastionNetworkNsgName')]", - "location": "[parameters('location')]", - "comments": "Wraps the AzureBastion subnet in this regional hub. Source: https://docs.microsoft.com/azure/bastion/bastion-nsg", - "properties": { - "securityRules": [ - { - "name": "AllowWebExperienceInBound", - "properties": { - "description": "Allow our users in. Update this to be as restrictive as possible.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "Internet", - "destinationPortRange": "443", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 100, - "direction": "Inbound" - } - }, - { - "name": "AllowControlPlaneInBound", - "properties": { - "description": "Service Requirement. Allow control plane access. Regional Tag not yet supported.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "GatewayManager", - "destinationPortRange": "443", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 110, - "direction": "Inbound" - } - }, - { - "name": "AllowHealthProbesInBound", - "properties": { - "description": "Service Requirement. Allow Health Probes.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "AzureLoadBalancer", - "destinationPortRange": "443", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 120, - "direction": "Inbound" - } - }, - { - "name": "AllowBastionHostToHostInBound", - "properties": { - "description": "Service Requirement. Allow Required Host to Host Communication.", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRanges": [ - "8080", - "5701" - ], - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 130, - "direction": "Inbound" - } - }, - { - "name": "DenyAllInBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Inbound" - } - }, - { - "name": "AllowSshToVnetOutBound", - "properties": { - "description": "Allow SSH out to the VNet", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "22", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 100, - "direction": "Outbound" - } - }, - { - "name": "AllowRdpToVnetOutBound", - "properties": { - "protocol": "Tcp", - "description": "Unused in this RI, but required for ARM validation.", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "3389", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 110, - "direction": "Outbound" - } - }, - { - "name": "AllowControlPlaneOutBound", - "properties": { - "description": "Required for control plane outbound. Regional prefix not yet supported", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "443", - "destinationAddressPrefix": "AzureCloud", - "access": "Allow", - "priority": 120, - "direction": "Outbound" - } - }, - { - "name": "AllowBastionHostToHostOutBound", - "properties": { - "description": "Service Requirement. Allow Required Host to Host Communication.", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRanges": [ - "8080", - "5701" - ], - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 130, - "direction": "Outbound" - } - }, - { - "name": "AllowBastionCertificateValidationOutBound", - "properties": { - "description": "Service Requirement. Allow Required Session and Certificate Validation.", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "80", - "destinationAddressPrefix": "Internet", - "access": "Allow", - "priority": 140, - "direction": "Outbound" - } - }, - { - "name": "DenyAllOutBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Outbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/virtualNetworks", - "apiVersion": "2020-05-01", - "name": "[variables('hubVnetName')]", - "location": "[parameters('location')]", - "comments": "This is this region's hub network.", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]" - ], - "properties": { - "addressSpace": { - "addressPrefixes": [ - "[parameters('hubVnetAddressSpace')]" - ] - }, - "subnets": [ - { - "name": "AzureFirewallSubnet", - "properties": { - "addressPrefix": "[parameters('azureFirewallSubnetAddressSpace')]" - } - }, - { - "name": "GatewaySubnet", - "properties": { - "addressPrefix": "[parameters('azureGatewaySubnetAddressSpace')]" - } - }, - { - "name": "AzureBastionSubnet", - "properties": { - "addressPrefix": "[parameters('azureBastionSubnetAddressSpace')]", - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]" - } - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/publicIpAddresses", - "apiVersion": "2020-05-01", - "name": "[variables('hubFwPipNames')[copyIndex()]]", - "location": "[parameters('location')]", - "comments": "This is a public IP for this regional hub's firewall. You'll want as many as necessary to avoid SNAT port exhaustion. Typical production hubs may require 20 IPs -- we only deploy three here as a reference.", - "sku": { - "name": "Standard" - }, - "properties": { - "publicIPAllocationMethod": "Static", - "idleTimeoutInMinutes": 4, - "publicIPAddressVersion": "IPv4" - }, - "copy": { - "name": "create-fw-pips", - "count": "[length(variables('hubFwPipNames'))]", - "mode": "Parallel" - } - }, - { - "type": "Microsoft.Network/publicIpAddresses", - "apiVersion": "2020-05-01", - "name": "[variables('bastionPipName')]", - "location": "[parameters('location')]", - "comments": "This is the public IP for this regional hub's Azure Bastion service.", - "sku": { - "name": "Standard" - }, - "properties": { - "publicIPAllocationMethod": "Static", - "idleTimeoutInMinutes": 4, - "publicIPAddressVersion": "IPv4" - } - }, - { - "type": "Microsoft.Network/bastionHosts", - "apiVersion": "2020-05-01", - "name": "[variables('bastionServiceName')]", - "location": "[parameters('location')]", - "comments": "This is this regional hub's Azure Bastion service.", - "dependsOn": [ - "[resourceId('Microsoft.Network/publicIpAddresses', variables('bastionPipName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]" - ], - "properties": { - "ipConfigurations": [ - { - "name": "hub-subnet", - "properties": { - "privateIPAllocationMethod": "Dynamic", - "subnet": { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('hubVnetName'), 'AzureBastionSubnet')]" - }, - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('bastionPipName'))]" - } - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.Network/bastionHosts', variables('bastionServiceName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "category": "BastionAuditLogs", - "enabled": true - } - ] - } - } - ] - }, -{ - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2021-02-01", - "name": "[variables('regionFlowLowStorageAccountName')]", - "location": "[parameters('location')]", - "sku": { - "name": "standard_LRS" - }, - "kind": "StorageV2", - "properties": { - "accessTier": "Hot", - "minimumTlsVersion": "TLS1_2", - "supportsHttpsTrafficOnly": true, - "allowBlobPublicAccess": false, - "allowSharedKeyAccess": false, - "networkAcls": { - "bypass": "AzureServices", - "defaultAction": "Deny", - "ipRules": [] - } - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [], - "metrics": [ - { - "category": "Transaction", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Storage/storageAccounts/blobServices/providers/diagnosticsettings", - "apiVersion": "2017-05-01-preview", - "name": "[concat(variables('regionFlowLowStorageAccountName'), '/default/Microsoft.Insights/default')]", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "category": "StorageRead", - "enabled": true - }, - { - "category": "StorageWrite", - "enabled": true - }, - { - "category": "StorageDelete", - "enabled": true - } - ], - "metrics": [ - { - "category": "Transaction", - "enabled": true - } - ] - } - }, - { - "type": "Microsoft.Network/ipGroups", - "apiVersion": "2020-05-01", - "name": "[variables('imageBuilderIpGroupName')]", - "location": "[parameters('location')]", - "comments": "This holds IP addresses of known AKS Jumpbox image building subnets in attached spokes.", - "properties": { - "ipAddresses": [ - "[reference(parameters('aksImageBuilderSubnetResourceId'), '2020-05-01').addressPrefix]" - ] - } - }, - { - "type": "Microsoft.Network/azureFirewalls", - "apiVersion": "2020-05-01", - "name": "[variables('hubFwName')]", - "location": "[parameters('location')]", - "comments": "This is the regional Azure Firewall that all regional spoke networks can egress through.", - "zones": [ - "1", - "2", - "3" - ], - "dependsOn": [ - "create-fw-pips", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]", - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "properties": { - "additionalProperties": { - "Network.DNS.EnableProxy": "true" - }, - "sku": { - "name": "AZFW_VNet", - "tier": "Standard" - }, - "threatIntelMode": "Deny", - "ipConfigurations": [ - { - "name": "[variables('hubFwPipNames')[0]]", - "properties": { - "subnet": { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('hubVnetName'), 'AzureFirewallSubnet')]" - }, - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('hubFwPipNames')[0])]" - } - } - }, - { - "name": "[variables('hubFwPipNames')[1]]", - "properties": { - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('hubFwPipNames')[1])]" - } - } - }, - { - "name": "[variables('hubFwPipNames')[2]]", - "properties": { - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('hubFwPipNames')[2])]" - } - } - } - ], - "natRuleCollections": [], - "networkRuleCollections": [ - { - "name": "allow-ntp", - "properties": { - "action": { - "type": "Allow" - }, - "priority": 100, - "rules": [ - { - "name": "ntp", - "description": "Network Time Protocol (NTP) time synchronization for image builder VMs.", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - "UDP" - ], - "destinationPorts": [ - "123" - ], - "destinationFqdns": [ - "ntp.ubuntu.com" - ] - } - ] - } - } - ], - "applicationRuleCollections": [ - { - "name": "AKSImageBuilder-Requirements", - "properties": { - "action": { - "type": "Allow" - }, - "priority": 400, - "rules": [ - { - "name": "to-azuremanagement", - "description": "This for AIB VMs to communicate with Azure management API.", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "management.azure.com" - ] - }, - { - "name": "to-blobstorage", - "description": "This is required as the Proxy VM and Packer VM both read and write from transient storage accounts (no ability to know what storage accounts before the process starts.)", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "*.blob.core.windows.net" - ] - }, - { - "name": "apt-get", - "description": "This is required as the Packer VM performs a package upgrade. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Http", - "port": 80 - }, - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "azure.archive.ubuntu.com", - "packages.microsoft.com", - "archive.ubuntu.com", - "security.ubuntu.com" - ] - }, - { - "name": "install-azcli", - "description": "This is required as the Packer VM needs to install Azure CLI. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "aka.ms", - "azurecliextensionsync.blob.core.windows.net", - "azurecliprod.blob.core.windows.net" - ] - }, - { - "name": "install-k8scli", - "description": "This is required as the Packer VM needs to install k8s cli tooling. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "objects.githubusercontent.com", - "storage.googleapis.com", - "api.github.com", - "github-releases.githubusercontent.com", - "github.com" - ] - }, - { - "name": "install-helm", - "description": "This is required as the Packer VM needs to install helm cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "raw.githubusercontent.com", - "get.helm.sh", - "github-releases.githubusercontent.com" - ] - }, - { - "name": "install-flux", - "description": "This is required as the Packer VMs needs to install flux cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "raw.githubusercontent.com", - "api.github.com", - "fluxcd.io", - "github-releases.githubusercontent.com" - ] - }, - { - "name": "install-open-service-mesh", - "description": "This is required as the Packer VMs needs to install open service mesh cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "github.com", - "github-releases.githubusercontent.com" - ] - }, - { - "name": "install-terraform", - "description": "This is required as the Packer VMs needs to install HashiCorp Terraform cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "releases.hashicorp.com" - ] - } - ] - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/azureFirewalls', variables('hubFwName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "categoryGroup": "allLogs", - "enabled": true - } - ], - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "condition": "[parameters('deployFlowLogResources')]", - "name": "connect-hub-regional-flowlogs", - "type": "Microsoft.Resources/deployments", - "resourceGroup": "networkWatcherRG", - "apiVersion": "2020-10-01", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]", - "[resourceId('Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/azureFirewalls', variables('hubFwName'))]" // This doesn't depend on the FW, but because network watchers are auto-deployed, this helps prevents a race condition between the vnet creation triggering a NetworkWatcher auto provisioning, and the referencing of it here. - ], - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "variables": {}, - "resources": [ - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]", - "storageId": "[resourceId(resourceGroup().name, 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[resourceId(resourceGroup().name, 'Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - } - ] - } - } - } - ], - "outputs": { - "hubVnetId": { - "value": "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]", - "type": "string" - } - } -} diff --git a/networking/hub-region.v2.bicep b/networking/hub-region.v2.bicep new file mode 100644 index 00000000..55b7da75 --- /dev/null +++ b/networking/hub-region.v2.bicep @@ -0,0 +1,1024 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('Subnet resource Ids for all AKS clusters nodepools in all attached spokes to allow necessary outbound traffic through the firewall') +param nodepoolSubnetResourceIds array + +@description('Subnet resource Id for the AKS image builder subnet') +@minLength(79) +param aksImageBuilderSubnetResourceId string + +@description('Subnet resource Id for the AKS jumpbox subnet') +@minLength(79) +param aksJumpboxSubnetResourceId string + +@allowed([ + 'australiaeast' + 'canadacentral' + 'centralus' + 'eastus' + 'eastus2' + 'westus2' + 'francecentral' + 'germanywestcentral' + 'northeurope' + 'southafricanorth' + 'southcentralus' + 'uksouth' + 'westeurope' + 'japaneast' + 'southeastasia' +]) +@description('The hub\'s regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support.') +@minLength(4) +param location string = 'eastus2' + +@description('A /24 to contain the regional firewall, management, and gateway subnet') +@minLength(10) +@maxLength(18) +param hubVnetAddressSpace string = '10.200.0.0/24' + +@description('A /26 under the VNet Address Space for the regional Azure Firewall') +@minLength(10) +@maxLength(18) +param azureFirewallSubnetAddressSpace string = '10.200.0.0/26' + +@description('A /27 under the VNet Address Space for our regional On-Prem Gateway') +@minLength(10) +@maxLength(18) +param azureGatewaySubnetAddressSpace string = '10.200.0.64/27' + +@description('A /27 under the VNet Address Space for regional Azure Bastion') +@minLength(10) +@maxLength(18) +param azureBastionSubnetAddressSpace string = '10.200.0.96/27' + +@description('Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing "false" to this parameter.') +param deployFlowLogResources bool = true + +/*** EXISTING RESOURCES ***/ + +@description('The resource group name containing virtual network in which Azure Image Builder will drop the compute into to perform the image build.') +resource rgSpokes 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { + scope: subscription() + name: split(aksImageBuilderSubnetResourceId, '/')[4] +} + +@description('AKS Spoke Virtual Network BU0001A0005-00') +resource aksImageBuilderVnet 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { + scope: rgSpokes + name: split(aksImageBuilderSubnetResourceId, '/')[8] +} + +@description('AKS ImageBuilder subnet') +resource aksImageBuilderSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-01-01' existing = { + parent: aksImageBuilderVnet + name: last(split(aksImageBuilderSubnetResourceId, '/')) +} + +@description('AKS Spoke Virtual Network BU0001A0005-01') +resource aksJumpboxVnet 'Microsoft.Network/virtualNetworks@2022-01-01' existing = { + scope: rgSpokes + name: split(aksJumpboxSubnetResourceId, '/')[8] +} + +@description('AKS Jumpbox subnet') +resource aksJumpboxSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-01-01' existing = { + parent: aksJumpboxVnet + name: last(split(aksJumpboxSubnetResourceId, '/')) +} + +@description('NetworkWatcher ResourceGroup; it contains regional Network Watchers') +resource networkWatcherResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = if (deployFlowLogResources) { + scope: subscription() + name: 'networkWatcherRG' +} + +/*** RESOURCES ***/ + +@description('This Log Analytics workspace stores logs from the regional hub network, its spokes, and bastion. Log analytics is a regional resource, as such there will be one workspace per hub (region)') +resource laHub 'Microsoft.OperationalInsights/workspaces@2021-06-01' = { + name: 'la-hub-${location}-${uniqueString(resourceGroup().id, 'vnet-${location}-hub')}' + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 90 + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + } +} + +@description('Enables Azure Sentinal integration.') +var laHubSolutionName = 'SecurityInsights(${laHub.name})' +resource laHubSolution 'Microsoft.OperationsManagement/solutions@2015-11-01-preview' = { + name: laHubSolutionName + location: location + plan: { + name: laHubSolutionName + promotionCode: '' + product: 'OMSGallery/SecurityInsights' + publisher: 'Microsoft' + } + properties: { + workspaceResourceId: laHub.id + } +} + +@description('Wraps the AzureBastion subnet in this regional hub. Source: https://docs.microsoft.com/azure/bastion/bastion-nsg') +resource nsgBastionSubnet 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-${location}-bastion' + location: location + properties: { + securityRules: [ + { + name: 'AllowWebExperienceInbound' + properties: { + description: 'Allow our users in. Update this to be as restrictive as possible.' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '443' + sourceAddressPrefix: 'Internet' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 100 + direction: 'Inbound' + } + } + { + name: 'AllowControlPlaneInbound' + properties: { + description: 'Service Requirement. Allow control plane access. Regional Tag not yet supported.' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '443' + sourceAddressPrefix: 'GatewayManager' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 110 + direction: 'Inbound' + } + } + { + name: 'AllowHealthProbesInbound' + properties: { + description: 'Service Requirement. Allow Health Probes.' + protocol: 'Tcp' + sourcePortRange: '*' + destinationPortRange: '443' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 120 + direction: 'Inbound' + } + } + { + name: 'AllowBastionHostToHostInbound' + properties: { + description: 'Service Requirement. Allow Required Host to Host Communication.' + protocol: '*' + sourcePortRange: '*' + destinationPortRanges: [ + '8080' + '5701' + ] + sourceAddressPrefix: 'VirtualNetwork' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 130 + direction: 'Inbound' + } + } + { + name: 'DenyAllInbound' + properties: { + description: 'No further inbound traffic allowed.' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Inbound' + } + } + { + name: 'AllowSshToVnetOutbound' + properties: { + description: 'Allow SSH out to the virtual network' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '22' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 100 + direction: 'Outbound' + } + } + { + name: 'AllowRdpToVnetOutbound' + properties: { + description: 'Allow RDP out to the virtual network' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '3389' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 110 + direction: 'Outbound' + } + } + { + name: 'AllowControlPlaneOutbound' + properties: { + description: 'Required for control plane outbound. Regional prefix not yet supported' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '443' + destinationAddressPrefix: 'AzureCloud' + access: 'Allow' + priority: 120 + direction: 'Outbound' + } + } + { + name: 'AllowBastionHostToHostOutbound' + properties: { + description: 'Service Requirement. Allow Required Host to Host Communication.' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRanges: [ + '8080' + '5701' + ] + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 130 + direction: 'Outbound' + } + } + { + name: 'AllowBastionCertificateValidationOutbound' + properties: { + description: 'Service Requirement. Allow Required Session and Certificate Validation.' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '80' + destinationAddressPrefix: 'Internet' + access: 'Allow' + priority: 140 + direction: 'Outbound' + } + } + { + name: 'DenyAllOutbound' + properties: { + description: 'No further outbound traffic allowed.' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Outbound' + } + } + ] + } +} + +resource nsgBastionSubnet_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: nsgBastionSubnet + properties: { + workspaceId: laHub.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('The regional hub network') +resource vnetHub 'Microsoft.Network/virtualNetworks@2021-05-01' = { + name: 'vnet-${location}-hub' + location: location + properties: { + addressSpace: { + addressPrefixes: [ + hubVnetAddressSpace + ] + } + subnets: [ + { + name: 'AzureFirewallSubnet' + properties: { + addressPrefix: azureFirewallSubnetAddressSpace + } + } + { + name: 'GatewaySubnet' + properties: { + addressPrefix: azureGatewaySubnetAddressSpace + } + } + { + name: 'AzureBastionSubnet' + properties: { + addressPrefix: azureBastionSubnetAddressSpace + networkSecurityGroup: { + id: nsgBastionSubnet.id + } + } + } + ] + } + + resource azureFirewallSubnet 'subnets' existing = { + name: 'AzureFirewallSubnet' + } + + resource azureBastionSubnet 'subnets' existing = { + name: 'AzureBastionSubnet' + } +} + +resource vnetHub_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: vnetHub + properties: { + workspaceId: laHub.id + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('Allocate three public IP addresses to the firewall') +var numFirewallIpAddressesToAssign = 3 +resource pipsAzureFirewall 'Microsoft.Network/publicIPAddresses@2021-05-01' = [for i in range(0, numFirewallIpAddressesToAssign): { + name: 'pip-fw-${location}-${padLeft(i, 2, '0')}' + location: location + sku: { + name: 'Standard' + } + zones: [ + '1' + '2' + '3' + ] + properties: { + publicIPAllocationMethod: 'Static' + idleTimeoutInMinutes: 4 + publicIPAddressVersion: 'IPv4' + } +}] + +@description('The public IP for the regional hub\'s Azure Bastion service.') +resource pipAzureBastion 'Microsoft.Network/publicIPAddresses@2021-05-01' = { + name: 'pip-ab-${location}' + location: location + sku: { + name: 'Standard' + } + zones: [ + '1' + '2' + '3' + ] + properties: { + publicIPAllocationMethod: 'Static' + idleTimeoutInMinutes: 4 + publicIPAddressVersion: 'IPv4' + } +} + +@description('This regional hub\'s Azure Bastion service.') +resource azureBastion 'Microsoft.Network/bastionHosts@2021-05-01' = { + name: 'ab-${location}' + location: location + properties: { + ipConfigurations: [ + { + name: 'hub-subnet' + properties: { + privateIPAllocationMethod: 'Dynamic' + subnet: { + id: vnetHub::azureBastionSubnet.id + } + publicIPAddress: { + id: pipAzureBastion.id + } + } + } + ] + } +} + +resource azureBastion_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: azureBastion + properties: { + workspaceId: laHub.id + logs: [ + { + category: 'BastionAuditLogs' + enabled: true + } + ] + } +} + +@description('Storage account to store the flow logs') +resource flowlogs_storageAccount 'Microsoft.Storage/storageAccounts@2021-08-01' = { + name: substring('stnfl${location}${uniqueString(resourceGroup().id)}', 0, 24) + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + accessTier: 'Hot' + minimumTlsVersion: 'TLS1_2' + supportsHttpsTrafficOnly: true + allowBlobPublicAccess: false + allowSharedKeyAccess: false + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + ipRules: [] + } + } + + resource blobStorage 'blobServices' existing = { + name: 'default' + } +} + +resource flowlogs_storageAccount_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: flowlogs_storageAccount + properties: { + workspaceId: laHub.id + logs: [] + metrics: [ + { + category: 'Transaction' + enabled: true + } + ] + } +} + +@description('This holds IP addresses of known AKS Jumpbox image building subnets in attached spokes.') +resource imageBuilder_ipgroup 'Microsoft.Network/ipGroups@2021-05-01' = { + name: 'ipg-${location}-AksJumpboxImageBuilders' + location: location + properties: { + ipAddresses: [ + aksImageBuilderSubnet.properties.addressPrefix + ] + } +} + +@description('This holds IP addresses of known AKS Jumpboxs in attached spokes.') +resource aksJumpbox_ipgroup 'Microsoft.Network/ipGroups@2021-05-01' = { + name: 'ipg-${location}-AksJumpboxes' + location: location + properties: { + ipAddresses: [ + aksJumpboxSubnet.properties.addressPrefix + ] + } +} + +@description('This holds IP addresses of known nodepool subnets in attached spokes.') +resource aks_ipgroup 'Microsoft.Network/ipGroups@2021-05-01' = { + name: 'ipg-${location}-AksNodepools' + location: location + properties: { + ipAddresses: [for nodepoolSubnetResourceId in nodepoolSubnetResourceIds: '${reference(nodepoolSubnetResourceId, '2020-05-01').addressPrefix}'] + } +} + +resource region_flowlog_storageAccount_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2017-05-01-preview' = { + name: 'default' + scope: flowlogs_storageAccount::blobStorage + properties: { + workspaceId: laHub.id + logs: [ + { + category: 'StorageRead' + enabled: true + } + { + category: 'StorageWrite' + enabled: true + } + { + category: 'StorageDelete' + enabled: true + } + ] + metrics: [ + { + category: 'Transaction' + enabled: true + } + ] + } +} + +@description('This is the regional Azure Firewall that all regional spoke networks can egress through.') +resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' = { + name: 'fw-${location}' + location: location + zones: [ + '1' + '2' + '3' + ] + properties: { + additionalProperties: { + 'Network.DNS.EnableProxy': 'true' + } + sku: { + tier: 'Premium' + name: 'AZFW_VNet' + } + threatIntelMode: 'Deny' + ipConfigurations: [for i in range(0, numFirewallIpAddressesToAssign): { + name: pipsAzureFirewall[i].name + properties: { + subnet: (0 == i) ? { + id: vnetHub::azureFirewallSubnet.id + } : null + publicIPAddress: { + id: pipsAzureFirewall[i].id + } + } + }] + natRuleCollections: [] + networkRuleCollections: [ + { + name: 'allow-ntp' + properties: { + action: { + type: 'Allow' + } + priority: 100 + rules: [ + { + name: 'ntp' + description: 'Network Time Protocol (NTP) time synchronization for image builder VMs and jumpboxes.' + sourceIpGroups: [ + imageBuilder_ipgroup.id + aksJumpbox_ipgroup.id + ] + protocols: [ + 'UDP' + ] + destinationPorts: [ + '123' + ] + destinationFqdns: [ + 'ntp.ubuntu.com' + ] + } + ] + } + } + ] + applicationRuleCollections: [ + { + name: 'AKS-Global-Requirements' + properties: { + action: { + type: 'Allow' + } + priority: 200 + rules: [ + { + name: 'runtime' + description: 'This covers all runtime requirements for AKS (sans addons). If you wish to use a more restricted set of values than what this provides, see: https://docs.microsoft.com/azure/firewall/protect-azure-kubernetes-service and https://docs.microsoft.com/azure/aks/limit-egress-traffic#required-outbound-network-rules-and-fqdns-for-aks-clusters and remove this entry' + sourceIpGroups: [ + aks_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + fqdnTags: [ + 'AzureKubernetesService' + ] + } + { + name: 'azure-monitor-addon' + description: 'All required for Azure Monitor for containers per https://docs.microsoft.com/azure/aks/limit-egress-traffic#azure-monitor-for-containers - Optionally you can restrict the ods and oms wildcards to JUST your cluster\'s log analytics instances.' + sourceIpGroups: [ + aks_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + '*.ods.opinsights.azure.com' + '*.oms.opinsights.azure.com' + '${location}.monitoring.azure.com' + ] + } + { + name: 'azure-policy-addon' +#disable-next-line no-hardcoded-env-urls + description: 'All required for Azure Policy per https://docs.microsoft.com/azure/aks/limit-egress-traffic#azure-policy. If not using the AzureKubernetesService fqdnTag, you must also add gov-prod-policy-data.trafficmanager.net, raw.githubusercontent.com, and dc.services.visualstudio.com' + sourceIpGroups: [ + aks_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ +#disable-next-line no-hardcoded-env-urls + 'data.policy.core.windows.net' +#disable-next-line no-hardcoded-env-urls + 'store.policy.core.windows.net' + ] + } + ] + } + } + { + name: 'Flux-Requirements' + properties: { + action: { + type: 'Allow' + } + priority: 300 + rules: [ + { + name: 'flux-to-github' + description: 'This address is required for Flux <-> Github repository with the desired cluster baseline configuration.' + sourceIpGroups: [ + aks_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'github.com' + 'api.github.com' + ] + } + ] + } + } + { + name: 'AKSImageBuilder-Requirements' + properties: { + action: { + type: 'Allow' + } + priority: 400 + rules: [ + { + name: 'to-azuremanagement' + description: 'This for AIB VMs to communicate with Azure management API.' + sourceIpGroups: [ + imageBuilder_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ +#disable-next-line no-hardcoded-env-urls + 'management.azure.com' + ] + } + { + name: 'to-blobstorage' + description: 'This is required as the Proxy VM and Packer VM both read and write from transient storage accounts (no ability to know what storage accounts before the process starts.)' + sourceIpGroups: [ + imageBuilder_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ +#disable-next-line no-hardcoded-env-urls + '*.blob.core.windows.net' + ] + } + { + name: 'apt-get' + description: 'This is required as the Packer VM performs a package upgrade. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + imageBuilder_ipgroup.id + ] + protocols: [ + { + protocolType: 'Http' + port: 80 + } + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'azure.archive.ubuntu.com' + 'packages.microsoft.com' + 'archive.ubuntu.com' + 'security.ubuntu.com' + ] + } + { + name: 'install-azcli' + description: 'This is required as the Packer VM needs to install Azure CLI. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + imageBuilder_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'aka.ms' +#disable-next-line no-hardcoded-env-urls + 'azurecliextensionsync.blob.core.windows.net' +#disable-next-line no-hardcoded-env-urls + 'azurecliprod.blob.core.windows.net' + ] + } + { + name: 'install-k8scli' + description: 'This is required as the Packer VM needs to install k8s cli tooling. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + imageBuilder_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'objects.githubusercontent.com' + 'storage.googleapis.com' + 'api.github.com' + 'github-releases.githubusercontent.com' + 'github.com' + ] + } + { + name: 'install-helm' + description: 'This is required as the Packer VM needs to install helm cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + imageBuilder_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'raw.githubusercontent.com' + 'get.helm.sh' + 'github-releases.githubusercontent.com' + ] + } + { + name: 'install-flux' + description: 'This is required as the Packer VMs needs to install flux cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + imageBuilder_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'raw.githubusercontent.com' + 'api.github.com' + 'fluxcd.io' + 'github-releases.githubusercontent.com' + ] + } + { + name: 'install-open-service-mesh' + description: 'This is required as the Packer VMs needs to install open service mesh cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + imageBuilder_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'github.com' + 'github-releases.githubusercontent.com' + ] + } + { + name: 'install-terraform' + description: 'This is required as the Packer VMs needs to install HashiCorp Terraform cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn\'t do this.]' + sourceIpGroups: [ + imageBuilder_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'releases.hashicorp.com' + ] + } + ] + } + } + { + name: 'aks-jumpbox' + properties: { + action: { + type: 'Allow' + } + priority: 450 + rules: [ + { + name: 'az-login' + description: 'Allow jumpboxes to perform az login.' + sourceIpGroups: [ + aks_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ +#disable-next-line no-hardcoded-env-urls + 'login.microsoftonline.com' + ] + } + { + name: 'az-management-api' + description: 'Allow jumpboxes to communicate with Azure management APIs.' + sourceIpGroups: [ + aks_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ +#disable-next-line no-hardcoded-env-urls + 'management.azure.com' + ] + } + { + name: 'az-cli-extensions' + description: 'Allow jumpboxes query az cli status and download extensions' + sourceIpGroups: [ + aks_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'aka.ms' +#disable-next-line no-hardcoded-env-urls + 'azurecliprod.blob.core.windows.net' +#disable-next-line no-hardcoded-env-urls + 'azcliextensionsync.blob.core.windows.net' + ] + } + { + name: 'github' + description: 'Allow pulling things down from GitHub. [Only a requirement of this walkthrough because we deploy some manifests that you clone from your repo.]' + sourceIpGroups: [ + aks_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + 'github.com' + '*.github.io' + 'raw.githubusercontent.com' + ] + } + { + name: 'azure-monitor-addon' + description: 'Required for Azure Monitor Extension on Jumpbox.' + sourceIpGroups: [ + aks_ipgroup.id + ] + protocols: [ + { + protocolType: 'Https' + port: 443 + } + ] + targetFqdns: [ + '*.ods.opinsights.azure.com' + '*.oms.opinsights.azure.com' + '${location}.monitoring.azure.com' + '*.agentsvc.azure-automation.net' + ] + } + ] + } + } + ] + } +} + +resource hubFirewall_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'default' + scope: hubFirewall + properties: { + workspaceId: laHub.id + logs: [ + { + categoryGroup: 'allLogs' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('Flow Logs deployment') +module regionalFlowlogsDeployment 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'connect-hub-regional-flowlogs' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgBastionSubnet.id + laHubId: laHub.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +/*** OUTPUTS ***/ + +output hubVnetId string = vnetHub.id +output storageAccountName string = flowlogs_storageAccount.name diff --git a/networking/hub-region.v2.json b/networking/hub-region.v2.json deleted file mode 100644 index 35c7729f..00000000 --- a/networking/hub-region.v2.json +++ /dev/null @@ -1,1130 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.2.0.0", - "parameters": { - "nodepoolSubnetResourceIds": { - "type": "array", - "metadata": { - "description": "Subnet resource Ids for all AKS clusters nodepools in all attached spokes to allow necessary outbound traffic through the firewall" - } - }, - "aksImageBuilderSubnetResourceId": { - "type": "string", - "metadata": { - "description": "Subnet resource Id for the AKS image builder subnet" - } - }, - "aksJumpboxSubnetResourceId": { - "type": "string", - "metadata": { - "description": "Subnet resource Id for the AKS jumpbox subnet" - } - }, - "location": { - "defaultValue": "eastus2", - "type": "string", - "allowedValues": [ - "australiaeast", - "canadacentral", - "centralus", - "eastus", - "eastus2", - "westus2", - "francecentral", - "germanywestcentral", - "northeurope", - "southafricanorth", - "southcentralus", - "uksouth", - "westeurope", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "The hub's regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support." - } - }, - "hubVnetAddressSpace": { - "defaultValue": "10.200.0.0/24", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /24 to contain the regional firewall, management, and gateway subnet" - } - }, - "azureFirewallSubnetAddressSpace": { - "defaultValue": "10.200.0.0/26", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /26 under the VNet Address Space for the regional Azure Firewall" - } - }, - "azureGatewaySubnetAddressSpace": { - "defaultValue": "10.200.0.64/27", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /27 under the VNet Address Space for our regional On-Prem Gateway" - } - }, - "azureBastionSubnetAddressSpace": { - "defaultValue": "10.200.0.96/27", - "type": "string", - "maxLength": 18, - "minLength": 10, - "metadata": { - "description": "A /27 under the VNet Address Space for regional Azure Bastion" - } - }, - "deployFlowLogResources": { - "defaultValue": true, - "type": "bool", - "metadata": { - "description": "Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing 'false' to this parameter." - } - } - }, - "variables": { - "aksIpGroupName": "[concat('ipg-', parameters('location'), '-AksNodepools')]", - "imageBuilderIpGroupName": "[concat('ipg-', parameters('location'), '-AksJumpboxImageBuilders')]", - "baseFwPipName": "[concat('pip-fw-', parameters('location'))]", - "hubFwPipNames": [ - "[concat(variables('baseFwPipName'), '-default')]", - "[concat(variables('baseFwPipName'), '-01')]", - "[concat(variables('baseFwPipName'), '-02')]" - ], - "aksJumpBoxIpGroupName": "[concat('ipg-', parameters('location'), '-AksJumpboxes')]", - "hubFwName": "[concat('fw-', parameters('location'))]", - "hubVNetName": "[concat('vnet-', parameters('location'), '-hub')]", - "bastionNetworkNsgName": "[concat('nsg-', parameters('location'), '-bastion')]", - "bastionPipName": "[concat('pip-ab-', parameters('location'))]", - "bastionServiceName": "[concat('ab-', parameters('location'))]", - "hubLaName": "[concat('la-hub-', parameters('location'), '-', uniqueString(resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))))]", - "regionFlowLowStorageAccountName": "[take(concat('stnfl', parameters('location'), uniqueString(resourceGroup().id)), 24)]", - "azureSentialLaSolutionName": "[concat('SecurityInsights(', variables('hubLaName'), ')')]" - }, - "resources": [ - { - "type": "Microsoft.OperationalInsights/workspaces", - "apiVersion": "2020-08-01", - "name": "[variables('hubLaName')]", - "location": "[parameters('location')]", - "comments": "This Log Analytics workspace stores logs from the regional hub network, its spokes, and bastion.", - "properties": { - "sku": { - "name": "PerGB2018" - }, - "retentionInDays": 90, - "publicNetworkAccessForIngestion": "Enabled", - "publicNetworkAccessForQuery": "Enabled" - } - }, - { - "type": "Microsoft.OperationsManagement/solutions", - "apiVersion": "2015-11-01-preview", - "name": "[variables('azureSentialLaSolutionName')]", - "location": "[parameters('location')]", - "comments": "Enables Azure Sentinal integration.", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "plan": { - "name": "[variables('azureSentialLaSolutionName')]", - "promotionCode": "", - "product": "OMSGallery/SecurityInsights", - "publisher": "Microsoft" - }, - "properties": { - "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - } - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[variables('bastionNetworkNsgName')]", - "location": "[parameters('location')]", - "comments": "Wraps the AzureBastion subnet in this regional hub. Source: https://docs.microsoft.com/azure/bastion/bastion-nsg", - "properties": { - "securityRules": [ - { - "name": "AllowWebExperienceInBound", - "properties": { - "description": "Allow our users in. Update this to be as restrictive as possible.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "Internet", - "destinationPortRange": "443", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 100, - "direction": "Inbound" - } - }, - { - "name": "AllowControlPlaneInBound", - "properties": { - "description": "Service Requirement. Allow control plane access. Regional Tag not yet supported.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "GatewayManager", - "destinationPortRange": "443", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 110, - "direction": "Inbound" - } - }, - { - "name": "AllowHealthProbesInBound", - "properties": { - "description": "Service Requirement. Allow Health Probes.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "AzureLoadBalancer", - "destinationPortRange": "443", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 120, - "direction": "Inbound" - } - }, - { - "name": "AllowBastionHostToHostInBound", - "properties": { - "description": "Service Requirement. Allow Required Host to Host Communication.", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRanges": [ - "8080", - "5701" - ], - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 130, - "direction": "Inbound" - } - }, - { - "name": "DenyAllInBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Inbound" - } - }, - { - "name": "AllowSshToVnetOutBound", - "properties": { - "description": "Allow SSH out to the VNet", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "22", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 100, - "direction": "Outbound" - } - }, - { - "name": "AllowRdpToVnetOutBound", - "properties": { - "protocol": "Tcp", - "description": "Unused in this RI, but required for ARM validation.", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "3389", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 110, - "direction": "Outbound" - } - }, - { - "name": "AllowControlPlaneOutBound", - "properties": { - "description": "Required for control plane outbound. Regional prefix not yet supported", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "443", - "destinationAddressPrefix": "AzureCloud", - "access": "Allow", - "priority": 120, - "direction": "Outbound" - } - }, - { - "name": "AllowBastionHostToHostOutBound", - "properties": { - "description": "Service Requirement. Allow Required Host to Host Communication.", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRanges": [ - "8080", - "5701" - ], - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 130, - "direction": "Outbound" - } - }, - { - "name": "AllowBastionCertificateValidationOutBound", - "properties": { - "description": "Service Requirement. Allow Required Session and Certificate Validation.", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "80", - "destinationAddressPrefix": "Internet", - "access": "Allow", - "priority": 140, - "direction": "Outbound" - } - }, - { - "name": "DenyAllOutBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Outbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/virtualNetworks", - "apiVersion": "2020-05-01", - "name": "[variables('hubVnetName')]", - "location": "[parameters('location')]", - "comments": "This is this region's hub network.", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]" - ], - "properties": { - "addressSpace": { - "addressPrefixes": [ - "[parameters('hubVnetAddressSpace')]" - ] - }, - "subnets": [ - { - "name": "AzureFirewallSubnet", - "properties": { - "addressPrefix": "[parameters('azureFirewallSubnetAddressSpace')]" - } - }, - { - "name": "GatewaySubnet", - "properties": { - "addressPrefix": "[parameters('azureGatewaySubnetAddressSpace')]" - } - }, - { - "name": "AzureBastionSubnet", - "properties": { - "addressPrefix": "[parameters('azureBastionSubnetAddressSpace')]", - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]" - } - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/publicIpAddresses", - "apiVersion": "2020-05-01", - "name": "[variables('hubFwPipNames')[copyIndex()]]", - "location": "[parameters('location')]", - "comments": "This is a public IP for this regional hub's firewall. You'll want as many as necessary to avoid SNAT port exhaustion. Typical production hubs may require 20 IPs -- we only deploy three here as a reference.", - "sku": { - "name": "Standard" - }, - "properties": { - "publicIPAllocationMethod": "Static", - "idleTimeoutInMinutes": 4, - "publicIPAddressVersion": "IPv4" - }, - "copy": { - "name": "create-fw-pips", - "count": "[length(variables('hubFwPipNames'))]", - "mode": "Parallel" - } - }, - { - "type": "Microsoft.Network/publicIpAddresses", - "apiVersion": "2020-05-01", - "name": "[variables('bastionPipName')]", - "location": "[parameters('location')]", - "comments": "This is the public IP for this regional hub's Azure Bastion service.", - "sku": { - "name": "Standard" - }, - "properties": { - "publicIPAllocationMethod": "Static", - "idleTimeoutInMinutes": 4, - "publicIPAddressVersion": "IPv4" - } - }, - { - "type": "Microsoft.Network/bastionHosts", - "apiVersion": "2020-05-01", - "name": "[variables('bastionServiceName')]", - "location": "[parameters('location')]", - "comments": "This is this regional hub's Azure Bastion service.", - "dependsOn": [ - "[resourceId('Microsoft.Network/publicIpAddresses', variables('bastionPipName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]" - ], - "properties": { - "ipConfigurations": [ - { - "name": "hub-subnet", - "properties": { - "privateIPAllocationMethod": "Dynamic", - "subnet": { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('hubVnetName'), 'AzureBastionSubnet')]" - }, - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('bastionPipName'))]" - } - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.Network/bastionHosts', variables('bastionServiceName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "category": "BastionAuditLogs", - "enabled": true - } - ] - } - } - ] - }, -{ - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2021-02-01", - "name": "[variables('regionFlowLowStorageAccountName')]", - "location": "[parameters('location')]", - "sku": { - "name": "standard_LRS" - }, - "kind": "StorageV2", - "properties": { - "accessTier": "Hot", - "minimumTlsVersion": "TLS1_2", - "supportsHttpsTrafficOnly": true, - "allowBlobPublicAccess": false, - "allowSharedKeyAccess": false, - "networkAcls": { - "bypass": "AzureServices", - "defaultAction": "Deny", - "ipRules": [] - } - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [], - "metrics": [ - { - "category": "Transaction", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Storage/storageAccounts/blobServices/providers/diagnosticsettings", - "apiVersion": "2017-05-01-preview", - "name": "[concat(variables('regionFlowLowStorageAccountName'), '/default/Microsoft.Insights/default')]", - "dependsOn": [ - "[resourceId('Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "category": "StorageRead", - "enabled": true - }, - { - "category": "StorageWrite", - "enabled": true - }, - { - "category": "StorageDelete", - "enabled": true - } - ], - "metrics": [ - { - "category": "Transaction", - "enabled": true - } - ] - } - }, - { - "type": "Microsoft.Network/ipGroups", - "apiVersion": "2020-05-01", - "name": "[variables('aksIpGroupName')]", - "location": "[parameters('location')]", - "comments": "This holds IP addresses of known nodepool subnets in attached spokes.", - "properties": { - "copy": [ - { - "name": "ipAddresses", - "count": "[length(parameters('nodepoolSubnetResourceIds'))]", - "input": "[reference(parameters('nodepoolSubnetResourceIds')[copyIndex('ipAddresses')], '2020-05-01').addressPrefix]" - } - ] - } - }, - { - "type": "Microsoft.Network/ipGroups", - "apiVersion": "2020-05-01", - "name": "[variables('imageBuilderIpGroupName')]", - "location": "[parameters('location')]", - "comments": "This holds IP addresses of known AKS Jumpbox image building subnets in attached spokes.", - "properties": { - "ipAddresses": [ - "[reference(parameters('aksImageBuilderSubnetResourceId'), '2020-05-01').addressPrefix]" - ] - } - }, - { - "type": "Microsoft.Network/ipGroups", - "apiVersion": "2020-05-01", - "name": "[variables('aksJumpBoxIpGroupName')]", - "location": "[parameters('location')]", - "comments": "This holds IP addresses of known AKS Jumpboxs in attached spokes.", - "properties": { - "ipAddresses": [ - "[reference(parameters('aksJumpboxSubnetResourceId'), '2020-05-01').addressPrefix]" - ] - } - }, - { - "type": "Microsoft.Network/azureFirewalls", - "apiVersion": "2020-05-01", - "name": "[variables('hubFwName')]", - "location": "[parameters('location')]", - "comments": "This is the regional Azure Firewall that all regional spoke networks can egress through.", - "zones": [ - "1", - "2", - "3" - ], - "dependsOn": [ - "create-fw-pips", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]", - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]", - "[resourceId('Microsoft.Network/ipGroups', variables('aksIpGroupName'))]", - "[resourceId('Microsoft.Network/ipGroups', variables('aksJumpBoxIpGroupName'))]" - ], - "properties": { - "additionalProperties": { - "Network.DNS.EnableProxy": "true" - }, - "sku": { - "name": "AZFW_VNet", - "tier": "Standard" - }, - "threatIntelMode": "Deny", - "ipConfigurations": [ - { - "name": "[variables('hubFwPipNames')[0]]", - "properties": { - "subnet": { - "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('hubVnetName'), 'AzureFirewallSubnet')]" - }, - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('hubFwPipNames')[0])]" - } - } - }, - { - "name": "[variables('hubFwPipNames')[1]]", - "properties": { - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('hubFwPipNames')[1])]" - } - } - }, - { - "name": "[variables('hubFwPipNames')[2]]", - "properties": { - "publicIPAddress": { - "id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('hubFwPipNames')[2])]" - } - } - } - ], - "natRuleCollections": [], - "networkRuleCollections": [ - { - "name": "allow-ntp", - "properties": { - "action": { - "type": "Allow" - }, - "priority": 100, - "rules": [ - { - "name": "ntp", - "description": "Network Time Protocol (NTP) time synchronization for image builder VMs and jumpboxes.", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]", - "[resourceId('Microsoft.Network/ipGroups', variables('aksJumpBoxIpGroupName'))]" - ], - "protocols": [ - "UDP" - ], - "destinationPorts": [ - "123" - ], - "destinationFqdns": [ - "ntp.ubuntu.com" - ] - } - ] - } - } - ], - "applicationRuleCollections": [ - { - "name": "AKS-Global-Requirements", - "properties": { - "action": { - "type": "Allow" - }, - "priority": 200, - "rules": [ - { - "name": "runtime", - "description": "This covers all runtime requirements for AKS (sans addons). If you wish to use a more restricted set of values than what this provides, see: https://docs.microsoft.com/azure/firewall/protect-azure-kubernetes-service and https://docs.microsoft.com/azure/aks/limit-egress-traffic#required-outbound-network-rules-and-fqdns-for-aks-clusters and remove this entry", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('aksIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "fqdnTags": [ - "AzureKubernetesService" - ] - }, - { - "name": "azure-monitor-addon", - "description": "All required for Azure Monitor for containers per https://docs.microsoft.com/azure/aks/limit-egress-traffic#azure-monitor-for-containers - Optionally you can restrict the ods and oms wildcards to JUST your clusters' log analytics instances.", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('aksIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "*.ods.opinsights.azure.com", - "*.oms.opinsights.azure.com", - "[concat(parameters('location'), '.monitoring.azure.com')]" - ] - }, - { - "name": "azure-policy-addon", - "description": "All required for Azure Policy per https://docs.microsoft.com/azure/aks/limit-egress-traffic#azure-policy. If not using the AzureKubernetesService fqdnTag, you must also add gov-prod-policy-data.trafficmanager.net, raw.githubusercontent.com, and dc.services.visualstudio.com", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('aksIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "data.policy.core.windows.net", - "store.policy.core.windows.net" - ] - } - ] - } - }, - { - "name": "Flux-Requirements", - "properties": { - "action": { - "type": "Allow" - }, - "priority": 300, - "rules": [ - { - "name": "flux-to-github", - "description": "This address is required for Flux <-> Github repository with the desired cluster baseline configuration.", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('aksIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "github.com", - "api.github.com" - ] - } - ] - } - }, - { - "name": "AKSImageBuilder-Requirements", - "properties": { - "action": { - "type": "Allow" - }, - "priority": 400, - "rules": [ - { - "name": "to-azuremanagement", - "description": "This for AIB VMs to communicate with Azure management API.", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "management.azure.com" - ] - }, - { - "name": "to-blobstorage", - "description": "This is required as the Proxy VM and Packer VM both read and write from transient storage accounts (no ability to know what storage accounts before the process starts.)", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "*.blob.core.windows.net" - ] - }, - { - "name": "apt-get", - "description": "This is required as the Packer VM performs a package upgrade. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Http", - "port": 80 - }, - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "azure.archive.ubuntu.com", - "packages.microsoft.com", - "archive.ubuntu.com", - "security.ubuntu.com" - ] - }, - { - "name": "install-azcli", - "description": "This is required as the Packer VM needs to install Azure CLI. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "aka.ms", - "azurecliextensionsync.blob.core.windows.net", - "azurecliprod.blob.core.windows.net" - ] - }, - { - "name": "install-k8scli", - "description": "This is required as the Packer VM needs to install k8s cli tooling. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "objects.githubusercontent.com", - "storage.googleapis.com", - "api.github.com", - "github-releases.githubusercontent.com", - "github.com" - ] - }, - { - "name": "install-helm", - "description": "This is required as the Packer VM needs to install helm cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "raw.githubusercontent.com", - "get.helm.sh", - "github-releases.githubusercontent.com" - ] - }, - { - "name": "install-flux", - "description": "This is required as the Packer VMs needs to install flux cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "raw.githubusercontent.com", - "api.github.com", - "fluxcd.io", - "github-releases.githubusercontent.com" - ] - }, - { - "name": "install-open-service-mesh", - "description": "This is required as the Packer VMs needs to install open service mesh cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "github.com", - "github-releases.githubusercontent.com" - ] - }, - { - "name": "install-terraform", - "description": "This is required as the Packer VMs needs to install HashiCorp Terraform cli. [Step performed in the referenced jump box building process. Not needed if your jump box building process doesn't do this.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('imageBuilderIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "releases.hashicorp.com" - ] - } - ] - } - }, - { - "name": "aks-jumpbox", - "properties": { - "action": { - "type": "Allow" - }, - "priority": 450, - "rules": [ - { - "name": "az-login", - "description": "Allow jumpboxes to perform az login.", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('aksJumpBoxIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "login.microsoftonline.com" - ] - }, - { - "name": "az-management-api", - "description": "Allow jumpboxes to communicate with Azure management APIs.", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('aksJumpBoxIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "management.azure.com" - ] - }, - { - "name": "az-cli-extensions", - "description": "Allow jumpboxes query az cli status and download extensions", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('aksJumpBoxIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "aka.ms", - "azurecliprod.blob.core.windows.net", - "azcliextensionsync.blob.core.windows.net" - ] - }, - { - "name": "github", - "description": "Allow pulling things down from GitHub. [Only a requirement of this walkthrough because we deploy some manifests that you clone from your repo.]", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('aksJumpBoxIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "github.com", - "*.github.io", - "raw.githubusercontent.com" - ] - }, - { - "name": "azure-monitor-addon", - "description": "Required for Azure Monitor Extension on Jumpbox.", - "sourceIpGroups": [ - "[resourceId('Microsoft.Network/ipGroups', variables('aksJumpBoxIpGroupName'))]" - ], - "protocols": [ - { - "protocolType": "Https", - "port": 443 - } - ], - "targetFqdns": [ - "*.ods.opinsights.azure.com", - "*.oms.opinsights.azure.com", - "[concat(parameters('location'), '.monitoring.azure.com')]", - "*.agentsvc.azure-automation.net" - ] - } - ] - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/default", - "dependsOn": [ - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/azureFirewalls', variables('hubFwName'))]" - ], - "properties": { - "workspaceId": "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "logs": [ - { - "categoryGroup": "allLogs", - "enabled": true - } - ], - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "condition": "[parameters('deployFlowLogResources')]", - "name": "connect-hub-regional-flowlogs", - "type": "Microsoft.Resources/deployments", - "resourceGroup": "networkWatcherRG", - "apiVersion": "2020-10-01", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]", - "[resourceId('Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]", - "[resourceId('Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "[resourceId('Microsoft.Network/azureFirewalls', variables('hubFwName'))]" // This doesn't depend on the FW, but because network watchers are auto-deployed, this helps prevents a race condition between the vnet creation triggering a NetworkWatcher auto provisioning, and the referencing of it here. - ], - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "variables": {}, - "resources": [ - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', variables('bastionNetworkNsgName'))]", - "storageId": "[resourceId(resourceGroup().name, 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[resourceId(resourceGroup().name, 'Microsoft.OperationalInsights/workspaces', variables('hubLaName'))]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - } - ] - } - } - } - ], - "outputs": { - "hubVnetId": { - "value": "[resourceId('Microsoft.Network/virtualNetworks', variables('hubVnetName'))]", - "type": "string" - } - } -} diff --git a/networking/modules/ClusterVNetShouldNotHaveNICwithpublicIP.bicep b/networking/modules/ClusterVNetShouldNotHaveNICwithpublicIP.bicep new file mode 100644 index 00000000..145fb25d --- /dev/null +++ b/networking/modules/ClusterVNetShouldNotHaveNICwithpublicIP.bicep @@ -0,0 +1,36 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('The id of the cluster\'s virtual network') +@minLength(12) +param clusterVNetId string + +/*** EXISTING RESOURCES ***/ + +resource policyResourceIdNoPublicIpsInVnet 'Microsoft.Authorization/policyDefinitions@2021-06-01' existing = { + scope: subscription() + name: guid(subscription().id, 'NoPublicIPsForNICsInVnet') +} + +/*** RESOURCES ***/ + +@description('Cluster VNet should never have a NIC with a public IP. - Policy Assignment') +resource policyAssignment 'Microsoft.Authorization/policyAssignments@2021-06-01' = { + name: guid(policyResourceIdNoPublicIpsInVnet.id, clusterVNetId) + properties: { + displayName: 'Network interfaces in cluster [${last(split(clusterVNetId, '/'))}] should not have public IPs' + description: 'Cluster VNet should never have a NIC with a public IP.' + policyDefinitionId: policyResourceIdNoPublicIpsInVnet.id + parameters: { + vnetResourceId: { + value: clusterVNetId + } + } + nonComplianceMessages: [ + { + message: 'No NICs with public IPs are allowed in the regulated environment spoke.' + } + ] + } +} diff --git a/networking/modules/flowlogsDeployment.bicep b/networking/modules/flowlogsDeployment.bicep new file mode 100644 index 00000000..745ef2e6 --- /dev/null +++ b/networking/modules/flowlogsDeployment.bicep @@ -0,0 +1,52 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('The flowLog\'s location') +@minLength(4) +param location string + +@description('The flowLog\'s target resourceId') +@minLength(79) +param targetResourceId string + +@description('The Id of the Log Analytics workspace that stores logs from the regional hub network') +@minLength(79) +param laHubId string + +@description('The Id of the Storage account to store the flow logs') +@minLength(79) +param flowLogsStorageId string + +/*** EXISTING RESOURCES ***/ + +resource networkWatcher 'Microsoft.Network/networkWatchers@2021-05-01' existing = { + name: 'NetworkWatcher_${location}' +} + +/*** RESOURCES ***/ + +resource flowlog 'Microsoft.Network/networkWatchers/flowLogs@2021-05-01' = { + parent: networkWatcher + name: 'fl${guid(targetResourceId)}' + location: location + properties: { + targetResourceId: targetResourceId + storageId: flowLogsStorageId + enabled: true + format: { + version: 2 + } + flowAnalyticsConfiguration: { + networkWatcherFlowAnalyticsConfiguration: { + enabled: true + workspaceResourceId: laHubId + trafficAnalyticsInterval: 10 + } + } + retentionPolicy: { + days: 365 + enabled: true + } + } +} diff --git a/networking/modules/hubsSpokesPeeringDeployment.bicep b/networking/modules/hubsSpokesPeeringDeployment.bicep new file mode 100644 index 00000000..2c255659 --- /dev/null +++ b/networking/modules/hubsSpokesPeeringDeployment.bicep @@ -0,0 +1,39 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('The hub\'s VNet resource Id') +@minLength(2) +param hubVNetResourceId string + +@description('The spokes\'s VNet name') +@minLength(2) +param spokesVNetName string + +@description('The spokes\'s resource group') +@minLength(1) +param rgSpokes string + +/*** EXISTING RESOURCES ***/ + +@description('The spoke\'s VNet') +resource spokesVNet 'Microsoft.Network/virtualNetworks@2021-05-01' existing = { + name: spokesVNetName + scope: resourceGroup(rgSpokes) +} + +/*** RESOURCES ***/ + +@description('Hub-to-spoke peering.') +resource hubsSpokesVirtualNetworkPeering 'Microsoft.Network/virtualNetworks/virtualNetworkPeerings@2021-05-01' = { + name: '${last(split(hubVNetResourceId, '/'))}/hub-to-${spokesVNetName}' + properties: { + remoteVirtualNetwork: { + id: spokesVNet.id + } + allowForwardedTraffic: false + allowGatewayTransit: false + allowVirtualNetworkAccess: true + useRemoteGateways: false + } +} diff --git a/networking/spoke-BU0001A0005-00.bicep b/networking/spoke-BU0001A0005-00.bicep new file mode 100644 index 00000000..bf75cf37 --- /dev/null +++ b/networking/spoke-BU0001A0005-00.bicep @@ -0,0 +1,317 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('The regional hub network to which this regional spoke will peer to.') +@minLength(79) +param hubVnetResourceId string + +@allowed([ + 'australiaeast' + 'canadacentral' + 'centralus' + 'eastus' + 'eastus2' + 'westus2' + 'francecentral' + 'germanywestcentral' + 'northeurope' + 'southafricanorth' + 'southcentralus' + 'uksouth' + 'westeurope' + 'japaneast' + 'southeastasia' + ]) +@description('The spokes\'s regional affinity, must be the same as the hub\'s location. All resources tied to this spoke will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support.') +param location string + +@description('Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing "false" to this parameter.') +param deployFlowLogResources bool = true + +/*** EXISTING RESOURCES ***/ + +@description('The resource group name containing virtual network in which the regional Azure Firewall is deployed.') +resource rgHubs 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { + scope: subscription() + name: split(hubVnetResourceId, '/')[4] +} + +@description('The regional Azure Firewall that all regional spoke networks can egress through.') +resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' existing = { + scope: rgHubs + name: 'fw-${location}' +} + +resource hubLaWorkspace 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' existing = { + scope: rgHubs + name: 'la-hub-${location}-${uniqueString(rgHubs.id, 'vnet-${location}-hub')}' +} + +@description('NetworkWatcher ResourceGroup; it contains regional Network Watchers') +resource networkWatcherResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = if (deployFlowLogResources) { + scope: subscription() + name: 'networkWatcherRG' +} + +@description('Storage account to store the flow logs') +resource flowlogs_storageAccount 'Microsoft.Storage/storageAccounts@2021-08-01' existing = { + scope: rgHubs + name: substring('stnfl${location}${uniqueString(rgHubs.id)}', 0, 24) +} + +/*** RESOURCES ***/ + +@description('Next hop to regional hub Azure Firewall') +resource afRouteTable 'Microsoft.Network/routeTables@2021-05-01' = { + name: 'route-to-${location}-hub-fw' + location: location + properties: { + routes: [ + { + name: 'r-nexthop-to-fw' + properties: { + nextHopType: 'VirtualAppliance' + addressPrefix: '0.0.0.0/0' + nextHopIpAddress: hubFirewall.properties.ipConfigurations[0].properties.privateIPAddress + } + } + ] + } +} + +@description('NSG on the jumpbox image builder subnet.') +resource nsgJumpboxImgbuilderSubnet 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-vnet-spoke-BU0001A0005-00-imageBuilder' + location: location + properties: { + securityRules: [ + { + name: 'AllowAzureLoadBalancer60001InBound' + properties: { + description: 'Allows heath probe traffic to AIB Proxy VM on 60001 (SSH)' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationPortRange: '60001' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 100 + direction: 'Inbound' + } + } + { + name: 'AllowVNet60001InBound' + properties: { + description: 'Allows traffic from AIB Service PrivateLink to AIB Proxy VM' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '60001' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 110 + direction: 'Inbound' + } + } + { + name: 'AllowVNet22InBound' + properties: { + description: 'Allows Packer VM to receive SSH traffic from AIB Proxy VM' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '22' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 120 + direction: 'Inbound' + } + } + { + name: 'DenyAllInBound' + properties: { + description: 'Deny remaining traffic.' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Inbound' + } + } + { + name: 'Allow443ToInternetOutBound' + properties: { + description: 'Allow VMs to communicate to Azure management APIs, Azure Storage, and perform install tasks.' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '443' + destinationAddressPrefix: 'Internet' + access: 'Allow' + priority: 100 + direction: 'Outbound' + } + } + { + name: 'Allow80ToInternetOutBound' + properties: { + description: 'Allow Packer VM to use apt-get to upgrade packages' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '80' + destinationAddressPrefix: 'Internet' + access: 'Allow' + priority: 102 + direction: 'Outbound' + } + } + { + name: 'AllowSshToVNetOutBound' + properties: { + description: 'Allow Proxy VM to communicate to Packer VM' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '22' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 110 + direction: 'Outbound' + } + } + { + name: 'DenyAllOutBound' + properties: { + description: 'Deny all remaining outbound traffic' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Outbound' + } + } + ] + } +} + +resource nsgJumpboxImgbuilderSubnet_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: nsgJumpboxImgbuilderSubnet + properties: { + workspaceId: hubLaWorkspace.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('This vnet is used exclusively for jumpbox image builds.') +resource imageBuilderVNet 'Microsoft.Network/virtualNetworks@2021-05-01' = { + name: 'vnet-spoke-BU0001A0005-00' + location: location + properties: { + addressSpace: { + addressPrefixes: [ + '10.241.0.0/28' + ] + } + subnets: [ + { + name: 'snet-imagebuilder' + properties: { + addressPrefix: '10.241.0.0/28' + routeTable: { + id: afRouteTable.id + } + networkSecurityGroup: { + id: nsgJumpboxImgbuilderSubnet.id + } + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Disabled' + } + } + ] + dhcpOptions: { + dnsServers: [ + hubFirewall.properties.ipConfigurations[0].properties.privateIPAddress + ] + } + } + + resource snetImageBuilder 'subnets' existing = { + name: 'snet-imagebuilder' + } +} + +@description('Peer to regional hub.') +resource imageBuilderVNetPeering 'Microsoft.Network/virtualNetworks/virtualNetworkPeerings@2021-05-01' = { + name: 'spoke-to-${last(split(hubVnetResourceId, '/'))}' + parent: imageBuilderVNet + properties: { + remoteVirtualNetwork: { + id: hubVnetResourceId + } + allowForwardedTraffic: false + allowVirtualNetworkAccess: true + allowGatewayTransit: false + useRemoteGateways: false + } +} + +resource imageBuilderVNet_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: imageBuilderVNet + properties: { + workspaceId: hubLaWorkspace.id + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('Flow Logs deployment') +module flowlogsDeployment 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'connect-spoke-bu0001A0005-00-flowlogs' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgJumpboxImgbuilderSubnet.id + laHubId: hubLaWorkspace.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +module hubsSpokesPeering 'modules/hubsSpokesPeeringDeployment.bicep' = { + name: 'hub-to-jumpboxVNet-peering' + scope: rgHubs + params: { + hubVNetResourceId: hubVnetResourceId + spokesVNetName: imageBuilderVNet.name + rgSpokes: resourceGroup().name + } + dependsOn: [ + imageBuilderVNetPeering + ] +} + +/*** OUTPUTS ***/ + +output imageBuilderSubnetResourceId string = imageBuilderVNet::snetImageBuilder.id diff --git a/networking/spoke-BU0001A0005-00.json b/networking/spoke-BU0001A0005-00.json deleted file mode 100644 index 375df41a..00000000 --- a/networking/spoke-BU0001A0005-00.json +++ /dev/null @@ -1,384 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "hubVnetResourceId": { - "type": "string", - "metadata": { - "description": "The regional hub network to which this regional spoke will peer to." - } - }, - "location": { - "type": "string", - "allowedValues": [ - "australiaeast", - "canadacentral", - "centralus", - "eastus", - "eastus2", - "westus2", - "francecentral", - "germanywestcentral", - "northeurope", - "southafricanorth", - "southcentralus", - "uksouth", - "westeurope", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "The spokes's regional affinity, must be the same as the hub's location. All resources tied to this spoke will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support." - } - }, - "deployFlowLogResources": { - "defaultValue": true, - "type": "bool", - "metadata": { - "description": "Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing 'false' to this parameter." - } - } - }, - "variables": { - "orgAppId": "BU0001A0005", - "imageBuilderVNetName": "[concat('vnet-spoke-', variables('orgAppId'), '-00')]", - "imageBuilderSubnetNsgName": "[concat('nsg-', variables('imageBuilderVNetName'), '-imageBuilder')]", - - "routeTableName": "[concat('route-to-', parameters('location'), '-hub-fw')]", - "hubRgName": "[split(parameters('hubVnetResourceId'),'/')[4]]", - "hubNetworkName": "[split(parameters('hubVnetResourceId'),'/')[8]]", - "hubFwResourceId": "[resourceId(variables('hubRgName'), 'Microsoft.Network/azureFirewalls', concat('fw-', parameters('location')))]", - "hubLaWorkspaceName": "[concat('la-hub-', parameters('location'), '-', uniqueString(parameters('hubVnetResourceId')))]", - "hubLaWorkspaceResourceId": "[resourceId(variables('hubRgName'), 'Microsoft.OperationalInsights/workspaces', variables('hubLaWorkspaceName'))]", - "toHubPeeringName": "[concat('spoke-to-', variables('hubNetworkName'))]", - "regionFlowLowStorageAccountName": "[take(concat('stnfl', parameters('location'), uniqueString(subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('hubRgName')))), 24)]" - }, - "resources": [ - { - "type": "Microsoft.Network/routeTables", - "apiVersion": "2020-05-01", - "name": "[variables('routeTableName')]", - "location": "[parameters('location')]", - "comments": "Next hop to regional hub Azure Firewall", - "properties": { - "routes": [ - { - "name": "r-nexthop-to-fw", - "properties": { - "nextHopType": "VirtualAppliance", - "addressPrefix": "0.0.0.0/0", - "nextHopIpAddress": "[reference(variables('hubFwResourceId'), '2020-05-01').ipConfigurations[0].properties.privateIpAddress]" - } - } - ] - } - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[variables('imageBuilderSubnetNsgName')]", - "location": "[parameters('location')]", - "comments": "NSG on the jumpbox image builder subnet.", - "properties": { - "securityRules": [ - { - "name": "AllowAzureLoadBalancer60001InBound", - "properties": { - "description": "Allows heath probe traffic to AIB Proxy VM on 60001 (SSH)", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "AzureLoadBalancer", - "destinationPortRange": "60001", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 100, - "direction": "Inbound" - } - }, - { - "name": "AllowVNet60001InBound", - "properties": { - "description": "Allows traffic from AIB Service PrivateLink to AIB Proxy VM", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "60001", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 110, - "direction": "Inbound" - } - }, - { - "name": "AllowVNet22InBound", - "properties": { - "description": "Allows Packer VM to receive SSH traffic from AIB Proxy VM", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "22", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 120, - "direction": "Inbound" - } - }, - { - "name": "DenyAllInBound", - "properties": { - "description": "Deny remaining traffic.", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Inbound" - } - }, - { - "name": "Allow443ToInternetOutBound", - "properties": { - "description": "Allow VMs to communicate to Azure management APIs, Azure Storage, and perform install tasks.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "443", - "destinationAddressPrefix": "Internet", - "access": "Allow", - "priority": 100, - "direction": "Outbound" - } - }, - { - "name": "Allow80ToInternetOutBound", - "properties": { - "description": "Allow Packer VM to use apt-get to upgrade packages", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "80", - "destinationAddressPrefix": "Internet", - "access": "Allow", - "priority": 102, - "direction": "Outbound" - } - }, - { - "name": "AllowSshToVNetOutBound", - "properties": { - "description": "Allow Proxy VM to communicate to Packer VM", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "22", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 110, - "direction": "Outbound" - } - }, - { - "name": "DenyAllOutBound", - "properties": { - "description": "Deny all remaining outbound traffic", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Outbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('imageBuilderSubnetNsgName'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/virtualNetworks", - "apiVersion": "2020-05-01", - "name": "[variables('imageBuilderVNetName')]", - "location": "[parameters('location')]", - "comments": "This vnet is used exclusively for jumpbox image builds.", - "dependsOn": [ - "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('imageBuilderSubnetNsgName'))]" - ], - "properties": { - "addressSpace": { - "addressPrefixes": [ - "10.241.0.0/28" - ] - }, - "subnets": [ - { - "name": "snet-imagebuilder", - "properties": { - "addressPrefix": "10.241.0.0/28", - "routeTable": { - "id": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]" - }, - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('imageBuilderSubnetNsgName'))]" - }, - "privateEndpointNetworkPolicies": "Enabled", - "privateLinkServiceNetworkPolicies": "Disabled" - } - } - ], - "dhcpOptions": { - "dnsServers": [ - "[reference(variables('hubFwResourceId'), '2020-05-01').ipConfigurations[0].properties.privateIpAddress]" - ] - } - }, - "resources": [ - { - "type": "virtualNetworkPeerings", - "apiVersion": "2020-05-01", - "name": "[variables('toHubPeeringName')]", - "comments": "Peer to regional hub.", - "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('imageBuilderVNetName'))]" - ], - "properties": { - "remoteVirtualNetwork": { - "id": "[parameters('hubVnetResourceId')]" - }, - "allowForwardedTraffic": false, - "allowVirtualNetworkAccess": true, - "allowGatewayTransit": false, - "useRemoteGateways": false - } - }, - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('imageBuilderVNetName'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Resources/deployments", - "apiVersion": "2020-06-01", - "name": "[concat('CreateHubTo', variables('imageBuilderVNetName'), 'Peer')]", - "resourceGroup": "[variables('hubRgName')]", - "comments": "Sub incremental deployment to perform hub-to-spoke peering", - "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks/virtualNetworkPeerings', variables('imageBuilderVNetName'), variables('toHubPeeringName'))]" - ], - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "resources": [ - { - "type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings", - "apiVersion": "2020-05-01", - "name": "[concat(variables('hubNetworkName'), '/hub-to-', variables('imageBuilderVNetName'))]", - "properties": { - "remoteVirtualNetwork": { - "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('imageBuilderVNetName'))]" - }, - "allowForwardedTraffic": false, - "allowGatewayTransit": false, - "allowVirtualNetworkAccess": true, - "useRemoteGateways": false - } - } - ] - } - } - }, - { - "condition": "[parameters('deployFlowLogResources')]", - "name": "connect-spoke-bu0001A0005-00-flowlogs", - "type": "Microsoft.Resources/deployments", - "resourceGroup": "networkWatcherRG", - "apiVersion": "2020-10-01", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', variables('imageBuilderSubnetNsgName'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('imageBuilderVNetName'))]" - ], - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "variables": {}, - "resources": [ - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', variables('imageBuilderSubnetNsgName'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', variables('imageBuilderSubnetNsgName'))]", - "storageId": "[resourceId(variables('hubRgName'), 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[variables('hubLaWorkspaceResourceId')]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - } - ] - } - } - } - ], - "outputs": { - "imageBuilderSubnetResourceId": { - "type": "string", - "value": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('imageBuilderVNetName'), 'snet-imagebuilder')]" - } - } -} diff --git a/networking/spoke-BU0001A0005-01.bicep b/networking/spoke-BU0001A0005-01.bicep new file mode 100644 index 00000000..f05793a6 --- /dev/null +++ b/networking/spoke-BU0001A0005-01.bicep @@ -0,0 +1,1023 @@ +targetScope = 'resourceGroup' + +/*** PARAMETERS ***/ + +@description('The regional hub network to which this regional spoke will peer to.') +@minLength(79) +param hubVnetResourceId string + +@allowed([ + 'australiaeast' + 'canadacentral' + 'centralus' + 'eastus' + 'eastus2' + 'westus2' + 'francecentral' + 'germanywestcentral' + 'northeurope' + 'southafricanorth' + 'southcentralus' + 'uksouth' + 'westeurope' + 'japaneast' + 'southeastasia' + ]) +@description('The spokes\'s regional affinity, must be the same as the hub\'s location. All resources tied to this spoke will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support.') +param location string + +@description('Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing "false" to this parameter.') +param deployFlowLogResources bool = true + +/*** VARIBALES ***/ + +@description('The organization\'s application ID') +var orgAppId = 'BU0001A0005' + +/*** EXISTING RESOURCES ***/ + +@description('The resource group name containing virtual network in which the regional Azure Firewall is deployed.') +resource rgHubs 'Microsoft.Resources/resourceGroups@2021-04-01' existing = { + scope: subscription() + name: split(hubVnetResourceId, '/')[4] +} + +@description('The regional Azure Firewall that all regional spoke networks can egress through.') +resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' existing = { + scope: rgHubs + name: 'fw-${location}' +} + +@description('The regional hub network') +resource hubsVnet 'Microsoft.Network/virtualNetworks@2021-05-01' existing = { + scope: rgHubs + name: 'vnet-${location}-hub' + + resource azureBastionSubnet 'subnets' existing = { + name: 'AzureBastionSubnet' + } +} + +@description('The hub\'s resource group') +resource hubLaWorkspace 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' existing = { + scope: rgHubs + name: 'la-hub-${location}-${uniqueString(rgHubs.id, 'vnet-${location}-hub')}' +} + +@description('NetworkWatcher ResourceGroup; it contains regional Network Watchers') +resource networkWatcherResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = if (deployFlowLogResources) { + scope: subscription() + name: 'networkWatcherRG' +} + +@description('Storage account to store the flow logs') +resource flowlogs_storageAccount 'Microsoft.Storage/storageAccounts@2021-08-01' existing = { + scope: rgHubs + name: substring('stnfl${location}${uniqueString(rgHubs.id)}', 0, 24) +} + +/*** RESOURCES ***/ + +@description('Next hop to regional hub Azure Firewall') +resource afRouteTable 'Microsoft.Network/routeTables@2021-05-01' = { + name: 'route-to-${location}-hub-fw' + location: location + properties: { + routes: [ + { + name: 'r-nexthop-to-fw' + properties: { + nextHopType: 'VirtualAppliance' + addressPrefix: '0.0.0.0/0' + nextHopIpAddress: hubFirewall.properties.ipConfigurations[0].properties.privateIPAddress + } + } + ] + } +} + +@description('NSG blocking all inbound traffic other than port 22 for jumpbox access.') +resource nsgAllowSshFromHubBastionInBound 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-vnet-spoke-${orgAppId}-01-management-ops' + location: location + properties: { + securityRules: [ + { + name: 'AllowSshFromHubBastionInBound' + properties: { + description: 'Allow our Azure Bastion users in.' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: hubsVnet::azureBastionSubnet.properties.addressPrefix + destinationPortRange: '22' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 100 + direction: 'Inbound' + } + } + { + name: 'DenyAllInBound' + properties: { + description: 'Deny remaining traffic.' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Inbound' + } + } + { + name: 'Allow443InternetOutBound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '443' + destinationAddressPrefix: 'Internet' + access: 'Allow' + priority: 100 + direction: 'Outbound' + } + } + { + name: 'Allow443VnetOutBound' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '443' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 110 + direction: 'Outbound' + } + } + { + name: 'DenyAllOutBound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Outbound' + } + } + ] + } +} + +resource nsgAllowSshFromHubBastionInBound_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: nsgAllowSshFromHubBastionInBound + properties: { + workspaceId: hubLaWorkspace.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('NSG on all AKS system nodepools. Feel free to constrict further both inbound and outbound!') +resource nsgAksSystemNodepools 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-vnet-spoke-${orgAppId}-01-system-nodepools' + location: location + properties: { + securityRules: [ + { + name: 'DenySshInBound' + properties: { + description: 'No SSH access allowed to nodes.' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '22' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 100 + direction: 'Inbound' + } + } + ] + } +} + +resource nsgAksSystemNodepools_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: nsgAksSystemNodepools + properties: { + workspaceId: hubLaWorkspace.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('NSG on the AKS in-scope nodepools. Feel free to constrict further both inbound and outbound!') +resource nsgAksInScopeNodepools 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-vnet-spoke-${orgAppId}-01-is-nodepools' + location: location + properties: { + securityRules: [ + { + name: 'DenySshInBound' + properties: { + description: 'No SSH access allowed to nodes.' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '22' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 100 + direction: 'Inbound' + } + } + ] + } +} + +resource nsgAksInScopeNodepools_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: nsgAksInScopeNodepools + properties: { + workspaceId: hubLaWorkspace.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('NSG on the AKS out-of-scope nodepools. Feel free to constrict further both inbound and outbound!') +resource nsgAksOutOfScopeNodepools 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-vnet-spoke-${orgAppId}-01-oos-nodepools' + location: location + properties: { + securityRules: [ + { + name: 'DenySshInBound' + properties: { + description: 'No SSH access allowed to nodes.' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '22' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 100 + direction: 'Inbound' + } + } + ] + } +} + +resource nsgAksOutOfScopeNodepools_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: nsgAksOutOfScopeNodepools + properties: { + workspaceId: hubLaWorkspace.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('Default NSG on the private link subnet. No traffic should be allowed out, and only Tcp/443 in. Key Vault and Container Registry is expected to be accessed in here.') +resource nsgAksPrivateLinkEndpoint 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-vnet-spoke-${orgAppId}-01-privatelinkendpoints' + location: location + properties: { + securityRules: [ + { + name: 'AllowAll443InFromVnet' + properties: { + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '443' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 100 + direction: 'Inbound' + } + } + { + name: 'DenyAllInBound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Inbound' + } + } + { + name: 'DenyAllOutBound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Outbound' + } + } + ] + } +} + +resource nsgAksPrivateLinkEndpoint_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: nsgAksPrivateLinkEndpoint + properties: { + workspaceId: hubLaWorkspace.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('Default NSG on the AKS ILB subnet. Feel free to constrict further!') +resource nsgAksDefaultILBSubnet 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-vnet-spoke-${orgAppId}-01-akslibs' + location: location + properties: { + securityRules: [ + ] + } +} + +resource nsgAksDefaultILBSubnet_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: nsgAksDefaultILBSubnet + properties: { + workspaceId: hubLaWorkspace.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('NSG on the App Gateway subnet.') +resource nsgAppGatewaySubnet 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-vnet-spoke-${orgAppId}-01-appgw' + location: location + properties: { + securityRules: [ + { + name: 'Allow443InBound' + properties: { + description: 'Allow ALL web traffic into 443. (If you wanted to allow-list specific IPs, this is where you\'d list them.)' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'Internet' + destinationPortRange: '443' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 100 + direction: 'Inbound' + } + } + { + name: 'AllowControlPlaneInBound' + properties: { + description: 'Allow Azure Control Plane in. (https://docs.microsoft.com/azure/application-gateway/configuration-infrastructure#network-security-groups)' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '65200-65535' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 110 + direction: 'Inbound' + } + } + { + name: 'AllowHealthProbesInBound' + properties: { + description: 'Allow Azure Health Probes in. (https://docs.microsoft.com/azure/application-gateway/configuration-infrastructure#network-security-groups)' + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationPortRange: '*' + destinationAddressPrefix: 'VirtualNetwork' + access: 'Allow' + priority: 120 + direction: 'Inbound' + } + } + { + name: 'DenyAllInBound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Deny' + priority: 1000 + direction: 'Inbound' + } + } + { + name: 'AllowAllOutBound' + properties: { + protocol: '*' + sourcePortRange: '*' + sourceAddressPrefix: '*' + destinationPortRange: '*' + destinationAddressPrefix: '*' + access: 'Allow' + priority: 1000 + direction: 'Outbound' + } + } + ] + } +} + +resource nsgAppGatewaySubnet_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: nsgAppGatewaySubnet + properties: { + workspaceId: hubLaWorkspace.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('NSG on the ACR docker subnet.') +resource nsgAcrDockerSubnet 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: 'nsg-vnet-spoke-${orgAppId}-01-acragents' + location: location + properties: { + securityRules: [ + { + name: 'AllowKeyVaultOutBound' + properties: { + description: 'Allow KeyVault Access' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '443' + destinationAddressPrefix: 'AzureKeyVault' + access: 'Allow' + priority: 100 + direction: 'Outbound' + } + } + { + name: 'AllowStorageOutBound' + properties: { + description: 'Allow Storage Access' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '443' + destinationAddressPrefix: 'Storage' + access: 'Allow' + priority: 110 + direction: 'Outbound' + } + } + { + name: 'AllowEventHubOutBound' + properties: { + description: 'Allow EventHub Access' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '443' + destinationAddressPrefix: 'EventHub' + access: 'Allow' + priority: 120 + direction: 'Outbound' + } + } + { + name: 'AllowAadOutBound' + properties: { + description: 'Allow Azure Active Directory Access' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '443' + destinationAddressPrefix: 'AzureActiveDirectory' + access: 'Allow' + priority: 130 + direction: 'Outbound' + } + } + { + name: 'AllowAzureMonitorBound' + properties: { + description: 'Allow Azure Active Directory Access' + protocol: 'Tcp' + sourcePortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationPortRange: '443' + destinationAddressPrefix: 'AzureMonitor' + access: 'Allow' + priority: 140 + direction: 'Outbound' + } + } + ] + } +} + +resource nsgAcrDockerSubnet_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: nsgAcrDockerSubnet + properties: { + workspaceId: hubLaWorkspace.id + logs: [ + { + category: 'NetworkSecurityGroupEvent' + enabled: true + } + { + category: 'NetworkSecurityGroupRuleCounter' + enabled: true + } + ] + } +} + +@description('cluster\'s virtual network. 65,536 (-reserved) IPs available to the workload, split across four subnets for AKS, one for App Gateway, and two for management.') +resource clusterVNet 'Microsoft.Network/virtualNetworks@2021-05-01' = { + name: 'vnet-spoke-${orgAppId}-01' + location: location + properties: { + addressSpace: { + addressPrefixes: [ + '10.240.0.0/16' + ] + } + subnets: [ + { + name: 'snet-cluster-systemnodepool' + properties: { + addressPrefix: '10.240.8.0/22' + routeTable: { + id: afRouteTable.id + } + networkSecurityGroup: { + id: nsgAksSystemNodepools.id + } + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Disabled' + } + } + { + name: 'snet-cluster-inscopenodepools' + properties: { + addressPrefix: '10.240.12.0/22' + networkSecurityGroup: { + id: nsgAksInScopeNodepools.id + } + routeTable: { + id: afRouteTable.id + } + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Disabled' + } + } + { + name: 'snet-cluster-outofscopenodepools' + properties: { + addressPrefix: '10.240.16.0/22' + networkSecurityGroup: { + id: nsgAksOutOfScopeNodepools.id + } + routeTable: { + id: afRouteTable.id + } + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Disabled' + } + } + { + name: 'snet-cluster-ingressservices' + properties: { + addressPrefix: '10.240.4.0/28' + networkSecurityGroup: { + id: nsgAksDefaultILBSubnet.id + } + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Disabled' + } + } + { + name: 'snet-applicationgateway' + properties: { + addressPrefix: '10.240.5.0/24' + networkSecurityGroup: { + id: nsgAppGatewaySubnet.id + } + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Disabled' + } + } + { + name: 'snet-management-ops' + properties: { + addressPrefix: '10.240.1.0/28' + routeTable: { + id: afRouteTable.id + } + networkSecurityGroup: { + id: nsgAllowSshFromHubBastionInBound.id + } + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Disabled' + } + } + { + name: 'snet-management-agents' + properties: { + addressPrefix: '10.240.2.0/26' + routeTable: { + id: afRouteTable.id + } + networkSecurityGroup: { + id: nsgAllowSshFromHubBastionInBound.id + } + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Disabled' + } + } + { + name: 'snet-management-acragents' + properties: { + addressPrefix: '10.240.251.0/28' + /*routeTable: { + id: afRouteTable.id + }*/ + networkSecurityGroup: { + id: nsgAcrDockerSubnet.id + } + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Disabled' + } + } + { + name: 'snet-privatelinkendpoints' + properties: { + addressPrefix: '10.240.250.0/28' + routeTable: { + id: afRouteTable.id + } + networkSecurityGroup: { + id: nsgAksPrivateLinkEndpoint.id + } + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Enabled' + } + } + ] + dhcpOptions: { + dnsServers: [ + hubFirewall.properties.ipConfigurations[0].properties.privateIPAddress + ] + } + } + dependsOn: [ + policyAssignmentNoPublicIpsInVnet + ] + + resource aksSystemNodepoolSubnet 'subnets' existing = { + name: 'snet-cluster-systemnodepool' + } + + resource aksSystemInScopeNodepoolsSubnet 'subnets' existing = { + name: 'snet-cluster-inscopenodepools' + } + + resource aksSystemOutOfScopeNodepoolsSubnet 'subnets' existing = { + name: 'snet-cluster-outofscopenodepools' + } + + resource aksManagementOpsSubnet 'subnets' existing = { + name: 'snet-management-ops' + } +} + +@description('Deploys subscription-level policy related to spoke deployment. ') +module policyAssignmentNoPublicIpsInVnet './modules/ClusterVNetShouldNotHaveNICwithpublicIP.bicep' = { + name: 'Apply-Subscription-Spoke-PipUsage-Policies-01' + params: { + clusterVNetId: resourceId('Microsoft.Network/virtualNetworks','vnet-spoke-${orgAppId}-01') + } +} + +@description('Peer to regional hub.') +resource clusterVNet_virtualNetworkPeering 'Microsoft.Network/virtualNetworks/virtualNetworkPeerings@2021-05-01' = { + name: 'spoke-to-${last(split(hubVnetResourceId, '/'))}' + parent: clusterVNet + properties: { + remoteVirtualNetwork: { + id: hubVnetResourceId + } + allowForwardedTraffic: false + allowVirtualNetworkAccess: true + allowGatewayTransit: false + useRemoteGateways: false + } +} + +resource clusterVNet_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'toHub' + scope: clusterVNet + properties: { + workspaceId: hubLaWorkspace.id + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +module hubsSpokesPeering 'modules/hubsSpokesPeeringDeployment.bicep' = { + name: 'hub-to-clusterVNet-peering' + scope: rgHubs + params: { + hubVNetResourceId: hubVnetResourceId + spokesVNetName: clusterVNet.name + rgSpokes: resourceGroup().name + } + dependsOn: [ + clusterVNet_virtualNetworkPeering + ] +} + +@description('Enables Azure Container Registry Private Link on vnet.') +resource acrPrivateDnsZones 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: 'privatelink.azurecr.io' + location: 'global' + properties: {} +} + +@description('Enables cluster vnet private zone DNS lookup - used by cluster vnet for direct DNS queries (ones not proxied via the hub).') +resource acrPrivateDnsZones_virtualNetworkLink_toCluster 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + name: 'to_${clusterVNet.name}' + parent: acrPrivateDnsZones + location: 'global' + properties: { + virtualNetwork: { + id: clusterVNet.id + } + registrationEnabled: false + } +} + +@description('Enabling hub vnet private zone DNS lookup for ACR - used by azure firewall\'s dns proxy.') +resource acrPrivateDnsZones_virtualNetworkLink_toHubVNet 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + name: 'to_${last(split(hubVnetResourceId, '/'))}' + parent: acrPrivateDnsZones + location: 'global' + properties: { + virtualNetwork: { + id: hubVnetResourceId + } + registrationEnabled: false + } +} + +@description('Enables AKS Private Link on vnet.') +resource aksPrivateDnsZones 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: 'privatelink.${location}.azmk8s.io' + location: 'global' + properties: {} +} + +@description('Enabling hub vnet private zone DNS lookup for private AKS - used by azure firewall\'s dns proxy.') +resource aksPrivateDnsZones_virtualNetworkLink_toClusterVNet 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + name: 'to_${last(split(hubVnetResourceId, '/'))}' + parent: aksPrivateDnsZones + location: 'global' + properties: { + virtualNetwork: { + id: clusterVNet.id + } + registrationEnabled: false + } +} + +@description('Enables Azure Key Vault Private Link support.') +resource akvPrivateDnsZones 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: 'privatelink.vaultcore.azure.net' + location: 'global' + properties: {} +} + +@description('Enables Azure Key Vault Private Link on cluster vnet.') +resource akvPrivateDnsZones_virtualNetworkLink_toCluster 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + name: 'to_${clusterVNet.name}' + parent: akvPrivateDnsZones + location: 'global' + properties: { + virtualNetwork: { + id: clusterVNet.id + } + registrationEnabled: false + } +} + +@description('Enables hub vnet private zone DNS lookup for AKV - used by azure firewall\'s dns proxy.') +resource akvPrivateDnsZones_virtualNetworkLink_toHubVNet 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + name: 'to_${last(split(hubVnetResourceId, '/'))}' + parent: akvPrivateDnsZones + location: 'global' + properties: { + virtualNetwork: { + id: hubVnetResourceId + } + registrationEnabled: false + } +} + +@description('Used as primary entry point for workload. Expected to be assigned to an Azure Application Gateway.') +resource pipPrimaryCluster 'Microsoft.Network/publicIPAddresses@2021-05-01' = { + name: 'pip-BU0001A0005-00' + location: location + sku: { + name: 'Standard' + } + zones: [ + '1' + '2' + '3' + ] + properties: { + publicIPAllocationMethod: 'Static' + idleTimeoutInMinutes: 4 + publicIPAddressVersion: 'IPv4' + } +} + +module flowlogsDeploymentAcrDockerSubnet 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'flowlogs-Deployment-AcrDockerSubnet-NSG' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgAcrDockerSubnet.id + laHubId: hubLaWorkspace.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +module flowlogsDeploymentAksDefaultILBSubnet 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'flowlogs-Deployment-AksDefaultILB-Subnet-NSG' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgAksDefaultILBSubnet.id + laHubId: hubLaWorkspace.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +module flowlogsDeploymentAppGatewaySubnet 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'flowlogs-Deployment-AppGateway-Subnet-NSG' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgAppGatewaySubnet.id + laHubId: hubLaWorkspace.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +module flowlogsDeploymentksInScopeNodepools 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'flowlogs-Deploymentks-InScopeNodepools-NSG' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgAksInScopeNodepools.id + laHubId: hubLaWorkspace.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +module flowlogsDeploymentAllowSshFromHubBastionInBound 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'flowlogs-Deployment-AllowSshFromHubBastionInBound-NSG' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgAllowSshFromHubBastionInBound.id + laHubId: hubLaWorkspace.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +module flowlogsDeploymentAksOutOfScopeNodepools 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'flowlogs-Deployment-AksOutOfScopeNodepools-NSG' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgAksOutOfScopeNodepools.id + laHubId: hubLaWorkspace.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +module flowlogsDeploymentAksPrivateLinkEndpoint 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'flowlogs-Deployment-AksPrivateLinkEndpoint-NSG' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgAksPrivateLinkEndpoint.id + laHubId: hubLaWorkspace.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +module flowlogsDeploymentAksSystemNodepools 'modules/flowlogsDeployment.bicep' = if (deployFlowLogResources) { + name: 'flowlogs-Deployment-AksSystemNodepools-NSG' + scope: networkWatcherResourceGroup + params: { + location: location + targetResourceId: nsgAksSystemNodepools.id + laHubId: hubLaWorkspace.id + flowLogsStorageId: flowlogs_storageAccount.id + } +} + +/*** OUTPUTS ***/ + +output clusterVnetResourceId string = clusterVNet.id + +output jumpboxSubnetResourceId string = clusterVNet::aksManagementOpsSubnet.id + +output nodepoolSubnetResourceIds array = [ + clusterVNet::aksSystemNodepoolSubnet.id + clusterVNet::aksSystemInScopeNodepoolsSubnet.id + clusterVNet::aksSystemOutOfScopeNodepoolsSubnet.id +] + +output appGwPublicIpAddress string = pipPrimaryCluster.properties.ipAddress diff --git a/networking/spoke-BU0001A0005-01.json b/networking/spoke-BU0001A0005-01.json deleted file mode 100644 index ab5cc3ec..00000000 --- a/networking/spoke-BU0001A0005-01.json +++ /dev/null @@ -1,1284 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "hubVnetResourceId": { - "type": "string", - "metadata": { - "description": "The regional hub network to which this regional spoke will peer to." - } - }, - "location": { - "type": "string", - "allowedValues": [ - "australiaeast", - "canadacentral", - "centralus", - "eastus", - "eastus2", - "westus2", - "francecentral", - "germanywestcentral", - "northeurope", - "southafricanorth", - "southcentralus", - "uksouth", - "westeurope", - "japaneast", - "southeastasia" - ], - "metadata": { - "description": "The spokes's regional affinity, must be the same as the hub's location. All resources tied to this spoke will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support." - } - }, - "deployFlowLogResources": { - "defaultValue": true, - "type": "bool", - "metadata": { - "description": "Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing 'false' to this parameter." - } - } - }, - "variables": { - "orgAppId": "BU0001A0005", - "clusterVNetName": "[concat('vnet-spoke-', variables('orgAppId'), '-01')]", - - "routeTableName": "[concat('route-to-', parameters('location'), '-hub-fw')]", - "hubRgName": "[split(parameters('hubVnetResourceId'),'/')[4]]", - "hubNetworkName": "[split(parameters('hubVnetResourceId'),'/')[8]]", - - "hubFwResourceId": "[resourceId(variables('hubRgName'), 'Microsoft.Network/azureFirewalls', concat('fw-', parameters('location')))]", - "hubLaWorkspaceName": "[concat('la-hub-', parameters('location'), '-', uniqueString(parameters('hubVnetResourceId')))]", - "hubLaWorkspaceResourceId": "[resourceId(variables('hubRgName'), 'Microsoft.OperationalInsights/workspaces', variables('hubLaWorkspaceName'))]", - "toHubPeeringName": "[concat('spoke-to-', variables('hubNetworkName'))]", - - "hubBastionSubnetResourceId": "[resourceId(variables('hubRgName'), 'Microsoft.Network/virtualNetworks/subnets', variables('hubNetworkName'), 'AzureBastionSubnet')]", - - "acrPrivateDnsZonesName": "privatelink.azurecr.io", - "aksPrivateDnsZonesName": "[concat('privatelink.', parameters('location'), '.azmk8s.io')]", - "akvPrivateDnsZonesName": "privatelink.vaultcore.azure.net", - - "primaryClusterPipName": "[concat('pip-', variables('orgAppId'), '-00')]", - - "policyResourceIdNoPublicIpsInVnet": "[subscriptionResourceId('Microsoft.Authorization/policyDefinitions', guid(subscription().id, 'NoPublicIPsForNICsInVnet'))]", - "clusterVnetResourceId": "[resourceId('Microsoft.Network/virtualNetworks', variables('clusterVNetName'))]", - "regionFlowLowStorageAccountName": "[take(concat('stnfl', parameters('location'), uniqueString(subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('hubRgName')))), 24)]" - }, - "resources": [ - { - "type": "Microsoft.Network/routeTables", - "apiVersion": "2020-05-01", - "name": "[variables('routeTableName')]", - "location": "[parameters('location')]", - "comments": "Next hop to regional hub Azure Firewall", - "properties": { - "routes": [ - { - "name": "r-nexthop-to-fw", - "properties": { - "nextHopType": "VirtualAppliance", - "addressPrefix": "0.0.0.0/0", - "nextHopIpAddress": "[reference(variables('hubFwResourceId'), '2020-05-01').ipConfigurations[0].properties.privateIpAddress]" - } - } - ] - } - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[concat('nsg-', variables('clusterVNetName'), '-management-ops')]", - "location": "[parameters('location')]", - "comments": "NSG blocking all inbound traffic other than port 22 for jumpbox access.", - "properties": { - "securityRules": [ - { - "name": "AllowSshFromHubBastionInBound", - "properties": { - "description": "Allow our Azure Bastion users in.", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "[reference(variables('hubBastionSubnetResourceId'), '2020-05-01').addressPrefix]", - "destinationPortRange": "22", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 100, - "direction": "Inbound" - } - }, - { - "name": "DenyAllInBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Inbound" - } - }, - { - "name": "Allow443InternetOutBound", - "properties": { - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "443", - "destinationAddressPrefix": "Internet", - "access": "Allow", - "priority": 100, - "direction": "Outbound" - } - }, - { - "name": "Allow443VnetOutBound", - "properties": { - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "443", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 110, - "direction": "Outbound" - } - }, - { - "name": "DenyAllOutBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Outbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-management-ops'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[concat('nsg-', variables('clusterVNetName'), '-system-nodepools')]", - "location": "[parameters('location')]", - "comments": "NSG on all AKS system nodepools. Feel free to constrict further both inbound and outbound!", - "properties": { - "securityRules": [ - { - "name": "DenySshInBound", - "properties": { - "description": "No SSH access allowed to nodes.", - "protocol": "TCP", - "sourcePortRange": "*", - "destinationPortRange": "22", - "sourceAddressPrefix": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 100, - "direction": "Inbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-system-nodepools'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[concat('nsg-', variables('clusterVNetName'), '-is-nodepools')]", - "location": "[parameters('location')]", - "comments": "NSG on the AKS in-scope nodepools. Feel free to constrict further both inbound and outbound!", - "properties": { - "securityRules": [ - { - "name": "DenySshInBound", - "properties": { - "description": "No SSH access allowed to nodes.", - "protocol": "TCP", - "sourcePortRange": "*", - "destinationPortRange": "22", - "sourceAddressPrefix": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 100, - "direction": "Inbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-is-nodepools'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[concat('nsg-', variables('clusterVNetName'), '-oos-nodepools')]", - "location": "[parameters('location')]", - "comments": "NSG on the AKS out-of-scope nodepools. Feel free to constrict further both inbound and outbound!", - "properties": { - "securityRules": [ - { - "name": "DenySshInBound", - "properties": { - "description": "No SSH access allowed to nodes.", - "protocol": "TCP", - "sourcePortRange": "*", - "destinationPortRange": "22", - "sourceAddressPrefix": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 100, - "direction": "Inbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-oos-nodepools'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[concat('nsg-', variables('clusterVNetName'), '-privatelinkendpoints')]", - "location": "[parameters('location')]", - "comments": "Default NSG on the private link subnet. No traffic should be allowed out, and only Tcp/443 in. Key Vault and Container Registry is expected to be accessed in here.", - "properties": { - "securityRules": [ - { - "name": "AllowAll443InFromVnet", - "properties": { - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "443", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 100, - "direction": "Inbound" - } - }, - { - "name": "DenyAllInBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Inbound" - } - }, - { - "name": "DenyAllOutBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Outbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-privatelinkendpoints'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[concat('nsg-', variables('clusterVNetName'), '-aksilbs')]", - "location": "[parameters('location')]", - "comments": "Default NSG on the AKS ILB subnet. Feel free to constrict further!", - "properties": { - "securityRules": [] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-aksilbs'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[concat('nsg-', variables('clusterVNetName'), '-appgw')]", - "location": "[parameters('location')]", - "comments": "NSG on the App Gateway subnet.", - "properties": { - "securityRules": [ - { - "name": "Allow443InBound", - "properties": { - "description": "Allow ALL web traffic into 443. (If you wanted to allow-list specific IPs, this is where you'd list them.)", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "Internet", - "destinationPortRange": "443", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 100, - "direction": "Inbound" - } - }, - { - "name": "AllowControlPlaneInBound", - "properties": { - "description": "Allow Azure Control Plane in. (https://docs.microsoft.com/azure/application-gateway/configuration-infrastructure#network-security-groups)", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "65200-65535", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 110, - "direction": "Inbound" - } - }, - { - "name": "AllowHealthProbesInBound", - "properties": { - "description": "Allow Azure Health Probes in. (https://docs.microsoft.com/azure/application-gateway/configuration-infrastructure#network-security-groups)", - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "AzureLoadBalancer", - "destinationPortRange": "*", - "destinationAddressPrefix": "VirtualNetwork", - "access": "Allow", - "priority": 120, - "direction": "Inbound" - } - }, - { - "name": "DenyAllInBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Deny", - "priority": 1000, - "direction": "Inbound" - } - }, - { - "name": "AllowAllOutBound", - "properties": { - "protocol": "*", - "sourcePortRange": "*", - "sourceAddressPrefix": "*", - "destinationPortRange": "*", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 1000, - "direction": "Outbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-appgw'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Network/networkSecurityGroups", - "apiVersion": "2020-05-01", - "name": "[concat('nsg-', variables('clusterVNetName'), '-acragents')]", - "location": "[parameters('location')]", - "comments": "NSG on the ACR docker subnet.", - "properties": { - "securityRules": [ - { - "name": "AllowKeyVaultOutBound", - "properties": { - "description": "Allow KeyVault Access", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "443", - "destinationAddressPrefix": "AzureKeyVault", - "access": "Allow", - "priority": 100, - "direction": "Outbound" - } - }, - { - "name": "AllowStorageOutBound", - "properties": { - "description": "Allow Storage Access", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "443", - "destinationAddressPrefix": "Storage", - "access": "Allow", - "priority": 110, - "direction": "Outbound" - } - }, - { - "name": "AllowEventHubOutBound", - "properties": { - "description": "Allow EventHub Access", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "443", - "destinationAddressPrefix": "EventHub", - "access": "Allow", - "priority": 120, - "direction": "Outbound" - } - }, - { - "name": "AllowAadOutBound", - "properties": { - "description": "Allow Azure Active Directory Access", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "443", - "destinationAddressPrefix": "AzureActiveDirectory", - "access": "Allow", - "priority": 130, - "direction": "Outbound" - } - }, - { - "name": "AllowAzureMonitorBound", - "properties": { - "description": "Allow Azure Active Directory Access", - "protocol": "Tcp", - "sourcePortRange": "*", - "sourceAddressPrefix": "VirtualNetwork", - "destinationPortRange": "443", - "destinationAddressPrefix": "AzureMonitor", - "access": "Allow", - "priority": 140, - "direction": "Outbound" - } - } - ] - }, - "resources": [ - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-acragents'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "logs": [ - { - "category": "NetworkSecurityGroupEvent", - "enabled": true - }, - { - "category": "NetworkSecurityGroupRuleCounter", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Resources/deployments", - "apiVersion": "2020-10-01", - "name": "Apply-Subscription-Spoke-PipUsage-Policies-01", - "comments": "Deploys subscription-level policy related to spoke deployment.", - "location": "[parameters('location')]", - "subscriptionId": "[subscription().subscriptionId]", - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/subscriptionDeploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "resources": [ - { - "type": "Microsoft.Authorization/policyAssignments", - "apiVersion": "2020-09-01", - "name": "[guid(variables('policyResourceIdNoPublicIpsInVnet'), variables('clusterVnetResourceId'))]", - "comments": "Cluster VNet should never have a NIC with a public IP.", - "properties": { - "displayName": "[concat('Network interfaces in [', variables('clusterVNetName'), '] should not have public IPs')]", - "description": "Cluster VNet should never have a NIC with a public IP.", - "scope": "[subscription().id]", - "notScopes": [], - "policyDefinitionId": "[variables('policyResourceIdNoPublicIpsInVnet')]", - "parameters": { - "vnetResourceId": { - "value": "[variables('clusterVnetResourceId')]" - } - }, - "nonComplianceMessages": [ - { - "message": "No NICs with public IPs are allowed in the regulated environment spoke." - } - ] - } - } - ] - } - } - }, - { - "type": "Microsoft.Network/virtualNetworks", - "apiVersion": "2020-05-01", - "name": "[variables('clusterVNetName')]", - "location": "[parameters('location')]", - "comments": "65,536 (-reserved) IPs available to the workload, split across four subnets for AKS, one for App Gateway, and two for management.", - "dependsOn": [ - "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-appgw'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-system-nodepools'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-is-nodepools'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-oos-nodepools'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-aksilbs'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-management-ops'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-privatelinkendpoints'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-acragents'))]", - "[subscriptionResourceId('Microsoft.Resources/deployments', 'Apply-Subscription-Spoke-PipUsage-Policies-01')]" - ], - "properties": { - "addressSpace": { - "addressPrefixes": [ - "10.240.0.0/16" - ] - }, - "subnets": [ - { - "name": "snet-cluster-systemnodepool", - "properties": { - "addressPrefix": "10.240.8.0/22", - "routeTable": { - "id": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]" - }, - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-system-nodepools'))]" - }, - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Disabled" - } - }, - { - "name": "snet-cluster-inscopenodepools", - "properties": { - "addressPrefix": "10.240.12.0/22", - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-is-nodepools'))]" - }, - "routeTable": { - "id": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]" - }, - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Disabled" - } - }, - { - "name": "snet-cluster-outofscopenodepools", - "properties": { - "addressPrefix": "10.240.16.0/22", - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-oos-nodepools'))]" - }, - "routeTable": { - "id": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]" - }, - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Disabled" - } - }, - { - "name": "snet-cluster-ingressservices", - "properties": { - "addressPrefix": "10.240.4.0/28", - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-aksilbs'))]" - }, - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Disabled" - } - }, - { - "name": "snet-applicationgateway", - "properties": { - "addressPrefix": "10.240.5.0/24", - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-appgw'))]" - }, - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Disabled" - } - }, - { - "name": "snet-management-ops", - "properties": { - "addressPrefix": "10.240.1.0/28", - "routeTable": { - "id": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]" - }, - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-management-ops'))]" - }, - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Disabled" - } - }, - { - "name": "snet-management-agents", - "properties": { - "addressPrefix": "10.240.2.0/26", - "routeTable": { - "id": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]" - }, - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-management-ops'))]" - }, - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Disabled" - } - }, - { - "name": "snet-management-acragents", - "properties": { - "addressPrefix": "10.240.251.0/28", - /*"routeTable": { - "id": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]" - },*/ - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-acragents'))]" - }, - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Disabled" - } - }, - { - "name": "snet-privatelinkendpoints", - "properties": { - "addressPrefix": "10.240.250.0/28", - "routeTable": { - "id": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]" - }, - "networkSecurityGroup": { - "id": "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-privatelinkendpoints'))]" - }, - "privateEndpointNetworkPolicies": "Disabled", - "privateLinkServiceNetworkPolicies": "Enabled" - } - } - ], - "dhcpOptions": { - "dnsServers": [ - "[reference(variables('hubFwResourceId'), '2020-05-01').ipConfigurations[0].properties.privateIpAddress]" - ] - } - }, - "resources": [ - { - "type": "virtualNetworkPeerings", - "apiVersion": "2020-05-01", - "name": "[variables('toHubPeeringName')]", - "comments": "Peer to regional hub.", - "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('clusterVNetName'))]" - ], - "properties": { - "remoteVirtualNetwork": { - "id": "[parameters('hubVnetResourceId')]" - }, - "allowForwardedTraffic": false, - "allowVirtualNetworkAccess": true, - "allowGatewayTransit": false, - "useRemoteGateways": false - } - }, - { - "type": "providers/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "name": "Microsoft.Insights/toHub", - "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('clusterVNetName'))]" - ], - "properties": { - "workspaceId": "[variables('hubLaWorkspaceResourceId')]", - "metrics": [ - { - "category": "AllMetrics", - "enabled": true - } - ] - } - } - ] - }, - { - "type": "Microsoft.Resources/deployments", - "apiVersion": "2020-06-01", - "name": "[concat('CreateHubTo', variables('clusterVNetName'), 'Peer')]", - "resourceGroup": "[variables('hubRgName')]", - "comments": "Sub incremental deployment to perform hub-to-spoke peering", - "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks/virtualNetworkPeerings', variables('clusterVNetName'), variables('toHubPeeringName'))]" - ], - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "resources": [ - { - "type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings", - "apiVersion": "2020-05-01", - "name": "[concat(variables('hubNetworkName'), '/hub-to-', variables('clusterVNetName'))]", - "properties": { - "remoteVirtualNetwork": { - "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('clusterVNetName'))]" - }, - "allowForwardedTraffic": false, - "allowGatewayTransit": false, - "allowVirtualNetworkAccess": true, - "useRemoteGateways": false - } - } - ] - } - } - }, - { - "type": "Microsoft.Network/privateDnsZones", - "apiVersion": "2020-06-01", - "name": "[variables('acrPrivateDnsZonesName')]", - "location": "global", - "comments": "Enabling Azure Container Registry Private Link on vnet.", - "properties": {}, - "resources": [ - { - "type": "virtualNetworkLinks", - "apiVersion": "2020-06-01", - "name": "[concat('to_', variables('clusterVNetName'))]", - "location": "global", - "comments": "Enabling cluster vnet private zone DNS lookup - used by cluster vnet for direct DNS queries (ones not proxied via the hub).", - "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('clusterVNetName'))]", - "[resourceId('Microsoft.Network/privateDnsZones', variables('acrPrivateDnsZonesName'))]" - ], - "properties": { - "virtualNetwork": { - "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('clusterVNetName'))]" - }, - "registrationEnabled": false - } - }, - { - "type": "virtualNetworkLinks", - "apiVersion": "2020-06-01", - "name": "[concat('to_', variables('hubNetworkName'))]", - "location": "global", - "comments": "Enabling hub vnet private zone DNS lookup for ACR - used by azure firewall's dns proxy.", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateDnsZones', variables('acrPrivateDnsZonesName'))]" - ], - "properties": { - "virtualNetwork": { - "id": "[parameters('hubVnetResourceId')]" - }, - "registrationEnabled": false - } - } - ] - }, - { - "type": "Microsoft.Network/privateDnsZones", - "apiVersion": "2020-06-01", - "name": "[variables('aksPrivateDnsZonesName')]", - "location": "global", - "comments": "Enabling AKS Private Link on vnet.", - "properties": {}, - "resources": [ - { - "type": "virtualNetworkLinks", - "apiVersion": "2020-06-01", - "name": "[concat('to_', variables('hubNetworkName'))]", - "location": "global", - "comments": "Enabling hub vnet private zone DNS lookup for private AKS - used by azure firewall's dns proxy.", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateDnsZones', variables('aksPrivateDnsZonesName'))]" - ], - "properties": { - "virtualNetwork": { - "id": "[parameters('hubVnetResourceId')]" - }, - "registrationEnabled": false - } - } - ] - }, - { - "type": "Microsoft.Network/privateDnsZones", - "apiVersion": "2018-09-01", - "name": "[variables('akvPrivateDnsZonesName')]", - "location": "global", - "comments": "Enabling Azure Key Vault Private Link support.", - "properties": {}, - "resources": [ - { - "type": "virtualNetworkLinks", - "apiVersion": "2020-06-01", - "name": "[concat('to_', variables('clusterVNetName'))]", - "location": "global", - "comments": "Enabling Azure Key Vault Private Link on cluster vnet.", - "dependsOn": [ - "[resourceId('Microsoft.Network/virtualNetworks', variables('clusterVNetName'))]", - "[resourceId('Microsoft.Network/privateDnsZones', variables('akvPrivateDnsZonesName'))]" - ], - "properties": { - "virtualNetwork": { - "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('clusterVNetName'))]" - }, - "registrationEnabled": false - } - }, - { - "type": "virtualNetworkLinks", - "apiVersion": "2020-06-01", - "name": "[concat('to_', variables('hubNetworkName'))]", - "location": "global", - "comments": "Enabling hub vnet private zone DNS lookup for ACR - used by azure firewall's dns proxy.", - "dependsOn": [ - "[resourceId('Microsoft.Network/privateDnsZones', variables('akvPrivateDnsZonesName'))]" - ], - "properties": { - "virtualNetwork": { - "id": "[parameters('hubVnetResourceId')]" - }, - "registrationEnabled": false - } - } - ] - }, - { - "type": "Microsoft.Network/publicIpAddresses", - "apiVersion": "2020-05-01", - "name": "[variables('primaryClusterPipName')]", - "location": "[parameters('location')]", - "comments": "Used as primary entry point for workload. Expected to be assigned to an Azure Application Gateway.", - "sku": { - "name": "Standard" - }, - "properties": { - "publicIPAllocationMethod": "Static", - "idleTimeoutInMinutes": 4, - "publicIPAddressVersion": "IPv4" - } - }, - { - "condition": "[parameters('deployFlowLogResources')]", - "name": "connect-spoke-bu0001A0005-01-flowlogs", - "type": "Microsoft.Resources/deployments", - "resourceGroup": "networkWatcherRG", - "apiVersion": "2020-10-01", - "dependsOn": [ - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-acragents'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-aksilbs'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-appgw'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-is-nodepools'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-management-ops'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-oos-nodepools'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-privatelinkendpoints'))]", - "[resourceId('Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-system-nodepools'))]", - "[resourceId('Microsoft.Network/virtualNetworks', variables('clusterVNetName'))]" - ], - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "variables": {}, - "resources": [ - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-acragents'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-acragents'))]", - "storageId": "[resourceId(variables('hubRgName'), 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[variables('hubLaWorkspaceResourceId')]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - }, - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-aksilbs'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-aksilbs'))]", - "storageId": "[resourceId(variables('hubRgName'), 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[variables('hubLaWorkspaceResourceId')]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - }, - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-appgw'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-appgw'))]", - "storageId": "[resourceId(variables('hubRgName'), 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[variables('hubLaWorkspaceResourceId')]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - }, - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-is-nodepools'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-is-nodepools'))]", - "storageId": "[resourceId(variables('hubRgName'), 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[variables('hubLaWorkspaceResourceId')]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - }, - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-management-ops'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-management-ops'))]", - "storageId": "[resourceId(variables('hubRgName'), 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[variables('hubLaWorkspaceResourceId')]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - }, - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-oos-nodepools'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-oos-nodepools'))]", - "storageId": "[resourceId(variables('hubRgName'), 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[variables('hubLaWorkspaceResourceId')]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - }, - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-privatelinkendpoints'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-privatelinkendpoints'))]", - "storageId": "[resourceId(variables('hubRgName'), 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[variables('hubLaWorkspaceResourceId')]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - }, - { - "name": "[concat('NetworkWatcher_', parameters('location'), '/fl', guid(resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-system-nodepools'))))]", - "type": "Microsoft.Network/networkWatchers/flowLogs", - "apiVersion": "2020-05-01", - "location": "[parameters('location')]", - "properties": { - "targetResourceId": "[resourceId(resourceGroup().name, 'Microsoft.Network/networkSecurityGroups', concat('nsg-', variables('clusterVNetName'), '-system-nodepools'))]", - "storageId": "[resourceId(variables('hubRgName'), 'Microsoft.Storage/storageAccounts', variables('regionFlowLowStorageAccountName'))]", - "enabled": true, - "format": { - "version": 2 - }, - "flowAnalyticsConfiguration": { - "networkWatcherFlowAnalyticsConfiguration": { - "enabled": true, - "workspaceResourceId": "[variables('hubLaWorkspaceResourceId')]", - "trafficAnalyticsInterval": 10 - } - }, - "retentionPolicy": { - "days": 365, - "enabled": true - } - } - } - ] - } - } - } - ], - "outputs": { - "clusterVnetResourceId": { - "value": "[resourceId('Microsoft.Network/virtualNetworks', variables('clusterVNetName'))]", - "type": "string" - }, - "nodepoolSubnetResourceIds": { - "value": "[createArray( - resourceId('Microsoft.Network/virtualNetworks/subnets', variables('clusterVNetName'), 'snet-cluster-systemnodepool'), - resourceId('Microsoft.Network/virtualNetworks/subnets', variables('clusterVNetName'), 'snet-cluster-inscopenodepools'), - resourceId('Microsoft.Network/virtualNetworks/subnets', variables('clusterVNetName'), 'snet-cluster-outofscopenodepools'))]", - "type": "array" - }, - "jumpboxSubnetResourceId": { - "value": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('clusterVNetName'), 'snet-management-ops')]", - "type": "string" - }, - "appGwPublicIpAddress": { - "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('primaryClusterPipName'))).ipAddress]", - "type": "string" - } - } -}