Merge pull request #5425 from dfe-analytical-services/EES-5446-preven…

…t-function-app-slot-swap-until-orchestrations-complete

Ees 5446 prevent function app slot swap until orchestrations complete

infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep

@@ -1,4 +1,4 @@
-import { ResourceNames, FirewallRule } from '../../types.bicep'
+import { ResourceNames, FirewallRule, IpRange } from '../../types.bicep'
 
 @description('Specifies common resource naming variables.')
 param resourceNames ResourceNames
@@ -15,8 +15,15 @@ param dataProcessorFunctionAppExists bool
 @description('Specifies the Application (Client) Id of a pre-existing App Registration used to represent the Data Processor Function App.')
 param dataProcessorAppRegistrationClientId string
 
-@description('Public API Storage : Firewall rules.')
-param storageFirewallRules FirewallRule[] = []
+@description('Specifies the principal id of the Azure DevOps SPN.')
+@secure()
+param devopsServicePrincipalId string
+
+@description('The IP address ranges that can access the Data Processor storage accounts.')
+param storageFirewallRules IpRange[]
+
+@description('The IP address ranges that can access the Data Processor Function App endpoints.')
+param functionAppFirewallRules FirewallRule[]
 
 @description('Whether to create or update Azure Monitor alerts during this deploy')
 param deployAlerts bool
@@ -36,7 +43,6 @@ resource adminAppServiceIdentity 'Microsoft.ManagedIdentity/identities@2023-01-3
 }
 
 var adminAppClientId = adminAppServiceIdentity.properties.clientId
-var adminAppPrincipalId = adminAppServiceIdentity.properties.principalId
 
 resource publicApiStorageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' existing = {
   name: resourceNames.publicApi.publicApiStorageAccount
@@ -72,15 +78,15 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = {
     applicationInsightsKey: applicationInsightsKey
     subnetId: outboundVnetSubnet.id
     privateEndpointSubnetId: inboundVnetSubnet.id
-    publicNetworkAccessEnabled: false
+    publicNetworkAccessEnabled: true
+    functionAppEndpointFirewallRules: functionAppFirewallRules
     entraIdAuthentication: {
       appRegistrationClientId: dataProcessorAppRegistrationClientId
       allowedClientIds: [
         adminAppClientId
+        devopsServicePrincipalId
       ]
-      allowedPrincipalIds: [
-        adminAppPrincipalId
-      ]
+      allowedPrincipalIds: []
       requireAuthentication: true
     }
     userAssignedManagedIdentityParams: {
@@ -98,9 +104,6 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = {
     }
     preWarmedInstanceCount: 1
     healthCheckPath: '/api/HealthCheck'
-    appSettings: {
-      App__MetaInsertBatchSize: 1000
-    }
     azureFileShares: [{
       storageName: resourceNames.publicApi.publicApiFileShare
       storageAccountKey: publicApiStorageAccount.listKeys().keys[0].value
@@ -151,3 +154,5 @@ module fileServiceAvailabilityAlerts '../../components/alerts/fileServices/avail
 output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name
 output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId
 output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath
+output url string = dataProcessorFunctionAppModule.outputs.url
+output stagingUrl string = dataProcessorFunctionAppModule.outputs.stagingUrl

infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep

@@ -1,4 +1,4 @@
-import { ResourceNames, FirewallRule } from '../../types.bicep'
+import { ResourceNames, IpRange } from '../../types.bicep'
 
 param resourceNames ResourceNames
 
@@ -9,7 +9,7 @@ param location string
 param publicApiDataFileShareQuota int
 
 @description('Public API Storage : Firewall rules.')
-param storageFirewallRules FirewallRule[]
+param storageFirewallRules IpRange[]
 
 @description('Specifies a set of tags with which to tag the resource in Azure.')
 param tagValues object

infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep

@@ -1,4 +1,4 @@
-import { ResourceNames, FirewallRule, PrincipalNameAndId } from '../../types.bicep'
+import { ResourceNames, IpRange, PrincipalNameAndId } from '../../types.bicep'
 
 @description('Specifies common resource naming variables.')
 param resourceNames ResourceNames
@@ -23,7 +23,7 @@ param storageSizeGB int = 32
 param autoGrowStatus string = 'Disabled'
 
 @description('Firewall rules.')
-param firewallRules FirewallRule[] = []
+param firewallRules IpRange[] = []
 
 @description('Specifies the subnet id that the PostgreSQL private endpoint will be attached to.')
 param privateEndpointSubnetId string

infrastructure/templates/public-api/application/shared/virtualNetwork.bicep

@@ -83,6 +83,9 @@ output adminAppServiceSubnetStartIpAddress string = parseCidr(adminSubnet.proper
 @description('The last usable IP address for the Admin App Service Subnet.')
 output adminAppServiceSubnetEndIpAddress string = parseCidr(adminSubnet.properties.addressPrefix).lastUsable
 
+@description('The IP address range for the Admin App Service Subnet.')
+output adminAppServiceSubnetCidr string = adminSubnet.properties.addressPrefix
+
 @description('The first usable IP address for the Publisher Function App Subnet.')
 output publisherFunctionAppSubnetStartIpAddress string = parseCidr(publisherSubnet.properties.addressPrefix).firstUsable
 

infrastructure/templates/public-api/ci/azure-pipelines.yml

@@ -1,15 +1,24 @@
 trigger: none
 
 parameters:
-  - name: deployContainerApp
-    displayName: Can we deploy the Container App yet? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added.
-    default: true
-  - name: updatePsqlFlexibleServer
+  - name: deploySharedPrivateDnsZones
+    displayName: Do the shared Private DNS Zones need creating or updating?
+    default: false
+  - name: deployPsqlFlexibleServer
     displayName: Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys.
     default: false
+  - name: deployContainerApp
+    displayName: Does the Public API Container App need creating or updating? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added.
+    default: true
+  - name: deployDataProcessor
+    displayName: Does the Data Processor need creating or updating?
+    default: true
   - name: deployAlerts
     displayName: Whether to create or update Azure Monitor alerts during this deploy.
     default: false
+  - name: awaitActiveOrchestrations
+    displayName: Should this deploy wait for active orchestrations in Function Apps to complete prior to deploying?
+    default: true
   - name: forceDeployToEnvironment
     displayName: Set to either dev or test to force a deploy to that environment from the chosen branch.
     type: string
@@ -41,12 +50,18 @@ variables:
     value: $[eq(variables['Build.SourceBranch'], 'refs/heads/master')]
   - name: vmImageName
     value: ubuntu-latest
+  - name: deploySharedPrivateDnsZones
+    value: ${{ parameters.deploySharedPrivateDnsZones }}
+  - name: deployPsqlFlexibleServer
+    value: ${{ parameters.deployPsqlFlexibleServer }}
   - name: deployContainerApp
     value: ${{ parameters.deployContainerApp }}
-  - name: updatePsqlFlexibleServer
-    value: ${{ parameters.updatePsqlFlexibleServer }}
+  - name: deployDataProcessor
+    value: ${{ parameters.deployDataProcessor }}
   - name: deployAlerts
     value: ${{ parameters.deployAlerts }}
+  - name: awaitActiveOrchestrations
+    value: ${{ parameters.awaitActiveOrchestrations }}
 
 pool:
   vmImage: $(vmImageName)

infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml

@@ -10,6 +10,7 @@ parameters:
 jobs:
   - deployment: DeployPublicDataProcessor
     displayName: Deploy Public Data Processor
+    condition: and(succeeded(), eq(variables.deployDataProcessor, true))
     dependsOn: ${{ parameters.dependsOn }}
     environment: ${{ parameters.environment }}
     strategy:
@@ -42,9 +43,11 @@ jobs:
                     --resource-group $(resourceGroupName) \
                     --slot staging \
                     --settings \
-                      "[email protected](VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \
+                      "App__MetaInsertBatchSize=1000" \
                       "App__EnableThemeDeletion=$(enableThemeDeletion)" \
+                      "[email protected](VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \
                       "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \
+                      "AzureWebJobs.TriggerLongRunningOrchestration.Disabled=true" \
                       "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)"
                   
                   az webapp config connection-string set \
@@ -63,28 +66,6 @@ jobs:
                     --settings \
                       "[email protected](VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))"
 
-            # TODO EES-5128
-            # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow
-            # DevOps to deploy the Data Processor Function App without having to temporarily
-            # make it publicly accessible.
-            - task: AzureCLI@2
-              displayName: Temporarily enable public network access before deploy
-              retryCountOnTaskFailure: 1
-              inputs:
-                azureSubscription: ${{ parameters.serviceConnection }}
-                scriptType: bash
-                scriptLocation: inlineScript
-                inlineScript: |
-                  set -e
-          
-                  az functionapp update \
-                    --name $(dataProcessorFunctionAppName) \
-                    --resource-group $(resourceGroupName) \
-                    --slot staging \
-                    --set \
-                      publicNetworkAccess=Enabled \
-                      siteConfig.publicNetworkAccess=Enabled
-
             # TODO EES-5128
             # Retry deploying the Function App in order to allow the staging slot the time to
             # fully restart after config and network settings have been updated prior to deploy.
@@ -112,29 +93,21 @@ jobs:
                     --resource-group $(resourceGroupName) \
                     --slot staging
 
-            # TODO EES-5128
-            # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow
-            # DevOps to deploy the Data Processor Function App without having to temporarily
-            # make it publicly accessible.
-            - task: AzureCLI@2
-              displayName: Disable public network access after deploy
-              retryCountOnTaskFailure: 1
-              condition: always()
-              inputs:
-                azureSubscription: ${{ parameters.serviceConnection }}
-                scriptType: bash
-                scriptLocation: inlineScript
-                inlineScript: |
-                  set -e
-          
-                  az functionapp update \
-                    --name $(dataProcessorFunctionAppName) \
-                    --resource-group $(resourceGroupName) \
-                    --slot staging \
-                    --set \
-                      publicNetworkAccess=Disabled \
-                      siteConfig.publicNetworkAccess=Disabled
+            - template: ../tasks/wait-for-endpoint-success.yml
+              parameters:
+                serviceConnection: ${{ parameters.serviceConnection }}
+                displayName: Waiting for staging slot to start successfully
+                accessTokenScope: $(dataProcessorAppRegistrationClientId)
+                endpoint: $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck
 
+            - template: ../tasks/wait-for-orchestrations-to-complete.yml
+              parameters:
+                serviceConnection: ${{ parameters.serviceConnection }}
+                displayName: Waiting for active orchestrations in the production slot to complete
+                accessTokenScope: $(dataProcessorAppRegistrationClientId)
+                endpoint: $(dataProcessorFunctionAppUrl)/api/StatusCheck
+                condition: eq(variables.awaitActiveOrchestrations, true)
+
             - task: AzureCLI@2
               displayName: Swap slots
               retryCountOnTaskFailure: 1
@@ -149,3 +122,10 @@ jobs:
                     --resource-group $(resourceGroupName) \
                     --slot staging \
                     --target-slot production
+
+            - template: ../tasks/wait-for-endpoint-success.yml
+              parameters:
+                serviceConnection: ${{ parameters.serviceConnection }}
+                displayName: Checking that production slot is healthy after slot swap
+                accessTokenScope: $(dataProcessorAppRegistrationClientId)
+                endpoint: $(dataProcessorFunctionAppUrl)/api/HealthCheck

infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml

@@ -35,8 +35,9 @@ jobs:
                 action: validate
                 serviceConnection: ${{ parameters.serviceConnection }}
                 parameterFile: $(paramFile)
+                deploySharedPrivateDnsZones: false
+                deployPsqlFlexibleServer: false
                 deployContainerApp: true
-                updatePsqlFlexibleServer: false
                 deployAlerts: false
                 dataProcessorExists: true
 
@@ -62,8 +63,9 @@ jobs:
                 action: create
                 serviceConnection: ${{ parameters.serviceConnection }}
                 parameterFile: $(paramFile)
+                deploySharedPrivateDnsZones: $(deploySharedPrivateDnsZones)
+                deployPsqlFlexibleServer: $(deployPsqlFlexibleServer)
                 deployContainerApp: $(deployContainerApp)
-                updatePsqlFlexibleServer: $(updatePsqlFlexibleServer)
                 deployAlerts: $(deployAlerts)
                 dataProcessorExists: $(dataProcessorExists)
 

infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml

@@ -11,14 +11,20 @@ parameters:
     type: string
   - name: parameterFile
     type: string
+  - name: deploySharedPrivateDnsZones
+    type: string
+  - name: deployPsqlFlexibleServer
+    type: string
   - name: deployContainerApp
     type: string
-  - name: updatePsqlFlexibleServer
+    default: true
+  - name: deployDataProcessor
     type: string
   - name: deployAlerts
     type: string
   - name: dataProcessorExists
     type: string
+    default: true
 
 steps:
   - task: AzureCLI@2
@@ -27,9 +33,10 @@ steps:
       azureSubscription: ${{ parameters.serviceConnection }}
       scriptType: bash
       scriptLocation: inlineScript
+      addSpnToEnvironment: true
       inlineScript: |
         set -e
-        
+
         az deployment group ${{ parameters.action }} \
           --name $(infraDeployName) \
           --resource-group $(resourceGroupName) \
@@ -40,14 +47,16 @@ steps:
               resourceTags='$(resourceTags)' \
               postgreSqlAdminName='$(postgreSqlAdminName)' \
               postgreSqlAdminPassword='$(postgreSqlAdminPassword)' \
-              postgreSqlFirewallRules='$(maintenanceFirewallRules)' \
               postgreSqlEntraIdAdminPrincipals='$(postgreSqlEntraIdAdminPrincipals)' \
-              storageFirewallRules='$(maintenanceFirewallRules)' \
+              maintenanceIpRanges='$(maintenanceFirewallRules)' \
               acrResourceGroupName='$(acrResourceGroupName)' \
               dockerImagesTag='$(resources.pipeline.MainBuild.runName)' \
+              deploySharedPrivateDnsZones=${{ parameters.deploySharedPrivateDnsZones }} \
+              deployPsqlFlexibleServer=${{ parameters.deployPsqlFlexibleServer }} \
               deployContainerApp=${{ parameters.deployContainerApp }} \
-              updatePsqlFlexibleServer=${{ parameters.updatePsqlFlexibleServer }} \
+              deployDataProcessor=${{ parameters.deployDataProcessor }} \
               deployAlerts=${{ parameters.deployAlerts }} \
               dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \
               dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \
-              apiAppRegistrationClientId='$(apiAppRegistrationClientId)'
+              apiAppRegistrationClientId='$(apiAppRegistrationClientId)' \
+              devopsServicePrincipalId="$servicePrincipalId"

infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml

@@ -0,0 +1,57 @@
+parameters:
+  - name: serviceConnection
+    type: string
+  - name: displayName
+    type: string
+    default: Waiting for a successful response from endpoint
+  - name: accessTokenScope
+    type: string
+    default: null
+  - name: pollingDelaySeconds
+    type: number
+    default: 5
+  - name: maxAttempts
+    type: number
+    default: 50
+  - name: endpoint
+    type: string
+
+steps:
+  - task: AzureCLI@2
+    displayName: ${{ parameters.displayName }}
+    inputs:
+      azureSubscription: ${{ parameters.serviceConnection }}
+      scriptType: bash
+      scriptLocation: inlineScript
+      inlineScript: |
+        
+        if [ -n "${{ parameters.accessTokenScope }}" ]; then
+          accessToken=`az account get-access-token \
+            --resource ${{ parameters.accessTokenScope }} \
+            --query "accessToken" \
+            -o tsv`
+        fi
+
+        for attempt in $(seq 1 ${{ parameters.maxAttempts }});
+        do
+
+          echo "Attempt number $attempt of ${{ parameters.maxAttempts }} - calling ${{ parameters.endpoint }} to check for successful response."
+          
+          if [ -n "$accessToken" ]; then
+            httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null ${{ parameters.endpoint }}`
+          else 
+            httpStatusCode=`curl --write-out '%{http_code}' -s --output /dev/null ${{ parameters.endpoint }}`
+          fi 
+
+          if (( $httpStatusCode >= 200 && $httpStatusCode <= 204 )); then
+            echo "Received successful response with status code $httpStatusCode."
+            exit 0
+          fi
+
+          echo "Received response with status code $httpStatusCode. Retrying in ${{ parameters.pollingDelaySeconds }} seconds."
+          sleep ${{ parameters.pollingDelaySeconds }}
+        
+        done
+
+        echo "Timed out waiting for successful response."
+        exit 1