From c7bad4050500977aa89513931548a0586d3e7a73 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:46:20 +0000 Subject: [PATCH 01/73] Update resource Microsoft.OperationalInsights/workspaces to 2023-09-01 --- .../public-api/components/containerAppEnvironment.bicep | 2 +- .../templates/public-api/components/logAnalyticsWorkspace.bicep | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/templates/public-api/components/containerAppEnvironment.bicep b/infrastructure/templates/public-api/components/containerAppEnvironment.bicep index 0a25cfe872b..09146a19b81 100644 --- a/infrastructure/templates/public-api/components/containerAppEnvironment.bicep +++ b/infrastructure/templates/public-api/components/containerAppEnvironment.bicep @@ -27,7 +27,7 @@ param tagValues object var containerAppEnvironmentName = '${subscription}-ees-cae' -resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2021-06-01' existing = { +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' existing = { name: logAnalyticsWorkspaceName } diff --git a/infrastructure/templates/public-api/components/logAnalyticsWorkspace.bicep b/infrastructure/templates/public-api/components/logAnalyticsWorkspace.bicep index a02cb739bd3..2290715f778 100644 --- a/infrastructure/templates/public-api/components/logAnalyticsWorkspace.bicep +++ b/infrastructure/templates/public-api/components/logAnalyticsWorkspace.bicep @@ -16,7 +16,7 @@ param tagValues object var logAnalyticsWorkspaceName = '${subscription}-ees-log' -resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2021-06-01' = { +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { name: logAnalyticsWorkspaceName location: location properties: { From 91255434a9896fd096c98c685d50e84ed112dbb9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 13:20:05 +0000 Subject: [PATCH 02/73] Update resource Microsoft.App/managedEnvironments to 2024-03-01 --- .../public-api/components/containerAppEnvironment.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/templates/public-api/components/containerAppEnvironment.bicep b/infrastructure/templates/public-api/components/containerAppEnvironment.bicep index a8ee0de1254..994088262be 100644 --- a/infrastructure/templates/public-api/components/containerAppEnvironment.bicep +++ b/infrastructure/templates/public-api/components/containerAppEnvironment.bicep @@ -39,7 +39,7 @@ resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2021-06 name: logAnalyticsWorkspaceName } -resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' = { +resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2024-03-01' = { name: containerAppEnvironmentName location: location properties: { From 7b53e605559bad4f27fad38a6a59aad788121e9a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 16:21:06 +0000 Subject: [PATCH 03/73] Update resource Microsoft.Network/virtualNetworks to 2023-11-01 --- .../templates/public-api/application/virtualNetwork.bicep | 2 +- infrastructure/templates/public-api/components/network.bicep | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/templates/public-api/application/virtualNetwork.bicep b/infrastructure/templates/public-api/application/virtualNetwork.bicep index 532d6689caf..407ed12a1fd 100644 --- a/infrastructure/templates/public-api/application/virtualNetwork.bicep +++ b/infrastructure/templates/public-api/application/virtualNetwork.bicep @@ -13,7 +13,7 @@ param dataProcessorFunctionAppNameSuffix string @description('Specifies the name suffix of the Container App Environment') param containerAppEnvironmentNameSuffix string -resource vNet 'Microsoft.Network/virtualNetworks@2023-09-01' existing = { +resource vNet 'Microsoft.Network/virtualNetworks@2023-11-01' existing = { name: vNetName } diff --git a/infrastructure/templates/public-api/components/network.bicep b/infrastructure/templates/public-api/components/network.bicep index 40434849de6..b792b7ce356 100644 --- a/infrastructure/templates/public-api/components/network.bicep +++ b/infrastructure/templates/public-api/components/network.bicep @@ -15,7 +15,7 @@ param tagValues object var vNetName = '${subscription}-vnet-${environment}' -resource vNet 'Microsoft.Network/virtualNetworks@2023-09-01' = { +resource vNet 'Microsoft.Network/virtualNetworks@2023-11-01' = { name: vNetName location: location properties: { From 1a40b980373a9ce4b0310f2ba95eda50544168be Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 18:11:22 +0000 Subject: [PATCH 04/73] Update resource Microsoft.Storage/storageAccounts/blobServices to 2023-04-01 --- infrastructure/templates/public-api/components/blobStore.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/templates/public-api/components/blobStore.bicep b/infrastructure/templates/public-api/components/blobStore.bicep index 0db29c4ea20..d83b7e8b512 100644 --- a/infrastructure/templates/public-api/components/blobStore.bicep +++ b/infrastructure/templates/public-api/components/blobStore.bicep @@ -14,7 +14,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' existing name: storageAccountName } -resource blobServices 'Microsoft.Storage/storageAccounts/blobServices@2023-01-01' = { +resource blobServices 'Microsoft.Storage/storageAccounts/blobServices@2023-04-01' = { name: blobStoreName parent: storageAccount properties: { From 445056bb8431da9df858e80dc2d850ef497e0b2b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 22:32:03 +0000 Subject: [PATCH 05/73] Update resource Microsoft.Storage/storageAccounts/fileServices to 2023-04-01 --- infrastructure/templates/public-api/components/fileShares.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/templates/public-api/components/fileShares.bicep b/infrastructure/templates/public-api/components/fileShares.bicep index c44bc2c968d..abdd4df0e7f 100644 --- a/infrastructure/templates/public-api/components/fileShares.bicep +++ b/infrastructure/templates/public-api/components/fileShares.bicep @@ -21,7 +21,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' existing name: storageAccountName } -resource fileService 'Microsoft.Storage/storageAccounts/fileServices@2023-01-01' = { +resource fileService 'Microsoft.Storage/storageAccounts/fileServices@2023-04-01' = { name: 'default' parent: storageAccount } From 971292617469dd38f1511716848e78b639cfe4b8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 9 May 2024 10:49:17 +0000 Subject: [PATCH 06/73] chore(deps): update resource microsoft.network/virtualnetworks/subnets to 2023-11-01 --- .../public-api/application/virtualNetwork.bicep | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/infrastructure/templates/public-api/application/virtualNetwork.bicep b/infrastructure/templates/public-api/application/virtualNetwork.bicep index efb18cedab4..b43a3f30da1 100644 --- a/infrastructure/templates/public-api/application/virtualNetwork.bicep +++ b/infrastructure/templates/public-api/application/virtualNetwork.bicep @@ -17,27 +17,27 @@ resource vNet 'Microsoft.Network/virtualNetworks@2023-09-01' existing = { name: vNetName } -resource adminSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' existing = { +resource adminSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = { name: '${subscription}-snet-ees-admin' parent: vNet } -resource publisherSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' existing = { +resource publisherSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = { name: '${subscription}-snet-ees-publisher' parent: vNet } -resource dataProcessorSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' existing = { +resource dataProcessorSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = { name: '${resourcePrefix}-snet-fa-${dataProcessorFunctionAppNameSuffix}' parent: vNet } -resource containerAppEnvironmentSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' existing = { +resource containerAppEnvironmentSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = { name: '${subscription}-ees-snet-cae-${containerAppEnvironmentNameSuffix}' parent: vNet } -resource psqlFlexibleServerSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' existing = { +resource psqlFlexibleServerSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = { name: '${subscription}-ees-snet-psql-flexibleserver' parent: vNet } From 803129c7231f60f92d07b18800c961f69119bcc4 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 9 May 2024 16:31:51 +0100 Subject: [PATCH 07/73] EES-5127 - adding in mounts for the Parquet file share into the Data Processor Function App and API Container App. --- .../api-infrastructure-pipeline.yml | 20 +++-- .../public-api/components/containerApp.bicep | 21 ++++++ .../components/containerAppEnvironment.bicep | 21 ++++++ .../public-api/components/functionApp.bicep | 23 ++++++ .../templates/public-api/main.bicep | 74 +++++++++++++------ .../Functions/ConnectionUtils.cs | 14 +++- .../Functions/HealthCheckFunctions.cs | 33 +++++++++ 7 files changed, 174 insertions(+), 32 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs diff --git a/infrastructure/templates/public-api/api-infrastructure-pipeline.yml b/infrastructure/templates/public-api/api-infrastructure-pipeline.yml index c452bb3faf7..ded863a07ea 100644 --- a/infrastructure/templates/public-api/api-infrastructure-pipeline.yml +++ b/infrastructure/templates/public-api/api-infrastructure-pipeline.yml @@ -1,11 +1,5 @@ trigger: none -resources: - pipelines: - - pipeline: EESBuildPipeline - source: Explore Education Statistics - trigger: none - parameters: - name: deployContainerApp displayName: Can we deploy the Container App yet? This is dependent on the user-assigned Managed Identity for the API Container App being created with the AcrPull role, and the database users added to PSQL. @@ -13,11 +7,14 @@ parameters: - name: updatePsqlFlexibleServer displayName: Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys. default: false - + - name: buildBranchToDeploy + displayName: Build branch to deploy. + default: 'refs/heads/dev' + variables: - group: Public API Infrastructure - common - name: isDev - value: $[eq(variables['Build.SourceBranch'], 'refs/heads/dev')] + value: $[eq(variables['Build.SourceBranch'], 'refs/heads/EES-5127-add-parquet-file-share-mounts')] - name: isTest value: $[eq(variables['Build.SourceBranch'], 'refs/heads/test')] - name: isMaster @@ -45,6 +42,13 @@ variables: - name: updatePsqlFlexibleServer value: ${{parameters.updatePsqlFlexibleServer}} +resources: + pipelines: + - pipeline: EESBuildPipeline + source: Explore Education Statistics + trigger: none + branch: ${{parameters.buildBranchToDeploy}} + pool: vmImage: $(vmImageName) diff --git a/infrastructure/templates/public-api/components/containerApp.bicep b/infrastructure/templates/public-api/components/containerApp.bicep index 1474c1b7004..3a5d1489dde 100644 --- a/infrastructure/templates/public-api/components/containerApp.bicep +++ b/infrastructure/templates/public-api/components/containerApp.bicep @@ -66,6 +66,25 @@ param managedIdentityName string @description('Id of the owning Container App Environment') param managedEnvironmentId string +@description('Volumes to mount within Containers - used in conjunction with "volumeMounts"') +param volumes { + name: string + storageType: string + storageName: string + mountOptions: string? + secrets: { + path: string + secretRef: string + }[]? +}[] = [] + +@description('Volume mount points within Containers - used in conjunction with "volumes"') +param volumeMounts { + mountPath: string + volumeName: string +}[] = [] + + var containerImageName = '${acrLoginServer}/${containerAppImageName}' var containerApplicationName = toLower('${resourcePrefix}-ca-${containerAppName}') @@ -115,6 +134,7 @@ resource containerApp 'Microsoft.App/containerApps@2023-05-01' = { cpu: json(cpuCore) memory: '${memorySize}Gi' } + volumeMounts: volumeMounts } ] scale: { @@ -131,6 +151,7 @@ resource containerApp 'Microsoft.App/containerApps@2023-05-01' = { } ] } + volumes: volumes } workloadProfileName: 'Consumption' } diff --git a/infrastructure/templates/public-api/components/containerAppEnvironment.bicep b/infrastructure/templates/public-api/components/containerAppEnvironment.bicep index a8ee0de1254..a8022820207 100644 --- a/infrastructure/templates/public-api/components/containerAppEnvironment.bicep +++ b/infrastructure/templates/public-api/components/containerAppEnvironment.bicep @@ -31,6 +31,15 @@ param tagValues object @description('Specifies a suffix to append to the full name of the Container App Environment') param containerAppEnvironmentNameSuffix string = '' +@description('Specifies an array of Azure Fileshares to be available for Container Apps hosted within this Container App Environment') +param azureFileStorages { + storageName: string + storageAccountKey: string + storageAccountName: string + fileShareName: string + accessMode: 'ReadWrite' | 'ReadOnly' +}[] + var containerAppEnvironmentName = empty(containerAppEnvironmentNameSuffix) ? '${subscription}-ees-cae' : '${subscription}-ees-cae-${containerAppEnvironmentNameSuffix}' @@ -58,6 +67,18 @@ resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' workloadProfiles: workloadProfiles } tags: tagValues + + resource azureFileStorage 'storages@2022-03-01' = [for storage in azureFileStorages: { + name: storage.storageName + properties: { + azureFile: { + accountKey: storage.storageAccountKey + accountName: storage.storageAccountName + shareName: storage.fileShareName + accessMode: storage.accessMode + } + } + }] } output containerAppEnvironmentName string = containerAppEnvironmentName diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 66cc80a0800..21a64daaaa0 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -54,6 +54,15 @@ param preWarmedInstanceCount int? @description('Specifies whether or not the Function App will always be on and not idle after periods of no traffic - must be compatible with the chosen hosting plan') param alwaysOn bool? +@description('Specifies additional Azure Storage Accounts to make available to this Function App') +param additionalAzureFileStorage { + storageName: string + storageAccountKey: string + storageAccountName: string + fileShareName: string + mountPath: string +}? + var appServicePlanName = '${resourcePrefix}-asp-${functionAppName}' var reserved = appServicePlanOS == 'Linux' var fullFunctionAppName = '${subscription}-ees-papi-fa-${functionAppName}' @@ -148,6 +157,20 @@ resource functionApp 'Microsoft.Web/sites@2023-01-01' = { tags: tagValues } +resource azureStorageAccount 'Microsoft.Web/sites/config@2021-01-15' = if (additionalAzureFileStorage != null) { + name: 'azurestorageaccounts' + parent: functionApp + properties: { + '${additionalAzureFileStorage!.storageName}': { + type: 'AzureFiles' + shareName: additionalAzureFileStorage!.fileShareName + mountPath: additionalAzureFileStorage!.mountPath + accountName: additionalAzureFileStorage!.storageAccountName + accessKey: additionalAzureFileStorage!.storageAccountKey + } + } +} + // We determine any pre-existing appsettings for both the production and the staging slots during this infrastructure // deploy and supply them as the most important appsettings. This prevents infrastructure deploys from overriding any // appsettings back to their original values by allowing existing ones to take precedence. diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index bdee54b112a..6b762e287fd 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -86,6 +86,9 @@ var keyVaultName = '${subscription}-kv-ees-01' var acrName = 'eesacr' var vNetName = '${subscription}-vnet-ees' var containerAppEnvironmentNameSuffix = '01' +var parquetFileShareMountName = 'parquet-fileshare-mount' +var parquetFileShareMountPath = '/data/public-api-parquet' +var parquetFileShareStorageName = 'parquet-fileshare-storage' var tagValues = union(resourceTags ?? {}, { Environment: environmentName @@ -98,13 +101,13 @@ resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-07-01' e } // Reference the existing core Storage Account as currently managed by the EES ARM template. -resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' existing = { +resource coreStorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' existing = { name: coreStorageAccountName scope: resourceGroup(resourceGroup().name) } -var storageAccountKey = storageAccount.listKeys().keys[0].value +var coreStorageAccountKey = coreStorageAccount.listKeys().keys[0].value var endpointSuffix = environment().suffixes.storage -var coreStorageConnectionString = 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};EndpointSuffix=${endpointSuffix};AccountKey=${storageAccountKey}' +var coreStorageConnectionString = 'DefaultEndpointsProtocol=https;AccountName=${coreStorageAccount.name};EndpointSuffix=${endpointSuffix};AccountKey=${coreStorageAccountKey}' // Reference the existing VNet as currently managed by the EES ARM template, and register new subnets for Bicep-controlled resources. module vNetModule 'application/virtualNetwork.bicep' = { @@ -138,22 +141,8 @@ module logAnalyticsWorkspaceModule 'components/logAnalyticsWorkspace.bicep' = { } } -// Create a generic Container App Environment for any Container Apps to use. -module containerAppEnvironmentModule 'components/containerAppEnvironment.bicep' = { - name: 'containerAppEnvironmentDeploy' - params: { - subscription: subscription - location: location - containerAppEnvironmentNameSuffix: containerAppEnvironmentNameSuffix - subnetId: vNetModule.outputs.containerAppEnvironmentSubnetRef - logAnalyticsWorkspaceName: logAnalyticsWorkspaceModule.outputs.logAnalyticsWorkspaceName - applicationInsightsKey: applicationInsightsModule.outputs.applicationInsightsKey - tagValues: tagValues - } -} - // Deploy File Share. -module fileShareModule 'components/fileShares.bicep' = { +module parquetFileShareModule 'components/fileShares.bicep' = { name: 'fileShareDeploy' params: { resourcePrefix: resourcePrefix @@ -196,17 +185,53 @@ resource apiContainerAppManagedIdentity 'Microsoft.ManagedIdentity/userAssignedI name: apiContainerAppManagedIdentityName } +// Create a generic Container App Environment for any Container Apps to use. +module containerAppEnvironmentModule 'components/containerAppEnvironment.bicep' = { + name: 'containerAppEnvironmentDeploy' + params: { + subscription: subscription + location: location + containerAppEnvironmentNameSuffix: containerAppEnvironmentNameSuffix + subnetId: vNetModule.outputs.containerAppEnvironmentSubnetRef + logAnalyticsWorkspaceName: logAnalyticsWorkspaceModule.outputs.logAnalyticsWorkspaceName + applicationInsightsKey: applicationInsightsModule.outputs.applicationInsightsKey + tagValues: tagValues + azureFileStorages: [ + { + storageName: parquetFileShareStorageName + storageAccountName: coreStorageAccountName + storageAccountKey: coreStorageAccountKey + fileShareName: parquetFileShareModule.outputs.fileShareName + accessMode: 'ReadWrite' + } + ] + } +} + // Deploy main Public API Container App. module apiContainerAppModule 'components/containerApp.bicep' = if (deployContainerApp) { name: 'apiContainerAppDeploy' params: { resourcePrefix: resourcePrefix location: location - containerAppName: apiContainerAppName + containerAppName: apiContainerAppName acrLoginServer: containerRegistry.properties.loginServer containerAppImageName: 'ees-public-api/api:${dockerImagesTag}' managedIdentityName: apiContainerAppManagedIdentity.name managedEnvironmentId: containerAppEnvironmentModule.outputs.containerAppEnvironmentId + volumeMounts: [ + { + volumeName: parquetFileShareMountName + mountPath: parquetFileShareMountPath + } + ] + volumes: [ + { + name: parquetFileShareMountName + storageType: 'AzureFile' + storageName: parquetFileShareStorageName + } + ] appSettings: [ { name: 'ConnectionStrings__PublicDataDb' @@ -231,7 +256,7 @@ module apiContainerAppModule 'components/containerApp.bicep' = if (deployContain } { name: 'ParquetFiles__BasePath' - value: 'data/public-api-parquet' + value: parquetFileShareMountPath } { // This property informs the Container App of the name of the Admin's system-assigned identity. @@ -282,6 +307,13 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { family: 'EP' } preWarmedInstanceCount: 1 + additionalAzureFileStorage: { + storageName: parquetFileShareStorageName + storageAccountKey: coreStorageAccountKey + storageAccountName: coreStorageAccountName + fileShareName: parquetFileShareModule.outputs.fileShareName + mountPath: parquetFileShareMountPath + } } } @@ -324,7 +356,7 @@ module storeAdminPsqlConnectionString 'components/keyVaultSecret.bicep' = { } } -var coreStorageConnectionStringSecretKey = 'coreStorageConnectionString' +var coreStorageConnectionStringSecretKey = 'ees-core-storage-connectionstring' module storeCoreStorageConnectionString 'components/keyVaultSecret.bicep' = { name: 'storeCoreStorageConnectionString' diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Functions/ConnectionUtils.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Functions/ConnectionUtils.cs index 972c2f3e853..9c770565f7d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Functions/ConnectionUtils.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Functions/ConnectionUtils.cs @@ -34,11 +34,19 @@ public static string GetPostgreSqlConnectionString(string name) private static string GetConnectionString(string name, string connectionTypeValue) { - // Attempt to get a connection string defined for running locally. - // Settings in the local.settings.json file are only used by Functions tools when running locally. + // Attempt to get a connection string defined using the double-underscore syntax as used by Azure to + // supply nested configuration. var connectionString = - Environment.GetEnvironmentVariable($"ConnectionStrings:{name}", EnvironmentVariableTarget.Process); + Environment.GetEnvironmentVariable($"ConnectionStrings__{name}"); + if (connectionString.IsNullOrEmpty()) + { + // Attempt to get a connection string defined for running locally. + // Settings in the local.settings.json file are only used by Functions tools when running locally. + connectionString = + Environment.GetEnvironmentVariable($"ConnectionStrings:{name}", EnvironmentVariableTarget.Process); + } + if (connectionString.IsNullOrEmpty()) { // Get the connection string from the Azure Functions App using the naming convention for type SQLAzure. diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs new file mode 100644 index 00000000000..033c663092f --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs @@ -0,0 +1,33 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using Microsoft.Azure.Functions.Worker; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class HealthCheckFunctions( + ILogger logger, + PublicDataDbContext publicDataDbContext) +{ + [Function(nameof(CountDataSets))] + public async Task CountDataSets( + [ActivityTrigger] object? input, + FunctionContext executionContext) + { + var message = $"Found {await publicDataDbContext.DataSets.CountAsync()} datasets."; + logger.LogInformation(message); + return message; + } + + [Function(nameof(ListFileShareContents))] + public async Task ListFileShareContents( + [ActivityTrigger] object? input, + FunctionContext executionContext) + { + var files = Directory.GetFiles("/data/public-api-parquet"); + var message = $"Found the following files:\n\n{files.JoinToString('\n')}"; + logger.LogInformation(message); + return message; + } +} From 09108e6f779099b43c25b3e3941616ddf6f0d06f Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 10 May 2024 14:55:02 +0100 Subject: [PATCH 08/73] EES-5127 - extracting mounting of Function App file share into separte az command --- .../api-infrastructure-pipeline.yml | 18 +++++++++++------- .../public-api/deploy-stage-template.yml | 19 +++++++++++++++++++ .../templates/public-api/main.bicep | 19 ++++++++++++++++++- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/infrastructure/templates/public-api/api-infrastructure-pipeline.yml b/infrastructure/templates/public-api/api-infrastructure-pipeline.yml index ded863a07ea..ec990cf9db5 100644 --- a/infrastructure/templates/public-api/api-infrastructure-pipeline.yml +++ b/infrastructure/templates/public-api/api-infrastructure-pipeline.yml @@ -1,5 +1,16 @@ trigger: none +resources: + pipelines: + - pipeline: EESBuildPipeline + source: Explore Education Statistics + trigger: + branches: + - refs/heads/dev + - refs/heads/test + - refs/heads/master + branch: ${{parameters.buildBranchToDeploy}} + parameters: - name: deployContainerApp displayName: Can we deploy the Container App yet? This is dependent on the user-assigned Managed Identity for the API Container App being created with the AcrPull role, and the database users added to PSQL. @@ -42,13 +53,6 @@ variables: - name: updatePsqlFlexibleServer value: ${{parameters.updatePsqlFlexibleServer}} -resources: - pipelines: - - pipeline: EESBuildPipeline - source: Explore Education Statistics - trigger: none - branch: ${{parameters.buildBranchToDeploy}} - pool: vmImage: $(vmImageName) diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index 7354fa929c7..f502187ef06 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -132,3 +132,22 @@ stages: --resource-group '$(resourceGroupName)' \ --slot 'staging' \ --target-slot 'production' + + - task: AzureCLI@2 + displayName: 'Deploy Data Processor Function App - attach Parquet Fileshare' + inputs: + azureSubscription: ${{parameters.serviceConnection}} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + + az webapp config storage-account add \ + --name '$(dataProcessorFunctionAppName)' \ + --resource-group '$(resourceGroupName)' \ + --custom-id '$(parquetFileShareStorageName)' \ + --storage-type AzureFiles \ + --account-name '$(coreStorageAccountName)' \ + --share-name '$(parquetFileShareStorageName)' \ + --access-key '`az keyvault secret show --name $(coreStorageAccessKeySecretKey) --vault-name $(keyVaultName) --query value --output tsv`' \ + --mount-path '$(parquetFileShareMountPath)' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 6b762e287fd..ccff786d572 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -87,7 +87,7 @@ var acrName = 'eesacr' var vNetName = '${subscription}-vnet-ees' var containerAppEnvironmentNameSuffix = '01' var parquetFileShareMountName = 'parquet-fileshare-mount' -var parquetFileShareMountPath = '/data/public-api-parquet' +var parquetFileShareMountPath = '/home/public-api-parquet' var parquetFileShareStorageName = 'parquet-fileshare-storage' var tagValues = union(resourceTags ?? {}, { @@ -369,7 +369,24 @@ module storeCoreStorageConnectionString 'components/keyVaultSecret.bicep' = { } } +var coreStorageAccessKeySecretKey = 'ees-core-storage-access-key' + +module storeCoreStorageAccessKey 'components/keyVaultSecret.bicep' = { + name: 'storeCoreStorageAccessKey' + params: { + keyVaultName: keyVaultName + isEnabled: true + secretName: coreStorageAccessKeySecretKey + secretValue: coreStorageAccountKey + contentType: 'text/plain' + } +} + output dataProcessorContentDbConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-contentdb' output dataProcessorPsqlConnectionStringSecretKey string = dataProcessorPsqlConnectionStringSecretKey output coreStorageConnectionStringSecretKey string = coreStorageConnectionStringSecretKey output keyVaultName string = keyVaultName +output parquetFileShareStorageName string = parquetFileShareStorageName +output coreStorageAccountName string = coreStorageAccountName +output coreStorageAccessKeySecretKey string = coreStorageAccessKeySecretKey +output parquetFileShareMountPath string = parquetFileShareMountPath From 4d6ea5d5bd28d45306ae42446c8a45ce1bb87707 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 10 May 2024 16:15:18 +0100 Subject: [PATCH 09/73] EES-5127 - removing secured vnet deployment settings from Function App --- .../api-infrastructure-pipeline.yml | 2 +- .../public-api/components/functionApp.bicep | 57 +------------------ .../public-api/deploy-stage-template.yml | 36 ++++++------ .../templates/public-api/main.bicep | 2 +- .../host.json | 1 + 5 files changed, 22 insertions(+), 76 deletions(-) diff --git a/infrastructure/templates/public-api/api-infrastructure-pipeline.yml b/infrastructure/templates/public-api/api-infrastructure-pipeline.yml index ec990cf9db5..c4d1cca9f20 100644 --- a/infrastructure/templates/public-api/api-infrastructure-pipeline.yml +++ b/infrastructure/templates/public-api/api-infrastructure-pipeline.yml @@ -25,7 +25,7 @@ parameters: variables: - group: Public API Infrastructure - common - name: isDev - value: $[eq(variables['Build.SourceBranch'], 'refs/heads/EES-5127-add-parquet-file-share-mounts')] + value: $[eq(variables['Build.SourceBranch'], 'refs/heads/EES-5127-add-parquet-file-share-mounts-remove-existing-fileshares')] - name: isTest value: $[eq(variables['Build.SourceBranch'], 'refs/heads/test')] - name: isMaster diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 21a64daaaa0..7ed2edc9f15 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -94,43 +94,11 @@ resource dedicatedStorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' properties: { supportsHttpsTrafficOnly: true defaultToOAuthAuthentication: true - networkAcls: { - bypass: 'AzureServices' - defaultAction: 'Deny' - virtualNetworkRules: [ - { - action: 'Allow' - id: subnetId - } - ] - } } } var dedicatedStorageAccountString = 'DefaultEndpointsProtocol=https;AccountName=${dedicatedStorageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${dedicatedStorageAccount.listKeys().keys[0].value}' -// When deploying to a Function App utilising a secured VNet to store its deployment files, -// unique file shares must be pre-generated and unique to each slot prior to deploying the -// Function App itself if we wish to use slot swapping. -// -// See the second paragraph of https://learn.microsoft.com/en-us/azure/azure-functions/functions-infrastructure-as-code?tabs=json%2Clinux%2Cdevops&pivots=premium-plan#secured-deployments. -var fileShareName1 = '${toLower(fullFunctionAppName)}-1' -var fileShareName2 = '${toLower(fullFunctionAppName)}-2' - -resource fileShare1 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { - name: '${dedicatedStorageAccountName}/default/${fileShareName1}' - dependsOn: [ - dedicatedStorageAccount - ] -} - -resource fileShare2 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { - name: '${dedicatedStorageAccountName}/default/${fileShareName2}' - dependsOn: [ - dedicatedStorageAccount - ] -} - resource functionApp 'Microsoft.Web/sites@2023-01-01' = { name: fullFunctionAppName location: location @@ -202,10 +170,7 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { // This property tells the Function App that the deployment code resides in this Storage account. WEBSITE_CONTENTAZUREFILECONNECTIONSTRING: dedicatedStorageAccountString - // These 2 properties indicate that the traffic which pulls down the deployment code for the Function App - // from Storage should go over the VNet and find their code in file shares within their linked Storage Account. - WEBSITE_CONTENTOVERVNET: 1 - vnetContentShareEnabled: true + WEBSITE_CONTENTSHARE: fullFunctionAppName // This setting is necessary in order to allow slot swapping to work without complaining that // "Storage volume is currently in R/O mode". @@ -225,32 +190,12 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { }) stagingOnlySettings: { SLOT_NAME: 'staging' - // When deploying to a Function App utilising a secured VNet to store its deployment files, - // unique file shares must be pre-generated and unique to each slot prior to deploying the - // Function App itself if we wish to use slot swapping. - // - // In conjunction with WEBSITE_CONTENTAZUREFILECONNECTIONSTRING, this property tells the - // Function App that the deployment code resides in this Storage account and within *this* - // file share. - // - // See the second paragraph of https://learn.microsoft.com/en-us/azure/azure-functions/functions-infrastructure-as-code?tabs=json%2Clinux%2Cdevops&pivots=premium-plan#secured-deployments. - // - // When slots are swapped, this fileshare configuration value will swap with the slots and thereby make the new - // code deployment location available to the other slot. - WEBSITE_CONTENTSHARE: fileShareName2 } prodOnlySettings: { SLOT_NAME: 'production' - // As above, this value is distinct from the initial staging slot equivalent and will swap to the other slot upon - // a slot swap operation. - WEBSITE_CONTENTSHARE: fileShareName1 } tagValues: tagValues } - dependsOn: [ - fileShare1 - fileShare2 - ] } // Allow Key Vault references passed as secure appsettings to be resolved by the Function App and its deployment slots. diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index f502187ef06..b8b5ab9a887 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -133,21 +133,21 @@ stages: --slot 'staging' \ --target-slot 'production' - - task: AzureCLI@2 - displayName: 'Deploy Data Processor Function App - attach Parquet Fileshare' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e - - az webapp config storage-account add \ - --name '$(dataProcessorFunctionAppName)' \ - --resource-group '$(resourceGroupName)' \ - --custom-id '$(parquetFileShareStorageName)' \ - --storage-type AzureFiles \ - --account-name '$(coreStorageAccountName)' \ - --share-name '$(parquetFileShareStorageName)' \ - --access-key '`az keyvault secret show --name $(coreStorageAccessKeySecretKey) --vault-name $(keyVaultName) --query value --output tsv`' \ - --mount-path '$(parquetFileShareMountPath)' +# - task: AzureCLI@2 +# displayName: 'Deploy Data Processor Function App - attach Parquet Fileshare' +# inputs: +# azureSubscription: ${{parameters.serviceConnection}} +# scriptType: bash +# scriptLocation: inlineScript +# inlineScript: | +# set -e +# +# az webapp config storage-account add \ +# --name '$(dataProcessorFunctionAppName)' \ +# --resource-group '$(resourceGroupName)' \ +# --custom-id '$(parquetFileShareStorageName)' \ +# --storage-type AzureFiles \ +# --account-name '$(coreStorageAccountName)' \ +# --share-name '$(parquetFileShareStorageName)' \ +# --access-key '@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageAccessKeySecretKey))' \ +# --mount-path '$(parquetFileShareMountPath)' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index ccff786d572..9e6221e40be 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -87,7 +87,7 @@ var acrName = 'eesacr' var vNetName = '${subscription}-vnet-ees' var containerAppEnvironmentNameSuffix = '01' var parquetFileShareMountName = 'parquet-fileshare-mount' -var parquetFileShareMountPath = '/home/public-api-parquet' +var parquetFileShareMountPath = '/data/public-api-parquet' var parquetFileShareStorageName = 'parquet-fileshare-storage' var tagValues = union(resourceTags ?? {}, { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json index 126c1f40f14..9556a83e0f5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json @@ -3,6 +3,7 @@ "extensions": { "durableTask": { "storageProvider": { + "hubName": "DataProcessorTaskHub", "type": "AzureStorage" } } From 80e076f27ea0f6fe2882b5e33e64ff892f60b756 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 10 May 2024 17:26:22 +0100 Subject: [PATCH 10/73] EES-5127 - adding dedicated storage accounts per slot to enable zero downtime durable function execution when using swap slots --- .../api-infrastructure-pipeline.yml | 16 +-- .../public-api/components/functionApp.bicep | 101 ++++++++++++++++-- .../public-api/deploy-stage-template.yml | 18 +++- .../templates/public-api/main.bicep | 20 ++-- .../host.json | 5 +- 5 files changed, 131 insertions(+), 29 deletions(-) diff --git a/infrastructure/templates/public-api/api-infrastructure-pipeline.yml b/infrastructure/templates/public-api/api-infrastructure-pipeline.yml index c4d1cca9f20..65146962fb1 100644 --- a/infrastructure/templates/public-api/api-infrastructure-pipeline.yml +++ b/infrastructure/templates/public-api/api-infrastructure-pipeline.yml @@ -57,19 +57,19 @@ pool: vmImage: $(vmImageName) stages: -- template: validate-stage-template.yml - parameters: - stageName: 'Validate_Against_Development' - condition: eq(variables.isDev, true) - environment: 'Development' - serviceConnection: $(serviceConnectionDevelopment) - parameterFile: $(devParamFile) +#- template: validate-stage-template.yml +# parameters: +# stageName: 'Validate_Against_Development' +# condition: eq(variables.isDev, true) +# environment: 'Development' +# serviceConnection: $(serviceConnectionDevelopment) +# parameterFile: $(devParamFile) - template: deploy-stage-template.yml parameters: stageName: 'Deploy_to_Development' condition: and(succeeded(), eq(variables.isDev, true)) - dependsOn: 'Validate_Against_Development' +# dependsOn: 'Validate_Against_Development' environment: 'Development' serviceConnection: $(serviceConnectionDevelopment) parameterFile: $(devParamFile) diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 7ed2edc9f15..012163d0c0d 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -77,15 +77,20 @@ resource appServicePlan 'Microsoft.Web/serverfarms@2023-01-01' = { } } -var dedicatedStorageAccountName = replace('${subscription}eessa${functionAppName}', '-', '') +// Configuring a single shared storage account for task management, and 2 individual storage accounts to be split +// between the production slot and staging slot. See +// https://learn.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-zero-downtime-deployment#status-check-with-slot +var durableManagementStorageAccountName = replace('${subscription}eessa${functionAppName}mg', '-', '') +var slot1StorageAccountName = replace('${subscription}eessa${functionAppName}s1', '-', '') +var slot2StorageAccountName = replace('${subscription}eessa${functionAppName}s2', '-', '') // Create a dedicated Storage Account for this Function App for it to store its deployment code and jobs in. // Grant the Function App access by whitelisting its subnet for inbound traffic. // // For performance, it is considered good practice for each Function App to have its own dedicated Storage Account. See // https://learn.microsoft.com/en-us/azure/azure-functions/storage-considerations?tabs=azure-cli#optimize-storage-performance. -resource dedicatedStorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { - name: dedicatedStorageAccountName +resource durableManagementStorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { + name: durableManagementStorageAccountName location: location sku: { name: 'Standard_LRS' @@ -94,10 +99,70 @@ resource dedicatedStorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' properties: { supportsHttpsTrafficOnly: true defaultToOAuthAuthentication: true + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + virtualNetworkRules: [ + { + action: 'Allow' + id: subnetId + } + ] + } + } +} + +var durableManagementStorageAccountString = 'DefaultEndpointsProtocol=https;AccountName=${durableManagementStorageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${durableManagementStorageAccount.listKeys().keys[0].value}' + +resource slot1StorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { + name: slot1StorageAccountName + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'Storage' + properties: { + supportsHttpsTrafficOnly: true + defaultToOAuthAuthentication: true + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + virtualNetworkRules: [ + { + action: 'Allow' + id: subnetId + } + ] + } + } +} + +var slot1StorageAccountString = 'DefaultEndpointsProtocol=https;AccountName=${slot1StorageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${slot1StorageAccount.listKeys().keys[0].value}' + +resource slot2StorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { + name: slot2StorageAccountName + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'Storage' + properties: { + supportsHttpsTrafficOnly: true + defaultToOAuthAuthentication: true + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + virtualNetworkRules: [ + { + action: 'Allow' + id: subnetId + } + ] + } } } -var dedicatedStorageAccountString = 'DefaultEndpointsProtocol=https;AccountName=${dedicatedStorageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${dedicatedStorageAccount.listKeys().keys[0].value}' +var slot2StorageAccountString = 'DefaultEndpointsProtocol=https;AccountName=${slot2StorageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${slot2StorageAccount.listKeys().keys[0].value}' resource functionApp 'Microsoft.Web/sites@2023-01-01' = { name: fullFunctionAppName @@ -139,6 +204,20 @@ resource azureStorageAccount 'Microsoft.Web/sites/config@2021-01-15' = if (addit } } +resource slot1StorageAccountFileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { + name: '${durableManagementStorageAccountName}/default/${fullFunctionAppName}1' + dependsOn: [ + durableManagementStorageAccount + ] +} + +resource slot2StorageAccountFileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { + name: '${durableManagementStorageAccountName}/default/${fullFunctionAppName}2' + dependsOn: [ + durableManagementStorageAccount + ] +} + // We determine any pre-existing appsettings for both the production and the staging slots during this infrastructure // deploy and supply them as the most important appsettings. This prevents infrastructure deploys from overriding any // appsettings back to their original values by allowing existing ones to take precedence. @@ -165,12 +244,16 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { commonSettings: union(settings, { // This tells the Function App where to store its "azure-webjobs-hosts" and "azure-webjobs-secrets" files. - AzureWebJobsStorage: dedicatedStorageAccountString + AzureWebJobsStorage: durableManagementStorageAccountString // This property tells the Function App that the deployment code resides in this Storage account. - WEBSITE_CONTENTAZUREFILECONNECTIONSTRING: dedicatedStorageAccountString + WEBSITE_CONTENTAZUREFILECONNECTIONSTRING: durableManagementStorageAccountString - WEBSITE_CONTENTSHARE: fullFunctionAppName + // These 2 properties indicate that the traffic which pulls down the deployment code for the Function App + // from Storage should go over the VNet and find their code in file shares within their linked Storage Account. + WEBSITE_CONTENTOVERVNET: 1 +// vnetContentShareEnabled: true + // WEBSITE_VNET_ROUTE_ALL: 1 // This setting is necessary in order to allow slot swapping to work without complaining that // "Storage volume is currently in R/O mode". @@ -190,9 +273,13 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { }) stagingOnlySettings: { SLOT_NAME: 'staging' + DurableManagementStorage: slot1StorageAccountString + WEBSITE_CONTENTSHARE: '${fullFunctionAppName}1' } prodOnlySettings: { SLOT_NAME: 'production' + DurableManagementStorage: slot2StorageAccountString + WEBSITE_CONTENTSHARE: '${fullFunctionAppName}2' } tagValues: tagValues } diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index b8b5ab9a887..98c17129d49 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -145,9 +145,23 @@ stages: # az webapp config storage-account add \ # --name '$(dataProcessorFunctionAppName)' \ # --resource-group '$(resourceGroupName)' \ -# --custom-id '$(parquetFileShareStorageName)' \ +# --custom-id '$(parquetFileShareName)' \ # --storage-type AzureFiles \ # --account-name '$(coreStorageAccountName)' \ -# --share-name '$(parquetFileShareStorageName)' \ # --access-key '@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageAccessKeySecretKey))' \ +# --share-name '$(parquetFileShareName)' \ # --mount-path '$(parquetFileShareMountPath)' +# +# - task: AzureCLI@2 +# displayName: 'Deploy Data Processor Function App - list storage' +# inputs: +# azureSubscription: ${{parameters.serviceConnection}} +# scriptType: bash +# scriptLocation: inlineScript +# inlineScript: | +# set -e +# +# az webapp config storage-account list \ +# --name '$(dataProcessorFunctionAppName)' \ +# --resource-group '$(resourceGroupName)' + diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 9e6221e40be..c080cafce6f 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -198,7 +198,7 @@ module containerAppEnvironmentModule 'components/containerAppEnvironment.bicep' tagValues: tagValues azureFileStorages: [ { - storageName: parquetFileShareStorageName + storageName: parquetFileShareModule.outputs.fileShareName storageAccountName: coreStorageAccountName storageAccountKey: coreStorageAccountKey fileShareName: parquetFileShareModule.outputs.fileShareName @@ -229,7 +229,7 @@ module apiContainerAppModule 'components/containerApp.bicep' = if (deployContain { name: parquetFileShareMountName storageType: 'AzureFile' - storageName: parquetFileShareStorageName + storageName: parquetFileShareModule.outputs.fileShareName } ] appSettings: [ @@ -307,13 +307,13 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { family: 'EP' } preWarmedInstanceCount: 1 - additionalAzureFileStorage: { - storageName: parquetFileShareStorageName - storageAccountKey: coreStorageAccountKey - storageAccountName: coreStorageAccountName - fileShareName: parquetFileShareModule.outputs.fileShareName - mountPath: parquetFileShareMountPath - } +// additionalAzureFileStorage: { +// storageName: parquetFileShareModule.outputs.fileShareName +// storageAccountKey: coreStorageAccountKey +// storageAccountName: coreStorageAccountName +// fileShareName: parquetFileShareModule.outputs.fileShareName +// mountPath: parquetFileShareMountPath +// } } } @@ -386,7 +386,7 @@ output dataProcessorContentDbConnectionStringSecretKey string = 'ees-publicapi-d output dataProcessorPsqlConnectionStringSecretKey string = dataProcessorPsqlConnectionStringSecretKey output coreStorageConnectionStringSecretKey string = coreStorageConnectionStringSecretKey output keyVaultName string = keyVaultName -output parquetFileShareStorageName string = parquetFileShareStorageName output coreStorageAccountName string = coreStorageAccountName output coreStorageAccessKeySecretKey string = coreStorageAccessKeySecretKey +output parquetFileShareName string = parquetFileShareModule.outputs.fileShareName output parquetFileShareMountPath string = parquetFileShareMountPath diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json index 9556a83e0f5..ac2d37bfc79 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json @@ -2,9 +2,10 @@ "version": "2.0", "extensions": { "durableTask": { + "hubName": "DataProcessorTaskHub", "storageProvider": { - "hubName": "DataProcessorTaskHub", - "type": "AzureStorage" + "type": "AzureStorage", + "connectionStringName": "DurableManagementStorage" } } }, From 824ae1349da07fa23ad19a105d601d7414623770 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 14 May 2024 14:58:28 +0100 Subject: [PATCH 11/73] EES-5127 - added brand new storage account to create Parquet file share within (figuring that the Core Storage account has compatibility issues being old now) --- .../public-api/components/fileShares.bicep | 8 +- .../public-api/deploy-stage-template.yml | 39 ++++---- .../templates/public-api/main.bicep | 94 +++++++++++++++---- 3 files changed, 102 insertions(+), 39 deletions(-) diff --git a/infrastructure/templates/public-api/components/fileShares.bicep b/infrastructure/templates/public-api/components/fileShares.bicep index c44bc2c968d..25577e5b8d6 100644 --- a/infrastructure/templates/public-api/components/fileShares.bicep +++ b/infrastructure/templates/public-api/components/fileShares.bicep @@ -29,10 +29,10 @@ resource fileService 'Microsoft.Storage/storageAccounts/fileServices@2023-01-01' resource fileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { name: shareName parent: fileService - properties: { - accessTier: fileShareAccessTier - shareQuota: fileShareQuota - } +// properties: { +// accessTier: fileShareAccessTier +// shareQuota: fileShareQuota +// } } output fileShareName string = fileShare.name diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index 98c17129d49..fad18c31ae9 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -105,6 +105,26 @@ stages: --settings \ "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" + - task: AzureCLI@2 + displayName: 'Deploy Data Processor Function App - attach Parquet Fileshare to staging slot' + inputs: + azureSubscription: ${{parameters.serviceConnection}} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + + az webapp config storage-account add \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --custom-id $(parquetFileShareName) \ + --storage-type AzureFiles \ + --account-name $(publicApiStorageAccountName) \ + --access-key `az keyvault secret show --name $(publicApiStorageAccessKeySecretKey) --vault-name $(keyVaultName) --query value --output tsv` \ + --share-name $(parquetFileShareName) \ + --mount-path $(parquetFileShareMountPath) \ + --slot 'staging' + - task: AzureCLI@2 displayName: 'Deploy Data Processor Function App - deploy to staging slot' inputs: @@ -132,25 +152,6 @@ stages: --resource-group '$(resourceGroupName)' \ --slot 'staging' \ --target-slot 'production' - -# - task: AzureCLI@2 -# displayName: 'Deploy Data Processor Function App - attach Parquet Fileshare' -# inputs: -# azureSubscription: ${{parameters.serviceConnection}} -# scriptType: bash -# scriptLocation: inlineScript -# inlineScript: | -# set -e -# -# az webapp config storage-account add \ -# --name '$(dataProcessorFunctionAppName)' \ -# --resource-group '$(resourceGroupName)' \ -# --custom-id '$(parquetFileShareName)' \ -# --storage-type AzureFiles \ -# --account-name '$(coreStorageAccountName)' \ -# --access-key '@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageAccessKeySecretKey))' \ -# --share-name '$(parquetFileShareName)' \ -# --mount-path '$(parquetFileShareMountPath)' # # - task: AzureCLI@2 # displayName: 'Deploy Data Processor Function App - list storage' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index c080cafce6f..2be675074e2 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -88,7 +88,7 @@ var vNetName = '${subscription}-vnet-ees' var containerAppEnvironmentNameSuffix = '01' var parquetFileShareMountName = 'parquet-fileshare-mount' var parquetFileShareMountPath = '/data/public-api-parquet' -var parquetFileShareStorageName = 'parquet-fileshare-storage' +var publicApiStorageAccountName = '${subscription}eespapisa' var tagValues = union(resourceTags ?? {}, { Environment: environmentName @@ -121,6 +121,51 @@ module vNetModule 'application/virtualNetwork.bicep' = { } } +resource publicApiStorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { + name: publicApiStorageAccountName + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'Storage' + properties: { + supportsHttpsTrafficOnly: true + defaultToOAuthAuthentication: true +// networkAcls: { +// bypass: 'AzureServices' +// defaultAction: 'Deny' +// virtualNetworkRules: [ +// { +// action: 'Allow' +// id: vNetModule.outputs.dataProcessorSubnetRef +// } +// { +// action: 'Allow' +// id: vNetModule.outputs.containerAppEnvironmentSubnetRef +// } +// ] +// } + } +} + +var publicApiStorageAccountKey = publicApiStorageAccount.listKeys().keys[0].value +var publicApiStorageConnectionString = 'DefaultEndpointsProtocol=https;AccountName=${publicApiStorageAccount.name};EndpointSuffix=${endpointSuffix};AccountKey=${publicApiStorageAccountKey}' + +// Deploy File Share. +module parquetFileShareModule 'components/fileShares.bicep' = { + name: 'fileShareDeploy' + params: { + resourcePrefix: resourcePrefix + fileShareName: 'data' + fileShareQuota: fileShareQuota + storageAccountName: publicApiStorageAccountName + fileShareAccessTier: 'TransactionOptimized' + } + dependsOn: [ + publicApiStorageAccount + ] +} + // Deploy a single shared Application Insights for all relevant Public API resources to use. module applicationInsightsModule 'components/appInsights.bicep' = { name: 'appInsightsDeploy' @@ -141,17 +186,6 @@ module logAnalyticsWorkspaceModule 'components/logAnalyticsWorkspace.bicep' = { } } -// Deploy File Share. -module parquetFileShareModule 'components/fileShares.bicep' = { - name: 'fileShareDeploy' - params: { - resourcePrefix: resourcePrefix - fileShareName: 'data' - fileShareQuota: fileShareQuota - storageAccountName: coreStorageAccountName - } -} - var formattedPostgreSqlFirewallRules = map(postgreSqlFirewallRules, rule => { name: replace(rule.name, ' ', '_') startIpAddress: rule.startIpAddress @@ -199,8 +233,8 @@ module containerAppEnvironmentModule 'components/containerAppEnvironment.bicep' azureFileStorages: [ { storageName: parquetFileShareModule.outputs.fileShareName - storageAccountName: coreStorageAccountName - storageAccountKey: coreStorageAccountKey + storageAccountName: publicApiStorageAccountName + storageAccountKey: publicApiStorageAccountKey fileShareName: parquetFileShareModule.outputs.fileShareName accessMode: 'ReadWrite' } @@ -309,8 +343,8 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { preWarmedInstanceCount: 1 // additionalAzureFileStorage: { // storageName: parquetFileShareModule.outputs.fileShareName -// storageAccountKey: coreStorageAccountKey -// storageAccountName: coreStorageAccountName +// storageAccountKey: publicApiStorageAccountKey +// storageAccountName: publicApiStorageAccountName // fileShareName: parquetFileShareModule.outputs.fileShareName // mountPath: parquetFileShareMountPath // } @@ -382,11 +416,39 @@ module storeCoreStorageAccessKey 'components/keyVaultSecret.bicep' = { } } +var publicApiStorageConnectionStringSecretKey = 'ees-core-storage-connectionstring' + +module storePublicApiStorageConnectionString 'components/keyVaultSecret.bicep' = { + name: 'storePublicApiStorageConnectionString' + params: { + keyVaultName: keyVaultName + isEnabled: true + secretName: publicApiStorageConnectionStringSecretKey + secretValue: publicApiStorageConnectionString + contentType: 'text/plain' + } +} + +var publicApiStorageAccessKeySecretKey = 'ees-publicapi-storage-access-key' + +module storePublicApiStorageAccessKey 'components/keyVaultSecret.bicep' = { + name: 'storePublicApiStorageAccessKey' + params: { + keyVaultName: keyVaultName + isEnabled: true + secretName: publicApiStorageAccessKeySecretKey + secretValue: publicApiStorageAccountKey + contentType: 'text/plain' + } +} + output dataProcessorContentDbConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-contentdb' output dataProcessorPsqlConnectionStringSecretKey string = dataProcessorPsqlConnectionStringSecretKey output coreStorageConnectionStringSecretKey string = coreStorageConnectionStringSecretKey output keyVaultName string = keyVaultName output coreStorageAccountName string = coreStorageAccountName output coreStorageAccessKeySecretKey string = coreStorageAccessKeySecretKey +output publicApiStorageAccountName string = publicApiStorageAccountName +output publicApiStorageAccessKeySecretKey string = publicApiStorageAccessKeySecretKey output parquetFileShareName string = parquetFileShareModule.outputs.fileShareName output parquetFileShareMountPath string = parquetFileShareMountPath From 01867fe6ee1fda0886d7c2797d626c7a6b1b427d Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 14 May 2024 19:15:35 +0100 Subject: [PATCH 12/73] EES-5127 - making file mount conditional on its existence already --- .../components/appServiceSlotConfig.bicep | 2 + .../public-api/components/functionApp.bicep | 2 + .../public-api/deploy-stage-template.yml | 38 ++++++++++++------- .../templates/public-api/main.bicep | 1 + 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep index 33e0371876c..eb7b37f47e3 100644 --- a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep +++ b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep @@ -52,6 +52,8 @@ resource functionSlotConfig 'Microsoft.Web/sites/config@2023-01-01' = { // infrastructure deploys do not reset appsettings back to original values and cause // unwanted updates to production appsettings prior to a slot swap deploy process being // ready to run. +// +// See https://blog.dotnetstudio.nl/posts/2021/04/merge-appsettings-with-bicep. var combinedStagingSettings = union(commonSettings, stagingOnlySettings, existingStagingAppSettings) var combinedProductionSettings = union(commonSettings, prodOnlySettings, existingProductionAppSettings) diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 012163d0c0d..f153558146f 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -221,6 +221,8 @@ resource slot2StorageAccountFileShare 'Microsoft.Storage/storageAccounts/fileSer // We determine any pre-existing appsettings for both the production and the staging slots during this infrastructure // deploy and supply them as the most important appsettings. This prevents infrastructure deploys from overriding any // appsettings back to their original values by allowing existing ones to take precedence. +// +// See https://blog.dotnetstudio.nl/posts/2021/04/merge-appsettings-with-bicep. var existingStagingAppSettings = functionAppExists ? list(resourceId('Microsoft.Web/sites/slots/config', functionApp.name, 'staging', 'appsettings'), '2021-03-01').properties : {} var existingProductionAppSettings = functionAppExists ? list(resourceId('Microsoft.Web/sites/config', functionApp.name, 'appsettings'), '2021-03-01').properties : {} diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index fad18c31ae9..7c05a46a716 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -52,6 +52,10 @@ stages: set -e dataProcessorExists=`az functionapp list --resource-group $(resourceGroupName) --query "[?name=='$(dataProcessorFunctionAppName)']" | jq '. != []'` + if [[ "$dataProcessorExists" == "true" ]]; then + echo "Data Processor Function App exists - combining existing appsettings with new ones" + fi + az deployment group create \ --name 'DeployPublicApiInfrastructure$(upstreamPipelineBuildNumber)' \ --resource-group $(resourceGroupName) \ @@ -66,7 +70,7 @@ stages: postgreSqlFirewallRules='$(maintenanceFirewallRules)' \ dockerImagesTag='$(upstreamPipelineBuildNumber)' \ deployContainerApp=$(deployContainerApp) \ - updatePsqlFlexibleServer=$(updatePsqlFlexibleServer) + updatePsqlFlexibleServer=$(updatePsqlFlexibleServer) \ dataProcessorFunctionAppExists=$dataProcessorExists - template: pipeline-variables-from-bicep-outputs-template.yml @@ -81,14 +85,14 @@ stages: scriptLocation: inlineScript inlineScript: | set -e - + az functionapp config appsettings set \ --name '$(dataProcessorFunctionAppName)' \ --resource-group '$(resourceGroupName)' \ --slot 'staging' \ --settings \ "CoreStorage=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" - + az webapp config connection-string set \ --name '$(dataProcessorFunctionAppName)' \ --resource-group '$(resourceGroupName)' \ @@ -114,16 +118,22 @@ stages: inlineScript: | set -e - az webapp config storage-account add \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --custom-id $(parquetFileShareName) \ - --storage-type AzureFiles \ - --account-name $(publicApiStorageAccountName) \ - --access-key `az keyvault secret show --name $(publicApiStorageAccessKeySecretKey) --vault-name $(keyVaultName) --query value --output tsv` \ - --share-name $(parquetFileShareName) \ - --mount-path $(parquetFileShareMountPath) \ - --slot 'staging' + fileShareMountExists=`az webapp config storage-account list --resource-group $(resourceGroupName) --name $(dataProcessorFunctionAppName) --slot staging | jq '. != []'` + + if [[ "$fileShareMountExists" == "false" ]]; then + + az webapp config storage-account add \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --custom-id $(parquetFileShareName) \ + --storage-type AzureFiles \ + --account-name $(publicApiStorageAccountName) \ + --access-key `az keyvault secret show --name $(publicApiStorageAccessKeySecretKey) --vault-name $(keyVaultName) --query value --output tsv` \ + --share-name $(parquetFileShareName) \ + --mount-path $(parquetFileShareMountPath) \ + --slot 'staging' + + fi - task: AzureCLI@2 displayName: 'Deploy Data Processor Function App - deploy to staging slot' @@ -165,4 +175,4 @@ stages: # az webapp config storage-account list \ # --name '$(dataProcessorFunctionAppName)' \ # --resource-group '$(resourceGroupName)' - +# diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 2be675074e2..bfd4282e418 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -215,6 +215,7 @@ module postgreSqlServerModule 'components/postgresqlDatabase.bicep' = if (update var psqlManagedIdentityConnectionStringTemplate = 'Server=${psqlServerFullName}.postgres.database.azure.com;Database=[database_name];Port=5432;User Id=[managed_identity_name];Password=[access_token]' +// TODO EES-5128 - move into the Container App module? resource apiContainerAppManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (deployContainerApp) { name: apiContainerAppManagedIdentityName } From 0be068620947c74e33d92a3fa20f2654d89f8b7a Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 15 May 2024 16:09:43 +0100 Subject: [PATCH 13/73] EES-5127 - retrying Bicep mechanism for attaching file shares to Function App --- .../public-api/deploy-stage-template.yml | 50 +++++++++---------- .../templates/public-api/main.bicep | 14 +++--- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index 7c05a46a716..80a7233ac6b 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -109,31 +109,31 @@ stages: --settings \ "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" - - task: AzureCLI@2 - displayName: 'Deploy Data Processor Function App - attach Parquet Fileshare to staging slot' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e - - fileShareMountExists=`az webapp config storage-account list --resource-group $(resourceGroupName) --name $(dataProcessorFunctionAppName) --slot staging | jq '. != []'` - - if [[ "$fileShareMountExists" == "false" ]]; then - - az webapp config storage-account add \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --custom-id $(parquetFileShareName) \ - --storage-type AzureFiles \ - --account-name $(publicApiStorageAccountName) \ - --access-key `az keyvault secret show --name $(publicApiStorageAccessKeySecretKey) --vault-name $(keyVaultName) --query value --output tsv` \ - --share-name $(parquetFileShareName) \ - --mount-path $(parquetFileShareMountPath) \ - --slot 'staging' - - fi +# - task: AzureCLI@2 +# displayName: 'Deploy Data Processor Function App - attach Parquet Fileshare to staging slot' +# inputs: +# azureSubscription: ${{parameters.serviceConnection}} +# scriptType: bash +# scriptLocation: inlineScript +# inlineScript: | +# set -e +# +# fileShareMountExists=`az webapp config storage-account list --resource-group $(resourceGroupName) --name $(dataProcessorFunctionAppName) --slot staging | jq '. != []'` +# +# if [[ "$fileShareMountExists" == "false" ]]; then +# +# az webapp config storage-account add \ +# --name $(dataProcessorFunctionAppName) \ +# --resource-group $(resourceGroupName) \ +# --custom-id $(parquetFileShareName) \ +# --storage-type AzureFiles \ +# --account-name $(publicApiStorageAccountName) \ +# --access-key `az keyvault secret show --name $(publicApiStorageAccessKeySecretKey) --vault-name $(keyVaultName) --query value --output tsv` \ +# --share-name $(parquetFileShareName) \ +# --mount-path $(parquetFileShareMountPath) \ +# --slot 'staging' +# +# fi - task: AzureCLI@2 displayName: 'Deploy Data Processor Function App - deploy to staging slot' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index bfd4282e418..66e29dc3b82 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -342,13 +342,13 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { family: 'EP' } preWarmedInstanceCount: 1 -// additionalAzureFileStorage: { -// storageName: parquetFileShareModule.outputs.fileShareName -// storageAccountKey: publicApiStorageAccountKey -// storageAccountName: publicApiStorageAccountName -// fileShareName: parquetFileShareModule.outputs.fileShareName -// mountPath: parquetFileShareMountPath -// } + additionalAzureFileStorage: { + storageName: parquetFileShareModule.outputs.fileShareName + storageAccountKey: publicApiStorageAccountKey + storageAccountName: publicApiStorageAccountName + fileShareName: parquetFileShareModule.outputs.fileShareName + mountPath: parquetFileShareMountPath + } } } From 755101083cfefe9f41a2f5d185ae490c0cc0720f Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 15 May 2024 17:06:00 +0100 Subject: [PATCH 14/73] EES-5127 - retrying Bicep mechanism for attaching file shares to Function App, through siteConfig --- .../public-api/components/functionApp.bicep | 27 ++++++++----------- .../templates/public-api/main.bicep | 4 +-- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index f153558146f..26b131e39a5 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -55,13 +55,13 @@ param preWarmedInstanceCount int? param alwaysOn bool? @description('Specifies additional Azure Storage Accounts to make available to this Function App') -param additionalAzureFileStorage { +param additionalAzureFileStorages { storageName: string storageAccountKey: string storageAccountName: string fileShareName: string mountPath: string -}? +}[] = [] var appServicePlanName = '${resourcePrefix}-asp-${functionAppName}' var reserved = appServicePlanOS == 'Linux' @@ -185,25 +185,20 @@ resource functionApp 'Microsoft.Web/sites@2023-01-01' = { preWarmedInstanceCount: preWarmedInstanceCount ?? null netFrameworkVersion: '8.0' linuxFxVersion: appServicePlanOS == 'Linux' ? 'DOTNET-ISOLATED|8.0' : null + azureStorageAccounts: reduce(additionalAzureFileStorages, {}, (cur, next) => union(cur, { + '${next.storageName}': { + type: 'AzureFiles' + shareName: next.fileShareName + mountPath: next.mountPath + accountName: next.storageAccountName + accessKey: next.storageAccountKey + } + })) } } tags: tagValues } -resource azureStorageAccount 'Microsoft.Web/sites/config@2021-01-15' = if (additionalAzureFileStorage != null) { - name: 'azurestorageaccounts' - parent: functionApp - properties: { - '${additionalAzureFileStorage!.storageName}': { - type: 'AzureFiles' - shareName: additionalAzureFileStorage!.fileShareName - mountPath: additionalAzureFileStorage!.mountPath - accountName: additionalAzureFileStorage!.storageAccountName - accessKey: additionalAzureFileStorage!.storageAccountKey - } - } -} - resource slot1StorageAccountFileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { name: '${durableManagementStorageAccountName}/default/${fullFunctionAppName}1' dependsOn: [ diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 66e29dc3b82..9a5970d2ebb 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -342,13 +342,13 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { family: 'EP' } preWarmedInstanceCount: 1 - additionalAzureFileStorage: { + additionalAzureFileStorages: [{ storageName: parquetFileShareModule.outputs.fileShareName storageAccountKey: publicApiStorageAccountKey storageAccountName: publicApiStorageAccountName fileShareName: parquetFileShareModule.outputs.fileShareName mountPath: parquetFileShareMountPath - } + }] } } From f17c3b02a122a0d856b09f44869289f5ed749767 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 15 May 2024 19:31:31 +0100 Subject: [PATCH 15/73] EES-5127 - trying siteConfig in common settings --- .../public-api/components/functionApp.bicep | 22 ++++++++++--------- .../templates/public-api/main.bicep | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 26b131e39a5..219948d87da 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -55,7 +55,7 @@ param preWarmedInstanceCount int? param alwaysOn bool? @description('Specifies additional Azure Storage Accounts to make available to this Function App') -param additionalAzureFileStorages { +param azureFileShares { storageName: string storageAccountKey: string storageAccountName: string @@ -185,15 +185,6 @@ resource functionApp 'Microsoft.Web/sites@2023-01-01' = { preWarmedInstanceCount: preWarmedInstanceCount ?? null netFrameworkVersion: '8.0' linuxFxVersion: appServicePlanOS == 'Linux' ? 'DOTNET-ISOLATED|8.0' : null - azureStorageAccounts: reduce(additionalAzureFileStorages, {}, (cur, next) => union(cur, { - '${next.storageName}': { - type: 'AzureFiles' - shareName: next.fileShareName - mountPath: next.mountPath - accountName: next.storageAccountName - accessKey: next.storageAccountKey - } - })) } } tags: tagValues @@ -267,6 +258,17 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { FUNCTIONS_EXTENSION_VERSION: '~4' FUNCTIONS_WORKER_RUNTIME: functionAppRuntime APPINSIGHTS_INSTRUMENTATIONKEY: applicationInsightsKey + siteConfig: { + azureStorageAccounts: reduce(azureFileShares, {}, (cur, next) => union(cur, { + '${next.storageName}': { + type: 'AzureFiles' + shareName: next.fileShareName + mountPath: next.mountPath + accountName: next.storageAccountName + accessKey: next.storageAccountKey + } + })) + } }) stagingOnlySettings: { SLOT_NAME: 'staging' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 9a5970d2ebb..4ebe6a497e6 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -342,7 +342,7 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { family: 'EP' } preWarmedInstanceCount: 1 - additionalAzureFileStorages: [{ + azureFileShares: [{ storageName: parquetFileShareModule.outputs.fileShareName storageAccountKey: publicApiStorageAccountKey storageAccountName: publicApiStorageAccountName From 4103d33f994c2d2101d4258129c85ce2a1f7755e Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 15 May 2024 19:45:20 +0100 Subject: [PATCH 16/73] EES-5127 - trying azureStorageAccounts in both main config and slot config --- .../components/appServiceSlotConfig.bicep | 23 ++++++++++++++++ .../public-api/components/functionApp.bicep | 26 +++++++++++-------- .../templates/public-api/main.bicep | 6 +++++ 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep index eb7b37f47e3..0b9b884f8f2 100644 --- a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep +++ b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep @@ -22,6 +22,15 @@ param existingStagingAppSettings object @description('Specifies any existing appsettings from the production slot') param existingProductionAppSettings object +@description('Specifies additional Azure Storage Accounts to make available to the staging slot') +param azureFileShares { + storageName: string + storageAccountKey: string + storageAccountName: string + fileShareName: string + mountPath: string +}[] = [] + @description('A set of tags with which to tag the resource in Azure') param tagValues object @@ -64,6 +73,20 @@ resource appStagingSlotSettings 'Microsoft.Web/sites/slots/config@2023-01-01' = properties: combinedStagingSettings } +resource azureStorageAccounts 'Microsoft.Web/sites/slots/config@2021-01-15' = { + name: 'azurestorageaccounts' + parent: stagingSlot + properties: reduce(azureFileShares, {}, (cur, next) => union(cur, { + '${next.storageName}': { + type: 'AzureFiles' + shareName: next.fileShareName + mountPath: next.mountPath + accountName: next.storageAccountName + accessKey: next.storageAccountKey + } + })) +} + @description('Set appsettings on production slot') resource appProductionSettings 'Microsoft.Web/sites/config@2023-01-01' = { name: '${appName}/appsettings' diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 219948d87da..407872acf06 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -190,6 +190,20 @@ resource functionApp 'Microsoft.Web/sites@2023-01-01' = { tags: tagValues } +resource azureStorageAccounts 'Microsoft.Web/sites/config@2021-01-15' = { + name: 'azurestorageaccounts' + parent: functionApp + properties: reduce(azureFileShares, {}, (cur, next) => union(cur, { + '${next.storageName}': { + type: 'AzureFiles' + shareName: next.fileShareName + mountPath: next.mountPath + accountName: next.storageAccountName + accessKey: next.storageAccountKey + } + })) +} + resource slot1StorageAccountFileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { name: '${durableManagementStorageAccountName}/default/${fullFunctionAppName}1' dependsOn: [ @@ -258,17 +272,6 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { FUNCTIONS_EXTENSION_VERSION: '~4' FUNCTIONS_WORKER_RUNTIME: functionAppRuntime APPINSIGHTS_INSTRUMENTATIONKEY: applicationInsightsKey - siteConfig: { - azureStorageAccounts: reduce(azureFileShares, {}, (cur, next) => union(cur, { - '${next.storageName}': { - type: 'AzureFiles' - shareName: next.fileShareName - mountPath: next.mountPath - accountName: next.storageAccountName - accessKey: next.storageAccountKey - } - })) - } }) stagingOnlySettings: { SLOT_NAME: 'staging' @@ -280,6 +283,7 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { DurableManagementStorage: slot2StorageAccountString WEBSITE_CONTENTSHARE: '${fullFunctionAppName}2' } + azureFileShares: azureFileShares tagValues: tagValues } } diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 4ebe6a497e6..e627e988fe4 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -79,6 +79,7 @@ var adminAppServiceFullName = '${subscription}-as-ees-admin' var publisherFunctionAppFullName = '${subscription}-fa-ees-publisher' var dataProcessorFunctionAppName = 'processor' var dataProcessorFunctionAppFullName = '${resourcePrefix}-fa-${dataProcessorFunctionAppName}' +var dataProcessorFunctionAppManagedIdentityName = '${resourcePrefix}-id-fa-${dataProcessorFunctionAppName}' var psqlServerName = 'psql-flexibleserver' var psqlServerFullName = '${subscription}-ees-${psqlServerName}' var coreStorageAccountName = '${subscription}saeescore' @@ -322,6 +323,11 @@ module apiContainerAppModule 'components/containerApp.bicep' = if (deployContain ] } +resource dataProcessorFunctionAppManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: dataProcessorFunctionAppManagedIdentityName + location: location +} + // Deploy Data Processor Function. module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { name: 'dataProcessorFunctionAppDeploy' From 67c7cd67bcaffb4013bf27bea67f950ef1a866ba Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 15 May 2024 20:08:51 +0100 Subject: [PATCH 17/73] EES-5127 - assigning Function App and Staging slot a new user-assigned managed identity --- .../components/appServiceSlotConfig.bicep | 18 ++++++++-- .../public-api/components/containerApp.bicep | 10 ++---- .../public-api/components/functionApp.bicep | 19 ++++++++-- .../public-api/deploy-stage-template.yml | 36 +++++++++---------- .../templates/public-api/main.bicep | 3 +- 5 files changed, 54 insertions(+), 32 deletions(-) diff --git a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep index 0b9b884f8f2..3924de02f14 100644 --- a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep +++ b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep @@ -4,6 +4,9 @@ param appName string @description('Specifies the location of the resources') param location string +@description('An existing Managed Identity\'s Resource Id with which to associate this Function App') +param stagingSlotUserAssignedManagedIdentityId string? + @description('Specifies the names of slot settings (settings that stick to their slots rather than swap)') param slotSpecificSettingKeys string[] @@ -34,13 +37,22 @@ param azureFileShares { @description('A set of tags with which to tag the resource in Azure') param tagValues object +var identity = stagingSlotUserAssignedManagedIdentityId != null + ? { + type: 'UserAssigned' + userAssignedIdentities: { + '${stagingSlotUserAssignedManagedIdentityId}': {} + } + } + : { + type: 'SystemAssigned' + } + @description('Create a staging slot') resource stagingSlot 'Microsoft.Web/sites/slots@2023-01-01' = { name: '${appName}/staging' location: location - identity: { - type: 'SystemAssigned' - } + identity: identity properties: { enabled: true httpsOnly: true diff --git a/infrastructure/templates/public-api/components/containerApp.bicep b/infrastructure/templates/public-api/components/containerApp.bicep index 3a5d1489dde..77925d88a2a 100644 --- a/infrastructure/templates/public-api/components/containerApp.bicep +++ b/infrastructure/templates/public-api/components/containerApp.bicep @@ -61,7 +61,7 @@ param appSettings { param tagValues object @description('An existing Managed Identity\'s Resource Id with which to associate this Container App') -param managedIdentityName string +param userAssignedManagedIdentityId string @description('Id of the owning Container App Environment') param managedEnvironmentId string @@ -88,17 +88,13 @@ param volumeMounts { var containerImageName = '${acrLoginServer}/${containerAppImageName}' var containerApplicationName = toLower('${resourcePrefix}-ca-${containerAppName}') -resource containerAppIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { - name: managedIdentityName -} - resource containerApp 'Microsoft.App/containerApps@2023-05-01' = { name: containerApplicationName location: location identity: { type: 'UserAssigned' userAssignedIdentities: { - '${containerAppIdentity.id}': {} + '${userAssignedManagedIdentityId}': {} } } properties: { @@ -120,7 +116,7 @@ resource containerApp 'Microsoft.App/containerApps@2023-05-01' = { registries: [ { server: acrLoginServer - identity: containerAppIdentity.id + identity: userAssignedManagedIdentityId } ] } diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 407872acf06..1a93af4e43c 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -39,6 +39,9 @@ param applicationInsightsKey string @description('Specifies the subnet id') param subnetId string +@description('An existing Managed Identity\'s Resource Id with which to associate this Function App') +param userAssignedManagedIdentityId string? + @description('Specifies the SKU for the Function App hosting plan') param sku object @@ -67,6 +70,17 @@ var appServicePlanName = '${resourcePrefix}-asp-${functionAppName}' var reserved = appServicePlanOS == 'Linux' var fullFunctionAppName = '${subscription}-ees-papi-fa-${functionAppName}' +var identity = userAssignedManagedIdentityId != null + ? { + type: 'UserAssigned' + userAssignedIdentities: { + '${userAssignedManagedIdentityId}': {} + } + } + : { + type: 'SystemAssigned' + } + resource appServicePlan 'Microsoft.Web/serverfarms@2023-01-01' = { name: appServicePlanName location: location @@ -168,9 +182,7 @@ resource functionApp 'Microsoft.Web/sites@2023-01-01' = { name: fullFunctionAppName location: location kind: 'functionapp' - identity: { - type: 'SystemAssigned' - } + identity: identity properties: { httpsOnly: true serverFarmId: appServicePlan.id @@ -235,6 +247,7 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { params: { appName: functionApp.name location: location + stagingSlotUserAssignedManagedIdentityId: userAssignedManagedIdentityId existingStagingAppSettings: existingStagingAppSettings existingProductionAppSettings: existingProductionAppSettings slotSpecificSettingKeys: [ diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index 80a7233ac6b..651faff7f0e 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -87,25 +87,25 @@ stages: set -e az functionapp config appsettings set \ - --name '$(dataProcessorFunctionAppName)' \ - --resource-group '$(resourceGroupName)' \ - --slot 'staging' \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --slot staging \ --settings \ "CoreStorage=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" az webapp config connection-string set \ - --name '$(dataProcessorFunctionAppName)' \ - --resource-group '$(resourceGroupName)' \ - --slot 'staging' \ - --connection-string-type 'SQLAzure' \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --slot staging \ + --connection-string-type SQLAzure \ --settings \ "ContentDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorContentDbConnectionStringSecretKey))" az webapp config connection-string set \ - --name '$(dataProcessorFunctionAppName)' \ - --resource-group '$(resourceGroupName)' \ - --slot 'staging' \ - --connection-string-type 'PostgreSQL' \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --slot staging \ + --connection-string-type PostgreSQL \ --settings \ "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" @@ -145,9 +145,9 @@ stages: set -e az functionapp deployment source config-zip \ --src '$(Pipeline.Workspace)/EESBuildPipeline/public-api-data-processor-$(upstreamPipelineBuildNumber)/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.zip' \ - --name '$(dataProcessorFunctionAppName)' \ - --resource-group '$(resourceGroupName)' \ - --slot 'staging' + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --slot staging - task: AzureCLI@2 displayName: 'Deploy Data Processor Function App - swap slots' @@ -158,10 +158,10 @@ stages: inlineScript: | set -e az functionapp deployment slot swap \ - --name '$(dataProcessorFunctionAppName)' \ - --resource-group '$(resourceGroupName)' \ - --slot 'staging' \ - --target-slot 'production' + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --slot staging \ + --target-slot production # # - task: AzureCLI@2 # displayName: 'Deploy Data Processor Function App - list storage' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index e627e988fe4..89ddb2d1f5c 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -253,7 +253,7 @@ module apiContainerAppModule 'components/containerApp.bicep' = if (deployContain containerAppName: apiContainerAppName acrLoginServer: containerRegistry.properties.loginServer containerAppImageName: 'ees-public-api/api:${dockerImagesTag}' - managedIdentityName: apiContainerAppManagedIdentity.name + userAssignedManagedIdentityId: apiContainerAppManagedIdentity.id managedEnvironmentId: containerAppEnvironmentModule.outputs.containerAppEnvironmentId volumeMounts: [ { @@ -339,6 +339,7 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { tagValues: tagValues applicationInsightsKey: applicationInsightsModule.outputs.applicationInsightsKey subnetId: vNetModule.outputs.dataProcessorSubnetRef + userAssignedManagedIdentityId: dataProcessorFunctionAppManagedIdentity.id functionAppExists: dataProcessorFunctionAppExists keyVaultName: keyVaultName functionAppRuntime: 'dotnet-isolated' From 95739c9bed0689c0ca41d457c727419d8da40ed1 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 15 May 2024 20:15:43 +0100 Subject: [PATCH 18/73] EES-5127 - assigning Function App and Staging slot a new user-assigned managed identity --- .../templates/public-api/components/appServiceSlotConfig.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep index 3924de02f14..0a93e6c63fd 100644 --- a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep +++ b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep @@ -105,4 +105,4 @@ resource appProductionSettings 'Microsoft.Web/sites/config@2023-01-01' = { properties: combinedProductionSettings } -output stagingSlotPrincipalId string = stagingSlot.identity.principalId +output stagingSlotPrincipalId string = stagingSlotUserAssignedManagedIdentityId ?? stagingSlot.identity.principalId From 41cc37017f0cf20881b99d3f52b42c4d81dc9dbd Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 15 May 2024 20:43:25 +0100 Subject: [PATCH 19/73] EES-5127 - assigning Function App and Staging slot a new user-assigned managed identity --- .../components/appServiceSlotConfig.bicep | 2 +- .../public-api/components/functionApp.bicep | 10 +- .../components/storageAccount.bicep | 31 ++++-- .../templates/public-api/main.bicep | 94 ++++--------------- 4 files changed, 48 insertions(+), 89 deletions(-) diff --git a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep index 0a93e6c63fd..898e5ffbdf2 100644 --- a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep +++ b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep @@ -105,4 +105,4 @@ resource appProductionSettings 'Microsoft.Web/sites/config@2023-01-01' = { properties: combinedProductionSettings } -output stagingSlotPrincipalId string = stagingSlotUserAssignedManagedIdentityId ?? stagingSlot.identity.principalId +output stagingSlotPrincipalId string = stagingSlotUserAssignedManagedIdentityId == null ? stagingSlot.identity.principalId : '' diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 1a93af4e43c..15ad4214679 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -302,14 +302,16 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { } // Allow Key Vault references passed as secure appsettings to be resolved by the Function App and its deployment slots. +// Where the staging slot's managed identity differs from the main slot's managed identity, add its id to the list. +var additionalStagingPrincipalId = userAssignedManagedIdentityId == null + ? [functionAppSlotSettings.outputs.stagingSlotPrincipalId] + : [] + module functionAppKeyVaultAccessPolicy 'keyVaultAccessPolicy.bicep' = { name: '${functionAppName}FunctionAppKeyVaultAccessPolicy' params: { keyVaultName: keyVaultName - principalIds: [ - functionApp.identity.principalId - functionAppSlotSettings.outputs.stagingSlotPrincipalId - ] + principalIds: union([functionApp.identity.principalId], additionalStagingPrincipalId) } } diff --git a/infrastructure/templates/public-api/components/storageAccount.bicep b/infrastructure/templates/public-api/components/storageAccount.bicep index 0a28bcb0f3a..f9e1c684bea 100644 --- a/infrastructure/templates/public-api/components/storageAccount.bicep +++ b/infrastructure/templates/public-api/components/storageAccount.bicep @@ -1,6 +1,3 @@ -@description('Specifies the Resource Prefix') -param resourcePrefix string - @description('Specifies the location for all resources.') param location string @@ -9,7 +6,7 @@ param location string param storageAccountName string @description('Storage Account Network Rules') -param storageSubnetRules array = [] +param allowedSubnetIds string[] = [] @description('Storage Account Network Firewall Rules') param storageFirewallRules array = [] @@ -27,17 +24,18 @@ param storageFirewallRules array = [] ]) param skuStorageResource string = 'Standard_LRS' +@description('Storage Account Name') +param keyVaultName string + @description('A set of tags with which to tag the resource in Azure') param tagValues object // Variables and created data -var storageName = replace('${resourcePrefix}sa${storageAccountName}', '-', '') -var connectionStringSecretName = '${resourcePrefix}-sa-${storageAccountName}-connectionString' var endpointSuffix = environment().suffixes.storage //Resources resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { - name: storageName + name: storageAccountName location: location kind: 'StorageV2' sku: { @@ -53,7 +51,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { value: ipRule action: 'Allow' }] - virtualNetworkRules: [for ipRule in storageSubnetRules: { + virtualNetworkRules: [for ipRule in allowedSubnetIds: { id: ipRule action: 'Allow' }] @@ -65,7 +63,8 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { var key = storageAccount.listKeys().keys[0].value var storageAccountConnectionString = 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};EndpointSuffix=${endpointSuffix};AccountKey=${key}' -//store connections string +var connectionStringSecretName = '${storageAccountName}-connectionString' + module storeADOConnectionStringToKeyVault './keyVaultSecret.bicep' = { name: 'saConnectionStringSecretDeploy' params: { @@ -77,6 +76,20 @@ module storeADOConnectionStringToKeyVault './keyVaultSecret.bicep' = { } } +var accessKeySecretName = '${storageAccountName}-connectionString' + +module storeAccessKeyToKeyVault './keyVaultSecret.bicep' = { + name: 'saAccessKeySecretDeploy' + params: { + keyVaultName: keyVaultName + isEnabled: true + secretValue: key + contentType: 'text/plain' + secretName: accessKeySecretName + } +} + //Outputs output storageAccountName string = storageAccount.name output connectionStringSecretName string = connectionStringSecretName +output accessKeySecretName string = accessKeySecretName diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 89ddb2d1f5c..4386e2095b2 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -110,6 +110,10 @@ var coreStorageAccountKey = coreStorageAccount.listKeys().keys[0].value var endpointSuffix = environment().suffixes.storage var coreStorageConnectionString = 'DefaultEndpointsProtocol=https;AccountName=${coreStorageAccount.name};EndpointSuffix=${endpointSuffix};AccountKey=${coreStorageAccountKey}' +resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = { + name: keyVaultName +} + // Reference the existing VNet as currently managed by the EES ARM template, and register new subnets for Bicep-controlled resources. module vNetModule 'application/virtualNetwork.bicep' = { name: 'networkDeploy' @@ -122,36 +126,21 @@ module vNetModule 'application/virtualNetwork.bicep' = { } } -resource publicApiStorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { - name: publicApiStorageAccountName - location: location - sku: { - name: 'Standard_LRS' - } - kind: 'Storage' - properties: { - supportsHttpsTrafficOnly: true - defaultToOAuthAuthentication: true -// networkAcls: { -// bypass: 'AzureServices' -// defaultAction: 'Deny' -// virtualNetworkRules: [ -// { -// action: 'Allow' -// id: vNetModule.outputs.dataProcessorSubnetRef -// } -// { -// action: 'Allow' -// id: vNetModule.outputs.containerAppEnvironmentSubnetRef -// } -// ] -// } +module publicApiStorageAccountModule 'components/storageAccount.bicep' = { + name: 'publicApiStorageAccountDeploy' + params: { + location: location + storageAccountName: publicApiStorageAccountName + allowedSubnetIds: [ + vNetModule.outputs.dataProcessorSubnetRef + vNetModule.outputs.containerAppEnvironmentSubnetRef + ] + skuStorageResource: 'Standard_LRS' + keyVaultName: keyVaultName + tagValues: tagValues } } -var publicApiStorageAccountKey = publicApiStorageAccount.listKeys().keys[0].value -var publicApiStorageConnectionString = 'DefaultEndpointsProtocol=https;AccountName=${publicApiStorageAccount.name};EndpointSuffix=${endpointSuffix};AccountKey=${publicApiStorageAccountKey}' - // Deploy File Share. module parquetFileShareModule 'components/fileShares.bicep' = { name: 'fileShareDeploy' @@ -163,7 +152,7 @@ module parquetFileShareModule 'components/fileShares.bicep' = { fileShareAccessTier: 'TransactionOptimized' } dependsOn: [ - publicApiStorageAccount + publicApiStorageAccountModule ] } @@ -236,7 +225,7 @@ module containerAppEnvironmentModule 'components/containerAppEnvironment.bicep' { storageName: parquetFileShareModule.outputs.fileShareName storageAccountName: publicApiStorageAccountName - storageAccountKey: publicApiStorageAccountKey + storageAccountKey: '@Microsoft.KeyVault(VaultName=${keyVaultName};SecretName=${publicApiStorageAccountModule.outputs.accessKeySecretName})' fileShareName: parquetFileShareModule.outputs.fileShareName accessMode: 'ReadWrite' } @@ -351,7 +340,7 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { preWarmedInstanceCount: 1 azureFileShares: [{ storageName: parquetFileShareModule.outputs.fileShareName - storageAccountKey: publicApiStorageAccountKey + storageAccountKey: '@Microsoft.KeyVault(VaultName=${keyVaultName};SecretName=${publicApiStorageAccountModule.outputs.accessKeySecretName})' storageAccountName: publicApiStorageAccountName fileShareName: parquetFileShareModule.outputs.fileShareName mountPath: parquetFileShareMountPath @@ -411,52 +400,7 @@ module storeCoreStorageConnectionString 'components/keyVaultSecret.bicep' = { } } -var coreStorageAccessKeySecretKey = 'ees-core-storage-access-key' - -module storeCoreStorageAccessKey 'components/keyVaultSecret.bicep' = { - name: 'storeCoreStorageAccessKey' - params: { - keyVaultName: keyVaultName - isEnabled: true - secretName: coreStorageAccessKeySecretKey - secretValue: coreStorageAccountKey - contentType: 'text/plain' - } -} - -var publicApiStorageConnectionStringSecretKey = 'ees-core-storage-connectionstring' - -module storePublicApiStorageConnectionString 'components/keyVaultSecret.bicep' = { - name: 'storePublicApiStorageConnectionString' - params: { - keyVaultName: keyVaultName - isEnabled: true - secretName: publicApiStorageConnectionStringSecretKey - secretValue: publicApiStorageConnectionString - contentType: 'text/plain' - } -} - -var publicApiStorageAccessKeySecretKey = 'ees-publicapi-storage-access-key' - -module storePublicApiStorageAccessKey 'components/keyVaultSecret.bicep' = { - name: 'storePublicApiStorageAccessKey' - params: { - keyVaultName: keyVaultName - isEnabled: true - secretName: publicApiStorageAccessKeySecretKey - secretValue: publicApiStorageAccountKey - contentType: 'text/plain' - } -} - output dataProcessorContentDbConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-contentdb' output dataProcessorPsqlConnectionStringSecretKey string = dataProcessorPsqlConnectionStringSecretKey output coreStorageConnectionStringSecretKey string = coreStorageConnectionStringSecretKey output keyVaultName string = keyVaultName -output coreStorageAccountName string = coreStorageAccountName -output coreStorageAccessKeySecretKey string = coreStorageAccessKeySecretKey -output publicApiStorageAccountName string = publicApiStorageAccountName -output publicApiStorageAccessKeySecretKey string = publicApiStorageAccessKeySecretKey -output parquetFileShareName string = parquetFileShareModule.outputs.fileShareName -output parquetFileShareMountPath string = parquetFileShareMountPath From 7687fa76e942f00c997ea9d95a9735d800d9332c Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 15 May 2024 21:48:20 +0100 Subject: [PATCH 20/73] EES-5127 - added service endpoint to allow CAE to connect to protected storage account --- .../public-api/components/functionApp.bicep | 24 ++++++++------ .../components/storageAccount.bicep | 8 ++--- .../public-api/deploy-stage-template.yml | 3 +- .../templates/public-api/main.bicep | 33 +++++++++++++------ infrastructure/templates/template.json | 5 +++ .../Functions/HealthCheckFunctions.cs | 22 ++++++++++--- .../ProcessorHostBuilder.cs | 7 +++- 7 files changed, 70 insertions(+), 32 deletions(-) diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 15ad4214679..c53ab316671 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -40,7 +40,11 @@ param applicationInsightsKey string param subnetId string @description('An existing Managed Identity\'s Resource Id with which to associate this Function App') -param userAssignedManagedIdentityId string? +param userAssignedManagedIdentityParams { + id: string + name: string + principalId: string +}? @description('Specifies the SKU for the Function App hosting plan') param sku object @@ -70,11 +74,11 @@ var appServicePlanName = '${resourcePrefix}-asp-${functionAppName}' var reserved = appServicePlanOS == 'Linux' var fullFunctionAppName = '${subscription}-ees-papi-fa-${functionAppName}' -var identity = userAssignedManagedIdentityId != null +var identity = userAssignedManagedIdentityParams != null ? { type: 'UserAssigned' userAssignedIdentities: { - '${userAssignedManagedIdentityId}': {} + '${userAssignedManagedIdentityParams!.id}': {} } } : { @@ -197,7 +201,9 @@ resource functionApp 'Microsoft.Web/sites@2023-01-01' = { preWarmedInstanceCount: preWarmedInstanceCount ?? null netFrameworkVersion: '8.0' linuxFxVersion: appServicePlanOS == 'Linux' ? 'DOTNET-ISOLATED|8.0' : null + keyVaultReferenceIdentity: userAssignedManagedIdentityParams != null ? userAssignedManagedIdentityParams!.id : null } + keyVaultReferenceIdentity: userAssignedManagedIdentityParams != null ? userAssignedManagedIdentityParams!.id : null } tags: tagValues } @@ -247,7 +253,7 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { params: { appName: functionApp.name location: location - stagingSlotUserAssignedManagedIdentityId: userAssignedManagedIdentityId + stagingSlotUserAssignedManagedIdentityId: userAssignedManagedIdentityParams!.id existingStagingAppSettings: existingStagingAppSettings existingProductionAppSettings: existingProductionAppSettings slotSpecificSettingKeys: [ @@ -303,18 +309,16 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { // Allow Key Vault references passed as secure appsettings to be resolved by the Function App and its deployment slots. // Where the staging slot's managed identity differs from the main slot's managed identity, add its id to the list. -var additionalStagingPrincipalId = userAssignedManagedIdentityId == null - ? [functionAppSlotSettings.outputs.stagingSlotPrincipalId] - : [] +var keyVaultPrincipalIds = userAssignedManagedIdentityParams != null + ? [userAssignedManagedIdentityParams!.principalId] + : [functionApp.identity.principalId, functionAppSlotSettings.outputs.stagingSlotPrincipalId] module functionAppKeyVaultAccessPolicy 'keyVaultAccessPolicy.bicep' = { name: '${functionAppName}FunctionAppKeyVaultAccessPolicy' params: { keyVaultName: keyVaultName - principalIds: union([functionApp.identity.principalId], additionalStagingPrincipalId) + principalIds: keyVaultPrincipalIds } } output functionAppName string = functionApp.name -output principalId string = functionApp.identity.principalId -output tenantId string = functionApp.identity.tenantId diff --git a/infrastructure/templates/public-api/components/storageAccount.bicep b/infrastructure/templates/public-api/components/storageAccount.bicep index f9e1c684bea..b5b94ffbeaf 100644 --- a/infrastructure/templates/public-api/components/storageAccount.bicep +++ b/infrastructure/templates/public-api/components/storageAccount.bicep @@ -1,7 +1,6 @@ @description('Specifies the location for all resources.') param location string -//Specific parameters for the resources @description('Storage Account Name') param storageAccountName string @@ -30,10 +29,8 @@ param keyVaultName string @description('A set of tags with which to tag the resource in Azure') param tagValues object -// Variables and created data var endpointSuffix = environment().suffixes.storage -//Resources resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { name: storageAccountName location: location @@ -63,7 +60,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { var key = storageAccount.listKeys().keys[0].value var storageAccountConnectionString = 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};EndpointSuffix=${endpointSuffix};AccountKey=${key}' -var connectionStringSecretName = '${storageAccountName}-connectionString' +var connectionStringSecretName = '${storageAccountName}-connection-string' module storeADOConnectionStringToKeyVault './keyVaultSecret.bicep' = { name: 'saConnectionStringSecretDeploy' @@ -76,7 +73,7 @@ module storeADOConnectionStringToKeyVault './keyVaultSecret.bicep' = { } } -var accessKeySecretName = '${storageAccountName}-connectionString' +var accessKeySecretName = '${storageAccountName}-access-key' module storeAccessKeyToKeyVault './keyVaultSecret.bicep' = { name: 'saAccessKeySecretDeploy' @@ -89,7 +86,6 @@ module storeAccessKeyToKeyVault './keyVaultSecret.bicep' = { } } -//Outputs output storageAccountName string = storageAccount.name output connectionStringSecretName string = connectionStringSecretName output accessKeySecretName string = accessKeySecretName diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index 651faff7f0e..80e4f2979ed 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -91,7 +91,8 @@ stages: --resource-group $(resourceGroupName) \ --slot staging \ --settings \ - "CoreStorage=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" + "CoreStorage=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ + "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" az webapp config connection-string set \ --name $(dataProcessorFunctionAppName) \ diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 4386e2095b2..882686f6329 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -78,7 +78,6 @@ var apiContainerAppManagedIdentityName = '${resourcePrefix}-id-${apiContainerApp var adminAppServiceFullName = '${subscription}-as-ees-admin' var publisherFunctionAppFullName = '${subscription}-fa-ees-publisher' var dataProcessorFunctionAppName = 'processor' -var dataProcessorFunctionAppFullName = '${resourcePrefix}-fa-${dataProcessorFunctionAppName}' var dataProcessorFunctionAppManagedIdentityName = '${resourcePrefix}-id-fa-${dataProcessorFunctionAppName}' var psqlServerName = 'psql-flexibleserver' var psqlServerFullName = '${subscription}-ees-${psqlServerName}' @@ -110,10 +109,6 @@ var coreStorageAccountKey = coreStorageAccount.listKeys().keys[0].value var endpointSuffix = environment().suffixes.storage var coreStorageConnectionString = 'DefaultEndpointsProtocol=https;AccountName=${coreStorageAccount.name};EndpointSuffix=${endpointSuffix};AccountKey=${coreStorageAccountKey}' -resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = { - name: keyVaultName -} - // Reference the existing VNet as currently managed by the EES ARM template, and register new subnets for Bicep-controlled resources. module vNetModule 'application/virtualNetwork.bicep' = { name: 'networkDeploy' @@ -141,6 +136,18 @@ module publicApiStorageAccountModule 'components/storageAccount.bicep' = { } } +// We need to look up the Public API Storage Account in order to get its access keys, as it's not possible to feed +// Key Vault secret references into the "storageAccountKey" values for Azure File Storage mounts. +// +// It would be possible to use KV references if restructuring main.bicep to make the creation of the Container App +// and Data Processortheir own sub-modules in the "application" folder. Then, we could use @secure() params +// and keyVaultResource.getSecret() to pass the secrets through. +resource publicApiStorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' existing = { + name: publicApiStorageAccountName +} + +var publicApiStorageAccountAccessKey = publicApiStorageAccount.listKeys().keys[0].value + // Deploy File Share. module parquetFileShareModule 'components/fileShares.bicep' = { name: 'fileShareDeploy' @@ -225,7 +232,7 @@ module containerAppEnvironmentModule 'components/containerAppEnvironment.bicep' { storageName: parquetFileShareModule.outputs.fileShareName storageAccountName: publicApiStorageAccountName - storageAccountKey: '@Microsoft.KeyVault(VaultName=${keyVaultName};SecretName=${publicApiStorageAccountModule.outputs.accessKeySecretName})' + storageAccountKey: publicApiStorageAccountAccessKey fileShareName: parquetFileShareModule.outputs.fileShareName accessMode: 'ReadWrite' } @@ -295,7 +302,7 @@ module apiContainerAppModule 'components/containerApp.bicep' = if (deployContain // It uses this to grant permissions to the Data Processor user in order for it to be able to access // tables in the "public_data" database successfully. name: 'DataProcessorFunctionAppIdentityName' - value: dataProcessorFunctionAppFullName + value: dataProcessorFunctionAppManagedIdentity.name } { // This property informs the Container App of the name of the Publisher's system-assigned identity. @@ -328,7 +335,11 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { tagValues: tagValues applicationInsightsKey: applicationInsightsModule.outputs.applicationInsightsKey subnetId: vNetModule.outputs.dataProcessorSubnetRef - userAssignedManagedIdentityId: dataProcessorFunctionAppManagedIdentity.id + userAssignedManagedIdentityParams: { + id: dataProcessorFunctionAppManagedIdentity.id + name: dataProcessorFunctionAppManagedIdentity.name + principalId: dataProcessorFunctionAppManagedIdentity.properties.principalId + } functionAppExists: dataProcessorFunctionAppExists keyVaultName: keyVaultName functionAppRuntime: 'dotnet-isolated' @@ -340,7 +351,7 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { preWarmedInstanceCount: 1 azureFileShares: [{ storageName: parquetFileShareModule.outputs.fileShareName - storageAccountKey: '@Microsoft.KeyVault(VaultName=${keyVaultName};SecretName=${publicApiStorageAccountModule.outputs.accessKeySecretName})' + storageAccountKey: publicApiStorageAccountAccessKey storageAccountName: publicApiStorageAccountName fileShareName: parquetFileShareModule.outputs.fileShareName mountPath: parquetFileShareMountPath @@ -356,7 +367,7 @@ module storeDataProcessorPsqlConnectionString 'components/keyVaultSecret.bicep' keyVaultName: keyVaultName isEnabled: true secretName: dataProcessorPsqlConnectionStringSecretKey - secretValue: replace(replace(psqlManagedIdentityConnectionStringTemplate, '[database_name]', 'public_data'), '[managed_identity_name]', dataProcessorFunctionAppFullName) + secretValue: replace(replace(psqlManagedIdentityConnectionStringTemplate, '[database_name]', 'public_data'), '[managed_identity_name]', dataProcessorFunctionAppManagedIdentity.name) contentType: 'text/plain' } } @@ -402,5 +413,7 @@ module storeCoreStorageConnectionString 'components/keyVaultSecret.bicep' = { output dataProcessorContentDbConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-contentdb' output dataProcessorPsqlConnectionStringSecretKey string = dataProcessorPsqlConnectionStringSecretKey +output dataProcessorFunctionAppManagedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId + output coreStorageConnectionStringSecretKey string = coreStorageConnectionStringSecretKey output keyVaultName string = keyVaultName diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index f057e1e6bbc..42563f7e66b 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -3495,6 +3495,11 @@ "name": "[variables('containerAppEnvironmentSubnetName')]", "properties": { "addressPrefix": "10.0.8.0/23", + "serviceEndpoints": [ + { + "service": "Microsoft.Storage" + } + ], "delegations": [ { "name": "environment", diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs index 033c663092f..c793335a15c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs @@ -1,4 +1,5 @@ using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Functions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using Microsoft.Azure.Functions.Worker; using Microsoft.EntityFrameworkCore; @@ -15,6 +16,9 @@ public async Task CountDataSets( [ActivityTrigger] object? input, FunctionContext executionContext) { + var connectionString = ConnectionUtils.GetPostgreSqlConnectionString("PublicDataDb"); + logger.LogInformation(connectionString); + var message = $"Found {await publicDataDbContext.DataSets.CountAsync()} datasets."; logger.LogInformation(message); return message; @@ -25,9 +29,19 @@ public async Task ListFileShareContents( [ActivityTrigger] object? input, FunctionContext executionContext) { - var files = Directory.GetFiles("/data/public-api-parquet"); - var message = $"Found the following files:\n\n{files.JoinToString('\n')}"; - logger.LogInformation(message); - return message; + logger.LogInformation("Attempting to read from file share"); + + try + { + var files = Directory.GetFiles("/data/public-api-parquet"); + var message = $"Found the following files in the file share:\n\n{files.JoinToString('\n')}"; + logger.LogInformation(message); + return message; + } + catch (Exception e) + { + logger.LogError(e, "Error encountered when attempting to list files in the file share"); + throw; + } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs index 9de1828a35a..2e5a9f7e725 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs @@ -73,7 +73,11 @@ public static IHostBuilder ConfigureProcessorHostBuilder(this IHostBuilder hostB { services.AddDbContext(options => { - var sqlServerTokenProvider = new DefaultAzureCredential(); + var sqlServerTokenProvider = new DefaultAzureCredential( + new DefaultAzureCredentialOptions + { + ManagedIdentityClientId = Environment.GetEnvironmentVariable("AZURE_CLIENT_ID") + }); var accessToken = sqlServerTokenProvider.GetToken( new TokenRequestContext(scopes: [ @@ -83,6 +87,7 @@ public static IHostBuilder ConfigureProcessorHostBuilder(this IHostBuilder hostB var connectionStringWithAccessToken = connectionString.Replace("[access_token]", accessToken); + Console.WriteLine(connectionStringWithAccessToken); var dbDataSource = new NpgsqlDataSourceBuilder(connectionStringWithAccessToken).Build(); options.UseNpgsql(dbDataSource); From ddc4691511e78580bea3fdcd0946c8b162ef35f8 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 16 May 2024 12:35:50 +0100 Subject: [PATCH 21/73] EES-5127 - attempting to avoid Conflicts when deploying multiple firewall rules --- .../public-api/components/fileShares.bicep | 8 +- .../public-api/components/functionApp.bicep | 109 ++++++------------ .../components/postgresqlDatabase.bicep | 18 +-- .../templates/public-api/main.bicep | 2 + 4 files changed, 53 insertions(+), 84 deletions(-) diff --git a/infrastructure/templates/public-api/components/fileShares.bicep b/infrastructure/templates/public-api/components/fileShares.bicep index 25577e5b8d6..c44bc2c968d 100644 --- a/infrastructure/templates/public-api/components/fileShares.bicep +++ b/infrastructure/templates/public-api/components/fileShares.bicep @@ -29,10 +29,10 @@ resource fileService 'Microsoft.Storage/storageAccounts/fileServices@2023-01-01' resource fileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { name: shareName parent: fileService -// properties: { -// accessTier: fileShareAccessTier -// shareQuota: fileShareQuota -// } + properties: { + accessTier: fileShareAccessTier + shareQuota: fileShareQuota + } } output fileShareName string = fileShare.name diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index c53ab316671..05e7a6f9e8c 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -107,81 +107,46 @@ var slot2StorageAccountName = replace('${subscription}eessa${functionAppName}s2' // // For performance, it is considered good practice for each Function App to have its own dedicated Storage Account. See // https://learn.microsoft.com/en-us/azure/azure-functions/storage-considerations?tabs=azure-cli#optimize-storage-performance. -resource durableManagementStorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { - name: durableManagementStorageAccountName - location: location - sku: { - name: 'Standard_LRS' - } - kind: 'Storage' - properties: { - supportsHttpsTrafficOnly: true - defaultToOAuthAuthentication: true - networkAcls: { - bypass: 'AzureServices' - defaultAction: 'Deny' - virtualNetworkRules: [ - { - action: 'Allow' - id: subnetId - } - ] - } + +// TODO EES-5128 - add private endpoints to allow VNet traffic to go directly to Storage Account over the VNet. +module durableManagementStorageAccountModule 'storageAccount.bicep' = { + name: '${durableManagementStorageAccountName}StorageAccountDeploy' + params: { + location: location + storageAccountName: durableManagementStorageAccountName + allowedSubnetIds: [subnetId] + skuStorageResource: 'Standard_LRS' + keyVaultName: keyVaultName + tagValues: tagValues } } -var durableManagementStorageAccountString = 'DefaultEndpointsProtocol=https;AccountName=${durableManagementStorageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${durableManagementStorageAccount.listKeys().keys[0].value}' - -resource slot1StorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { - name: slot1StorageAccountName - location: location - sku: { - name: 'Standard_LRS' - } - kind: 'Storage' - properties: { - supportsHttpsTrafficOnly: true - defaultToOAuthAuthentication: true - networkAcls: { - bypass: 'AzureServices' - defaultAction: 'Deny' - virtualNetworkRules: [ - { - action: 'Allow' - id: subnetId - } - ] - } +// TODO EES-5128 - add private endpoints to allow VNet traffic to go directly to Storage Account over the VNet. +module slot1StorageAccountModule 'storageAccount.bicep' = { + name: '${slot1StorageAccountName}StorageAccountDeploy' + params: { + location: location + storageAccountName: slot1StorageAccountName + allowedSubnetIds: [subnetId] + skuStorageResource: 'Standard_LRS' + keyVaultName: keyVaultName + tagValues: tagValues } } -var slot1StorageAccountString = 'DefaultEndpointsProtocol=https;AccountName=${slot1StorageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${slot1StorageAccount.listKeys().keys[0].value}' - -resource slot2StorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { - name: slot2StorageAccountName - location: location - sku: { - name: 'Standard_LRS' - } - kind: 'Storage' - properties: { - supportsHttpsTrafficOnly: true - defaultToOAuthAuthentication: true - networkAcls: { - bypass: 'AzureServices' - defaultAction: 'Deny' - virtualNetworkRules: [ - { - action: 'Allow' - id: subnetId - } - ] - } +// TODO EES-5128 - add private endpoints to allow VNet traffic to go directly to Storage Account over the VNet. +module slot2StorageAccountModule 'storageAccount.bicep' = { + name: '${slot2StorageAccountName}StorageAccountDeploy' + params: { + location: location + storageAccountName: slot2StorageAccountName + allowedSubnetIds: [subnetId] + skuStorageResource: 'Standard_LRS' + keyVaultName: keyVaultName + tagValues: tagValues } } -var slot2StorageAccountString = 'DefaultEndpointsProtocol=https;AccountName=${slot2StorageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${slot2StorageAccount.listKeys().keys[0].value}' - resource functionApp 'Microsoft.Web/sites@2023-01-01' = { name: fullFunctionAppName location: location @@ -225,14 +190,14 @@ resource azureStorageAccounts 'Microsoft.Web/sites/config@2021-01-15' = { resource slot1StorageAccountFileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { name: '${durableManagementStorageAccountName}/default/${fullFunctionAppName}1' dependsOn: [ - durableManagementStorageAccount + durableManagementStorageAccountModule ] } resource slot2StorageAccountFileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { name: '${durableManagementStorageAccountName}/default/${fullFunctionAppName}2' dependsOn: [ - durableManagementStorageAccount + durableManagementStorageAccountModule ] } @@ -265,10 +230,10 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { commonSettings: union(settings, { // This tells the Function App where to store its "azure-webjobs-hosts" and "azure-webjobs-secrets" files. - AzureWebJobsStorage: durableManagementStorageAccountString + AzureWebJobsStorage: '@Microsoft.KeyVault(VaultName=${keyVaultName};SecretName=${durableManagementStorageAccountModule.outputs.connectionStringSecretName})' // This property tells the Function App that the deployment code resides in this Storage account. - WEBSITE_CONTENTAZUREFILECONNECTIONSTRING: durableManagementStorageAccountString + WEBSITE_CONTENTAZUREFILECONNECTIONSTRING: '@Microsoft.KeyVault(VaultName=${keyVaultName};SecretName=${durableManagementStorageAccountModule.outputs.connectionStringSecretName})' // These 2 properties indicate that the traffic which pulls down the deployment code for the Function App // from Storage should go over the VNet and find their code in file shares within their linked Storage Account. @@ -294,12 +259,12 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { }) stagingOnlySettings: { SLOT_NAME: 'staging' - DurableManagementStorage: slot1StorageAccountString + DurableManagementStorage: '@Microsoft.KeyVault(VaultName=${keyVaultName};SecretName=${slot1StorageAccountModule.outputs.connectionStringSecretName})' WEBSITE_CONTENTSHARE: '${fullFunctionAppName}1' } prodOnlySettings: { SLOT_NAME: 'production' - DurableManagementStorage: slot2StorageAccountString + DurableManagementStorage: '@Microsoft.KeyVault(VaultName=${keyVaultName};SecretName=${slot2StorageAccountModule.outputs.connectionStringSecretName})' WEBSITE_CONTENTSHARE: '${fullFunctionAppName}2' } azureFileShares: azureFileShares diff --git a/infrastructure/templates/public-api/components/postgresqlDatabase.bicep b/infrastructure/templates/public-api/components/postgresqlDatabase.bicep index c9b2d3bad3d..6f8eb54d713 100644 --- a/infrastructure/templates/public-api/components/postgresqlDatabase.bicep +++ b/infrastructure/templates/public-api/components/postgresqlDatabase.bicep @@ -119,17 +119,19 @@ resource postgreSQLDatabase 'Microsoft.DBforPostgreSQL/flexibleServers@2023-03-0 name: name }] - resource rules 'firewallRules' = [for rule in firewallRules: { - name: rule.name - properties: { - startIpAddress: rule.startIpAddress - endIpAddress: rule.endIpAddress - } - }] - tags: tagValues } +@batchSize(1) +resource rules 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2022-12-01' = [for rule in firewallRules: { + name: rule.name + parent: postgreSQLDatabase + properties: { + startIpAddress: rule.startIpAddress + endIpAddress: rule.endIpAddress + } +}] + var privateLinkDnsZoneName = 'privatelink.postgres.database.azure.com' var privateEndpointName = '${databaseServerName}-plink' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 882686f6329..2367133e6f1 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -121,6 +121,8 @@ module vNetModule 'application/virtualNetwork.bicep' = { } } +// TODO EES-5128 - add private endpoints to allow VNet traffic to go directly to Storage Account over the VNet. +// Currently supported by subnet whitelisting and Storage service endpoints being enabled on the whitelisted subnets. module publicApiStorageAccountModule 'components/storageAccount.bicep' = { name: 'publicApiStorageAccountDeploy' params: { From e3aaf70143a54f52ebff6cd4235111588cb51cfa Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 16 May 2024 13:32:12 +0100 Subject: [PATCH 22/73] EES-5127 - removing debug code around conenction strings --- .../Functions/HealthCheckFunctions.cs | 25 ++++++++++--------- .../ProcessorHostBuilder.cs | 4 ++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs index c793335a15c..f4989e8a9b6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs @@ -1,5 +1,3 @@ -using GovUk.Education.ExploreEducationStatistics.Common.Extensions; -using GovUk.Education.ExploreEducationStatistics.Common.Functions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using Microsoft.Azure.Functions.Worker; using Microsoft.EntityFrameworkCore; @@ -16,16 +14,13 @@ public async Task CountDataSets( [ActivityTrigger] object? input, FunctionContext executionContext) { - var connectionString = ConnectionUtils.GetPostgreSqlConnectionString("PublicDataDb"); - logger.LogInformation(connectionString); - var message = $"Found {await publicDataDbContext.DataSets.CountAsync()} datasets."; logger.LogInformation(message); return message; } - [Function(nameof(ListFileShareContents))] - public async Task ListFileShareContents( + [Function(nameof(CheckForFileshareMount))] + public async Task CheckForFileshareMount( [ActivityTrigger] object? input, FunctionContext executionContext) { @@ -33,14 +28,20 @@ public async Task ListFileShareContents( try { - var files = Directory.GetFiles("/data/public-api-parquet"); - var message = $"Found the following files in the file share:\n\n{files.JoinToString('\n')}"; - logger.LogInformation(message); - return message; + var fileShareMountExists = Directory.Exists("/data/public-api-parquet"); + + if (fileShareMountExists) + { + logger.LogInformation("Successfully found the file share mount"); + } + else + { + logger.LogError("Unable to find the file share mount"); + } } catch (Exception e) { - logger.LogError(e, "Error encountered when attempting to list files in the file share"); + logger.LogError(e, "Error encountered when attempting to find the file share mount"); throw; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs index 2e5a9f7e725..207760d6ced 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs @@ -76,6 +76,9 @@ public static IHostBuilder ConfigureProcessorHostBuilder(this IHostBuilder hostB var sqlServerTokenProvider = new DefaultAzureCredential( new DefaultAzureCredentialOptions { + // Unlike Container Apps and App Services, DefaultAzureCredential does not pick up + // the "AZURE_CLIENT_ID" environment variable automatically when operating within + // a Function App. We therefore provide it manually. ManagedIdentityClientId = Environment.GetEnvironmentVariable("AZURE_CLIENT_ID") }); var accessToken = sqlServerTokenProvider.GetToken( @@ -87,7 +90,6 @@ public static IHostBuilder ConfigureProcessorHostBuilder(this IHostBuilder hostB var connectionStringWithAccessToken = connectionString.Replace("[access_token]", accessToken); - Console.WriteLine(connectionStringWithAccessToken); var dbDataSource = new NpgsqlDataSourceBuilder(connectionStringWithAccessToken).Build(); options.UseNpgsql(dbDataSource); From 5a88f58a8849fb9d37b0d6d7fa7bfc4e1f6ea51e Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 16 May 2024 13:40:48 +0100 Subject: [PATCH 23/73] EES-5127 - resolving errors around multiple storage account module deploys sharing the same deployment names when storing secrets --- .../templates/public-api/components/storageAccount.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/templates/public-api/components/storageAccount.bicep b/infrastructure/templates/public-api/components/storageAccount.bicep index b5b94ffbeaf..d71f068c906 100644 --- a/infrastructure/templates/public-api/components/storageAccount.bicep +++ b/infrastructure/templates/public-api/components/storageAccount.bicep @@ -63,7 +63,7 @@ var storageAccountConnectionString = 'DefaultEndpointsProtocol=https;AccountName var connectionStringSecretName = '${storageAccountName}-connection-string' module storeADOConnectionStringToKeyVault './keyVaultSecret.bicep' = { - name: 'saConnectionStringSecretDeploy' + name: '${storageAccountName}ConnectionStringSecretDeploy' params: { keyVaultName: keyVaultName isEnabled: true @@ -76,7 +76,7 @@ module storeADOConnectionStringToKeyVault './keyVaultSecret.bicep' = { var accessKeySecretName = '${storageAccountName}-access-key' module storeAccessKeyToKeyVault './keyVaultSecret.bicep' = { - name: 'saAccessKeySecretDeploy' + name: '${storageAccountName}AccessKeySecretDeploy' params: { keyVaultName: keyVaultName isEnabled: true From 92700404fae35c2d02f540654e394d32d9464b8a Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 16 May 2024 13:54:07 +0100 Subject: [PATCH 24/73] EES-5127 - cleanup and restoration of trimmed-down pipelines before merging back to Dev. --- .../api-infrastructure-pipeline.yml | 49 +++++++++++-------- .../public-api/deploy-stage-template.yml | 40 --------------- 2 files changed, 28 insertions(+), 61 deletions(-) diff --git a/infrastructure/templates/public-api/api-infrastructure-pipeline.yml b/infrastructure/templates/public-api/api-infrastructure-pipeline.yml index 65146962fb1..bac494475d5 100644 --- a/infrastructure/templates/public-api/api-infrastructure-pipeline.yml +++ b/infrastructure/templates/public-api/api-infrastructure-pipeline.yml @@ -1,5 +1,20 @@ trigger: none +parameters: + - name: deployContainerApp + displayName: Can we deploy the Container App yet? This is dependent on the user-assigned Managed Identity for the API Container App being created with the AcrPull role, and the database users added to PSQL. + default: true + - name: updatePsqlFlexibleServer + displayName: Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys. + default: false + +# This param is helpful for debugging to allow the selection of a particular branch from which to base a deploy from this pipeline. +# This should be removed in the long term in favour of using the "Resources" selection from the "Run pipeline" dialog. +# +# - name: buildBranchToDeploy +# displayName: Build branch to deploy. This allows a person who is manually running the pipeline to specify the use of the latest EESBuildPipeline build that was run against that branch. +# default: 'Branch from latest pipeline run' + resources: pipelines: - pipeline: EESBuildPipeline @@ -9,23 +24,15 @@ resources: - refs/heads/dev - refs/heads/test - refs/heads/master - branch: ${{parameters.buildBranchToDeploy}} +# This param is helpful for debugging to allow the selection of a particular branch from which to base a deploy from this pipeline. +# This should be removed in the long term in favour of using the "Resources" selection from the "Run pipeline" dialog. +# +# branch: ${{ replace(parameters.buildBranchToDeploy, 'Branch from latest pipeline run', '') }} -parameters: - - name: deployContainerApp - displayName: Can we deploy the Container App yet? This is dependent on the user-assigned Managed Identity for the API Container App being created with the AcrPull role, and the database users added to PSQL. - default: true - - name: updatePsqlFlexibleServer - displayName: Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys. - default: false - - name: buildBranchToDeploy - displayName: Build branch to deploy. - default: 'refs/heads/dev' - variables: - group: Public API Infrastructure - common - name: isDev - value: $[eq(variables['Build.SourceBranch'], 'refs/heads/EES-5127-add-parquet-file-share-mounts-remove-existing-fileshares')] + value: $[eq(variables['Build.SourceBranch'], 'refs/heads/dev')] - name: isTest value: $[eq(variables['Build.SourceBranch'], 'refs/heads/test')] - name: isMaster @@ -57,19 +64,19 @@ pool: vmImage: $(vmImageName) stages: -#- template: validate-stage-template.yml -# parameters: -# stageName: 'Validate_Against_Development' -# condition: eq(variables.isDev, true) -# environment: 'Development' -# serviceConnection: $(serviceConnectionDevelopment) -# parameterFile: $(devParamFile) +- template: validate-stage-template.yml + parameters: + stageName: 'Validate_Against_Development' + condition: eq(variables.isDev, true) + environment: 'Development' + serviceConnection: $(serviceConnectionDevelopment) + parameterFile: $(devParamFile) - template: deploy-stage-template.yml parameters: stageName: 'Deploy_to_Development' condition: and(succeeded(), eq(variables.isDev, true)) -# dependsOn: 'Validate_Against_Development' + dependsOn: 'Validate_Against_Development' environment: 'Development' serviceConnection: $(serviceConnectionDevelopment) parameterFile: $(devParamFile) diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index 80e4f2979ed..70fea2452ba 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -110,32 +110,6 @@ stages: --settings \ "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" -# - task: AzureCLI@2 -# displayName: 'Deploy Data Processor Function App - attach Parquet Fileshare to staging slot' -# inputs: -# azureSubscription: ${{parameters.serviceConnection}} -# scriptType: bash -# scriptLocation: inlineScript -# inlineScript: | -# set -e -# -# fileShareMountExists=`az webapp config storage-account list --resource-group $(resourceGroupName) --name $(dataProcessorFunctionAppName) --slot staging | jq '. != []'` -# -# if [[ "$fileShareMountExists" == "false" ]]; then -# -# az webapp config storage-account add \ -# --name $(dataProcessorFunctionAppName) \ -# --resource-group $(resourceGroupName) \ -# --custom-id $(parquetFileShareName) \ -# --storage-type AzureFiles \ -# --account-name $(publicApiStorageAccountName) \ -# --access-key `az keyvault secret show --name $(publicApiStorageAccessKeySecretKey) --vault-name $(keyVaultName) --query value --output tsv` \ -# --share-name $(parquetFileShareName) \ -# --mount-path $(parquetFileShareMountPath) \ -# --slot 'staging' -# -# fi - - task: AzureCLI@2 displayName: 'Deploy Data Processor Function App - deploy to staging slot' inputs: @@ -163,17 +137,3 @@ stages: --resource-group $(resourceGroupName) \ --slot staging \ --target-slot production -# -# - task: AzureCLI@2 -# displayName: 'Deploy Data Processor Function App - list storage' -# inputs: -# azureSubscription: ${{parameters.serviceConnection}} -# scriptType: bash -# scriptLocation: inlineScript -# inlineScript: | -# set -e -# -# az webapp config storage-account list \ -# --name '$(dataProcessorFunctionAppName)' \ -# --resource-group '$(resourceGroupName)' -# From d5d21f7f6778b5701d3d66dcb5c77e9351ccc7ff Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 16 May 2024 14:13:59 +0100 Subject: [PATCH 25/73] EES-5127 - restoring content-over-vnet variables for final test before merging to dev --- .../public-api/components/containerAppEnvironment.bicep | 4 ++-- .../templates/public-api/components/functionApp.bicep | 6 ++++-- infrastructure/templates/public-api/main.bicep | 9 +++++---- .../Functions/HealthCheckFunctions.cs | 4 ++-- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/infrastructure/templates/public-api/components/containerAppEnvironment.bicep b/infrastructure/templates/public-api/components/containerAppEnvironment.bicep index a8022820207..f738cc06883 100644 --- a/infrastructure/templates/public-api/components/containerAppEnvironment.bicep +++ b/infrastructure/templates/public-api/components/containerAppEnvironment.bicep @@ -31,14 +31,14 @@ param tagValues object @description('Specifies a suffix to append to the full name of the Container App Environment') param containerAppEnvironmentNameSuffix string = '' -@description('Specifies an array of Azure Fileshares to be available for Container Apps hosted within this Container App Environment') +@description('Specifies an array of Azure File Shares to be available for Container Apps hosted within this Container App Environment') param azureFileStorages { storageName: string storageAccountKey: string storageAccountName: string fileShareName: string accessMode: 'ReadWrite' | 'ReadOnly' -}[] +}[] = [] var containerAppEnvironmentName = empty(containerAppEnvironmentNameSuffix) ? '${subscription}-ees-cae' diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 05e7a6f9e8c..583da4ab02e 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -238,8 +238,10 @@ module functionAppSlotSettings 'appServiceSlotConfig.bicep' = { // These 2 properties indicate that the traffic which pulls down the deployment code for the Function App // from Storage should go over the VNet and find their code in file shares within their linked Storage Account. WEBSITE_CONTENTOVERVNET: 1 -// vnetContentShareEnabled: true - // WEBSITE_VNET_ROUTE_ALL: 1 + vnetContentShareEnabled: true + + // This property instructs the Function App to direct all outbound traffic over the VNet. + WEBSITE_VNET_ROUTE_ALL: 1 // This setting is necessary in order to allow slot swapping to work without complaining that // "Storage volume is currently in R/O mode". diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 2367133e6f1..32bf4f309f6 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -138,11 +138,12 @@ module publicApiStorageAccountModule 'components/storageAccount.bicep' = { } } -// We need to look up the Public API Storage Account in order to get its access keys, as it's not possible to feed -// Key Vault secret references into the "storageAccountKey" values for Azure File Storage mounts. +// TODO EES-5128 - we're currently needing to look up the Public API Storage Account in order to get its access keys, +// as it's not possible to feed Key Vault secret references into the "storageAccountKey" values for Azure File Storage +// mounts. // // It would be possible to use KV references if restructuring main.bicep to make the creation of the Container App -// and Data Processortheir own sub-modules in the "application" folder. Then, we could use @secure() params +// and Data Processor their own sub-modules in the "application" folder. Then, we could use @secure() params // and keyVaultResource.getSecret() to pass the secrets through. resource publicApiStorageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' existing = { name: publicApiStorageAccountName @@ -214,7 +215,6 @@ module postgreSqlServerModule 'components/postgresqlDatabase.bicep' = if (update var psqlManagedIdentityConnectionStringTemplate = 'Server=${psqlServerFullName}.postgres.database.azure.com;Database=[database_name];Port=5432;User Id=[managed_identity_name];Password=[access_token]' -// TODO EES-5128 - move into the Container App module? resource apiContainerAppManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (deployContainerApp) { name: apiContainerAppManagedIdentityName } @@ -267,6 +267,7 @@ module apiContainerAppModule 'components/containerApp.bicep' = if (deployContain } ] appSettings: [ + // TODO EES-5128 - replace this with a Key Vault reference string. { name: 'ConnectionStrings__PublicDataDb' value: replace(replace(psqlManagedIdentityConnectionStringTemplate, '[database_name]', 'public_data'), '[managed_identity_name]', apiContainerAppManagedIdentityName) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs index f4989e8a9b6..2742987c5a1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs @@ -19,8 +19,8 @@ public async Task CountDataSets( return message; } - [Function(nameof(CheckForFileshareMount))] - public async Task CheckForFileshareMount( + [Function(nameof(CheckForFileShareMount))] + public async Task CheckForFileShareMount( [ActivityTrigger] object? input, FunctionContext executionContext) { From df9e8e5e3ef58703553f8363fd04e7ec20ae363a Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 21 May 2024 13:25:44 +0100 Subject: [PATCH 26/73] EES-5127 - responding to PR comments. Using configuration to look up Parquet filepath in health check function. Renaming variables. --- .../templates/public-api/components/storageAccount.bicep | 8 ++++---- .../Functions/HealthCheckFunctions.cs | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/infrastructure/templates/public-api/components/storageAccount.bicep b/infrastructure/templates/public-api/components/storageAccount.bicep index d71f068c906..7207777e842 100644 --- a/infrastructure/templates/public-api/components/storageAccount.bicep +++ b/infrastructure/templates/public-api/components/storageAccount.bicep @@ -44,12 +44,12 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { networkAcls: { bypass: 'AzureServices' defaultAction: 'Deny' - ipRules: [for ipRule in storageFirewallRules: { - value: ipRule + ipRules: [for firewallRule in storageFirewallRules: { + value: firewallRule action: 'Allow' }] - virtualNetworkRules: [for ipRule in allowedSubnetIds: { - id: ipRule + virtualNetworkRules: [for subnetId in allowedSubnetIds: { + id: subnetId action: 'Allow' }] } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs index 2742987c5a1..56004541a4c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs @@ -1,13 +1,16 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Options; using Microsoft.Azure.Functions.Worker; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; public class HealthCheckFunctions( ILogger logger, - PublicDataDbContext publicDataDbContext) + PublicDataDbContext publicDataDbContext, + IOptions parquetFileOptions) { [Function(nameof(CountDataSets))] public async Task CountDataSets( @@ -28,9 +31,7 @@ public async Task CheckForFileShareMount( try { - var fileShareMountExists = Directory.Exists("/data/public-api-parquet"); - - if (fileShareMountExists) + if (Directory.Exists(parquetFileOptions.Value.BasePath)) { logger.LogInformation("Successfully found the file share mount"); } From 95df2c9b769348a1fa5a31258dabb030a0423ca9 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 22 May 2024 10:54:13 +0100 Subject: [PATCH 27/73] EES-5127 - disabling public access to Data Processor functions --- .../templates/public-api/components/functionApp.bicep | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 583da4ab02e..c62c875160e 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -39,6 +39,9 @@ param applicationInsightsKey string @description('Specifies the subnet id') param subnetId string +@description('Specifis whether this Function App is accessible from the public internet') +param publicNetworkAccessEnabled bool = false + @description('An existing Managed Identity\'s Resource Id with which to associate this Function App') param userAssignedManagedIdentityParams { id: string @@ -169,6 +172,7 @@ resource functionApp 'Microsoft.Web/sites@2023-01-01' = { keyVaultReferenceIdentity: userAssignedManagedIdentityParams != null ? userAssignedManagedIdentityParams!.id : null } keyVaultReferenceIdentity: userAssignedManagedIdentityParams != null ? userAssignedManagedIdentityParams!.id : null + publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' } tags: tagValues } From 64c9b32db585a2f2299ffaf635e2c2f58f75109a Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Tue, 7 May 2024 16:05:45 +0100 Subject: [PATCH 28/73] EES-4856 Add data csv preview lines --- .../Services/BlobStorageService.cs | 2 +- .../DataSetFilesControllerTests.cs | 142 +++++++++++++++++- ...ucationStatistics.Content.Api.Tests.csproj | 7 + .../appsettings.IntegrationTest.json | 21 +++ .../DataSetFileService.cs | 36 ++++- .../DataSetFileViewModel.cs | 4 +- .../DataGuidanceDataSetService.cs | 3 +- 7 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/appsettings.IntegrationTest.json diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs index 8de908d03f2..eecc9b613a3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs @@ -216,7 +216,7 @@ public async Task UploadFile( _logger.LogInformation("Uploading file to blob {containerName}/{path}", containerName, path); - await blob.UploadAsync( + await blob.UploadAsync( // @MarkFix fails to find container here? path: tempFilePath, httpHeaders: new BlobHttpHeaders { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index a0fbbfcacc6..f69e2fbffb1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -1,12 +1,16 @@ #nullable enable using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net.Http; using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Common; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using GovUk.Education.ExploreEducationStatistics.Common.Services; +using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Tests; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; @@ -25,13 +29,18 @@ using GovUk.Education.ExploreEducationStatistics.Content.Services; using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Content.ViewModels; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.WebUtilities; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using MockQueryable.Moq; using Moq; +using Testcontainers.Azurite; using Xunit; using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; +using File = GovUk.Education.ExploreEducationStatistics.Content.Model.File; using ReleaseVersion = GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion; namespace GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.Controllers; @@ -1845,7 +1854,27 @@ public async Task FetchDataSetDetails_Success() .WithSubjectId(Guid.NewGuid()) ); - var client = BuildApp() + var publicBlobStorageService = new PublicBlobStorageService( + logger: new Logger(new LoggerFactory()), + configuration: await CreateConfigurtionWithAzurite()); + + var formFile = CreateDataCsvFormFile(""" + Headers + Line 1 + Line 2 + Line 3 + Line 4 + Line 5 + Line 6 + """); + + await publicBlobStorageService.UploadFile( + containerName: BlobContainers.PublicReleaseFiles, + releaseFile.PublicPath(), + formFile); + + var client = BuildApp( + publicBlobStorageService: publicBlobStorageService) .AddContentDbTestData(context => { context.ReleaseFiles.Add(releaseFile); @@ -1912,6 +1941,16 @@ public async Task FetchDataSetDetails_Success() .Select(i => i.Label) .ToList(), viewModel.Meta.Indicators); + + viewModel.File.DataCsvPreviewLines // @MarkFix Assert.Equal here? + .AssertDeepEqualTo([ + "Headers", + "Line 1", + "Line 2", + "Line 3", + "Line 4", + "Line 5", + ]); } [Fact] @@ -1944,7 +1983,19 @@ public async Task FetchDataSetFiltersOrdered_Success() new FilterMeta { Id = filter2Id, Label = "Filter 2", ColumnName = "filter_2", }, ]))); - var client = BuildApp() + var publicBlobStorageService = new PublicBlobStorageService( + logger: new Logger(new LoggerFactory()), + configuration: await CreateConfigurtionWithAzurite()); + + var formFile = CreateDataCsvFormFile(string.Empty); + + await publicBlobStorageService.UploadFile( + containerName: BlobContainers.PublicReleaseFiles, + releaseFile.PublicPath(), + formFile); + + var client = BuildApp( + publicBlobStorageService: publicBlobStorageService) .AddContentDbTestData(context => { context.ReleaseFiles.Add(releaseFile); @@ -1997,7 +2048,19 @@ public async Task FetchDataSetIndicatorsOrdered_Success() new IndicatorMeta { Id = indicator4Id, Label = "Indicator 4", ColumnName = "indicator_4", }, ]))); - var client = BuildApp() + var publicBlobStorageService = new PublicBlobStorageService( + logger: new Logger(new LoggerFactory()), + configuration: await CreateConfigurtionWithAzurite()); + + var formFile = CreateDataCsvFormFile(string.Empty); + + await publicBlobStorageService.UploadFile( + containerName: BlobContainers.PublicReleaseFiles, + releaseFile.PublicPath(), + formFile); + + var client = BuildApp( + publicBlobStorageService: publicBlobStorageService) .AddContentDbTestData(context => { context.ReleaseFiles.Add(releaseFile); @@ -2098,7 +2161,19 @@ public async Task AmendmentNotPublished_ReturnsOk() .WithReleaseVersion(publication.ReleaseVersions[2]) // the draft version .WithFile(file); - var client = BuildApp() + var publicBlobStorageService = new PublicBlobStorageService( + logger: new Logger(new LoggerFactory()), + configuration: await CreateConfigurtionWithAzurite()); + + var formFile = CreateDataCsvFormFile(string.Empty); + + await publicBlobStorageService.UploadFile( + containerName: BlobContainers.PublicReleaseFiles, + releaseFile1.PublicPath(), + formFile); + + var client = BuildApp( + publicBlobStorageService: publicBlobStorageService) .AddContentDbTestData(context => { context.ReleaseFiles.AddRange(releaseFile0, releaseFile1, releaseFile2); @@ -2149,7 +2224,9 @@ public async Task DataSetFileRemovedOnAmendment_ReturnsNotFound() } private WebApplicationFactory BuildApp( - ContentDbContext? contentDbContext = null) + ContentDbContext? contentDbContext = null, + //StatisticsDbContext? statisticsDbContext = null, + IPublicBlobStorageService? publicBlobStorageService = null) { return TestApp .ResetDbContexts() @@ -2157,10 +2234,63 @@ private WebApplicationFactory BuildApp( { services.AddTransient(s => new ReleaseVersionRepository( contentDbContext ?? s.GetRequiredService())); + services.AddTransient(s => + publicBlobStorageService ?? new PublicBlobStorageService( + s.GetRequiredService>(), + new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + // Use appsettings.IntegrationTest.json to prevent passing due to data-storage docker + // container running separately. appsettings.IntegrationTest.json doesn't use ports + // 10000/10001 for blobs/queues + .AddJsonFile("appsettings.IntegrationTest.json", optional: false) + .AddEnvironmentVariables() + .Build())); + //services.AddTransient(s => new FootnoteRepository( + // statisticsDbContext ?? s.GetRequiredService())); services.AddTransient( s => new DataSetFileService( contentDbContext ?? s.GetRequiredService(), - s.GetRequiredService())); + s.GetRequiredService(), + s.GetRequiredService())); }); } + + private async Task CreateConfigurtionWithAzurite() + { + var azuriteContainer = new AzuriteBuilder() + .WithImage("mcr.microsoft.com/azure-storage/azurite:3.27.0") + .WithHostname("data-storage-test") + .Build(); + await azuriteContainer.StartAsync(); + + var configuration = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.IntegrationTest.json", optional: false) + .AddEnvironmentVariables() + .Build(); + configuration["PublicStorage"] = azuriteContainer.GetConnectionString(); + + return configuration; + } + + private IFormFile CreateDataCsvFormFile(string content) + { + var memoryStream = new MemoryStream(); + var writer = new StreamWriter(memoryStream); + writer.Write(content); + writer.Flush(); + memoryStream.Position = 0; + var headerDictionary = new HeaderDictionary(); + headerDictionary["ContentType"] = "text/csv"; + + return new FormFile( + memoryStream, + 0, + memoryStream.Length, + "id_from_form", + "dataCsv.csv") + { + Headers = headerDictionary, + }; + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.csproj b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.csproj index e511ec53685..670be601bac 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.csproj +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.csproj @@ -18,6 +18,7 @@ + all @@ -32,4 +33,10 @@ + + + Always + + + diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/appsettings.IntegrationTest.json b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/appsettings.IntegrationTest.json new file mode 100644 index 00000000000..8c0552d156d --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/appsettings.IntegrationTest.json @@ -0,0 +1,21 @@ +{ + "enableSwagger": true, + "PublicStorage": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://data-storage:10210/devstoreaccount1;QueueEndpoint=http://data-storage:10211/devstoreaccount1;", + "ConnectionStrings": { + "ContentDb": "Server=db;Database=content;User=content;Password=Your_Password123;TrustServerCertificate=True;", + "StatisticsDb": "Server=db;Database=statistics;User=content;Password=Your_Password123;TrustServerCertificate=True;" + }, + "Logging": { + "LogLevel": { + "Default": "Debug", + "System": "Information", + "Microsoft": "Warning", + "Microsoft.EntityFrameworkCore.Database.Command": "Warning" + } + }, + "MemoryCache": { + "Overrides": { + "DurationInSeconds": 2 + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index 4c46f3200df..2d5a004313e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -6,8 +6,10 @@ using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Common; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using GovUk.Education.ExploreEducationStatistics.Content.Model; @@ -18,6 +20,7 @@ using GovUk.Education.ExploreEducationStatistics.Content.Requests; using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Content.ViewModels; +using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using static GovUk.Education.ExploreEducationStatistics.Common.Model.SortDirection; @@ -31,13 +34,19 @@ public class DataSetFileService : IDataSetFileService { private readonly ContentDbContext _contentDbContext; private readonly IReleaseVersionRepository _releaseVersionRepository; + private readonly IPublicBlobStorageService _publicBlobStorageService; + //private readonly IFootnoteRepository _footnoteRepository; public DataSetFileService( ContentDbContext contentDbContext, - IReleaseVersionRepository releaseVersionRepository) + IReleaseVersionRepository releaseVersionRepository, + IPublicBlobStorageService publicBlobStorageService) + //IFootnoteRepository footnoteRepository) { _contentDbContext = contentDbContext; _releaseVersionRepository = releaseVersionRepository; + _publicBlobStorageService = publicBlobStorageService; + //_footnoteRepository = footnoteRepository; } public async Task>> ListDataSetFiles( @@ -159,6 +168,12 @@ public async Task> GetDataSetFile( return new NotFoundResult(); } + var dataCsvPreviewLines = await GetDataCsvPreviewLines(releaseFile); + + // Fetch variable names and descriptions from DataSetFileMeta + + // Fetch footnotes + return new DataSetFileViewModel { Id = releaseFile.File.DataSetFileId!.Value, // we ensure this is set when fetching releaseFile @@ -187,6 +202,7 @@ public async Task> GetDataSetFile( Id = releaseFile.FileId, Name = releaseFile.File.Filename, Size = releaseFile.File.DisplaySize(), + DataCsvPreviewLines = dataCsvPreviewLines, SubjectId = releaseFile.File.SubjectId!.Value, }, Meta = BuildDataSetFileMetaViewModel( @@ -226,6 +242,24 @@ private static DataSetFileMetaViewModel BuildDataSetFileMetaViewModel( }; } + private async Task> GetDataCsvPreviewLines(ReleaseFile releaseFile) + { + List dataCsvPreviewLines = new(); + var datafileStreamProvider = () => _publicBlobStorageService.StreamBlob( + containerName: BlobContainers.PublicReleaseFiles, + path: releaseFile.PublicPath()); + using var dataFileReader = new StreamReader(await datafileStreamProvider.Invoke()); + + for (var i = 0; i < 6; i++) + { + var line = await dataFileReader.ReadLineAsync(); + if (line == null) { break; } + dataCsvPreviewLines.Add(line); + } + + return dataCsvPreviewLines; + } + private static List GetOrderedFilters(List metaFilters, List? filterSequenceEntries) { var filterSequence = filterSequenceEntries? diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs index e5fca02b087..db417a1246a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs @@ -56,7 +56,9 @@ public record DataSetFileFileViewModel public required string Name { get; init; } - public required string Size { get; init; } + public required string Size { get; init; } = string.Empty; + + public required List DataCsvPreviewLines { get; init; } = new(); public required Guid SubjectId { get; init; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Services/DataGuidanceDataSetService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Services/DataGuidanceDataSetService.cs index da2c9c16180..fd2a703a82d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Services/DataGuidanceDataSetService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Services/DataGuidanceDataSetService.cs @@ -98,6 +98,7 @@ public async Task> ListGeographicLevels(Guid subjectId, private async Task> ListVariables(Guid subjectId, CancellationToken cancellationToken = default) { + // @MarkFix get these from DataSetFileMeta var filters = await _statisticsDbContext.Filter .Where(filter => filter.SubjectId == subjectId) .Select(filter => @@ -114,7 +115,7 @@ private async Task> ListVariables(Guid subjectId, .ToList(); } - private async Task> ListFootnotes(Guid releaseVersionId, Guid subjectId) + private async Task> ListFootnotes(Guid releaseVersionId, Guid subjectId) // @MarkFix Get your footnotes here! { var footnotes = await _footnoteRepository.GetFootnotes(releaseVersionId: releaseVersionId, subjectId: subjectId); From cba0340991409431922664a61a199c09478b0181 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Tue, 14 May 2024 11:04:54 +0100 Subject: [PATCH 29/73] EES-4856 Add footnotes --- .../Services/BlobStorageService.cs | 2 +- .../DataSetFilesControllerTests.cs | 86 ++++++++++++++++++- .../Fixtures/FileGeneratorExtensions.cs | 1 - .../DataSetFileService.cs | 14 +-- .../DataSetFileViewModel.cs | 3 + 5 files changed, 95 insertions(+), 11 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs index eecc9b613a3..8de908d03f2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs @@ -216,7 +216,7 @@ public async Task UploadFile( _logger.LogInformation("Uploading file to blob {containerName}/{path}", containerName, path); - await blob.UploadAsync( // @MarkFix fails to find container here? + await blob.UploadAsync( path: tempFilePath, httpHeaders: new BlobHttpHeaders { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index f69e2fbffb1..d77380e3f41 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -29,6 +29,12 @@ using GovUk.Education.ExploreEducationStatistics.Content.Services; using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Content.ViewModels; +using GovUk.Education.ExploreEducationStatistics.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository; +using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Data.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Data.Model.Tests.Utils; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.WebUtilities; @@ -2079,6 +2085,77 @@ await publicBlobStorageService.UploadFile( Assert.Equal("Indicator 4", viewModel.Meta.Indicators[3]); } + [Fact] + public async Task FetchDataSetFootnotes_Success() + { + Publication publication = _fixture.DefaultPublication() + .WithReleases( + _fixture.DefaultRelease(publishedVersions: 1) + .Generate(1)) + .WithTopic(_fixture.DefaultTopic() + .WithTheme(_fixture.DefaultTheme())); + + var subject = _fixture.DefaultSubject() + .Generate(); + + var releaseFile = _fixture.DefaultReleaseFile() + .WithReleaseVersion(publication.ReleaseVersions[0]) + .WithFile(_fixture.DefaultFile() + .WithSubjectId(subject.Id)) + .Generate(); + + var statsReleaseVersion = new Data.Model.ReleaseVersion { Id = releaseFile.ReleaseVersionId }; + + var releaseFootnote1 = _fixture.DefaultReleaseFootnote() + .WithReleaseVersion(statsReleaseVersion) + .WithFootnote(_fixture.DefaultFootnote() + .WithSubjects(new List { subject })) + .Generate(); + + var filter = _fixture.DefaultFilter() + .WithSubject(subject); + + var releaseFootnote2 = _fixture.DefaultReleaseFootnote() + .WithReleaseVersion(statsReleaseVersion) + .WithFootnote(_fixture.DefaultFootnote() + .WithFilters(new List { filter })) + .Generate(); + + var publicBlobStorageService = new PublicBlobStorageService( + logger: new Logger(new LoggerFactory()), + configuration: await CreateConfigurtionWithAzurite()); + + var formFile = CreateDataCsvFormFile(string.Empty); + + await publicBlobStorageService.UploadFile( + containerName: BlobContainers.PublicReleaseFiles, + releaseFile.PublicPath(), + formFile); + + var client = BuildApp( + publicBlobStorageService: publicBlobStorageService) + .AddContentDbTestData(context => + { + context.ReleaseFiles.Add(releaseFile); + }) + .AddStatisticsDbTestData(context => + { + context.ReleaseFootnote.AddRange(releaseFootnote1, releaseFootnote2); + }) + .CreateClient(); + + var uri = $"/api/data-set-files/{releaseFile.File.DataSetFileId}"; + + var response = await client.GetAsync(uri); + var viewModel = response.AssertOk(); + + var footnotes = viewModel.Footnotes; + Assert.Equal(2, footnotes.Count); + + Assert.Equal("Footnote 0 :: Content", footnotes[0].Label); + Assert.Equal("Footnote 1 :: Content", footnotes[1].Label); + } + [Fact] public async Task NoDataSetFile_ReturnsNotFound() { @@ -2225,7 +2302,7 @@ public async Task DataSetFileRemovedOnAmendment_ReturnsNotFound() private WebApplicationFactory BuildApp( ContentDbContext? contentDbContext = null, - //StatisticsDbContext? statisticsDbContext = null, + StatisticsDbContext? statisticsDbContext = null, IPublicBlobStorageService? publicBlobStorageService = null) { return TestApp @@ -2245,13 +2322,14 @@ private WebApplicationFactory BuildApp( .AddJsonFile("appsettings.IntegrationTest.json", optional: false) .AddEnvironmentVariables() .Build())); - //services.AddTransient(s => new FootnoteRepository( - // statisticsDbContext ?? s.GetRequiredService())); + services.AddTransient(s => new FootnoteRepository( + statisticsDbContext ?? s.GetRequiredService())); services.AddTransient( s => new DataSetFileService( contentDbContext ?? s.GetRequiredService(), s.GetRequiredService(), - s.GetRequiredService())); + s.GetRequiredService(), + s.GetRequiredService())); }); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs index cf34ecc558e..5d838366ade 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs @@ -1,5 +1,4 @@ using System; -using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index 2d5a004313e..c06294b1b0c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -21,6 +21,7 @@ using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Content.ViewModels; using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Data.Services.Utils; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using static GovUk.Education.ExploreEducationStatistics.Common.Model.SortDirection; @@ -35,18 +36,18 @@ public class DataSetFileService : IDataSetFileService private readonly ContentDbContext _contentDbContext; private readonly IReleaseVersionRepository _releaseVersionRepository; private readonly IPublicBlobStorageService _publicBlobStorageService; - //private readonly IFootnoteRepository _footnoteRepository; + private readonly IFootnoteRepository _footnoteRepository; public DataSetFileService( ContentDbContext contentDbContext, IReleaseVersionRepository releaseVersionRepository, - IPublicBlobStorageService publicBlobStorageService) - //IFootnoteRepository footnoteRepository) + IPublicBlobStorageService publicBlobStorageService, + IFootnoteRepository footnoteRepository) { _contentDbContext = contentDbContext; _releaseVersionRepository = releaseVersionRepository; _publicBlobStorageService = publicBlobStorageService; - //_footnoteRepository = footnoteRepository; + _footnoteRepository = footnoteRepository; } public async Task>> ListDataSetFiles( @@ -172,7 +173,9 @@ public async Task> GetDataSetFile( // Fetch variable names and descriptions from DataSetFileMeta - // Fetch footnotes + var footnotes = await _footnoteRepository.GetFootnotes( + releaseFile.ReleaseVersionId, + releaseFile.File.SubjectId); return new DataSetFileViewModel { @@ -209,6 +212,7 @@ public async Task> GetDataSetFile( releaseFile.File.DataSetFileMeta, releaseFile.FilterSequence, releaseFile.IndicatorSequence), + Footnotes = FootnotesViewModelBuilder.BuildFootnotes(footnotes), Api = BuildDataSetFileApiViewModel(releaseFile.File) }; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs index db417a1246a..619bfa1420b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs @@ -1,4 +1,5 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model; +using GovUk.Education.ExploreEducationStatistics.Data.ViewModels; using Newtonsoft.Json; using Newtonsoft.Json.Converters; @@ -18,6 +19,8 @@ public record DataSetFileViewModel public required DataSetFileMetaViewModel Meta { get; init; } + public required List Footnotes { get; init; } = []; + public DataSetFileApiViewModel? Api { get; set; } } From 0acd002d652bedc4825f157b1f126246b5d26204 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Tue, 14 May 2024 14:15:16 +0100 Subject: [PATCH 30/73] EES-4856 Add variables --- .../DataSetFilesControllerTests.cs | 67 +++++++++++++++++++ .../DataSetFileService.cs | 17 ++++- .../DataSetFileViewModel.cs | 3 + .../DataGuidanceDataSetService.cs | 3 +- 4 files changed, 87 insertions(+), 3 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index d77380e3f41..9516ca35fec 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -2085,6 +2085,73 @@ await publicBlobStorageService.UploadFile( Assert.Equal("Indicator 4", viewModel.Meta.Indicators[3]); } + [Fact] + public async Task FetchVariables_Success() + { + Publication publication = _fixture.DefaultPublication() + .WithReleases( + _fixture.DefaultRelease(publishedVersions: 1) + .Generate(1)) + .WithTopic(_fixture.DefaultTopic() + .WithTheme(_fixture.DefaultTheme())); + + ReleaseFile releaseFile = _fixture.DefaultReleaseFile() + .WithReleaseVersion(publication.ReleaseVersions[0]) + .WithFile(_fixture.DefaultFile() + .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() + .WithFilters([ + new FilterMeta { Label = "Filter 1", ColumnName = "A_filter_1", Hint = "hint", }, + new FilterMeta { Label = "Filter 2", ColumnName = "G_filter_2", }, + new FilterMeta { Label = "Filter 3", ColumnName = "C_filter_3", Hint = "Another hint", }, + ]) + .WithIndicators([ + new IndicatorMeta { Label = "Indicator 3", ColumnName = "B_indicator_3", }, + new IndicatorMeta { Label = "Indicator 2", ColumnName = "E_indicator_2", }, + new IndicatorMeta { Label = "Indicator 1", ColumnName = "D_indicator_1", }, + new IndicatorMeta { Label = "Indicator 4", ColumnName = "F_indicator_4", }, + ]))); + + var publicBlobStorageService = new PublicBlobStorageService( + logger: new Logger(new LoggerFactory()), + configuration: await CreateConfigurtionWithAzurite()); + + var formFile = CreateDataCsvFormFile(string.Empty); + + await publicBlobStorageService.UploadFile( + containerName: BlobContainers.PublicReleaseFiles, + releaseFile.PublicPath(), + formFile); + + var client = BuildApp( + publicBlobStorageService: publicBlobStorageService) + .AddContentDbTestData(context => + { + context.ReleaseFiles.Add(releaseFile); + }) + .CreateClient(); + + var uri = $"/api/data-set-files/{releaseFile.File.DataSetFileId}"; + + var response = await client.GetAsync(uri); + var viewModel = response.AssertOk(); + + Assert.Equal(7, viewModel.Variables.Count); + Assert.Equal("A_filter_1", viewModel.Variables[0].Value); + Assert.Equal("Filter 1 - hint", viewModel.Variables[0].Label); + Assert.Equal("B_indicator_3", viewModel.Variables[1].Value); + Assert.Equal("Indicator 3", viewModel.Variables[1].Label); + Assert.Equal("C_filter_3", viewModel.Variables[2].Value); + Assert.Equal("Filter 3 - Another hint", viewModel.Variables[2].Label); + Assert.Equal("D_indicator_1", viewModel.Variables[3].Value); + Assert.Equal("Indicator 1", viewModel.Variables[3].Label); + Assert.Equal("E_indicator_2", viewModel.Variables[4].Value); + Assert.Equal("Indicator 2", viewModel.Variables[4].Label); + Assert.Equal("F_indicator_4", viewModel.Variables[5].Value); + Assert.Equal("Indicator 4", viewModel.Variables[5].Label); + Assert.Equal("G_filter_2", viewModel.Variables[6].Value); + Assert.Equal("Filter 2", viewModel.Variables[6].Label); + } + [Fact] public async Task FetchDataSetFootnotes_Success() { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index c06294b1b0c..01fc16766d6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -171,7 +171,7 @@ public async Task> GetDataSetFile( var dataCsvPreviewLines = await GetDataCsvPreviewLines(releaseFile); - // Fetch variable names and descriptions from DataSetFileMeta + var variables = GetVariables(releaseFile.File.DataSetFileMeta!); var footnotes = await _footnoteRepository.GetFootnotes( releaseFile.ReleaseVersionId, @@ -212,6 +212,7 @@ public async Task> GetDataSetFile( releaseFile.File.DataSetFileMeta, releaseFile.FilterSequence, releaseFile.IndicatorSequence), + Variables = variables, Footnotes = FootnotesViewModelBuilder.BuildFootnotes(footnotes), Api = BuildDataSetFileApiViewModel(releaseFile.File) }; @@ -264,6 +265,20 @@ private async Task> GetDataCsvPreviewLines(ReleaseFile releaseFile) return dataCsvPreviewLines; } + private List GetVariables(DataSetFileMeta meta) + { + var filterVariables = meta.Filters + .Select(filter => new LabelValue( + string.IsNullOrWhiteSpace(filter.Hint) ? filter.Label : $"{filter.Label} - {filter.Hint}", + filter.ColumnName)) + .ToList(); + var indicatorVariables = meta.Indicators + .Select(indicator => new LabelValue(indicator.Label, indicator.ColumnName)); + return filterVariables.Concat(indicatorVariables) + .OrderBy(variable => variable.Value) + .ToList(); + } + private static List GetOrderedFilters(List metaFilters, List? filterSequenceEntries) { var filterSequence = filterSequenceEntries? diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs index 619bfa1420b..ed780c66d86 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs @@ -1,3 +1,4 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Data.ViewModels; using Newtonsoft.Json; @@ -21,6 +22,8 @@ public record DataSetFileViewModel public required List Footnotes { get; init; } = []; + public required List Variables { get; init; } = []; + public DataSetFileApiViewModel? Api { get; set; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Services/DataGuidanceDataSetService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Services/DataGuidanceDataSetService.cs index fd2a703a82d..da2c9c16180 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Services/DataGuidanceDataSetService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Services/DataGuidanceDataSetService.cs @@ -98,7 +98,6 @@ public async Task> ListGeographicLevels(Guid subjectId, private async Task> ListVariables(Guid subjectId, CancellationToken cancellationToken = default) { - // @MarkFix get these from DataSetFileMeta var filters = await _statisticsDbContext.Filter .Where(filter => filter.SubjectId == subjectId) .Select(filter => @@ -115,7 +114,7 @@ private async Task> ListVariables(Guid subjectId, .ToList(); } - private async Task> ListFootnotes(Guid releaseVersionId, Guid subjectId) // @MarkFix Get your footnotes here! + private async Task> ListFootnotes(Guid releaseVersionId, Guid subjectId) { var footnotes = await _footnoteRepository.GetFootnotes(releaseVersionId: releaseVersionId, subjectId: subjectId); From c2376bb600c964a6f8048f40d78a0a343d41993c Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Tue, 14 May 2024 15:51:36 +0100 Subject: [PATCH 31/73] EES-4856 Create DataSetFileCsvPreviewViewModel --- .../DataSetFilesControllerTests.cs | 146 ++++++++++++++---- .../Fixtures/FileGeneratorExtensions.cs | 1 + .../DataSetFileService.cs | 40 +++-- .../DataSetFileMetaViewModel.cs | 6 + .../DataSetFileViewModel.cs | 2 +- ...reateInitialDataSetVersionFunctionTests.cs | 1 + 6 files changed, 153 insertions(+), 43 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index 9516ca35fec..3df246c36e8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -1864,15 +1864,15 @@ public async Task FetchDataSetDetails_Success() logger: new Logger(new LoggerFactory()), configuration: await CreateConfigurtionWithAzurite()); - var formFile = CreateDataCsvFormFile(""" - Headers - Line 1 - Line 2 - Line 3 - Line 4 - Line 5 - Line 6 - """); + var formFile = CreateDataCsvFormFile("""" + column_1,column_2,column_3 + 1,2,3 + 2,"3,4",5 + 3,"""4",5 + 4,,6 + 5,6,7 + 6,7,8 + """"); await publicBlobStorageService.UploadFile( containerName: BlobContainers.PublicReleaseFiles, @@ -1948,15 +1948,80 @@ await publicBlobStorageService.UploadFile( .ToList(), viewModel.Meta.Indicators); - viewModel.File.DataCsvPreviewLines // @MarkFix Assert.Equal here? - .AssertDeepEqualTo([ - "Headers", - "Line 1", - "Line 2", - "Line 3", - "Line 4", - "Line 5", - ]); + Assert.Equal(3, viewModel.File.DataCsvPreview.Headers.Count); + viewModel.File.DataCsvPreview.Headers + .AssertDeepEqualTo(["column_1", "column_2", "column_3"]); + + Assert.Equal(5, viewModel.File.DataCsvPreview.Rows.Count); + + Assert.Equal(3, viewModel.File.DataCsvPreview.Rows[0].Count); + viewModel.File.DataCsvPreview.Rows[0].AssertDeepEqualTo(["1", "2", "3"]); + + Assert.Equal(3, viewModel.File.DataCsvPreview.Rows[1].Count); + viewModel.File.DataCsvPreview.Rows[1].AssertDeepEqualTo(["2", "3,4", "5"]); + + Assert.Equal(3, viewModel.File.DataCsvPreview.Rows[2].Count); + viewModel.File.DataCsvPreview.Rows[2].AssertDeepEqualTo(["3", "\"4", "5"]); + + Assert.Equal(3, viewModel.File.DataCsvPreview.Rows[3].Count); + viewModel.File.DataCsvPreview.Rows[3].AssertDeepEqualTo(["4", "", "6"]); + + Assert.Equal(3, viewModel.File.DataCsvPreview.Rows[4].Count); + viewModel.File.DataCsvPreview.Rows[4].AssertDeepEqualTo(["5", "6", "7"]); + } + + [Fact] + public async Task FetchCsvWithSingleDataRow_Success() + { + Publication publication = _fixture.DefaultPublication() + .WithReleases( + _fixture.DefaultRelease(publishedVersions: 1) + .Generate(1)) + .WithTopic(_fixture.DefaultTopic() + .WithTheme(_fixture.DefaultTheme())); + + ReleaseFile releaseFile = _fixture.DefaultReleaseFile() + .WithReleaseVersion(publication.ReleaseVersions[0]) + .WithFile(_fixture.DefaultFile() + .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta())); + + var publicBlobStorageService = new PublicBlobStorageService( + logger: new Logger(new LoggerFactory()), + configuration: await CreateConfigurtionWithAzurite()); + + var formFile = CreateDataCsvFormFile(""" + column_1,column_2,column_3 + 1,2,3 + """); + + await publicBlobStorageService.UploadFile( + containerName: BlobContainers.PublicReleaseFiles, + releaseFile.PublicPath(), + formFile); + + var client = BuildApp( + publicBlobStorageService: publicBlobStorageService) + .AddContentDbTestData(context => + { + context.ReleaseFiles.Add(releaseFile); + }) + .CreateClient(); + + var uri = $"/api/data-set-files/{releaseFile.File.DataSetFileId}"; + + var response = await client.GetAsync(uri); + var viewModel = response.AssertOk(); + + Assert.Equal(releaseFile.Name, viewModel.Title); + Assert.Equal(releaseFile.Summary, viewModel.Summary); + + Assert.Equal(3, viewModel.File.DataCsvPreview.Headers.Count); + viewModel.File.DataCsvPreview.Headers + .AssertDeepEqualTo(["column_1", "column_2", "column_3"]); + + var row = Assert.Single(viewModel.File.DataCsvPreview.Rows); + Assert.Equal(3, row.Count); + row.AssertDeepEqualTo(["1", "2", "3"]); } [Fact] @@ -1993,7 +2058,10 @@ public async Task FetchDataSetFiltersOrdered_Success() logger: new Logger(new LoggerFactory()), configuration: await CreateConfigurtionWithAzurite()); - var formFile = CreateDataCsvFormFile(string.Empty); + var formFile = CreateDataCsvFormFile(""" + column_1 + 1 + """); await publicBlobStorageService.UploadFile( containerName: BlobContainers.PublicReleaseFiles, @@ -2058,7 +2126,10 @@ public async Task FetchDataSetIndicatorsOrdered_Success() logger: new Logger(new LoggerFactory()), configuration: await CreateConfigurtionWithAzurite()); - var formFile = CreateDataCsvFormFile(string.Empty); + var formFile = CreateDataCsvFormFile(""" + column_1 + 1 + """); await publicBlobStorageService.UploadFile( containerName: BlobContainers.PublicReleaseFiles, @@ -2100,22 +2171,25 @@ public async Task FetchVariables_Success() .WithFile(_fixture.DefaultFile() .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() .WithFilters([ - new FilterMeta { Label = "Filter 1", ColumnName = "A_filter_1", Hint = "hint", }, - new FilterMeta { Label = "Filter 2", ColumnName = "G_filter_2", }, - new FilterMeta { Label = "Filter 3", ColumnName = "C_filter_3", Hint = "Another hint", }, + new FilterMeta { Id = Guid.NewGuid(), Label = "Filter 1", ColumnName = "A_filter_1", Hint = "hint", }, + new FilterMeta { Id = Guid.NewGuid(), Label = "Filter 2", ColumnName = "G_filter_2", }, + new FilterMeta { Id = Guid.NewGuid(), Label = "Filter 3", ColumnName = "C_filter_3", Hint = "Another hint", }, ]) .WithIndicators([ - new IndicatorMeta { Label = "Indicator 3", ColumnName = "B_indicator_3", }, - new IndicatorMeta { Label = "Indicator 2", ColumnName = "E_indicator_2", }, - new IndicatorMeta { Label = "Indicator 1", ColumnName = "D_indicator_1", }, - new IndicatorMeta { Label = "Indicator 4", ColumnName = "F_indicator_4", }, + new IndicatorMeta { Id = Guid.NewGuid(), Label = "Indicator 3", ColumnName = "B_indicator_3", }, + new IndicatorMeta { Id = Guid.NewGuid(), Label = "Indicator 2", ColumnName = "E_indicator_2", }, + new IndicatorMeta { Id = Guid.NewGuid(), Label = "Indicator 1", ColumnName = "D_indicator_1", }, + new IndicatorMeta { Id = Guid.NewGuid(), Label = "Indicator 4", ColumnName = "F_indicator_4", }, ]))); var publicBlobStorageService = new PublicBlobStorageService( logger: new Logger(new LoggerFactory()), configuration: await CreateConfigurtionWithAzurite()); - var formFile = CreateDataCsvFormFile(string.Empty); + var formFile = CreateDataCsvFormFile(""" + column_1 + 1 + """); await publicBlobStorageService.UploadFile( containerName: BlobContainers.PublicReleaseFiles, @@ -2192,7 +2266,10 @@ public async Task FetchDataSetFootnotes_Success() logger: new Logger(new LoggerFactory()), configuration: await CreateConfigurtionWithAzurite()); - var formFile = CreateDataCsvFormFile(string.Empty); + var formFile = CreateDataCsvFormFile(""" + column_1 + 1 + """); await publicBlobStorageService.UploadFile( containerName: BlobContainers.PublicReleaseFiles, @@ -2309,7 +2386,10 @@ public async Task AmendmentNotPublished_ReturnsOk() logger: new Logger(new LoggerFactory()), configuration: await CreateConfigurtionWithAzurite()); - var formFile = CreateDataCsvFormFile(string.Empty); + var formFile = CreateDataCsvFormFile(""" + column_1 + 1 + """); await publicBlobStorageService.UploadFile( containerName: BlobContainers.PublicReleaseFiles, @@ -2383,9 +2463,9 @@ private WebApplicationFactory BuildApp( s.GetRequiredService>(), new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) - // Use appsettings.IntegrationTest.json to prevent passing due to data-storage docker - // container running separately. appsettings.IntegrationTest.json doesn't use ports - // 10000/10001 for blobs/queues + // Use appsettings.IntegrationTest.json to prevent tests passing due to data-storage docker + // container running separately: appsettings.IntegrationTest.json uses different ports + // to those used by the data-storage docker container - i.e. not 10000/10001 .AddJsonFile("appsettings.IntegrationTest.json", optional: false) .AddEnvironmentVariables() .Build())); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs index 5d838366ade..36ee6c93865 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs @@ -19,6 +19,7 @@ public static InstanceSetters SetDefaults(this InstanceSetters sette .SetDefault(f => f.Id) .SetDefault(f => f.RootPath) .SetDefault(f => f.DataSetFileId) + .SetDefault(f => f.SubjectId) .SetContentLength(1024 * 1024) .SetContentType("text/csv") .SetType(FileType.Data) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index 01fc16766d6..e5b25239177 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -1,11 +1,13 @@ #nullable enable using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; +using CsvHelper; using GovUk.Education.ExploreEducationStatistics.Common; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -169,7 +171,7 @@ public async Task> GetDataSetFile( return new NotFoundResult(); } - var dataCsvPreviewLines = await GetDataCsvPreviewLines(releaseFile); + var dataCsvPreview = await GetDataCsvPreview(releaseFile); var variables = GetVariables(releaseFile.File.DataSetFileMeta!); @@ -205,7 +207,7 @@ public async Task> GetDataSetFile( Id = releaseFile.FileId, Name = releaseFile.File.Filename, Size = releaseFile.File.DisplaySize(), - DataCsvPreviewLines = dataCsvPreviewLines, + DataCsvPreview = dataCsvPreview, SubjectId = releaseFile.File.SubjectId!.Value, }, Meta = BuildDataSetFileMetaViewModel( @@ -247,22 +249,42 @@ private static DataSetFileMetaViewModel BuildDataSetFileMetaViewModel( }; } - private async Task> GetDataCsvPreviewLines(ReleaseFile releaseFile) + private async Task GetDataCsvPreview(ReleaseFile releaseFile) { - List dataCsvPreviewLines = new(); var datafileStreamProvider = () => _publicBlobStorageService.StreamBlob( containerName: BlobContainers.PublicReleaseFiles, path: releaseFile.PublicPath()); + using var dataFileReader = new StreamReader(await datafileStreamProvider.Invoke()); + using var csvReader = new CsvReader(dataFileReader, CultureInfo.InvariantCulture); + await csvReader.ReadAsync(); + csvReader.ReadHeader(); + var headers = csvReader.HeaderRecord?.ToList() ?? new List(); + + using var csvDataReader = new CsvDataReader(csvReader); + var rows = new List>(); + var lastLine = false; // assume one line of data in CSV - for (var i = 0; i < 6; i++) + // Fetch first five data rows only. If there are less, fetch what you can + for (var i = 0; i < 5 && !lastLine; i++) { - var line = await dataFileReader.ReadLineAsync(); - if (line == null) { break; } - dataCsvPreviewLines.Add(line); + var cellsPerRow = csvDataReader.FieldCount; + + var row = Enumerable + .Range(0, cellsPerRow) + .Select(csvReader.GetField) + .ToList(); + + rows.Add(row); + + lastLine = !await csvReader.ReadAsync(); } - return dataCsvPreviewLines; + return new DataSetFileCsvPreviewViewModel + { + Headers = headers, + Rows = rows, + }; } private List GetVariables(DataSetFileMeta meta) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileMetaViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileMetaViewModel.cs index cfb74b1a9c2..3b0a0c3fdaa 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileMetaViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileMetaViewModel.cs @@ -8,6 +8,12 @@ public record DataSetFileMetaViewModel public required List Indicators { get; init; } } +public record DataSetFileCsvPreviewViewModel +{ + public List Headers { get; init; } = new(); + public List> Rows { get; init; } = new(); +} + public record DataSetFileTimePeriodRangeViewModel { public required string From { get; init; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs index ed780c66d86..db32566c13a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs @@ -64,7 +64,7 @@ public record DataSetFileFileViewModel public required string Size { get; init; } = string.Empty; - public required List DataCsvPreviewLines { get; init; } = new(); + public required DataSetFileCsvPreviewViewModel DataCsvPreview { get; init; } = new(); public required Guid SubjectId { get; init; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateInitialDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateInitialDataSetVersionFunctionTests.cs index e1d1393e773..a492018d745 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateInitialDataSetVersionFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateInitialDataSetVersionFunctionTests.cs @@ -40,6 +40,7 @@ public async Task Success() .WithFiles(_fixture.DefaultFile() .ForIndex(0, s => s.SetType(FileType.Data)) .ForIndex(1, s => s.SetType(FileType.Metadata)) + .WithSubjectId(Guid.NewGuid()) .Generate(2)) .GenerateList() .ToTuple2(); From a9e9a97d26c775bbe4919bd4e93f16d8a70b67a8 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Thu, 16 May 2024 11:18:30 +0100 Subject: [PATCH 32/73] EES-4856 Move Variables to DataSetFileFileViewModel and update frontend interfaces --- .../DataSetFilesControllerTests.cs | 30 +++++++++---------- .../DataSetFileService.cs | 2 +- .../DataSetFileViewModel.cs | 4 +-- .../data-catalogue/__data__/testDataSets.ts | 3 ++ .../src/services/dataSetFileService.ts | 15 ++++++++-- 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index 3df246c36e8..263357923ab 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -2209,21 +2209,21 @@ await publicBlobStorageService.UploadFile( var response = await client.GetAsync(uri); var viewModel = response.AssertOk(); - Assert.Equal(7, viewModel.Variables.Count); - Assert.Equal("A_filter_1", viewModel.Variables[0].Value); - Assert.Equal("Filter 1 - hint", viewModel.Variables[0].Label); - Assert.Equal("B_indicator_3", viewModel.Variables[1].Value); - Assert.Equal("Indicator 3", viewModel.Variables[1].Label); - Assert.Equal("C_filter_3", viewModel.Variables[2].Value); - Assert.Equal("Filter 3 - Another hint", viewModel.Variables[2].Label); - Assert.Equal("D_indicator_1", viewModel.Variables[3].Value); - Assert.Equal("Indicator 1", viewModel.Variables[3].Label); - Assert.Equal("E_indicator_2", viewModel.Variables[4].Value); - Assert.Equal("Indicator 2", viewModel.Variables[4].Label); - Assert.Equal("F_indicator_4", viewModel.Variables[5].Value); - Assert.Equal("Indicator 4", viewModel.Variables[5].Label); - Assert.Equal("G_filter_2", viewModel.Variables[6].Value); - Assert.Equal("Filter 2", viewModel.Variables[6].Label); + Assert.Equal(7, viewModel.File.Variables.Count); + Assert.Equal("A_filter_1", viewModel.File.Variables[0].Value); + Assert.Equal("Filter 1 - hint", viewModel.File.Variables[0].Label); + Assert.Equal("B_indicator_3", viewModel.File.Variables[1].Value); + Assert.Equal("Indicator 3", viewModel.File.Variables[1].Label); + Assert.Equal("C_filter_3", viewModel.File.Variables[2].Value); + Assert.Equal("Filter 3 - Another hint", viewModel.File.Variables[2].Label); + Assert.Equal("D_indicator_1", viewModel.File.Variables[3].Value); + Assert.Equal("Indicator 1", viewModel.File.Variables[3].Label); + Assert.Equal("E_indicator_2", viewModel.File.Variables[4].Value); + Assert.Equal("Indicator 2", viewModel.File.Variables[4].Label); + Assert.Equal("F_indicator_4", viewModel.File.Variables[5].Value); + Assert.Equal("Indicator 4", viewModel.File.Variables[5].Label); + Assert.Equal("G_filter_2", viewModel.File.Variables[6].Value); + Assert.Equal("Filter 2", viewModel.File.Variables[6].Label); } [Fact] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index e5b25239177..1b48b1379f7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -208,13 +208,13 @@ public async Task> GetDataSetFile( Name = releaseFile.File.Filename, Size = releaseFile.File.DisplaySize(), DataCsvPreview = dataCsvPreview, + Variables = variables, SubjectId = releaseFile.File.SubjectId!.Value, }, Meta = BuildDataSetFileMetaViewModel( releaseFile.File.DataSetFileMeta, releaseFile.FilterSequence, releaseFile.IndicatorSequence), - Variables = variables, Footnotes = FootnotesViewModelBuilder.BuildFootnotes(footnotes), Api = BuildDataSetFileApiViewModel(releaseFile.File) }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs index db32566c13a..0142556feca 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs @@ -22,8 +22,6 @@ public record DataSetFileViewModel public required List Footnotes { get; init; } = []; - public required List Variables { get; init; } = []; - public DataSetFileApiViewModel? Api { get; set; } } @@ -66,5 +64,7 @@ public record DataSetFileFileViewModel public required DataSetFileCsvPreviewViewModel DataCsvPreview { get; init; } = new(); + public required List Variables { get; init; } = []; + public required Guid SubjectId { get; init; } } diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts index b03691401a4..88658250c82 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts @@ -117,6 +117,8 @@ export const testDataSetFile: DataSetFile = { id: 'file-id', name: 'file name', size: 'file size', + dataCsvPreview: { headers: ['column_1'], rows: [['1']] }, + variables: [{ value: 'column_1', label: 'Column 1 is for something' }], subjectId: 'subject-id', }, release: { @@ -144,6 +146,7 @@ export const testDataSetFile: DataSetFile = { geographicLevels: ['Local authority', 'National'], indicators: ['Indicator 1', 'Indicator 2'], }, + footnotes: [{ id: 'footnote-1', label: 'Footnote 1' }], }; export const testDataSetWithApi: DataSetFile = { diff --git a/src/explore-education-statistics-frontend/src/services/dataSetFileService.ts b/src/explore-education-statistics-frontend/src/services/dataSetFileService.ts index fd1e06927d4..85616f24e09 100644 --- a/src/explore-education-statistics-frontend/src/services/dataSetFileService.ts +++ b/src/explore-education-statistics-frontend/src/services/dataSetFileService.ts @@ -7,7 +7,17 @@ export interface DataSetFile { id: string; title: string; summary: string; - file: { id: string; name: string; size: string; subjectId: string }; + file: { + id: string; + name: string; + size: string; + dataCsvPreview: { + headers: string[]; + rows: string[][]; + }; + variables: { label: string; value: string }[]; + subjectId: string; + }; release: { id: string; isLatestPublishedRelease: boolean; @@ -22,7 +32,6 @@ export interface DataSetFile { title: string; type: ReleaseType; }; - api?: DataSetFileApi; meta: { geographicLevels: string[]; timePeriodRange: { @@ -32,6 +41,8 @@ export interface DataSetFile { filters: string[]; indicators: string[]; }; + footnotes: { id: string; label: string }[]; + api?: DataSetFileApi; } export interface DataSetFileSummary { From e13ddd12e74d03a28da9b9f8221eba3a789972e6 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Thu, 16 May 2024 14:02:16 +0100 Subject: [PATCH 33/73] EES-4856 Move Meta to DataSetFileFileViewModel --- .../DataSetFilesControllerTests.cs | 26 +++++++++---------- .../DataSetFileService.cs | 8 +++--- .../DataSetFileViewModel.cs | 4 +-- .../data-catalogue/__data__/testDataSets.ts | 18 ++++++------- .../components/DataSetFileDetails.tsx | 4 ++- .../src/services/dataSetFileService.ts | 18 ++++++------- 6 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index 263357923ab..87984e7d114 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -1925,7 +1925,7 @@ await publicBlobStorageService.UploadFile( Assert.Equal(dataSetFileMeta!.GeographicLevels .Select(gl => gl.GetEnumLabel()) .ToList(), - viewModel.Meta.GeographicLevels); + viewModel.File.Meta.GeographicLevels); Assert.Equal(new DataSetFileTimePeriodRangeViewModel { @@ -1936,17 +1936,17 @@ await publicBlobStorageService.UploadFile( "2001", TimeIdentifier.CalendarYear), }, - viewModel.Meta.TimePeriodRange); + viewModel.File.Meta.TimePeriodRange); Assert.Equal(dataSetFileMeta.Filters .Select(f => f.Label) .ToList(), - viewModel.Meta.Filters); + viewModel.File.Meta.Filters); Assert.Equal(dataSetFileMeta.Indicators .Select(i => i.Label) .ToList(), - viewModel.Meta.Indicators); + viewModel.File.Meta.Indicators); Assert.Equal(3, viewModel.File.DataCsvPreview.Headers.Count); viewModel.File.DataCsvPreview.Headers @@ -2081,10 +2081,10 @@ await publicBlobStorageService.UploadFile( var response = await client.GetAsync(uri); var viewModel = response.AssertOk(); - Assert.Equal(3, viewModel.Meta.Filters.Count); - Assert.Equal("Filter 1", viewModel.Meta.Filters[0]); - Assert.Equal("Filter 2", viewModel.Meta.Filters[1]); - Assert.Equal("Filter 3", viewModel.Meta.Filters[2]); + Assert.Equal(3, viewModel.File.Meta.Filters.Count); + Assert.Equal("Filter 1", viewModel.File.Meta.Filters[0]); + Assert.Equal("Filter 2", viewModel.File.Meta.Filters[1]); + Assert.Equal("Filter 3", viewModel.File.Meta.Filters[2]); } [Fact] @@ -2149,11 +2149,11 @@ await publicBlobStorageService.UploadFile( var response = await client.GetAsync(uri); var viewModel = response.AssertOk(); - Assert.Equal(4, viewModel.Meta.Indicators.Count); - Assert.Equal("Indicator 1", viewModel.Meta.Indicators[0]); - Assert.Equal("Indicator 2", viewModel.Meta.Indicators[1]); - Assert.Equal("Indicator 3", viewModel.Meta.Indicators[2]); - Assert.Equal("Indicator 4", viewModel.Meta.Indicators[3]); + Assert.Equal(4, viewModel.File.Meta.Indicators.Count); + Assert.Equal("Indicator 1", viewModel.File.Meta.Indicators[0]); + Assert.Equal("Indicator 2", viewModel.File.Meta.Indicators[1]); + Assert.Equal("Indicator 3", viewModel.File.Meta.Indicators[2]); + Assert.Equal("Indicator 4", viewModel.File.Meta.Indicators[3]); } [Fact] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index 1b48b1379f7..6dfdfb2c16a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -207,14 +207,14 @@ public async Task> GetDataSetFile( Id = releaseFile.FileId, Name = releaseFile.File.Filename, Size = releaseFile.File.DisplaySize(), + Meta = BuildDataSetFileMetaViewModel( + releaseFile.File.DataSetFileMeta, + releaseFile.FilterSequence, + releaseFile.IndicatorSequence), DataCsvPreview = dataCsvPreview, Variables = variables, SubjectId = releaseFile.File.SubjectId!.Value, }, - Meta = BuildDataSetFileMetaViewModel( - releaseFile.File.DataSetFileMeta, - releaseFile.FilterSequence, - releaseFile.IndicatorSequence), Footnotes = FootnotesViewModelBuilder.BuildFootnotes(footnotes), Api = BuildDataSetFileApiViewModel(releaseFile.File) }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs index 0142556feca..f52c51b43cf 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs @@ -18,8 +18,6 @@ public record DataSetFileViewModel public required DataSetFileReleaseViewModel Release { get; init; } - public required DataSetFileMetaViewModel Meta { get; init; } - public required List Footnotes { get; init; } = []; public DataSetFileApiViewModel? Api { get; set; } @@ -62,6 +60,8 @@ public record DataSetFileFileViewModel public required string Size { get; init; } = string.Empty; + public required DataSetFileMetaViewModel Meta { get; init; } + public required DataSetFileCsvPreviewViewModel DataCsvPreview { get; init; } = new(); public required List Variables { get; init; } = []; diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts index 88658250c82..a23d65c475f 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts @@ -117,6 +117,15 @@ export const testDataSetFile: DataSetFile = { id: 'file-id', name: 'file name', size: 'file size', + meta: { + timePeriodRange: { + from: '2023', + to: '2024', + }, + filters: ['Filter 1', 'Filter 2'], + geographicLevels: ['Local authority', 'National'], + indicators: ['Indicator 1', 'Indicator 2'], + }, dataCsvPreview: { headers: ['column_1'], rows: [['1']] }, variables: [{ value: 'column_1', label: 'Column 1 is for something' }], subjectId: 'subject-id', @@ -137,15 +146,6 @@ export const testDataSetFile: DataSetFile = { }, summary: 'Data set 1 summary', title: 'Data set 1', - meta: { - timePeriodRange: { - from: '2023', - to: '2024', - }, - filters: ['Filter 1', 'Filter 2'], - geographicLevels: ['Local authority', 'National'], - indicators: ['Indicator 1', 'Indicator 2'], - }, footnotes: [{ id: 'footnote-1', label: 'Footnote 1' }], }; diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileDetails.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileDetails.tsx index 863b70bdf68..e88a0a56576 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileDetails.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileDetails.tsx @@ -21,7 +21,9 @@ interface Props { export default function DataSetFileDetails({ dataSetFile }: Props) { const { release, - meta: { timePeriodRange, filters, geographicLevels, indicators }, + file: { + meta: { timePeriodRange, filters, geographicLevels, indicators }, + }, title, } = dataSetFile; diff --git a/src/explore-education-statistics-frontend/src/services/dataSetFileService.ts b/src/explore-education-statistics-frontend/src/services/dataSetFileService.ts index 85616f24e09..c18ae5850f2 100644 --- a/src/explore-education-statistics-frontend/src/services/dataSetFileService.ts +++ b/src/explore-education-statistics-frontend/src/services/dataSetFileService.ts @@ -11,6 +11,15 @@ export interface DataSetFile { id: string; name: string; size: string; + meta: { + geographicLevels: string[]; + timePeriodRange: { + from: string; + to: string; + }; + filters: string[]; + indicators: string[]; + }; dataCsvPreview: { headers: string[]; rows: string[][]; @@ -32,15 +41,6 @@ export interface DataSetFile { title: string; type: ReleaseType; }; - meta: { - geographicLevels: string[]; - timePeriodRange: { - from: string; - to: string; - }; - filters: string[]; - indicators: string[]; - }; footnotes: { id: string; label: string }[]; api?: DataSetFileApi; } From a8db475af0db206155d2fa310908f79f5169f694 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Mon, 20 May 2024 14:32:19 +0100 Subject: [PATCH 34/73] EES-4856 Changes after resolving merge conflict --- .../Controllers/DataSetFilesControllerTests.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index 87984e7d114..08256b3b9f1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -1857,7 +1857,6 @@ public async Task FetchDataSetDetails_Success() )) .WithPublicApiDataSetId(Guid.NewGuid()) .WithPublicApiDataSetVersion(major: 1, minor: 0) - .WithSubjectId(Guid.NewGuid()) ); var publicBlobStorageService = new PublicBlobStorageService( @@ -2046,7 +2045,6 @@ public async Task FetchDataSetFiltersOrdered_Success() new FilterSequenceEntry(filter3Id, new List()), ]) .WithFile(_fixture.DefaultFile() - .WithSubjectId(Guid.NewGuid()) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() .WithFilters([ new FilterMeta { Id = filter3Id, Label = "Filter 3", ColumnName = "filter_3", }, @@ -2113,7 +2111,6 @@ public async Task FetchDataSetIndicatorsOrdered_Success() new List { indicator3Id, indicator4Id }) ]) .WithFile(_fixture.DefaultFile() - .WithSubjectId(Guid.NewGuid()) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() .WithIndicators([ new IndicatorMeta { Id = indicator3Id, Label = "Indicator 3", ColumnName = "indicator_3", }, @@ -2367,7 +2364,6 @@ public async Task AmendmentNotPublished_ReturnsOk() .WithTheme(_fixture.DefaultTheme())); File file = _fixture.DefaultFile() - .WithSubjectId(Guid.NewGuid()) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta()); ReleaseFile releaseFile0 = _fixture.DefaultReleaseFile() From 4dec95cc08e4302baaccfcab8a1a4cbe44fa7014 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 24 May 2024 12:11:19 +0100 Subject: [PATCH 35/73] Update public API ports to avoid collisions with IDP container --- .../Properties/launchSettings.json | 2 +- src/explore-education-statistics-frontend/.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Properties/launchSettings.json b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Properties/launchSettings.json index 948a54761aa..464d392c830 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Properties/launchSettings.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Properties/launchSettings.json @@ -22,7 +22,7 @@ "dotnetRunMessages": true, "launchBrowser": true, "launchUrl": "docs", - "applicationUrl": "https://0.0.0.0:5031;http://0.0.0.0:5030;https://[::1]:5031;http://[::1]:5030", + "applicationUrl": "https://0.0.0.0:5041;http://0.0.0.0:5040;https://[::1]:5041;http://[::1]:5040", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } diff --git a/src/explore-education-statistics-frontend/.env b/src/explore-education-statistics-frontend/.env index 3821e38e9f0..c7868f6dccf 100644 --- a/src/explore-education-statistics-frontend/.env +++ b/src/explore-education-statistics-frontend/.env @@ -1,6 +1,6 @@ CONTENT_API_BASE_URL=http://localhost:5010/api DATA_API_BASE_URL=http://localhost:5000/api -PUBLIC_API_BASE_URL=http://localhost:5030/api/v1.0 +PUBLIC_API_BASE_URL=http://localhost:5040/api/v1.0 PUBLIC_API_DOCS_URL=TODO-GUIDANCE-URL NOTIFICATION_API_BASE_URL=http://localhost:7073/api GA_TRACKING_ID= From bbbc46857a0cfe016573680cee1662995e321c8f Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 24 May 2024 14:39:43 +0100 Subject: [PATCH 36/73] Update public API ports to avoid collisions with Windows service host --- .../Properties/launchSettings.json | 2 +- src/explore-education-statistics-frontend/.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Properties/launchSettings.json b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Properties/launchSettings.json index 464d392c830..a4f7eca00dc 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Properties/launchSettings.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Properties/launchSettings.json @@ -22,7 +22,7 @@ "dotnetRunMessages": true, "launchBrowser": true, "launchUrl": "docs", - "applicationUrl": "https://0.0.0.0:5041;http://0.0.0.0:5040;https://[::1]:5041;http://[::1]:5040", + "applicationUrl": "https://0.0.0.0:5051;http://0.0.0.0:5050;https://[::1]:5051;http://[::1]:5050", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } diff --git a/src/explore-education-statistics-frontend/.env b/src/explore-education-statistics-frontend/.env index c7868f6dccf..e97acc35c5b 100644 --- a/src/explore-education-statistics-frontend/.env +++ b/src/explore-education-statistics-frontend/.env @@ -1,6 +1,6 @@ CONTENT_API_BASE_URL=http://localhost:5010/api DATA_API_BASE_URL=http://localhost:5000/api -PUBLIC_API_BASE_URL=http://localhost:5040/api/v1.0 +PUBLIC_API_BASE_URL=http://localhost:5050/api/v1.0 PUBLIC_API_DOCS_URL=TODO-GUIDANCE-URL NOTIFICATION_API_BASE_URL=http://localhost:7073/api GA_TRACKING_ID= From 4cc1447af20eb011f71c8d6e402e201cbdb240e3 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Tue, 21 May 2024 10:22:11 +0100 Subject: [PATCH 37/73] EES-4856 Changes in response to comments --- .../Bau/DataSetFileMetaMigrationController.cs | 2 +- .../DataSetFilesControllerTests.cs | 58 +++++++++++++++---- .../DataSetFileMeta.cs | 2 +- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Bau/DataSetFileMetaMigrationController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Bau/DataSetFileMetaMigrationController.cs index f4240009323..57c3c7679cb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Bau/DataSetFileMetaMigrationController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Bau/DataSetFileMetaMigrationController.cs @@ -54,7 +54,7 @@ public async Task MigrateReleaseSeries( f.DataSetFileMeta != null && f.Type == FileType.Data) .ToListAsync(cancellationToken: cancellationToken)) - .Where(f => f.DataSetFileMeta.TimeIdentifier != null); + .Where(f => f.DataSetFileMeta.Years != null); if (num != null) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index 08256b3b9f1..3cd9df56e80 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -1859,9 +1859,14 @@ public async Task FetchDataSetDetails_Success() .WithPublicApiDataSetVersion(major: 1, minor: 0) ); + var azuriteContainer = await GetAzuriteContainer(); + + var configuration = CreateConfiguration(); + configuration["PublicStorage"] = azuriteContainer.GetConnectionString(); + var publicBlobStorageService = new PublicBlobStorageService( logger: new Logger(new LoggerFactory()), - configuration: await CreateConfigurtionWithAzurite()); + configuration: configuration); var formFile = CreateDataCsvFormFile("""" column_1,column_2,column_3 @@ -1984,9 +1989,14 @@ public async Task FetchCsvWithSingleDataRow_Success() .WithFile(_fixture.DefaultFile() .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta())); + var azuriteContainer = await GetAzuriteContainer(); + + var configuration = CreateConfiguration(); + configuration["PublicStorage"] = azuriteContainer.GetConnectionString(); + var publicBlobStorageService = new PublicBlobStorageService( logger: new Logger(new LoggerFactory()), - configuration: await CreateConfigurtionWithAzurite()); + configuration: configuration); var formFile = CreateDataCsvFormFile(""" column_1,column_2,column_3 @@ -2052,9 +2062,14 @@ public async Task FetchDataSetFiltersOrdered_Success() new FilterMeta { Id = filter2Id, Label = "Filter 2", ColumnName = "filter_2", }, ]))); + var azuriteContainer = await GetAzuriteContainer(); + + var configuration = CreateConfiguration(); + configuration["PublicStorage"] = azuriteContainer.GetConnectionString(); + var publicBlobStorageService = new PublicBlobStorageService( logger: new Logger(new LoggerFactory()), - configuration: await CreateConfigurtionWithAzurite()); + configuration: configuration); var formFile = CreateDataCsvFormFile(""" column_1 @@ -2119,9 +2134,14 @@ public async Task FetchDataSetIndicatorsOrdered_Success() new IndicatorMeta { Id = indicator4Id, Label = "Indicator 4", ColumnName = "indicator_4", }, ]))); + var azuriteContainer = await GetAzuriteContainer(); + + var configuration = CreateConfiguration(); + configuration["PublicStorage"] = azuriteContainer.GetConnectionString(); + var publicBlobStorageService = new PublicBlobStorageService( logger: new Logger(new LoggerFactory()), - configuration: await CreateConfigurtionWithAzurite()); + configuration: configuration); var formFile = CreateDataCsvFormFile(""" column_1 @@ -2179,9 +2199,14 @@ public async Task FetchVariables_Success() new IndicatorMeta { Id = Guid.NewGuid(), Label = "Indicator 4", ColumnName = "F_indicator_4", }, ]))); + var azuriteContainer = await GetAzuriteContainer(); + + var configuration = CreateConfiguration(); + configuration["PublicStorage"] = azuriteContainer.GetConnectionString(); + var publicBlobStorageService = new PublicBlobStorageService( logger: new Logger(new LoggerFactory()), - configuration: await CreateConfigurtionWithAzurite()); + configuration: configuration); var formFile = CreateDataCsvFormFile(""" column_1 @@ -2259,9 +2284,14 @@ public async Task FetchDataSetFootnotes_Success() .WithFilters(new List { filter })) .Generate(); - var publicBlobStorageService = new PublicBlobStorageService( + var azuriteContainer = await GetAzuriteContainer(); + + var configuration = CreateConfiguration(); + configuration["PublicStorage"] = azuriteContainer.GetConnectionString(); + + var publicBlobStorageService = new PublicBlobStorageService( logger: new Logger(new LoggerFactory()), - configuration: await CreateConfigurtionWithAzurite()); + configuration: configuration); var formFile = CreateDataCsvFormFile(""" column_1 @@ -2378,9 +2408,14 @@ public async Task AmendmentNotPublished_ReturnsOk() .WithReleaseVersion(publication.ReleaseVersions[2]) // the draft version .WithFile(file); + var azuriteContainer = await GetAzuriteContainer(); + + var configuration = CreateConfiguration(); + configuration["PublicStorage"] = azuriteContainer.GetConnectionString(); + var publicBlobStorageService = new PublicBlobStorageService( logger: new Logger(new LoggerFactory()), - configuration: await CreateConfigurtionWithAzurite()); + configuration: configuration); var formFile = CreateDataCsvFormFile(""" column_1 @@ -2476,20 +2511,23 @@ private WebApplicationFactory BuildApp( }); } - private async Task CreateConfigurtionWithAzurite() + private async Task GetAzuriteContainer() { var azuriteContainer = new AzuriteBuilder() .WithImage("mcr.microsoft.com/azure-storage/azurite:3.27.0") .WithHostname("data-storage-test") .Build(); await azuriteContainer.StartAsync(); + return azuriteContainer; + } + private IConfiguration CreateConfiguration() + { var configuration = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.IntegrationTest.json", optional: false) .AddEnvironmentVariables() .Build(); - configuration["PublicStorage"] = azuriteContainer.GetConnectionString(); return configuration; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileMeta.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileMeta.cs index 2c10a0a1008..850d3295fda 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileMeta.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileMeta.cs @@ -18,7 +18,7 @@ public class DataSetFileMeta public TimeIdentifier? TimeIdentifier { get; set; } // EES-4918 to remove [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] - public List? Years { get; set; } = new(); // EES-4918 to remove + public List? Years { get; set; } // EES-4918 to remove public required TimePeriodRangeMeta TimePeriodRange { get; set; } From a69fdc2548c3db77c383497ceb46a640bc76e6ab Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 24 May 2024 19:35:04 +0100 Subject: [PATCH 38/73] EES-5162 Add UUID v7s to public data model IDs --- .../Database/UuidV7ValueGenerator.cs | 17 +++++++++++++ ...n.ExploreEducationStatistics.Common.csproj | 1 + .../Utils/UuidUtils.cs | 24 +++++++++++++++++++ .../Change.cs | 4 +++- .../ChangeSet.cs | 16 +++++++++++++ .../DataSet.cs | 4 ++++ .../DataSetVersion.cs | 7 +++++- .../DataSetVersionImport.cs | 4 ++++ 8 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Common/Database/UuidV7ValueGenerator.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Common/Utils/UuidUtils.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Database/UuidV7ValueGenerator.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Database/UuidV7ValueGenerator.cs new file mode 100644 index 00000000000..ff2f1da4900 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Database/UuidV7ValueGenerator.cs @@ -0,0 +1,17 @@ +#nullable enable +using System; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.ValueGeneration; + +namespace GovUk.Education.ExploreEducationStatistics.Common.Database; + +/// +/// Generates UUID v7s that are compatible with non-MSSQL databases (e.g. Postgres). +/// +public class UuidV7ValueGenerator : ValueGenerator +{ + public override Guid Next(EntityEntry entry) => UuidUtils.UuidV7(); + + public override bool GeneratesTemporaryValues => false; +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/GovUk.Education.ExploreEducationStatistics.Common.csproj b/src/GovUk.Education.ExploreEducationStatistics.Common/GovUk.Education.ExploreEducationStatistics.Common.csproj index 6226d01aa96..7b9b8b76c49 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/GovUk.Education.ExploreEducationStatistics.Common.csproj +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/GovUk.Education.ExploreEducationStatistics.Common.csproj @@ -25,6 +25,7 @@ + diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Utils/UuidUtils.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Utils/UuidUtils.cs new file mode 100644 index 00000000000..2791844451f --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Utils/UuidUtils.cs @@ -0,0 +1,24 @@ +#nullable enable +using System; +using Medo; + +namespace GovUk.Education.ExploreEducationStatistics.Common.Utils; + +public static class UuidUtils +{ + /// + /// Generate a UUID v7 compatible with non-MSSQL databases e.g. Postgres. + /// + /// + /// Use if you are using an MSSQL database. + /// + public static Guid UuidV7() => Uuid7.NewGuid(); + + /// + /// Generate a UUID v7 compatible with MSSQL databases. + /// + /// + /// Use if you are using non-MSSQL databases. + /// + public static Guid UuidV7MsSql() => Uuid7.NewGuidMsSql(); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Change.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Change.cs index 87a7e5eb218..486be1011c7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Change.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Change.cs @@ -1,8 +1,10 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Utils; + namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model; public class Change { - public Guid Identifier { get; set; } = Guid.NewGuid(); + public Guid Identifier { get; set; } = UuidUtils.UuidV7(); public required ChangeType Type { get; set; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/ChangeSet.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/ChangeSet.cs index af48e4340bf..040ed55becb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/ChangeSet.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/ChangeSet.cs @@ -1,4 +1,5 @@ using GovUk.Education.ExploreEducationStatistics.Common.Converters; +using GovUk.Education.ExploreEducationStatistics.Common.Database; using GovUk.Education.ExploreEducationStatistics.Common.Model; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; @@ -26,6 +27,9 @@ internal class Config : IEntityTypeConfiguration { public void Configure(EntityTypeBuilder builder) { + builder.Property(cs => cs.Id) + .HasValueGenerator(); + builder.OwnsMany(cs => cs.Changes, cs => { cs.ToJson(); @@ -44,6 +48,9 @@ internal class Config : IEntityTypeConfiguration { public void Configure(EntityTypeBuilder builder) { + builder.Property(cs => cs.Id) + .HasValueGenerator(); + builder.OwnsMany(cs => cs.Changes, cs => { cs.ToJson(); @@ -62,6 +69,9 @@ public class Config : IEntityTypeConfiguration { public void Configure(EntityTypeBuilder builder) { + builder.Property(cs => cs.Id) + .HasValueGenerator(); + builder.OwnsMany(cs => cs.Changes, cs => { cs.ToJson(); @@ -88,6 +98,9 @@ internal class Config : IEntityTypeConfiguration { public void Configure(EntityTypeBuilder builder) { + builder.Property(cs => cs.Id) + .HasValueGenerator(); + builder.OwnsMany(cs => cs.Changes, cs => { cs.ToJson(); @@ -109,6 +122,9 @@ internal class Config : IEntityTypeConfiguration { public void Configure(EntityTypeBuilder builder) { + builder.Property(cs => cs.Id) + .HasValueGenerator(); + builder.OwnsMany(cs => cs.Changes, cs => { cs.ToJson(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSet.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSet.cs index ceae31f0ae5..51722636dcb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSet.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSet.cs @@ -1,3 +1,4 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Database; using GovUk.Education.ExploreEducationStatistics.Common.Model; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; @@ -42,6 +43,9 @@ internal class Config : IEntityTypeConfiguration { public void Configure(EntityTypeBuilder builder) { + builder.Property(ds => ds.Id) + .HasValueGenerator(); + builder.Property(ds => ds.Status).HasConversion(); builder diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersion.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersion.cs index 23b5442bf0a..031cd83ce37 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersion.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersion.cs @@ -1,4 +1,5 @@ using GovUk.Education.ExploreEducationStatistics.Common.Converters; +using GovUk.Education.ExploreEducationStatistics.Common.Database; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using Microsoft.EntityFrameworkCore; @@ -77,7 +78,11 @@ internal class Config : IEntityTypeConfiguration { public void Configure(EntityTypeBuilder builder) { - builder.Property(dsv => dsv.Status).HasConversion(); + builder.Property(dsv => dsv.Id) + .HasValueGenerator(); + + builder.Property(dsv => dsv.Status) + .HasConversion(); builder.OwnsOne(v => v.MetaSummary, ms => { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImport.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImport.cs index 7089783df88..85c9cadbc44 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImport.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImport.cs @@ -1,3 +1,4 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Database; using GovUk.Education.ExploreEducationStatistics.Common.Model; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; @@ -26,6 +27,9 @@ internal class Config : IEntityTypeConfiguration { public void Configure(EntityTypeBuilder builder) { + builder.Property(i => i.Id) + .HasValueGenerator(); + builder.Property(i => i.Stage) .HasConversion(); From e6e0e7bf358156086d8239da75509bb62d0d4c2b Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 24 May 2024 22:22:57 +0100 Subject: [PATCH 39/73] EES-5162 Add transaction around `DataSetService.CreateDataSetVersion` --- .../Model/Either.cs | 22 ++++++++++++ .../Services/DataSetService.cs | 35 +++++++++++++------ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Either.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Either.cs index 4ddfadd771b..861903c0bd8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Either.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Either.cs @@ -132,6 +132,13 @@ public static async Task IsRight(this Task> OnSuccessDo( + this Task> task, + Action successTask) + { + return await task.OnSuccessDo(_ => successTask()); + } + public static async Task> OnSuccessDo( this Task> task, Func successTask) @@ -139,6 +146,21 @@ public static async Task> OnSuccessDo await successTask()); } + public static async Task> OnSuccessDo( + this Task> task, + Action successTask) + { + var firstResult = await task; + + if (firstResult.IsLeft) + { + return firstResult.Left; + } + + successTask(firstResult.Right); + return firstResult.Right; + } + public static async Task> OnSuccessDo( this Task> task, Func successTask) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs index 0d8ae6b5981..79ab483d39c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs @@ -1,3 +1,4 @@ +using System.Transactions; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Validators; @@ -26,16 +27,30 @@ PublicDataDbContext publicDataDbContext Guid instanceId, CancellationToken cancellationToken = default) { - return await GetReleaseFile(request.ReleaseFileId, cancellationToken) - .OnSuccess(async releaseFile => await ValidateReleaseFile(releaseFile, cancellationToken) - .OnSuccess(async () => await CreateDataSet(releaseFile, cancellationToken)) - .OnSuccess(async dataSet => - await CreateDataSetVersion(dataSet, releaseFile, cancellationToken)) - .OnSuccessDo(async dataSetVersion => - await CreateDataSetVersionImport(dataSetVersion, instanceId, cancellationToken)) - .OnSuccessDo(async dataSetVersion => - await UpdateFilePublicDataSetVersionId(releaseFile, dataSetVersion, cancellationToken)) - .OnSuccess(dataSetVersion => (dataSetId: dataSetVersion.DataSetId, dataSetVersionId: dataSetVersion.Id))); + var strategy = contentDbContext.Database.CreateExecutionStrategy(); + + return await strategy.ExecuteAsync(async () => + { + using var transactionScope = new TransactionScope( + TransactionScopeOption.Required, + new TransactionOptions + { + IsolationLevel = IsolationLevel.ReadCommitted + }, + TransactionScopeAsyncFlowOption.Enabled); + + return await GetReleaseFile(request.ReleaseFileId, cancellationToken) + .OnSuccess(async releaseFile => await ValidateReleaseFile(releaseFile, cancellationToken) + .OnSuccess(async () => await CreateDataSet(releaseFile, cancellationToken)) + .OnSuccess(async dataSet => + await CreateDataSetVersion(dataSet, releaseFile, cancellationToken)) + .OnSuccessDo(async dataSetVersion => + await CreateDataSetVersionImport(dataSetVersion, instanceId, cancellationToken)) + .OnSuccessDo(async dataSetVersion => + await UpdateFilePublicDataSetVersionId(releaseFile, dataSetVersion, cancellationToken)) + .OnSuccessDo(transactionScope.Complete) + .OnSuccess(dataSetVersion => (dataSetId: dataSetVersion.DataSetId, dataSetVersionId: dataSetVersion.Id))); + }); } private async Task> GetReleaseFile( From 32a0afcc3079e2b9120c02a98a63784034cdb7e7 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Fri, 24 May 2024 12:33:47 +0100 Subject: [PATCH 40/73] EES-5101: Add Published column to ReleaseFiles table, inc. migration script to populate column. --- ...AddReleaseFilesPublishedColumn.Designer.cs | 2191 +++++++++++++++++ ..._EES4770_AddReleaseFilesPublishedColumn.cs | 35 + .../ContentDbContextModelSnapshot.cs | 5 +- .../ReleaseFile.cs | 45 +- 4 files changed, 2253 insertions(+), 23 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240503112512_EES4770_AddReleaseFilesPublishedColumn.Designer.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240503112512_EES4770_AddReleaseFilesPublishedColumn.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240503112512_EES4770_AddReleaseFilesPublishedColumn.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240503112512_EES4770_AddReleaseFilesPublishedColumn.Designer.cs new file mode 100644 index 00000000000..2d2eeb0cace --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240503112512_EES4770_AddReleaseFilesPublishedColumn.Designer.cs @@ -0,0 +1,2191 @@ +// +using System; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + [DbContext(typeof(ContentDbContext))] + [Migration("20240503112512_EES4770_AddReleaseFilesPublishedColumn")] + partial class EES4770_AddReleaseFilesPublishedColumn + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContactTelNo") + .HasColumnType("nvarchar(max)"); + + b.Property("TeamEmail") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TeamName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Contacts"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.FreeTextRank", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Rank") + .HasColumnType("int"); + + b.ToTable((string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("LegacyCreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("Resolved") + .HasColumnType("datetime2"); + + b.Property("ResolvedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ResolvedById"); + + b.ToTable("Comment"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentSectionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Locked") + .HasColumnType("datetime2"); + + b.Property("LockedById") + .IsConcurrencyToken() + .HasColumnType("uniqueidentifier"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentSectionId"); + + b.HasIndex("LockedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentBlock", (string)null); + + b.HasDiscriminator("Type").HasValue("ContentBlock"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Caption") + .HasColumnType("nvarchar(max)"); + + b.Property("Heading") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentSections"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestDraftVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("LatestDraftVersionId") + .IsUnique() + .HasFilter("[LatestDraftVersionId] IS NOT NULL"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("DataBlocks", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier") + .HasColumnName("DataBlockId"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("DataBlockVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("ExpectedImportedRows") + .HasColumnType("int"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("GeographicLevels") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ImportedRows") + .HasColumnType("int"); + + b.Property("LastProcessedRowIndex") + .HasColumnType("int"); + + b.Property("MetaFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("StagePercentageComplete") + .HasColumnType("int"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("TotalRows") + .HasColumnType("int"); + + b.Property("ZipFileId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId") + .IsUnique(); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("FileId"), new[] { "Status" }); + + b.HasIndex("MetaFileId") + .IsUnique(); + + b.HasIndex("ZipFileId") + .IsUnique() + .HasFilter("[ZipFileId] IS NOT NULL"); + + b.ToTable("DataImports"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataImportId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("DataImportId"); + + b.ToTable("DataImportErrors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("EmbedBlocks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DataBlockId") + .IsUnique(); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("FeaturedTables"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentLength") + .HasColumnType("bigint"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileMeta") + .HasColumnType("nvarchar(max)"); + + b.Property("DataSetFileVersion") + .HasColumnType("int"); + + b.Property("Filename") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("PublicDataSetVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReplacedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReplacingId") + .HasColumnType("uniqueidentifier"); + + b.Property("RootPath") + .HasColumnType("uniqueidentifier"); + + b.Property("SourceId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicDataSetVersionId"); + + b.HasIndex("ReplacedById") + .IsUnique() + .HasFilter("[ReplacedById] IS NOT NULL"); + + b.HasIndex("ReplacingId") + .IsUnique() + .HasFilter("[ReplacingId] IS NOT NULL"); + + b.HasIndex("SourceId"); + + b.HasIndex("Type"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.ToTable("GlossaryEntries"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("GuidanceText") + .HasColumnType("nvarchar(max)"); + + b.Property("GuidanceTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Trend") + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("KeyStatistics"); + + b.UseTptMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("OwningPublicationSlug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OwningPublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("Methodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DisplayDate") + .HasColumnType("datetime2"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("MethodologyNotes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("MethodologyVersionId", "Slug"); + + b.ToTable("MethodologyRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("AlternativeSlug") + .HasColumnType("nvarchar(max)"); + + b.Property("AlternativeTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("PublishingStrategy") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ScheduledWithReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyId"); + + b.HasIndex("PreviousVersionId"); + + b.HasIndex("ScheduledWithReleaseVersionId"); + + b.ToTable("MethodologyVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Annexes") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("MethodologyVersionId"); + + b.ToTable("MethodologyVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Permalink", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataSetTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MigratedFromLegacy") + .HasColumnType("bit"); + + b.Property("PublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("SubjectId"); + + b.ToTable("Permalinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseSeries") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .IsRequired() + .HasMaxLength(160) + .HasColumnType("nvarchar(160)"); + + b.Property("SupersededById") + .HasColumnType("uniqueidentifier"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TopicId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContactId"); + + b.HasIndex("LatestPublishedReleaseVersionId") + .IsUnique() + .HasFilter("[LatestPublishedReleaseVersionId] IS NOT NULL"); + + b.HasIndex("SupersededById"); + + b.HasIndex("TopicId"); + + b.ToTable("Publications"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("Owner") + .HasColumnType("bit"); + + b.HasKey("PublicationId", "MethodologyId"); + + b.HasIndex("MethodologyId"); + + b.ToTable("PublicationMethodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("PublicationId", "Slug"); + + b.ToTable("PublicationRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.ToTable("Releases"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("FilterSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("IndicatorSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("ReleaseFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("ReleaseStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataGuidance") + .HasColumnType("nvarchar(max)"); + + b.Property("NextReleaseDate") + .HasColumnType("nvarchar(max)"); + + b.Property("NotifiedOn") + .HasColumnType("datetime2"); + + b.Property("NotifySubscribers") + .HasColumnType("bit"); + + b.Property("PreReleaseAccessList") + .HasColumnType("nvarchar(max)"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublishScheduled") + .HasColumnType("datetime2"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("RelatedInformation") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseName") + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(6) + .HasColumnType("nvarchar(6)"); + + b.Property("Type") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("UpdatePublishedDate") + .HasColumnType("bit"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("ReleaseId"); + + b.HasIndex("Type"); + + b.HasIndex("PreviousVersionId", "Version"); + + b.ToTable("ReleaseVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Themes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Topic", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("ThemeId") + .HasColumnType("uniqueidentifier"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("ThemeId"); + + b.ToTable("Topics"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("On") + .HasColumnType("datetime2"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("Update"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstName") + .HasColumnType("nvarchar(max)"); + + b.Property("LastName") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.ToTable("UserPublicationInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("UserId"); + + b.ToTable("UserPublicationRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EmailSent") + .HasColumnType("bit"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("UserReleaseInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UserId"); + + b.ToTable("UserReleaseRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Charts") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Charts"); + + b.Property("Heading") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Heading"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Query") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Query"); + + b.Property("Source") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Table") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Table"); + + b.HasDiscriminator().HasValue("DataBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("EmbedBlockId") + .HasColumnType("uniqueidentifier") + .HasColumnName("EmbedBlockId"); + + b.HasIndex("EmbedBlockId") + .IsUnique() + .HasFilter("[EmbedBlockId] IS NOT NULL"); + + b.HasDiscriminator().HasValue("EmbedBlockLink"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.HtmlBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("HtmlBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MarkDownBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("MarkDownBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.HasIndex("DataBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.ToTable("KeyStatisticsDataBlock", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("Statistic") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.ToTable("KeyStatisticsText", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", "ContentBlock") + .WithMany("Comments") + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "ResolvedBy") + .WithMany() + .HasForeignKey("ResolvedById"); + + b.Navigation("ContentBlock"); + + b.Navigation("CreatedBy"); + + b.Navigation("ResolvedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", "ContentSection") + .WithMany("Content") + .HasForeignKey("ContentSectionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "LockedBy") + .WithMany() + .HasForeignKey("LockedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentSection"); + + b.Navigation("LockedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Content") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestDraftVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestDraftVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestPublishedVersionId"); + + b.Navigation("LatestDraftVersion"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "ContentBlock") + .WithMany() + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("DataBlockVersions") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "FileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "MetaFile") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "MetaFileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ZipFile") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "ZipFileId") + .OnDelete(DeleteBehavior.Restrict); + + b.Navigation("File"); + + b.Navigation("MetaFile"); + + b.Navigation("ZipFile"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "DataImport") + .WithMany("Errors") + .HasForeignKey("DataImportId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataImport"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", "DataBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("FeaturedTables") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedBy") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Replacing") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacingId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Source") + .WithMany() + .HasForeignKey("SourceId"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReplacedBy"); + + b.Navigation("Replacing"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("CreatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("KeyStatistics") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "LatestPublishedVersionId"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany("Notes") + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Versions") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ScheduledWithReleaseVersion") + .WithMany() + .HasForeignKey("ScheduledWithReleaseVersionId"); + + b.Navigation("CreatedBy"); + + b.Navigation("Methodology"); + + b.Navigation("PreviousVersion"); + + b.Navigation("ScheduledWithReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", null) + .WithOne("MethodologyContent") + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", "MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", "Contact") + .WithMany() + .HasForeignKey("ContactId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "LatestPublishedReleaseVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "LatestPublishedReleaseVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "SupersededBy") + .WithMany() + .HasForeignKey("SupersededById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Topic", "Topic") + .WithMany("Publications") + .HasForeignKey("TopicId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ExternalMethodology", "ExternalMethodology", b1 => + { + b1.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b1.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.HasKey("PublicationId"); + + b1.ToTable("ExternalMethodology", (string)null); + + b1.WithOwner() + .HasForeignKey("PublicationId"); + }); + + b.Navigation("Contact"); + + b.Navigation("ExternalMethodology"); + + b.Navigation("LatestPublishedReleaseVersion"); + + b.Navigation("SupersededBy"); + + b.Navigation("Topic"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Publications") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Methodologies") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Methodology"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("ReleaseStatuses") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("ReleaseVersions") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") + .WithMany("Versions") + .HasForeignKey("ReleaseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("PreviousVersion"); + + b.Navigation("Publication"); + + b.Navigation("Release"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Topic", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", "Theme") + .WithMany("Topics") + .HasForeignKey("ThemeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Theme"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Updates") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("Publication"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", "EmbedBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", "EmbedBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("EmbedBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithMany() + .HasForeignKey("DataBlockId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Navigation("Comments"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Navigation("Content"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Navigation("Errors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Navigation("Publications"); + + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Navigation("MethodologyContent") + .IsRequired(); + + b.Navigation("Notes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Navigation("Methodologies"); + + b.Navigation("ReleaseVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Navigation("Content"); + + b.Navigation("DataBlockVersions"); + + b.Navigation("FeaturedTables"); + + b.Navigation("KeyStatistics"); + + b.Navigation("ReleaseStatuses"); + + b.Navigation("Updates"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Navigation("Topics"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Topic", b => + { + b.Navigation("Publications"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240503112512_EES4770_AddReleaseFilesPublishedColumn.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240503112512_EES4770_AddReleaseFilesPublishedColumn.cs new file mode 100644 index 00000000000..4327b801ba1 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240503112512_EES4770_AddReleaseFilesPublishedColumn.cs @@ -0,0 +1,35 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using System; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + /// + public partial class EES4770_AddReleaseFilesPublishedColumn : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "Published", + table: "ReleaseFiles", + type: "datetime2", + nullable: true); + + migrationBuilder.Sql(@" + UPDATE ReleaseFiles + SET ReleaseFiles.Published = ReleaseVersions.Published + FROM ReleaseFiles + JOIN ReleaseVersions ON ReleaseVersions.Id = ReleaseFiles.ReleaseVersionId"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "Published", + table: "ReleaseFiles"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs index 9a57ffbb146..64c62339af7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs @@ -1,4 +1,4 @@ -// +// using System; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using Microsoft.EntityFrameworkCore; @@ -936,6 +936,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Order") .HasColumnType("int"); + b.Property("Published") + .HasColumnType("datetime2"); + b.Property("ReleaseVersionId") .HasColumnType("uniqueidentifier"); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseFile.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseFile.cs index 119f360a6b7..11928e38914 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseFile.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseFile.cs @@ -2,39 +2,40 @@ using System; using System.Collections.Generic; -namespace GovUk.Education.ExploreEducationStatistics.Content.Model +namespace GovUk.Education.ExploreEducationStatistics.Content.Model; + +public class ReleaseFile { - public class ReleaseFile - { - public Guid Id { get; set; } + public Guid Id { get; set; } + + public ReleaseVersion ReleaseVersion { get; set; } = null!; - public ReleaseVersion ReleaseVersion { get; set; } = null!; + public Guid ReleaseVersionId { get; set; } - public Guid ReleaseVersionId { get; set; } + public File File { get; set; } = null!; - public File File { get; set; } = null!; + public Guid FileId { get; set; } - public Guid FileId { get; set; } + public string? Name { get; set; } - public string? Name { get; set; } + public string? Summary { get; set; } - public string? Summary { get; set; } + public int Order { get; set; } - public int Order { get; set; } + public List? FilterSequence { get; set; } - public List? FilterSequence { get; set; } + public List? IndicatorSequence { get; set; } - public List? IndicatorSequence { get; set; } - } + public DateTime? Published { get; set; } +} - public abstract record SequenceEntry(TEntry Id, List ChildSequence); +public abstract record SequenceEntry(TEntry Id, List ChildSequence); - public record FilterSequenceEntry(Guid Id, List ChildSequence) : - SequenceEntry(Id, ChildSequence); +public record FilterSequenceEntry(Guid Id, List ChildSequence) : + SequenceEntry(Id, ChildSequence); - public record FilterGroupSequenceEntry(Guid Id, List ChildSequence) : - SequenceEntry(Id, ChildSequence); +public record FilterGroupSequenceEntry(Guid Id, List ChildSequence) : + SequenceEntry(Id, ChildSequence); - public record IndicatorGroupSequenceEntry(Guid Id, List ChildSequence) : - SequenceEntry(Id, ChildSequence); -} +public record IndicatorGroupSequenceEntry(Guid Id, List ChildSequence) : + SequenceEntry(Id, ChildSequence); From 43c4f894ddaa1aa8afdb9d6f23652f39c14ed510 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Fri, 24 May 2024 12:35:51 +0100 Subject: [PATCH 41/73] EES-5101: Copy ReleaseFile published date on create amendment. --- .../Services/ReleaseAmendmentService.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseAmendmentService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseAmendmentService.cs index 8a0391bb5a5..26a27178e76 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseAmendmentService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseAmendmentService.cs @@ -1,9 +1,4 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; @@ -18,6 +13,11 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; using ReleaseVersion = GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion; using Unit = GovUk.Education.ExploreEducationStatistics.Common.Model.Unit; @@ -623,6 +623,7 @@ private async Task> CopyFileLinks(ReleaseVe Summary = originalFile.Summary, FilterSequence = originalFile.FilterSequence, IndicatorSequence = originalFile.IndicatorSequence, + Published = originalFile.Published, }) .ToList(); From 057fc3e5811cd453bcc87f8468cad9d4a780de86 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Fri, 24 May 2024 12:36:38 +0100 Subject: [PATCH 42/73] EES-5101: Remove unused extension method. --- .../Extensions/FindExtensions.cs | 109 ++++++------------ 1 file changed, 36 insertions(+), 73 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/FindExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/FindExtensions.cs index a4c814ae51c..c3452e6737b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/FindExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/FindExtensions.cs @@ -1,84 +1,47 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; -using System.Reflection; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Metadata; -namespace GovUk.Education.ExploreEducationStatistics.Common.Extensions +namespace GovUk.Education.ExploreEducationStatistics.Common.Extensions; + +/** + * Provide extension to filter by primary key generically. + * https://stackoverflow.com/a/55272426 + */ +public static class FindExtensions { - /** - * Provide extension to filter by primary key generically. - * https://stackoverflow.com/a/55272426 - */ - public static class FindExtensions + public static IQueryable FindByPrimaryKey(this IQueryable queryable, + DbContext context, object key) where TEntity : class { - public static IQueryable FindByPrimaryKey(this IQueryable queryable, - DbContext context, object key) where TEntity : class - { - return FindByPrimaryKey(queryable, context, new[] {key}); - } - - public static IQueryable FindByPrimaryKey(this IQueryable queryable, - DbContext context, object[] key) where TEntity : class - { - return queryable.Where(context.FindByPrimaryKeyPredicate(key)); - } - - public static IQueryable FindAll(this IQueryable queryable, - DbContext dbContext, - params object[] keyValues) where TEntity : class - { - var entityType = dbContext.Model.FindEntityType(typeof(TEntity)); - var primaryKey = entityType.FindPrimaryKey(); - if (primaryKey.Properties.Count != 1) - throw new NotSupportedException("Only a single primary key is supported"); - - var pkProperty = primaryKey.Properties[0]; - var pkPropertyType = pkProperty.ClrType; - - foreach (var keyValue in keyValues) - { - if (!pkPropertyType.IsAssignableFrom(keyValue.GetType())) - throw new ArgumentException($"Key value '{keyValue}' is not of the right type"); - } - - var pkMemberInfo = typeof(TEntity).GetProperty(pkProperty.Name); - if (pkMemberInfo == null) - throw new ArgumentException("Type does not contain the primary key as an accessible property"); - - var parameter = Expression.Parameter(typeof(TEntity), "e"); - var body = Expression.Call(null, ContainsMethod, - Expression.Constant(keyValues), - Expression.Convert(Expression.MakeMemberAccess(parameter, pkMemberInfo), typeof(object))); - var predicateExpression = Expression.Lambda>(body, parameter); - - return queryable.Where(predicateExpression); - } + return FindByPrimaryKey(queryable, context, new[] { key }); + } - // TODO Precompile expression so this doesn't happen every time - private static Expression> FindByPrimaryKeyPredicate(this DbContext dbContext, object[] id) - { - var keyProperties = dbContext.GetPrimaryKeyProperties(); - var parameter = Expression.Parameter(typeof(T), "e"); - var body = keyProperties - .Select((p, i) => Expression.Equal( - Expression.Property(parameter, p.Name), - Expression.Convert( - Expression.PropertyOrField(Expression.Constant(new {id = id[i]}), "id"), - p.ClrType))) - .Aggregate(Expression.AndAlso); - return Expression.Lambda>(body, parameter); - } + public static IQueryable FindByPrimaryKey(this IQueryable queryable, + DbContext context, object[] key) where TEntity : class + { + return queryable.Where(context.FindByPrimaryKeyPredicate(key)); + } - private static readonly MethodInfo ContainsMethod = typeof(Enumerable).GetMethods() - .FirstOrDefault(m => m.Name == "Contains" && m.GetParameters().Length == 2) - .MakeGenericMethod(typeof(object)); + // TODO Precompile expression so this doesn't happen every time + private static Expression> FindByPrimaryKeyPredicate(this DbContext dbContext, object[] id) + { + var keyProperties = dbContext.GetPrimaryKeyProperties(); + var parameter = Expression.Parameter(typeof(T), "e"); + var body = keyProperties + .Select((p, i) => Expression.Equal( + Expression.Property(parameter, p.Name), + Expression.Convert( + Expression.PropertyOrField(Expression.Constant(new { id = id[i] }), "id"), + p.ClrType))) + .Aggregate(Expression.AndAlso); + return Expression.Lambda>(body, parameter); + } - private static IReadOnlyList GetPrimaryKeyProperties(this DbContext dbContext) - { - return dbContext.Model.FindEntityType(typeof(T)).FindPrimaryKey().Properties; - } + private static IReadOnlyList GetPrimaryKeyProperties(this DbContext dbContext) + { + return dbContext.Model.FindEntityType(typeof(T)).FindPrimaryKey().Properties; } -} \ No newline at end of file +} From d07659dc8678edb8076075192ae81d9973d22962 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Fri, 24 May 2024 12:39:54 +0100 Subject: [PATCH 43/73] EES-5101: Add Published date update to Publisher function. --- .../Services/ReleaseService.cs | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ReleaseService.cs index 09cbc8929ab..dcf55d60cef 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ReleaseService.cs @@ -1,14 +1,13 @@ -#nullable enable -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Publisher.Services.Interfaces; using Microsoft.EntityFrameworkCore; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Publisher.Extensions.PublisherExtensions; namespace GovUk.Education.ExploreEducationStatistics.Publisher.Services @@ -18,7 +17,8 @@ public class ReleaseService : IReleaseService private readonly ContentDbContext _contentDbContext; private readonly IReleaseVersionRepository _releaseVersionRepository; - public ReleaseService(ContentDbContext contentDbContext, + public ReleaseService( + ContentDbContext contentDbContext, IReleaseVersionRepository releaseVersionRepository) { _contentDbContext = contentDbContext; @@ -84,14 +84,38 @@ public async Task CompletePublishing(Guid releaseVersionId, DateTime actualPubli .SingleAsync(rv => rv.Id == releaseVersionId); _contentDbContext.ReleaseVersions.Update(releaseVersion); - releaseVersion.Published = - await _releaseVersionRepository.GetPublishedDate(releaseVersion.Id, actualPublishedDate); + + var publishedDate = await _releaseVersionRepository.GetPublishedDate(releaseVersion.Id, actualPublishedDate); + + releaseVersion.Published = publishedDate; + + await UpdateReleaseFilePublishedDate(releaseVersion, publishedDate); await UpdatePublishedDataBlockVersions(releaseVersion); await _contentDbContext.SaveChangesAsync(); } + private async Task UpdateReleaseFilePublishedDate( + ReleaseVersion releaseVersion, + DateTime publishedDate) + { + var dataReleaseFiles = _contentDbContext.ReleaseFiles + .Where(releaseFile => releaseFile.ReleaseVersionId == releaseVersion.Id) + .Include(rf => rf.File); + + if (releaseVersion.PreviousVersion is null) + { + await dataReleaseFiles.ForEachAsync(releaseFile => releaseFile.Published = publishedDate); + } + else + { + await dataReleaseFiles + .Where(rf => rf.Published == null) + .ForEachAsync(releaseFile => releaseFile.Published = DateTime.UtcNow); + } + } + private async Task UpdatePublishedDataBlockVersions(ReleaseVersion releaseVersion) { // Update all of the DataBlockParents to point their "LatestPublishedVersions" to the "latest" versions From efed835d767e16ac54023bc6eff5ae9ba4078aad Mon Sep 17 00:00:00 2001 From: Sam Biram <63285990+sambiramairelogic@users.noreply.github.com> Date: Tue, 28 May 2024 09:30:07 +0100 Subject: [PATCH 44/73] EES-5161 Add noindex,nofollow tags to all pages in non-production environments (#4887) --- src/explore-education-statistics-admin/public/index.html | 1 + .../src/components/PageMeta.tsx | 3 +++ .../src/modules/subscriptions/ConfirmSubscriptionPage.tsx | 1 - .../src/modules/subscriptions/ConfirmUnsubscriptionPage.tsx | 1 - 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/explore-education-statistics-admin/public/index.html b/src/explore-education-statistics-admin/public/index.html index e0e67c4783e..dfa230e5d6b 100644 --- a/src/explore-education-statistics-admin/public/index.html +++ b/src/explore-education-statistics-admin/public/index.html @@ -5,6 +5,7 @@ + + {process.env.APP_ENV !== 'Production' && ( + + )} {/* */} diff --git a/src/explore-education-statistics-frontend/src/modules/subscriptions/ConfirmSubscriptionPage.tsx b/src/explore-education-statistics-frontend/src/modules/subscriptions/ConfirmSubscriptionPage.tsx index 47bf0e0a33e..b7563038d63 100644 --- a/src/explore-education-statistics-frontend/src/modules/subscriptions/ConfirmSubscriptionPage.tsx +++ b/src/explore-education-statistics-frontend/src/modules/subscriptions/ConfirmSubscriptionPage.tsx @@ -63,7 +63,6 @@ const ConfirmSubscriptionPage: NextPage = ({ > - {confirmedSubscription ? ( = ({ > - {unsubscribedSubscription ? ( Date: Tue, 28 May 2024 09:32:24 +0100 Subject: [PATCH 45/73] EES-5160 - Brief #12 - Reroute any url with a hostname beginning www (#4886) --- .../server.js | 65 ++++++++++++------- .../tests/general_public/redirects.robot | 28 ++++++++ tests/robot-tests/tests/libs/common.robot | 6 ++ tests/robot-tests/tests/libs/utilities.py | 6 ++ 4 files changed, 82 insertions(+), 23 deletions(-) diff --git a/src/explore-education-statistics-frontend/server.js b/src/explore-education-statistics-frontend/server.js index fb340e16297..ae29ca2cd23 100644 --- a/src/explore-education-statistics-frontend/server.js +++ b/src/explore-education-statistics-frontend/server.js @@ -72,21 +72,53 @@ async function startServer() { const server = express(); function replaceLastOccurrence(input, pattern, replacement) { - if ( - input === undefined || - input === null || - input.length === 0 || - !input.endsWith(pattern) - ) { + if (!input || !input.endsWith(pattern)) { return input; } return `${input.slice(0, -pattern.length)}${replacement}`; } + /** + * @returns An absolute URL if redirection is required; undefined otherwise. + */ + function getRedirectUrl(request) { + let redirectPath = request.path; + + // Redirect URLs with trailing slash to equivalent without slash with 301 + redirectPath = replaceLastOccurrence(redirectPath, '/', ''); + + // Bots are adding /1000 and Google isn't yet omitting them from its index automatically, + // so we should redirect away from them + redirectPath = replaceLastOccurrence(redirectPath, '/1000', ''); + + // Search wrongly indexed pages from Google Search Console for matches + // Redirect away if found to (eventually) clear routes from index + const seoRedirect = seoRedirects.find( + seoRedirectPath => seoRedirectPath.from === request.url, + ); + if (seoRedirect) { + redirectPath = seoRedirect.to; + } + + const redirectionRequired = + request.hostname.startsWith('www') || redirectPath !== request.path; + + if (redirectionRequired) { + // Restore any search parameters on original request + const requestUrl = new URL(request.url, url); + const redirectUrl = new URL(`${redirectPath}${requestUrl.search}`, url); + + return redirectUrl.href; + } + + return undefined; + } + server.use((req, res, nextFunc) => { + // Return early for speed if redirect definitely isn't required if ( - req.url === '/' || + (req.url === '/' && !req.hostname.startsWith('www')) || req.url.startsWith('/assets') || req.url.startsWith('/_next') ) { @@ -94,22 +126,9 @@ async function startServer() { return undefined; } - let newUri = req.url; - // Redirect URLs with trailing slash to equivalent without slash with 301 - newUri = replaceLastOccurrence(newUri, '/', ''); - - // These are generated by spam sites. We could redirect them too to reduce Google's indexed 404 pages, - // but it seems current advice is actually to ignore these: - // https://support.google.com/webmasters/thread/253569816/google-crawler-adding-a-1000-to-the-end-of-a-ton-of-urls?hl=en - // newUri = replaceLastOccurance(newUri, '/1000', ''); - - const urlMatch = seoRedirects.find(source => source.from === req.url); - if (urlMatch !== undefined) { - newUri = urlMatch.to; - } - - if (newUri !== req.url) { - return res.redirect(301, newUri); + const redirectUrl = getRedirectUrl(req); + if (redirectUrl) { + return res.redirect(301, redirectUrl); } nextFunc(); diff --git a/tests/robot-tests/tests/general_public/redirects.robot b/tests/robot-tests/tests/general_public/redirects.robot index 5278ef38fa2..2b3e0a08f0c 100644 --- a/tests/robot-tests/tests/general_public/redirects.robot +++ b/tests/robot-tests/tests/general_public/redirects.robot @@ -42,3 +42,31 @@ Verify that routes without an absolute path still permit trailing slashes user navigates to public frontend %{PUBLIC_URL} user waits until page contains Explore education statistics user checks url equals %{PUBLIC_URL}/ + +Verify that routes with www are redirected without them + user navigates to public frontend with www %{PUBLIC_URL}/ + user waits until page contains Explore education statistics + user checks url equals %{PUBLIC_URL}/ + + user navigates to public frontend %{PUBLIC_URL_WITH}/data-catalogue/ + user waits until page contains Browse our open data + user checks url equals %{PUBLIC_URL}/data-catalogue + +Verify that routes with /1000 are redirected without them + user navigates to public frontend %{PUBLIC_URL}/data-catalogue/1000 + user waits until page contains Browse our open data + user checks url equals %{PUBLIC_URL}/data-catalogue + +Verify that routes with search parameters retain them + user navigates to public frontend %{PUBLIC_URL}/data-catalogue?foo=bar&baz=zod + user waits until page contains Browse our open data + user checks url equals %{PUBLIC_URL}/data-catalogue?foo=bar&baz=zod + +Verify that multiple rules work together + user navigates to public frontend %{PUBLIC_URL}/data-catalogue/1000?foo=bar&baz=zod + user waits until page contains Browse our open data + user checks url equals %{PUBLIC_URL}/data-catalogue?foo=bar&baz=zod + + user navigates to public frontend %{PUBLIC_URL}/data-catalogue/1000/?foo=bar + user waits until page contains Browse our open data + user checks url equals %{PUBLIC_URL}/data-catalogue?foo=bar diff --git a/tests/robot-tests/tests/libs/common.robot b/tests/robot-tests/tests/libs/common.robot index ee287517361..f7450281374 100644 --- a/tests/robot-tests/tests/libs/common.robot +++ b/tests/robot-tests/tests/libs/common.robot @@ -920,6 +920,12 @@ user navigates to public frontend enable basic auth headers go to ${URL} +user navigates to public frontend with www + [Arguments] ${URL}=%{PUBLIC_URL} + enable basic auth headers + ${www_url}= get www url ${URL} + go to ${www_url} + user navigates to [Arguments] ${URL} enable basic auth headers diff --git a/tests/robot-tests/tests/libs/utilities.py b/tests/robot-tests/tests/libs/utilities.py index c3cb7e26c24..f5f00ecb255 100644 --- a/tests/robot-tests/tests/libs/utilities.py +++ b/tests/robot-tests/tests/libs/utilities.py @@ -360,3 +360,9 @@ def _get_parent_webelement_from_locator(parent_locator: object, timeout: int = N return parent_locator else: raise_assertion_error(f"Parent locator was neither a str or a WebElement - {parent_locator}") + + +def get_www_url(publicUrl: str): + protocol, hostnameAndPort = publicUrl.split("://") + + return protocol + "://www." + hostnameAndPort From 8500747403a36f4a23ee9e412809596890049a30 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Fri, 24 May 2024 12:54:52 +0100 Subject: [PATCH 46/73] EES-5101: Add "Last updated" date field to data catalogue frontend. --- .../Api/Statistics/FootnoteControllerTests.cs | 14 ++++----- .../DataSetFileService.cs | 18 +++++++----- .../DataSetFileSummaryViewModel.cs | 4 ++- .../DataSetFileViewModel.cs | 2 ++ .../Controllers/PublicationControllerTests.cs | 9 +++--- .../ReleaseService.cs | 12 ++++---- .../SubjectViewModel.cs | 6 +++- .../__tests__/ReleaseContentPage.test.tsx | 1 + .../ReleaseDataBlockEditPage.test.tsx | 1 + .../__tests__/ReleaseTableToolPage.test.tsx | 2 ++ .../__tests__/DataBlockPageTabs.test.tsx | 1 + .../PreReleaseTableToolPage.test.tsx | 1 + .../src/prototypes/data/tableToolData.tsx | 29 +++++++++++++++++++ .../components/__tests__/DataSetStep.test.tsx | 2 ++ .../__tests__/TableToolWizard.test.tsx | 2 ++ .../src/services/tableBuilderService.ts | 1 + .../data-catalogue/DataSetFilePage.tsx | 6 ++++ .../data-catalogue/__data__/testDataSets.ts | 4 +++ .../__tests__/DataCataloguePage.test.tsx | 3 ++ .../components/DataSetFileSummary.tsx | 4 +++ .../components/DownloadStep.tsx | 12 ++++++-- .../__tests__/DownloadStep.test.tsx | 3 ++ .../__tests__/TableToolPage.test.tsx | 1 + .../src/services/dataSetFileService.ts | 2 ++ 24 files changed, 111 insertions(+), 29 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Statistics/FootnoteControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Statistics/FootnoteControllerTests.cs index f4c1cb95958..3d6d64ad757 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Statistics/FootnoteControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Statistics/FootnoteControllerTests.cs @@ -1,8 +1,4 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Controllers.Api.Statistics; using GovUk.Education.ExploreEducationStatistics.Admin.Requests; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; @@ -13,7 +9,10 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.ViewModels; using Moq; -using Xunit; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; using IReleaseService = GovUk.Education.ExploreEducationStatistics.Data.Services.Interfaces.IReleaseService; using Unit = GovUk.Education.ExploreEducationStatistics.Common.Model.Unit; @@ -30,7 +29,7 @@ public class FootnoteControllerTests public FootnoteControllerTests() { - var subjectIds = new[] {Guid.NewGuid(), Guid.NewGuid()}; + var subjectIds = new[] { Guid.NewGuid(), Guid.NewGuid() }; var footnote = new Footnote { @@ -105,7 +104,8 @@ public FootnoteControllerTests() Id = Guid.NewGuid(), FileName = "test.csv", Size = "1 Mb" - } + }, + lastUpdated: DateTime.Now ) ) .ToList() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index 6dfdfb2c16a..ddade6b5af4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -1,12 +1,4 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Linq.Expressions; -using System.Threading; -using System.Threading.Tasks; using CsvHelper; using GovUk.Education.ExploreEducationStatistics.Common; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; @@ -26,6 +18,14 @@ using GovUk.Education.ExploreEducationStatistics.Data.Services.Utils; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Common.Model.SortDirection; using static GovUk.Education.ExploreEducationStatistics.Content.Requests.DataSetsListRequestSortBy; using File = GovUk.Education.ExploreEducationStatistics.Content.Model.File; @@ -131,6 +131,7 @@ private static Expression, DataSetFileSumm LatestData = result.Value.ReleaseVersionId == result.Value.ReleaseVersion.Publication.LatestPublishedReleaseVersionId, Published = result.Value.ReleaseVersion.Published!.Value, + LastUpdated = result.Value.Published!.Value, Api = BuildDataSetFileApiViewModel(result.Value.File), Meta = BuildDataSetFileMetaViewModel( result.Value.File.DataSetFileMeta, @@ -194,6 +195,7 @@ public async Task> GetDataSetFile( releaseFile.ReleaseVersion.Publication.LatestPublishedReleaseVersionId == releaseFile.ReleaseVersionId, Published = releaseFile.ReleaseVersion.Published!.Value, + LastUpdated = releaseFile.Published!.Value, Publication = new DataSetFilePublicationViewModel { Id = releaseFile.ReleaseVersion.PublicationId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileSummaryViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileSummaryViewModel.cs index 438f1e5082d..81510ab2303 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileSummaryViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileSummaryViewModel.cs @@ -1,4 +1,4 @@ -using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; +using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; namespace GovUk.Education.ExploreEducationStatistics.Content.ViewModels; @@ -28,6 +28,8 @@ public record DataSetFileSummaryViewModel public required DateTime Published { get; init; } + public DateTime LastUpdated { get; init; } + public required DataSetFileMetaViewModel Meta { get; init; } public DataSetFileApiViewModel? Api { get; init; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs index f52c51b43cf..2b33236100d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs @@ -49,6 +49,8 @@ public record DataSetFileReleaseViewModel public required DateTime Published { get; init; } + public DateTime LastUpdated { get; init; } + public required DataSetFilePublicationViewModel Publication { get; init; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/PublicationControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/PublicationControllerTests.cs index 63ea66800c8..faacc75d32c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/PublicationControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/PublicationControllerTests.cs @@ -1,7 +1,4 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; @@ -15,6 +12,9 @@ using GovUk.Education.ExploreEducationStatistics.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.ViewModels; using Moq; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; using Xunit; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; using static Moq.MockBehavior; @@ -173,7 +173,8 @@ public void SubjectViewModel_SerializeAndDeserialize() Type = FileType.Ancillary, FileName = "Filename", UserName = "UserName" - }); + }, + DateTime.Now); var converted = DeserializeObject(SerializeObject(original)); converted.AssertDeepEqualTo(original); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Services/ReleaseService.cs index 02462c7b6c7..ac793da7134 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Services/ReleaseService.cs @@ -1,8 +1,4 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; @@ -11,12 +7,15 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; using GovUk.Education.ExploreEducationStatistics.Content.Security.Extensions; -using GovUk.Education.ExploreEducationStatistics.Data.Model; using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.ViewModels; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; using ReleaseVersion = GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion; namespace GovUk.Education.ExploreEducationStatistics.Data.Services @@ -92,7 +91,8 @@ private async Task> GetSubjects(Guid releaseVersionId, Li geographicLevels: await _dataGuidanceDataSetService.ListGeographicLevels(rs.SubjectId), filters: await GetFilters(rs.SubjectId, releaseFile.FilterSequence), indicators: await GetIndicators(rs.SubjectId, releaseFile.IndicatorSequence), - file: releaseFile.ToFileInfo() + file: releaseFile.ToFileInfo(), + lastUpdated: releaseFile.Published ); } )) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/SubjectViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/SubjectViewModel.cs index 6b48d946384..91cf782ead4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/SubjectViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/SubjectViewModel.cs @@ -22,6 +22,8 @@ public record SubjectViewModel public FileInfo File { get; } + public DateTime? LastUpdated { get; } + public SubjectViewModel( Guid id, string name, @@ -31,7 +33,8 @@ public SubjectViewModel( List geographicLevels, List filters, List indicators, - FileInfo file) + FileInfo file, + DateTime? lastUpdated) { Id = id; Name = name; @@ -42,5 +45,6 @@ public SubjectViewModel( Filters = filters; Indicators = indicators; File = file; + LastUpdated = lastUpdated; } } diff --git a/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx index fccfcad7640..fb2b322f03f 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx @@ -368,6 +368,7 @@ describe('ReleaseContentPage', () => { }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, ]; diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseDataBlockEditPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseDataBlockEditPage.test.tsx index e198edbede5..50296215b5b 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseDataBlockEditPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseDataBlockEditPage.test.tsx @@ -171,6 +171,7 @@ describe('ReleaseDataBlockEditPage', () => { }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, ]; diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseTableToolPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseTableToolPage.test.tsx index 0af0821611d..2835840381b 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseTableToolPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseTableToolPage.test.tsx @@ -82,6 +82,7 @@ describe('ReleaseTableToolPage', () => { }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: 'subject-2', @@ -102,6 +103,7 @@ describe('ReleaseTableToolPage', () => { }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, ]); diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/__tests__/DataBlockPageTabs.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/__tests__/DataBlockPageTabs.test.tsx index ecbceeccb54..f4f4d1ca5b7 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/__tests__/DataBlockPageTabs.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/__tests__/DataBlockPageTabs.test.tsx @@ -82,6 +82,7 @@ describe('DataBlockPageTabs', () => { }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, ]; diff --git a/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseTableToolPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseTableToolPage.test.tsx index 6e7479be63c..1c6992bf2b4 100644 --- a/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseTableToolPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseTableToolPage.test.tsx @@ -250,6 +250,7 @@ describe('PreReleaseTableToolPage', () => { }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, ]; diff --git a/src/explore-education-statistics-admin/src/prototypes/data/tableToolData.tsx b/src/explore-education-statistics-admin/src/prototypes/data/tableToolData.tsx index 67c6d592109..72928648ee7 100644 --- a/src/explore-education-statistics-admin/src/prototypes/data/tableToolData.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/data/tableToolData.tsx @@ -28,6 +28,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '8bfceddc-e6da-4b26-998e-08d987eb588f', @@ -50,6 +51,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '7e47015e-bf0e-4b42-9990-08d987eb588f', @@ -71,6 +73,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: 'f92c494b-571f-4dda-99a0-08d987eb588f', @@ -92,6 +95,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '2c9b1ba9-b79a-4cba-99a4-08d987eb588f', @@ -115,6 +119,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: 'c8bd097a-c011-4693-999e-08d987eb588f', @@ -137,6 +142,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: 'c28569ca-4a59-44cd-9996-08d987eb588f', @@ -158,6 +164,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '0f2f9b9b-85c8-4441-998c-08d987eb588f', @@ -180,6 +187,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '949f4689-729f-425d-99ac-08d987eb588f', @@ -201,6 +209,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '03027d63-f185-49c8-bccc-08d98e1873d8', @@ -222,6 +231,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: 'ef301961-dbfc-44a8-99b0-08d987eb588f', @@ -243,6 +253,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '2ed75fdd-c6b2-433a-99b4-08d987eb588f', @@ -266,6 +277,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: 'ecd8b9bb-822d-4840-20d1-08d98d84ca80', @@ -288,6 +300,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '655d0d18-d3e4-4c1d-99be-08d987eb588f', @@ -311,6 +324,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '84609c65-55d8-49d7-bc5a-08d98e1873d8', @@ -333,6 +347,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '74eb4c5b-b276-4d28-f2c7-08d99857c8b0', @@ -355,6 +370,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '432a7ff4-e7ef-46a3-bc5c-08d98e1873d8', @@ -377,6 +393,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '7f16dcfc-6491-40ae-bc58-08d98e1873d8', @@ -399,6 +416,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: 'd843a8fc-0fc3-4e10-bcec-08d98e1873d8', @@ -421,6 +439,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '28f10859-b469-494a-99c0-08d987eb588f', @@ -443,6 +462,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: 'b6af0e66-78fc-4a12-bcf2-08d98e1873d8', @@ -465,6 +485,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '34c61111-6958-4b47-99c2-08d987eb588f', @@ -487,6 +508,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '4ab9b578-8326-4c0e-bcf4-08d98e1873d8', @@ -509,6 +531,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '71a006ac-9c4e-421d-bc8c-08d98e1873d8', @@ -531,6 +554,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '82c7e75b-1317-41a1-bcf6-08d98e1873d8', @@ -553,6 +577,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: 'cf86bc9a-9d5c-4fd8-bcf0-08d98e1873d8', @@ -575,6 +600,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '103bab64-2d9f-44b7-bcf8-08d98e1873d8', @@ -597,6 +623,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '9b1e3140-5171-4edd-f2dd-08d99857c8b0', @@ -619,6 +646,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: '1e916eb9-6206-46ee-9df5-08d9924737d1', @@ -641,6 +669,7 @@ export const subjects: Subject[] = [ }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, ]; diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/DataSetStep.test.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/DataSetStep.test.tsx index 8aa924e90d2..91566da4bee 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/DataSetStep.test.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/DataSetStep.test.tsx @@ -32,6 +32,7 @@ describe('DataSetStep', () => { }, filters: ['School type'], indicators: ['Headcount', 'Percent'], + lastUpdated: '2023-12-01', }, { id: 'subject-2', @@ -52,6 +53,7 @@ describe('DataSetStep', () => { }, filters: ['Ethnicity', 'FSM'], indicators: ['Authorised absence rate'], + lastUpdated: '2023-12-01', }, ]; diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TableToolWizard.test.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TableToolWizard.test.tsx index 7260415c387..da7c322367c 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TableToolWizard.test.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TableToolWizard.test.tsx @@ -90,6 +90,7 @@ describe('TableToolWizard', () => { }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, { id: 'subject-2', @@ -110,6 +111,7 @@ describe('TableToolWizard', () => { }, filters: ['Filter 1'], indicators: ['Indicator 1'], + lastUpdated: '2023-12-01', }, ]; diff --git a/src/explore-education-statistics-common/src/services/tableBuilderService.ts b/src/explore-education-statistics-common/src/services/tableBuilderService.ts index 3a84c441487..319740b4062 100644 --- a/src/explore-education-statistics-common/src/services/tableBuilderService.ts +++ b/src/explore-education-statistics-common/src/services/tableBuilderService.ts @@ -87,6 +87,7 @@ export interface Subject { file: FileInfo; filters: string[]; indicators: string[]; + lastUpdated: string; } export interface FeaturedTable { diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataSetFilePage.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataSetFilePage.tsx index d37a032f517..b62044e396a 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataSetFilePage.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataSetFilePage.tsx @@ -168,6 +168,12 @@ export default function DataSetFilePage({ {release.published} +
+ Last updated{' '} + + {release.lastUpdated} + +
); const button = screen.getByRole('button', { name: 'Test button' }); + expect(button).not.toBeAriaDisabled(); expect(button).toBeEnabled(); await userEvent.click(button); expect(handleClick).toHaveBeenCalledTimes(1); - expect(button).toBeDisabled(); - // Button is still disabled - expect(button).toBeDisabled(); + expect(button).toBeAriaDisabled(); + expect(button).toBeEnabled(); }); - test('enabled if current `onClick` handler is processing and `disableDoubleClick` is false', async () => { + test('not aria-disabled if current `onClick` handler is processing and `preventDoubleClick` is false', async () => { const handleClick = jest.fn(async () => delay(100)); render( - , ); const button = screen.getByRole('button', { name: 'Test button' }); + expect(button).not.toBeAriaDisabled(); expect(button).toBeEnabled(); await userEvent.click(button); expect(handleClick).toHaveBeenCalledTimes(1); - expect(button).toBeEnabled(); - // Button is still enabled + expect(button).not.toBeAriaDisabled(); expect(button).toBeEnabled(); }); - test('enabled once the current `onClick` handler has finished', async () => { + test('not aria-disabled once the current `onClick` handler has finished', async () => { const handleClick = jest.fn(async () => delay(100)); render(); const button = screen.getByRole('button', { name: 'Test button' }); + expect(button).not.toBeAriaDisabled(); + expect(button).toBeEnabled(); + await userEvent.click(button); expect(handleClick).toHaveBeenCalledTimes(1); - expect(button).toBeDisabled(); + + expect(button).toBeAriaDisabled(); + expect(button).toBeEnabled(); // Task has completed, so button is now enabled await waitFor(() => { + expect(button).not.toBeAriaDisabled(); expect(button).toBeEnabled(); }); }); diff --git a/src/explore-education-statistics-common/src/components/__tests__/ButtonText.test.tsx b/src/explore-education-statistics-common/src/components/__tests__/ButtonText.test.tsx index 47e4ded6f3d..f3b19da5886 100644 --- a/src/explore-education-statistics-common/src/components/__tests__/ButtonText.test.tsx +++ b/src/explore-education-statistics-common/src/components/__tests__/ButtonText.test.tsx @@ -43,64 +43,70 @@ describe('ButtonText', () => { const button = screen.getByRole('button', { name: 'Test button' }); - expect(button).not.toBeDisabled(); + expect(button).toBeDisabled(); expect(button).toBeAriaDisabled(); }); - test('disabled if current `onClick` handler is processing', async () => { + test('aria-disabled if current `onClick` handler is processing', async () => { const handleClick = jest.fn(async () => delay(100)); render(Test button); const button = screen.getByRole('button', { name: 'Test button' }); + expect(button).not.toBeAriaDisabled(); expect(button).toBeEnabled(); await userEvent.click(button); expect(handleClick).toHaveBeenCalledTimes(1); - expect(button).toBeDisabled(); - // Button is still disabled - expect(button).toBeDisabled(); + expect(button).toBeAriaDisabled(); + expect(button).toBeEnabled(); }); - test('enabled if current `onClick` handler is processing and `disableDoubleClick` is false', async () => { + test('not aria-disabled if current `onClick` handler is processing and `preventDoubleClick` is false', async () => { const handleClick = jest.fn(async () => delay(100)); render( - + Test button , ); const button = screen.getByRole('button', { name: 'Test button' }); + expect(button).not.toBeAriaDisabled(); expect(button).toBeEnabled(); await userEvent.click(button); expect(handleClick).toHaveBeenCalledTimes(1); - expect(button).toBeEnabled(); - // Button is still enabled + expect(button).not.toBeAriaDisabled(); expect(button).toBeEnabled(); }); - test('enabled once the current `onClick` handler has finished', async () => { + test('not aria-disabled once the current `onClick` handler has finished', async () => { const handleClick = jest.fn(async () => delay(100)); render(Test button); const button = screen.getByRole('button', { name: 'Test button' }); + expect(button).not.toBeAriaDisabled(); + expect(button).toBeEnabled(); + await userEvent.click(button); expect(handleClick).toHaveBeenCalledTimes(1); - expect(button).toBeDisabled(); + + expect(button).toBeAriaDisabled(); + expect(button).toBeEnabled(); // Task has completed, so button is now enabled await waitFor(() => { + expect(button).not.toBeAriaDisabled(); expect(button).toBeEnabled(); }); }); diff --git a/src/explore-education-statistics-common/src/hooks/__tests__/useButton.test.ts b/src/explore-education-statistics-common/src/hooks/__tests__/useButton.test.ts index f5c456453ac..f74d8b15adc 100644 --- a/src/explore-education-statistics-common/src/hooks/__tests__/useButton.test.ts +++ b/src/explore-education-statistics-common/src/hooks/__tests__/useButton.test.ts @@ -12,8 +12,6 @@ describe('useButton', () => { id: 'id', testId: 'test id', type: 'submit', - underline: false, - variant: 'warning', }), ); expect(result.current.children).toBe('button text'); @@ -23,8 +21,6 @@ describe('useButton', () => { expect(result.current.id).toBe('id'); expect(result.current['data-testid']).toBe('test id'); expect(result.current.type).toBe('submit'); - expect(result.current.underline).toBe(false); - expect(result.current.variant).toBe('warning'); }); test('returns correct props when `disabled = true`', () => { @@ -58,6 +54,6 @@ describe('useButton', () => { }), ); expect(result.current['aria-disabled']).toBe(true); - expect(result.current.disabled).toBe(undefined); + expect(result.current.disabled).toBe(true); }); }); diff --git a/src/explore-education-statistics-common/src/hooks/useButton.ts b/src/explore-education-statistics-common/src/hooks/useButton.ts index d5935533a16..6f55d967ad0 100644 --- a/src/explore-education-statistics-common/src/hooks/useButton.ts +++ b/src/explore-education-statistics-common/src/hooks/useButton.ts @@ -1,6 +1,6 @@ import useMountedRef from '@common/hooks/useMountedRef'; import useToggle from '@common/hooks/useToggle'; -import { MouseEventHandler, ReactNode, useCallback } from 'react'; +import { MouseEvent, MouseEventHandler, ReactNode, useCallback } from 'react'; export interface ButtonOptions { ariaControls?: string; @@ -9,13 +9,11 @@ export interface ButtonOptions { children: ReactNode; className?: string; disabled?: boolean; - disableDoubleClick?: boolean; id?: string; + preventDoubleClick?: boolean; testId?: string; - onClick?: MouseEventHandler; type?: 'button' | 'submit' | 'reset'; - underline?: boolean; - variant?: 'secondary' | 'warning'; + onClick?: (event: MouseEvent) => void | Promise; } export default function useButton({ @@ -24,46 +22,38 @@ export default function useButton({ ariaExpanded, children, className, - disabled = false, - disableDoubleClick = true, + disabled, id, + preventDoubleClick = true, testId, type = 'button', - underline = true, - variant, onClick, }: ButtonOptions) { const [isClicking, toggleClicking] = useToggle(false); const isMountedRef = useMountedRef(); + const isDisabled = ariaDisabled || disabled || isClicking; + const handleClick: MouseEventHandler = useCallback( async event => { - if (ariaDisabled || disabled) { + if (isDisabled) { + event.preventDefault(); return; } - if (disableDoubleClick) { + if (preventDoubleClick) { toggleClicking.on(); } await onClick?.(event); - if (disableDoubleClick && isMountedRef.current) { + if (preventDoubleClick && isMountedRef.current) { toggleClicking.off(); } }, - [ - ariaDisabled, - disabled, - disableDoubleClick, - isMountedRef, - onClick, - toggleClicking, - ], + [isDisabled, preventDoubleClick, onClick, isMountedRef, toggleClicking], ); - const isDisabled = ariaDisabled || disabled || isClicking; - return { 'aria-controls': ariaControls, 'aria-disabled': isDisabled, @@ -71,12 +61,9 @@ export default function useButton({ children, className, 'data-testid': testId, - disabled: ariaDisabled ? undefined : disabled || isClicking, + disabled, id, - isDisabled, type, - underline, - variant, onClick: handleClick, }; } From 87859e38cc06af5db2c3bab06d42f1cddc9e03ee Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Tue, 21 May 2024 15:42:54 +0100 Subject: [PATCH 63/73] EES-4366 Prevent accidental multiple form submissions --- .../src/components/form/Form.tsx | 15 ++++++-- .../components/form/__tests__/Form.test.tsx | 34 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/explore-education-statistics-common/src/components/form/Form.tsx b/src/explore-education-statistics-common/src/components/form/Form.tsx index 6887be9cb73..1e26076d589 100644 --- a/src/explore-education-statistics-common/src/components/form/Form.tsx +++ b/src/explore-education-statistics-common/src/components/form/Form.tsx @@ -55,7 +55,13 @@ export default function Form({ const isMounted = useMountedRef(); const { - formState: { errors, submitCount, touchedFields, isSubmitted }, + formState: { + errors, + submitCount, + touchedFields, + isSubmitted, + isSubmitting, + }, handleSubmit: submit, } = useFormContext(); @@ -105,11 +111,16 @@ export default function Form({ const handleSubmit = useCallback( async (event: FormEvent) => { event.preventDefault(); + + if (isSubmitting) { + return; + } + toggleSummaryFocus.off(); await submit(async data => onSubmit(data))(event); }, - [submit, toggleSummaryFocus, onSubmit], + [isSubmitting, toggleSummaryFocus, submit, onSubmit], ); return ( diff --git a/src/explore-education-statistics-common/src/components/form/__tests__/Form.test.tsx b/src/explore-education-statistics-common/src/components/form/__tests__/Form.test.tsx index 30c583ad650..4c5cbf9dc12 100644 --- a/src/explore-education-statistics-common/src/components/form/__tests__/Form.test.tsx +++ b/src/explore-education-statistics-common/src/components/form/__tests__/Form.test.tsx @@ -2,6 +2,7 @@ import { createServerValidationErrorMock } from '@common-test/createAxiosErrorMo import FormProvider from '@common/components/form/FormProvider'; import Form from '@common/components/form/Form'; import SubmitError from '@common/components/form/util/SubmitError'; +import delay from '@common/utils/delay'; import Yup from '@common/validation/yup'; import { render, screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; @@ -166,6 +167,39 @@ describe('Form', () => { }); }); + test('prevents multiple `onSubmit` calls until submission completes', async () => { + const handleSubmit = jest.fn(() => delay(200)); + + render( + + {({ formState }) => ( +
+ The form + + {formState.isSubmitted &&

Submitted

} +
+ )} + , + ); + + await userEvent.click(screen.getByRole('button', { name: 'Submit' })); + await userEvent.click(screen.getByRole('button', { name: 'Submit' })); + await userEvent.click(screen.getByRole('button', { name: 'Submit' })); + + expect(handleSubmit).toHaveBeenCalledTimes(1); + + expect(await screen.findByText('Submitted')).toBeInTheDocument(); + + expect(handleSubmit).toHaveBeenCalledTimes(1); + }); + test('renders submit error with default message when error thrown', async () => { const { container } = render( Date: Tue, 21 May 2024 15:45:27 +0100 Subject: [PATCH 64/73] EES-4366 Convert release routes to protected routes --- .../pages/release/ReleasePageContainer.tsx | 28 +++++++++++++++---- .../src/routes/releaseRoutes.ts | 4 +-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx b/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx index 8a57b573253..5b7d66dc30f 100644 --- a/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx @@ -2,6 +2,8 @@ import NavBar from '@admin/components/NavBar'; import Page from '@admin/components/Page'; import PageTitle from '@admin/components/PageTitle'; import PreviousNextLinks from '@admin/components/PreviousNextLinks'; +import ProtectedRoute from '@admin/components/ProtectedRoute'; +import { useAuthContext } from '@admin/contexts/AuthContext'; import { ReleaseContextProvider } from '@admin/pages/release/contexts/ReleaseContext'; import { getReleaseApprovalStatusLabel } from '@admin/pages/release/utils/releaseSummaryUtil'; import { @@ -29,12 +31,12 @@ import releaseService from '@admin/services/releaseService'; import LoadingSpinner from '@common/components/LoadingSpinner'; import Tag from '@common/components/Tag'; import useAsyncHandledRetry from '@common/hooks/useAsyncHandledRetry'; -import React from 'react'; -import { generatePath, Route, RouteComponentProps, Switch } from 'react-router'; +import React, { useMemo } from 'react'; +import { generatePath, RouteComponentProps, Switch } from 'react-router'; import { publicationReleasesRoute } from '@admin/routes/publicationRoutes'; import { PublicationRouteParams } from '@admin/routes/routes'; -const navRoutes = [ +const allNavRoutes = [ releaseSummaryRoute, releaseDataRoute, releaseFootnotesRoute, @@ -45,7 +47,7 @@ const navRoutes = [ ]; const routes = [ - ...navRoutes, + ...allNavRoutes, releaseAncillaryFilesRoute, releaseAncillaryFileRoute, releaseDataFileRoute, @@ -69,6 +71,9 @@ const ReleasePageContainer = ({ location, }: RouteComponentProps) => { const { publicationId, releaseId } = match.params; + + const { user } = useAuthContext(); + const { value: release, setState: setRelease, @@ -78,6 +83,15 @@ const ReleasePageContainer = ({ [releaseId], ); + const navRoutes = useMemo(() => { + return allNavRoutes.filter(route => { + return ( + user?.permissions && + (!route.protectionAction || route.protectionAction(user.permissions)) + ); + }); + }, [user?.permissions]); + const currentRouteIndex = navRoutes.findIndex( route => @@ -157,11 +171,13 @@ const ReleasePageContainer = ({ */} + {getReleaseApprovalStatusLabel(release.approvalStatus)} {release.amendment && ( Amendment )} {release.live && Live} + ({ title: route.title, @@ -172,6 +188,7 @@ const ReleasePageContainer = ({ }))} label="Release" /> + { @@ -180,10 +197,11 @@ const ReleasePageContainer = ({ > {routes.map(route => ( - + ))} + {currentRouteIndex > -1 && ( Date: Tue, 21 May 2024 15:48:18 +0100 Subject: [PATCH 65/73] EES-4366 Add initial API data sets admin page --- .../pages/release/ReleasePageContainer.tsx | 2 + .../api-data-sets/ReleaseApiDataSetsPage.tsx | 122 +++++++++ .../__tests__/ReleaseApiDataSetsPage.test.tsx | 258 ++++++++++++++++++ .../components/ApiDataSetCreateForm.tsx | 71 +++++ .../components/ApiDataSetCreateModal.tsx | 93 +++++++ .../DraftApiDataSetsTable.module.scss | 29 ++ .../components/DraftApiDataSetsTable.tsx | 128 +++++++++ .../LiveApiDataSetsTable.module.scss | 9 + .../components/LiveApiDataSetsTable.tsx | 93 +++++++ .../__tests__/ApiDataSetCreateForm.test.tsx | 151 ++++++++++ .../__tests__/ApiDataSetCreateModal.test.tsx | 127 +++++++++ .../__tests__/DraftApiDataSetsTable.test.tsx | 253 +++++++++++++++++ .../__tests__/LiveApiDataSetsTable.test.tsx | 159 +++++++++++ .../utils/getVersionStatusColour.ts | 25 ++ .../utils/getVersionStatusText.ts | 14 + .../src/queries/apiDataSetCandidateQueries.ts | 14 + .../src/queries/apiDataSetQueries.ts | 13 + .../src/routes/releaseRoutes.ts | 18 ++ .../services/apiDataSetCandidateService.ts | 18 ++ .../src/services/apiDataSetService.ts | 109 ++++++++ .../test/render.tsx | 2 +- 21 files changed, 1707 insertions(+), 1 deletion(-) create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetsPage.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetsPage.test.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateForm.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.module.scss create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.module.scss create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateForm.test.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/DraftApiDataSetsTable.test.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/LiveApiDataSetsTable.test.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusColour.ts create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusText.ts create mode 100644 src/explore-education-statistics-admin/src/queries/apiDataSetCandidateQueries.ts create mode 100644 src/explore-education-statistics-admin/src/queries/apiDataSetQueries.ts create mode 100644 src/explore-education-statistics-admin/src/services/apiDataSetCandidateService.ts create mode 100644 src/explore-education-statistics-admin/src/services/apiDataSetService.ts diff --git a/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx b/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx index 5b7d66dc30f..07d0a95c184 100644 --- a/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx @@ -26,6 +26,7 @@ import { releaseSummaryEditRoute, releaseSummaryRoute, releaseTableToolRoute, + releaseApiDataSetsRoute, } from '@admin/routes/releaseRoutes'; import releaseService from '@admin/services/releaseService'; import LoadingSpinner from '@common/components/LoadingSpinner'; @@ -44,6 +45,7 @@ const allNavRoutes = [ releaseContentRoute, releaseStatusRoute, releasePreReleaseAccessRoute, + releaseApiDataSetsRoute, ]; const routes = [ diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetsPage.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetsPage.tsx new file mode 100644 index 00000000000..a25770ff836 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetsPage.tsx @@ -0,0 +1,122 @@ +import { useAuthContext } from '@admin/contexts/AuthContext'; +import ApiDataSetCreateModal from '@admin/pages/release/api-data-sets/components/ApiDataSetCreateModal'; +import DraftApiDataSetsTable, { + DraftApiDataSetSummary, +} from '@admin/pages/release/api-data-sets/components/DraftApiDataSetsTable'; +import LiveApiDataSetsTable, { + LiveApiDataSetSummary, +} from '@admin/pages/release/api-data-sets/components/LiveApiDataSetsTable'; +import { useReleaseContext } from '@admin/pages/release/contexts/ReleaseContext'; +import apiDataSetQueries from '@admin/queries/apiDataSetQueries'; +import InsetText from '@common/components/InsetText'; +import LoadingSpinner from '@common/components/LoadingSpinner'; +import WarningMessage from '@common/components/WarningMessage'; +import { useQuery } from '@tanstack/react-query'; +import React from 'react'; + +export default function ReleaseApiDataSetsPage() { + const { release } = useReleaseContext(); + const { user } = useAuthContext(); + + const { data: dataSets = [], isLoading: hasDataSetsLoading } = useQuery({ + ...apiDataSetQueries.list(release.publicationId), + refetchInterval: 20_000, + }); + + const draftDataSets = dataSets.filter( + dataSet => dataSet.draftVersion, + ) as DraftApiDataSetSummary[]; + + const liveDataSets = dataSets.filter( + dataSet => dataSet.latestLiveVersion && !dataSet.draftVersion, + ) as LiveApiDataSetSummary[]; + + const canUpdateRelease = release.approvalStatus !== 'Approved'; + + return ( + <> +

API data sets

+ + +

Before you start

+ +

+ API data sets are data sets that can be consumed by third-party + applications via the platform's public API. +

+ +

+ An API data set should ideally be a long-lived data series where the + data structure is expected to remain stable between each release. New + versions of the API data set containing any updates can be published + with future releases. +

+ +

+ If the structure of your data set will not be stable with future + releases, you are advised to avoid using an API data set. Users can + still download and explore your data by creating their own tables. +

+ + {!release.published && ( + + Changes will not be made in the public API until this release has + been published. + + )} + + + + {/* TODO: Update when non-BAU users can create API data sets */} + {user?.permissions.isBauUser && ( + <> + {canUpdateRelease ? ( + + ) : ( + + This release has been approved and API data sets can no longer + be created for it. + + )} + + )} + + {dataSets.length > 0 ? ( + <> + {draftDataSets.length > 0 && ( + <> +

Draft API data sets

+ + + + )} + + {liveDataSets.length > 0 && ( + <> +

Current live API data sets

+ + + + )} + + ) : ( + + No API data sets have been created for this publication. + + )} + + + ); +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetsPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetsPage.test.tsx new file mode 100644 index 00000000000..3b55569546f --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetsPage.test.tsx @@ -0,0 +1,258 @@ +import { AuthContextTestProvider, User } from '@admin/contexts/AuthContext'; +import { testRelease } from '@admin/pages/release/__data__/testRelease'; +import ReleaseApiDataSetsPage from '@admin/pages/release/api-data-sets/ReleaseApiDataSetsPage'; +import { ReleaseContextProvider } from '@admin/pages/release/contexts/ReleaseContext'; +import { + releaseApiDataSetsRoute, + ReleaseRouteParams, +} from '@admin/routes/releaseRoutes'; +import _apiDataSetCandidateService, { + ApiDataSetCandidate, +} from '@admin/services/apiDataSetCandidateService'; +import _apiDataSetService, { + ApiDataSetSummary, +} from '@admin/services/apiDataSetService'; +import { GlobalPermissions } from '@admin/services/authService'; +import { Release } from '@admin/services/releaseService'; +import render, { CustomRenderResult } from '@common-test/render'; +import { screen, within } from '@testing-library/react'; +import { generatePath, MemoryRouter, Route } from 'react-router-dom'; + +jest.mock('@admin/services/apiDataSetService'); +jest.mock('@admin/services/apiDataSetCandidateService'); + +const apiDataSetCandidateService = jest.mocked(_apiDataSetCandidateService); +const apiDataSetService = jest.mocked(_apiDataSetService); + +describe('ReleaseApiDataSetsPage', () => { + const testBauUser: User = { + id: 'user-id-1', + name: 'BAU user', + permissions: { + isBauUser: true, + } as GlobalPermissions, + }; + + const testAnalystUser: User = { + id: 'user-id-1', + name: 'Analyst user', + permissions: { + isBauUser: false, + } as GlobalPermissions, + }; + + const testDataSets: ApiDataSetSummary[] = [ + { + id: 'data-set-1', + title: 'Data set 1 title', + summary: 'Data set 1 summary', + status: 'Published', + draftVersion: { + id: 'version-1', + version: '1.0', + status: 'Draft', + type: 'Major', + }, + }, + { + id: 'data-set-2', + title: 'Data set 2 title', + summary: 'Data set 2 summary', + status: 'Published', + draftVersion: { + id: 'version-2', + version: '1.1', + status: 'Draft', + type: 'Minor', + }, + latestLiveVersion: { + id: 'version-3', + version: '1.0', + status: 'Published', + type: 'Major', + published: '2024-05-01T09:30:00+00:00', + }, + }, + { + id: 'data-set-3', + title: 'Data set 3 title', + summary: 'Data set 3 summary', + status: 'Published', + latestLiveVersion: { + id: 'version-4', + version: '1.0', + status: 'Published', + type: 'Major', + published: '2024-05-01T09:30:00+00:00', + }, + }, + ]; + + const testCandidates: ApiDataSetCandidate[] = [ + { + releaseFileId: 'release-file-1', + title: 'Test data set 1', + }, + { + releaseFileId: 'release-file-2', + title: 'Test data set 2', + }, + ]; + + test('renders draft and live data set tables correctly', async () => { + apiDataSetCandidateService.listCandidates.mockResolvedValue([]); + apiDataSetService.listDataSets.mockResolvedValue(testDataSets); + + renderPage(); + + expect(await screen.findByText('Draft API data sets')).toBeInTheDocument(); + + const draftsTable = within(screen.getByTestId('draft-api-data-sets')); + + const draftRows = draftsTable.getAllByRole('row'); + expect(draftRows).toHaveLength(3); + + const draftRow1Cells = within(draftRows[1]).getAllByRole('cell'); + expect(draftRow1Cells[0]).toHaveTextContent('v1.0'); + expect(draftRow1Cells[1]).toHaveTextContent('N/A'); + expect(draftRow1Cells[2]).toHaveTextContent('Data set 1 title'); + + const draftRow2Cells = within(draftRows[2]).getAllByRole('cell'); + expect(draftRow2Cells[0]).toHaveTextContent('v1.1'); + expect(draftRow2Cells[1]).toHaveTextContent('v1.0'); + expect(draftRow2Cells[2]).toHaveTextContent('Data set 2 title'); + + expect(screen.getByText('Current live API data sets')).toBeInTheDocument(); + + const liveTable = within(screen.getByTestId('live-api-data-sets')); + + const liveRows = liveTable.getAllByRole('row'); + expect(liveRows).toHaveLength(2); + + const liveRows1Cells = within(liveRows[1]).getAllByRole('cell'); + expect(liveRows1Cells[0]).toHaveTextContent('v1.0'); + expect(liveRows1Cells[1]).toHaveTextContent('Data set 3 title'); + }); + + test('renders correctly when no data sets exist', async () => { + apiDataSetCandidateService.listCandidates.mockResolvedValue([]); + apiDataSetService.listDataSets.mockResolvedValue(testDataSets); + + renderPage(); + + expect(await screen.findByText('Draft API data sets')).toBeInTheDocument(); + expect(screen.getByText('Current live API data sets')).toBeInTheDocument(); + + expect( + screen.queryByText( + 'No API data sets have been created for this publication.', + ), + ).not.toBeInTheDocument(); + }); + + test('renders info message when release has not been published', async () => { + apiDataSetCandidateService.listCandidates.mockResolvedValue([]); + apiDataSetService.listDataSets.mockResolvedValue([]); + + renderPage(); + + expect( + await screen.findByText( + 'Changes will not be made in the public API until this release has been published.', + ), + ).toBeInTheDocument(); + }); + + test('renders warning message when release has been approved and data sets cannot be created', async () => { + apiDataSetCandidateService.listCandidates.mockResolvedValue([]); + apiDataSetService.listDataSets.mockResolvedValue([]); + + renderPage({ + release: { + ...testRelease, + approvalStatus: 'Approved', + }, + }); + + expect( + await screen.findByText( + 'This release has been approved and API data sets can no longer be created for it.', + ), + ).toBeInTheDocument(); + + expect( + screen.queryByRole('button', { name: 'Create API data set' }), + ).not.toBeInTheDocument(); + }); + + test('does not render create button when user is not BAU', async () => { + apiDataSetCandidateService.listCandidates.mockResolvedValue([]); + apiDataSetService.listDataSets.mockResolvedValue([]); + + renderPage({ user: testAnalystUser }); + + expect( + await screen.findByText( + 'No API data sets have been created for this publication.', + ), + ).toBeInTheDocument(); + + expect(screen.queryByText('Create API data set')).not.toBeInTheDocument(); + }); + + test('clicking the create button opens modal form to create API data set', async () => { + apiDataSetCandidateService.listCandidates.mockResolvedValue(testCandidates); + apiDataSetService.listDataSets.mockResolvedValue([]); + + const { user } = renderPage(); + + expect(await screen.findByText('Create API data set')).toBeInTheDocument(); + + await user.click( + screen.getByRole('button', { name: 'Create API data set' }), + ); + + expect( + await screen.findByText('Create a new API data set'), + ).toBeInTheDocument(); + + const modal = within(screen.getByRole('dialog')); + + expect( + modal.getByRole('heading', { name: 'Create a new API data set' }), + ).toBeInTheDocument(); + + expect(modal.getByLabelText('Data set')).toBeInTheDocument(); + + expect( + modal.getByRole('button', { name: 'Confirm new API data set' }), + ).toBeInTheDocument(); + }); + + function renderPage(options?: { + release?: Release; + user?: User; + }): CustomRenderResult { + const { release = testRelease, user = testBauUser } = options ?? {}; + + return render( + + + (releaseApiDataSetsRoute.path, { + publicationId: testRelease.publicationId, + releaseId: testRelease.id, + }), + ]} + > + + + + , + ); + } +}); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateForm.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateForm.tsx new file mode 100644 index 00000000000..dc6c1f082fa --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateForm.tsx @@ -0,0 +1,71 @@ +import { ApiDataSetCandidate } from '@admin/services/apiDataSetCandidateService'; +import Button from '@common/components/Button'; +import ButtonGroup from '@common/components/ButtonGroup'; +import ButtonText from '@common/components/ButtonText'; +import { Form, FormFieldSelect } from '@common/components/form'; +import FormProvider from '@common/components/form/FormProvider'; +import LoadingSpinner from '@common/components/LoadingSpinner'; +import Yup from '@common/validation/yup'; +import React, { useMemo } from 'react'; +import { ObjectSchema } from 'yup'; + +export interface ApiDataSetCreateFormValues { + releaseFileId: string; +} + +export interface ApiDataSetCreateFormProps { + dataSetCandidates: ApiDataSetCandidate[]; + onCancel: () => void; + onSubmit: (values: ApiDataSetCreateFormValues) => void; +} + +export default function ApiDataSetCreateForm({ + dataSetCandidates, + onCancel, + onSubmit, +}: ApiDataSetCreateFormProps) { + const validationSchema = useMemo< + ObjectSchema + >(() => { + return Yup.object({ + releaseFileId: Yup.string().required('Choose a data set'), + }); + }, []); + + return ( + + {({ formState }) => { + return ( +
+ + name="releaseFileId" + label="Data set" + options={dataSetCandidates.map(candidate => ({ + label: candidate.title, + value: candidate.releaseFileId, + }))} + placeholder="Choose a data set" + /> + + + + + Cancel + + + + + ); + }} +
+ ); +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx new file mode 100644 index 00000000000..09fd9eade9c --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx @@ -0,0 +1,93 @@ +import Link from '@admin/components/Link'; +import ApiDataSetCreateForm, { + ApiDataSetCreateFormValues, +} from '@admin/pages/release/api-data-sets/components/ApiDataSetCreateForm'; +import apiDataSetCandidateQueries from '@admin/queries/apiDataSetCandidateQueries'; +import { + releaseDataRoute, + ReleaseRouteParams, +} from '@admin/routes/releaseRoutes'; +import apiDataSetService from '@admin/services/apiDataSetService'; +import Button from '@common/components/Button'; +import Modal from '@common/components/Modal'; +import WarningMessage from '@common/components/WarningMessage'; +import useToggle from '@common/hooks/useToggle'; +import { useQuery } from '@tanstack/react-query'; +import React from 'react'; +import { generatePath } from 'react-router-dom'; + +interface Props { + publicationId: string; + releaseId: string; +} + +export default function ApiDataSetCreateModal({ + publicationId, + releaseId, +}: Props) { + const [isOpen, toggleOpen] = useToggle(false); + + const { + data: dataSetCandidates = [], + isLoading, + refetch, + } = useQuery(apiDataSetCandidateQueries.list(releaseId)); + + const handleTriggerClick = async () => { + toggleOpen.on(); + await refetch(); + }; + + const handleSubmit = async ({ + releaseFileId, + }: ApiDataSetCreateFormValues) => { + await apiDataSetService.createDataSet({ + releaseFileId, + }); + }; + + if (isLoading) { + return null; + } + + return ( + Create API data set + } + > +

+ Select a data set to become an API data set. This will be made available + for third-party applications to consume via the public API. +

+ + {dataSetCandidates.length > 0 ? ( + + ) : ( + <> + + No API data sets can be created as there are no candidates data + files available. New candidate data files can be uploaded in the{' '} + (releaseDataRoute.path, { + publicationId, + releaseId, + })} + > + Data and files + {' '} + section. + + + + + )} + + ); +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.module.scss b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.module.scss new file mode 100644 index 00000000000..638c675806a --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.module.scss @@ -0,0 +1,29 @@ +@import '~govuk-frontend/dist/govuk/base'; + +.table { + thead th { + vertical-align: bottom; + } + + td { + padding-bottom: govuk-spacing(4); + padding-top: govuk-spacing(4); + vertical-align: middle; + } + + @include govuk-media-query($from: tablet) { + td { + padding-bottom: govuk-spacing(5); + padding-top: govuk-spacing(5); + } + } +} + +.rowHighlight { + border-left: 5px solid govuk-colour('red'); +} + +.versionTag { + text-align: center; + width: 100%; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.tsx new file mode 100644 index 00000000000..319195d2fbe --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.tsx @@ -0,0 +1,128 @@ +import Link from '@admin/components/Link'; +import getVersionStatusTagColour from '@admin/pages/release/api-data-sets/utils/getVersionStatusColour'; +import getVersionStatusText from '@admin/pages/release/api-data-sets/utils/getVersionStatusText'; +import { + releaseApiDataSetDetailsRoute, + ReleaseDataSetRouteParams, +} from '@admin/routes/releaseRoutes'; +import { + ApiDataSetDraftVersionSummary, + ApiDataSetSummary, +} from '@admin/services/apiDataSetService'; +import ButtonGroup from '@common/components/ButtonGroup'; +import ButtonText from '@common/components/ButtonText'; +import InsetText from '@common/components/InsetText'; +import Tag from '@common/components/Tag'; +import VisuallyHidden from '@common/components/VisuallyHidden'; +import classNames from 'classnames'; +import orderBy from 'lodash/orderBy'; +import React from 'react'; +import { generatePath } from 'react-router-dom'; +import styles from './DraftApiDataSetsTable.module.scss'; + +export interface DraftApiDataSetSummary extends ApiDataSetSummary { + draftVersion: ApiDataSetDraftVersionSummary; +} + +interface Props { + dataSets: DraftApiDataSetSummary[]; + publicationId: string; + releaseId: string; +} + +export default function DraftApiDataSetsTable({ + dataSets, + publicationId, + releaseId, +}: Props) { + if (!dataSets.length) { + return No draft API data sets for this publication.; + } + + const hasLiveDataSets = dataSets.some(dataSet => dataSet.latestLiveVersion); + const orderedDataSets = orderBy(dataSets, dataSet => dataSet.title); + + return ( + + + + + {hasLiveDataSets && } + + + + + + + {orderedDataSets.map( + ({ draftVersion, latestLiveVersion, ...dataSet }) => ( + + + {hasLiveDataSets ? ( + + ) : null} + + + + + ), + )} + +
+ Draft version + Live versionNameStatusActions
+ + {`v${draftVersion.version}`} + + + + {latestLiveVersion?.version + ? `v${latestLiveVersion?.version}` + : 'N/A'} + + {dataSet.title} + + {getVersionStatusText(draftVersion.status)} + + + + {draftVersion.status !== 'Processing' && ( + + Remove draft + for {dataSet.title} + + )} + ( + releaseApiDataSetDetailsRoute.path, + { + publicationId, + releaseId, + dataSetId: dataSet.id, + }, + )} + > + View / edit draft + for {dataSet.title} + + +
+ ); +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.module.scss b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.module.scss new file mode 100644 index 00000000000..297444aede3 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.module.scss @@ -0,0 +1,9 @@ +@import '~govuk-frontend/dist/govuk/base'; + +.table { + td { + padding-bottom: govuk-spacing(4); + padding-top: govuk-spacing(4); + vertical-align: middle; + } +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.tsx new file mode 100644 index 00000000000..0548f49e88d --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.tsx @@ -0,0 +1,93 @@ +import Link from '@admin/components/Link'; +import { + releaseApiDataSetDetailsRoute, + ReleaseDataSetRouteParams, +} from '@admin/routes/releaseRoutes'; +import { + ApiDataSetLiveVersionSummary, + ApiDataSetSummary, +} from '@admin/services/apiDataSetService'; +import Button from '@common/components/Button'; +import ButtonGroup from '@common/components/ButtonGroup'; +import InsetText from '@common/components/InsetText'; +import Tag from '@common/components/Tag'; +import VisuallyHidden from '@common/components/VisuallyHidden'; +import orderBy from 'lodash/orderBy'; +import React from 'react'; +import { generatePath } from 'react-router-dom'; +import styles from './LiveApiDataSetsTable.module.scss'; + +export interface LiveApiDataSetSummary extends ApiDataSetSummary { + latestLiveVersion: ApiDataSetLiveVersionSummary; +} + +interface Props { + // TODO: EES-4374 Remove when new versions can be created + canCreateNewVersions?: boolean; + canUpdateRelease?: boolean; + dataSets: LiveApiDataSetSummary[]; + publicationId: string; + releaseId: string; +} + +export default function LiveApiDataSetsTable({ + canCreateNewVersions, + canUpdateRelease, + dataSets, + publicationId, + releaseId, +}: Props) { + if (!dataSets.length) { + return No live API data sets for this publication.; + } + + const orderedDataSets = orderBy(dataSets, dataSet => dataSet.title); + + return ( + + + + + + + + + + {orderedDataSets.map(({ latestLiveVersion, ...dataSet }) => ( + + + + + + ))} + +
VersionNameActions
+ {`v${latestLiveVersion.version}`} + {dataSet.title} + + ( + releaseApiDataSetDetailsRoute.path, + { + publicationId, + releaseId, + dataSetId: dataSet.id, + }, + )} + > + View details + for {dataSet.title} + + {canUpdateRelease && canCreateNewVersions && ( + + )} + +
+ ); +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateForm.test.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateForm.test.tsx new file mode 100644 index 00000000000..6eca941006d --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateForm.test.tsx @@ -0,0 +1,151 @@ +import ApiDataSetCreateForm, { + ApiDataSetCreateFormProps, +} from '@admin/pages/release/api-data-sets/components/ApiDataSetCreateForm'; +import { ApiDataSetCandidate } from '@admin/services/apiDataSetCandidateService'; +import render from '@common-test/render'; +import { screen, waitFor, within } from '@testing-library/react'; +import noop from 'lodash/noop'; + +describe('ApiDataSetCreateForm', () => { + const testCandidates: ApiDataSetCandidate[] = [ + { + releaseFileId: 'release-file-id-1', + title: 'Test data set 1', + }, + { + releaseFileId: 'release-file-id-2', + title: 'Test data set 2', + }, + { + releaseFileId: 'release-file-id-3', + title: 'Test data set 2', + }, + ]; + + test('renders form correctly', () => { + render( + , + ); + + const options = within(screen.getByLabelText('Data set')).getAllByRole( + 'option', + ); + + expect(options).toHaveLength(4); + + expect(options[0]).toHaveValue(''); + expect(options[0]).toHaveTextContent('Choose a data set'); + + expect(options[1]).toHaveValue(testCandidates[0].releaseFileId); + expect(options[1]).toHaveTextContent(testCandidates[0].title); + + expect(options[2]).toHaveValue(testCandidates[1].releaseFileId); + expect(options[2]).toHaveTextContent(testCandidates[1].title); + + expect(options[3]).toHaveValue(testCandidates[2].releaseFileId); + expect(options[3]).toHaveTextContent(testCandidates[2].title); + }); + + test('shows validation error when no data set selected', async () => { + const { user } = render( + , + ); + + await user.click(screen.getByLabelText('Data set')); + await user.tab(); + + expect( + await screen.findByText('Choose a data set', { + selector: '#apiDataSetCreateForm-releaseFileId-error', + }), + ).toBeInTheDocument(); + }); + + test('shows validation error if submitted without selecting data set', async () => { + const { user } = render( + , + ); + + await user.click( + screen.getByRole('button', { name: 'Confirm new API data set' }), + ); + + expect( + await screen.findByText('Choose a data set', { + selector: '#apiDataSetCreateForm-releaseFileId-error', + }), + ).toBeInTheDocument(); + }); + + test('submitting form successfully calls `onSubmit` handler', async () => { + const handleSubmit = jest.fn(); + + const { user } = render( + , + ); + + await user.selectOptions( + screen.getByLabelText('Data set'), + 'Test data set 2', + ); + + expect(handleSubmit).not.toHaveBeenCalled(); + + await user.click( + screen.getByRole('button', { name: 'Confirm new API data set' }), + ); + + await waitFor(() => { + expect(handleSubmit).toHaveBeenCalledTimes(1); + expect(handleSubmit).toHaveBeenCalledWith< + Parameters + >({ + releaseFileId: testCandidates[1].releaseFileId, + }); + }); + }); + + test('submitting form with validation error does not call `onSubmit` handler', async () => { + const handleSubmit = jest.fn(); + + const { user } = render( + , + ); + + expect(handleSubmit).not.toHaveBeenCalled(); + + await user.click( + screen.getByRole('button', { name: 'Confirm new API data set' }), + ); + + expect( + await screen.findByText('Choose a data set', { + selector: '#apiDataSetCreateForm-releaseFileId-error', + }), + ).toBeInTheDocument(); + + await waitFor(() => { + expect(handleSubmit).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx new file mode 100644 index 00000000000..0b289950f33 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx @@ -0,0 +1,127 @@ +import ApiDataSetCreateModal from '@admin/pages/release/api-data-sets/components/ApiDataSetCreateModal'; +import _apiDataSetCandidateService, { + ApiDataSetCandidate, +} from '@admin/services/apiDataSetCandidateService'; +import _apiDataSetService from '@admin/services/apiDataSetService'; +import baseRender from '@common-test/render'; +import { screen, waitFor } from '@testing-library/react'; +import { ReactElement } from 'react'; +import { MemoryRouter } from 'react-router-dom'; + +jest.mock('@admin/services/apiDataSetService'); +jest.mock('@admin/services/apiDataSetCandidateService'); + +const apiDataSetCandidateService = jest.mocked(_apiDataSetCandidateService); +const apiDataSetService = jest.mocked(_apiDataSetService); + +describe('ApiDataSetCreateModal', () => { + const testCandidates: ApiDataSetCandidate[] = [ + { + releaseFileId: 'release-file-1', + title: 'Test data set 1', + }, + { + releaseFileId: 'release-file-2', + title: 'Test data set 2', + }, + ]; + + test('renders warning message in modal when no candidates', async () => { + apiDataSetCandidateService.listCandidates.mockResolvedValue([]); + + const { user } = render( + , + ); + + await user.click( + await screen.findByRole('button', { name: 'Create API data set' }), + ); + + expect( + await screen.findByText( + /No API data sets can be created as there are no candidates data files available/, + ), + ).toBeInTheDocument(); + + expect(screen.getByRole('button', { name: 'Close' })).toBeInTheDocument(); + }); + + test('renders form in modal when there are candidates', async () => { + apiDataSetCandidateService.listCandidates.mockResolvedValue(testCandidates); + + const { user } = render( + , + ); + + expect(await screen.findByText('Create API data set')).toBeInTheDocument(); + + await user.click( + screen.getByRole('button', { name: 'Create API data set' }), + ); + + expect( + await screen.findByText('Create a new API data set'), + ).toBeInTheDocument(); + + expect( + screen.getByRole('heading', { name: 'Create a new API data set' }), + ).toBeInTheDocument(); + + expect(screen.getByLabelText('Data set')).toBeInTheDocument(); + + expect( + screen.getByRole('button', { name: 'Confirm new API data set' }), + ).toBeInTheDocument(); + + expect( + screen.queryByRole('button', { name: 'Close' }), + ).not.toBeInTheDocument(); + }); + + test('submitting the form calls correct service', async () => { + apiDataSetCandidateService.listCandidates.mockResolvedValue(testCandidates); + + const { user } = render( + , + ); + + expect(await screen.findByText('Create API data set')).toBeInTheDocument(); + + await user.click( + screen.getByRole('button', { name: 'Create API data set' }), + ); + + await user.selectOptions( + await screen.findByLabelText('Data set'), + testCandidates[0].releaseFileId, + ); + + expect(apiDataSetService.createDataSet).not.toHaveBeenCalled(); + + await user.click( + screen.getByRole('button', { name: 'Confirm new API data set' }), + ); + + await waitFor(() => { + expect(apiDataSetService.createDataSet).toHaveBeenCalledTimes(1); + expect(apiDataSetService.createDataSet).toHaveBeenCalledWith< + Parameters + >({ + releaseFileId: testCandidates[0].releaseFileId, + }); + }); + }); + + function render(ui: ReactElement) { + return baseRender({ui}); + } +}); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/DraftApiDataSetsTable.test.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/DraftApiDataSetsTable.test.tsx new file mode 100644 index 00000000000..2f262e3d523 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/DraftApiDataSetsTable.test.tsx @@ -0,0 +1,253 @@ +import DraftApiDataSetsTable, { + DraftApiDataSetSummary, +} from '@admin/pages/release/api-data-sets/components/DraftApiDataSetsTable'; +import { render as baseRender, screen, within } from '@testing-library/react'; +import { ReactNode } from 'react'; +import { MemoryRouter } from 'react-router-dom'; + +describe('DraftApiDataSetsTable', () => { + const testDataSets: DraftApiDataSetSummary[] = [ + { + id: 'data-set-4', + title: 'Data set 4 title', + summary: 'Data set 4 summary', + status: 'Published', + draftVersion: { + id: 'version-6', + version: '1.0', + status: 'Draft', + type: 'Major', + }, + }, + { + id: 'data-set-3', + title: 'Data set 3 title', + summary: 'Data set 3 summary', + status: 'Published', + draftVersion: { + id: 'version-5', + version: '1.0', + status: 'Processing', + type: 'Major', + }, + }, + { + id: 'data-set-2', + title: 'Data set 2 title', + summary: 'Data set 2 summary', + status: 'Published', + draftVersion: { + id: 'version-4', + version: '2.0', + status: 'Draft', + type: 'Major', + }, + latestLiveVersion: { + published: '2024-02-01T09:30:00+00:00', + id: 'version-3', + version: '1.0', + status: 'Published', + type: 'Major', + }, + }, + { + id: 'data-set-1', + title: 'Data set 1 title', + summary: 'Data set 1 summary', + status: 'Published', + draftVersion: { + id: 'version-2', + version: '1.1', + status: 'Mapping', + type: 'Minor', + }, + latestLiveVersion: { + published: '2024-02-01T09:30:00+00:00', + id: 'version-1', + version: '1.0', + status: 'Published', + type: 'Major', + }, + }, + { + id: 'data-set-6', + title: 'Data set 6 title', + summary: 'Data set 6 summary', + status: 'Published', + draftVersion: { + id: 'version-8', + version: '1.0', + status: 'Cancelled', + type: 'Major', + }, + }, + { + id: 'data-set-5', + title: 'Data set 5 title', + summary: 'Data set 5 summary', + status: 'Published', + draftVersion: { + id: 'version-7', + version: '1.0', + status: 'Failed', + type: 'Major', + }, + }, + ]; + + test('renders draft data set rows correctly', () => { + render( + , + ); + + const baseDataSetUrl = + '/publication/publication-1/release/release-1/api-data-sets'; + + const rows = within(screen.getByRole('table')).getAllByRole('row'); + + expect(rows).toHaveLength(7); + + // Row 1 + + const row1Cells = within(rows[1]).getAllByRole('cell'); + + expect(row1Cells[0]).toHaveTextContent('v1.1'); + expect(row1Cells[1]).toHaveTextContent('v1.0'); + expect(row1Cells[2]).toHaveTextContent('Data set 1 title'); + expect(row1Cells[3]).toHaveTextContent('Action required'); + + expect( + within(row1Cells[4]).getByRole('link', { + name: 'View / edit draft for Data set 1 title', + }), + ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-1`); + expect( + within(row1Cells[4]).getByRole('button', { + name: 'Remove draft for Data set 1 title', + }), + ).toBeInTheDocument(); + + // Row 2 + + const row2Cells = within(rows[2]).getAllByRole('cell'); + + expect(row2Cells[0]).toHaveTextContent('v2.0'); + expect(row2Cells[1]).toHaveTextContent('v1.0'); + expect(row2Cells[2]).toHaveTextContent('Data set 2 title'); + expect(row2Cells[3]).toHaveTextContent('Ready'); + + expect( + within(row2Cells[4]).getByRole('link', { + name: 'View / edit draft for Data set 2 title', + }), + ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-2`); + + expect( + within(row2Cells[4]).getByRole('button', { + name: 'Remove draft for Data set 2 title', + }), + ).toBeInTheDocument(); + + // Row 3 + + const row3Cells = within(rows[3]).getAllByRole('cell'); + + expect(row3Cells[0]).toHaveTextContent('v1.0'); + expect(row3Cells[1]).toHaveTextContent('N/A'); + expect(row3Cells[2]).toHaveTextContent('Data set 3 title'); + expect(row3Cells[3]).toHaveTextContent('Processing'); + + expect( + within(row3Cells[4]).getByRole('link', { + name: 'View / edit draft for Data set 3 title', + }), + ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-3`); + expect( + within(row3Cells[4]).queryByRole('button', { name: /Remove draft/ }), + ).not.toBeInTheDocument(); + + // Row 4 + + const row4Cells = within(rows[4]).getAllByRole('cell'); + + expect(row4Cells[0]).toHaveTextContent('v1.0'); + expect(row4Cells[1]).toHaveTextContent('N/A'); + expect(row4Cells[2]).toHaveTextContent('Data set 4 title'); + expect(row4Cells[3]).toHaveTextContent('Ready'); + + expect( + within(row4Cells[4]).getByRole('link', { + name: 'View / edit draft for Data set 4 title', + }), + ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-4`); + + expect( + within(row4Cells[4]).getByRole('button', { + name: 'Remove draft for Data set 4 title', + }), + ).toBeInTheDocument(); + + // Row 5 + + const row5Cells = within(rows[5]).getAllByRole('cell'); + + expect(row5Cells[0]).toHaveTextContent('v1.0'); + expect(row5Cells[1]).toHaveTextContent('N/A'); + expect(row5Cells[2]).toHaveTextContent('Data set 5 title'); + expect(row5Cells[3]).toHaveTextContent('Failed'); + + expect( + within(row5Cells[4]).getByRole('link', { + name: 'View / edit draft for Data set 5 title', + }), + ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-5`); + + expect( + within(row5Cells[4]).getByRole('button', { + name: 'Remove draft for Data set 5 title', + }), + ).toBeInTheDocument(); + + // Row 6 + + const row6Cells = within(rows[6]).getAllByRole('cell'); + + expect(row6Cells[0]).toHaveTextContent('v1.0'); + expect(row6Cells[1]).toHaveTextContent('N/A'); + expect(row6Cells[2]).toHaveTextContent('Data set 6 title'); + expect(row6Cells[3]).toHaveTextContent('Cancelled'); + + expect( + within(row6Cells[4]).getByRole('link', { + name: 'View / edit draft for Data set 6 title', + }), + ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-6`); + + expect( + within(row6Cells[4]).getByRole('button', { + name: 'Remove draft for Data set 6 title', + }), + ).toBeInTheDocument(); + }); + + test('renders message when no data sets', () => { + render( + , + ); + + expect(screen.getByText(/No draft API data sets/)).toBeInTheDocument(); + expect(screen.queryByRole('table')).not.toBeInTheDocument(); + }); + + function render(element: ReactNode) { + return baseRender({element}); + } +}); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/LiveApiDataSetsTable.test.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/LiveApiDataSetsTable.test.tsx new file mode 100644 index 00000000000..cc862d37a25 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/LiveApiDataSetsTable.test.tsx @@ -0,0 +1,159 @@ +import LiveApiDataSetsTable, { + LiveApiDataSetSummary, +} from '@admin/pages/release/api-data-sets/components/LiveApiDataSetsTable'; +import { render as baseRender, screen, within } from '@testing-library/react'; +import { ReactNode } from 'react'; +import { MemoryRouter } from 'react-router-dom'; + +describe('LiveApiDataSetsTable', () => { + const testDataSets: LiveApiDataSetSummary[] = [ + { + id: 'data-set-2', + title: 'Data set 2 title', + summary: 'Data set 2 summary', + status: 'Published', + latestLiveVersion: { + published: '2024-02-01T09:30:00+00:00', + id: 'version-2', + version: '1.0', + status: 'Published', + type: 'Major', + }, + }, + { + id: 'data-set-1', + title: 'Data set 1 title', + summary: 'Data set 1 summary', + status: 'Published', + latestLiveVersion: { + published: '2024-02-01T09:30:00+00:00', + id: 'version-1', + version: '2.0', + status: 'Published', + type: 'Major', + }, + }, + { + id: 'data-set-3', + title: 'Data set 3 title', + summary: 'Data set 3 summary', + status: 'Published', + latestLiveVersion: { + published: '2024-02-01T09:30:00+00:00', + id: 'version-3', + version: '1.2', + status: 'Published', + type: 'Minor', + }, + }, + ]; + + test('renders live data set rows correctly', () => { + render( + , + ); + + const baseDataSetUrl = + '/publication/publication-1/release/release-1/api-data-sets'; + + const rows = within(screen.getByRole('table')).getAllByRole('row'); + + expect(rows).toHaveLength(4); + + // Row 1 + + const row1Cells = within(rows[1]).getAllByRole('cell'); + + expect(row1Cells[0]).toHaveTextContent('v2.0'); + expect(row1Cells[1]).toHaveTextContent('Data set 1 title'); + + expect( + within(row1Cells[2]).getByRole('link', { + name: 'View details for Data set 1 title', + }), + ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-1`); + + expect( + within(row1Cells[2]).getByRole('button', { + name: 'Create new version for Data set 1 title', + }), + ).toBeInTheDocument(); + + // Row 2 + + const row2Cells = within(rows[2]).getAllByRole('cell'); + + expect(row2Cells[0]).toHaveTextContent('v1.0'); + expect(row2Cells[1]).toHaveTextContent('Data set 2 title'); + + expect( + within(row2Cells[2]).getByRole('link', { + name: 'View details for Data set 2 title', + }), + ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-2`); + + expect( + within(row2Cells[2]).getByRole('button', { + name: 'Create new version for Data set 2 title', + }), + ).toBeInTheDocument(); + + // Row 3 + + const row3Cells = within(rows[3]).getAllByRole('cell'); + + expect(row3Cells[0]).toHaveTextContent('v1.2'); + expect(row3Cells[1]).toHaveTextContent('Data set 3 title'); + + expect( + within(row3Cells[2]).getByRole('link', { + name: 'View details for Data set 3 title', + }), + ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-3`); + + expect( + within(row3Cells[2]).getByRole('button', { + name: 'Create new version for Data set 3 title', + }), + ).toBeInTheDocument(); + }); + + test("does not render 'Create new version' buttons when release cannot be updated", () => { + render( + , + ); + + expect( + screen.queryAllByRole('button', { name: /Create new version/ }), + ).toHaveLength(0); + }); + + test('renders message when no data sets', () => { + render( + , + ); + + expect(screen.getByText(/No live API data sets/)).toBeInTheDocument(); + expect(screen.queryByRole('table')).not.toBeInTheDocument(); + }); + + function render(element: ReactNode) { + return baseRender({element}); + } +}); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusColour.ts b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusColour.ts new file mode 100644 index 00000000000..9e34a287579 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusColour.ts @@ -0,0 +1,25 @@ +import { DataSetVersionStatus } from '@admin/services/apiDataSetService'; +import { TagProps } from '@common/components/Tag'; + +export default function getVersionStatusTagColour( + status: DataSetVersionStatus, +): TagProps['colour'] { + switch (status) { + case 'Published': + return 'blue'; + case 'Deprecated': + return 'light-blue'; + case 'Withdrawn': + return 'grey'; + case 'Draft': + return 'green'; + case 'Cancelled': + case 'Failed': + case 'Mapping': + return 'red'; + case 'Processing': + return 'yellow'; + default: + return 'grey'; + } +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusText.ts b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusText.ts new file mode 100644 index 00000000000..19e2866300d --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusText.ts @@ -0,0 +1,14 @@ +import { DataSetVersionStatus } from '@admin/services/apiDataSetService'; + +export default function getVersionStatusText( + status: DataSetVersionStatus, +): string { + switch (status) { + case 'Draft': + return 'Ready'; + case 'Mapping': + return 'Action required'; + default: + return status; + } +} diff --git a/src/explore-education-statistics-admin/src/queries/apiDataSetCandidateQueries.ts b/src/explore-education-statistics-admin/src/queries/apiDataSetCandidateQueries.ts new file mode 100644 index 00000000000..e48485af4cf --- /dev/null +++ b/src/explore-education-statistics-admin/src/queries/apiDataSetCandidateQueries.ts @@ -0,0 +1,14 @@ +import apiDataSetCandidateService from '@admin/services/apiDataSetCandidateService'; +import { createQueryKeys } from '@lukemorales/query-key-factory'; + +const apiDataSetCandidateQueries = createQueryKeys('apiDataSetCandidates', { + list(releaseVersionId: string) { + return { + queryKey: [releaseVersionId], + queryFn: () => + apiDataSetCandidateService.listCandidates(releaseVersionId), + }; + }, +}); + +export default apiDataSetCandidateQueries; diff --git a/src/explore-education-statistics-admin/src/queries/apiDataSetQueries.ts b/src/explore-education-statistics-admin/src/queries/apiDataSetQueries.ts new file mode 100644 index 00000000000..d02e85d1111 --- /dev/null +++ b/src/explore-education-statistics-admin/src/queries/apiDataSetQueries.ts @@ -0,0 +1,13 @@ +import apiDataSetService from '@admin/services/apiDataSetService'; +import { createQueryKeys } from '@lukemorales/query-key-factory'; + +const apiDataSetQueries = createQueryKeys('apiDataSetQueries', { + list(publicationId: string) { + return { + queryKey: [publicationId], + queryFn: () => apiDataSetService.listDataSets(publicationId), + }; + }, +}); + +export default apiDataSetQueries; diff --git a/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts b/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts index 3e69825a46b..0964e00055e 100644 --- a/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts +++ b/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts @@ -1,4 +1,5 @@ import { ProtectedRouteProps } from '@admin/components/ProtectedRoute'; +import ReleaseApiDataSetsPage from '@admin/pages/release/api-data-sets/ReleaseApiDataSetsPage'; import ReleaseContentPage from '@admin/pages/release/content/ReleaseContentPage'; import ReleaseDataFilePage from '@admin/pages/release/data/ReleaseDataFilePage'; import ReleaseAncillaryFilePage from '@admin/pages/release/data/ReleaseAncillaryFilePage'; @@ -42,6 +43,10 @@ export type ReleaseFootnoteRouteParams = ReleaseRouteParams & { footnoteId: string; }; +export type ReleaseDataSetRouteParams = ReleaseRouteParams & { + dataSetId: string; +}; + export interface ReleaseRouteProps extends ProtectedRouteProps { title: string; path: string; @@ -154,3 +159,16 @@ export const releasePreReleaseAccessRoute: ReleaseRouteProps = { title: 'Pre-release access', component: ReleasePreReleaseAccessPage, }; + +export const releaseApiDataSetsRoute: ReleaseRouteProps = { + path: '/publication/:publicationId/release/:releaseId/api-data-sets', + title: 'API data sets', + component: ReleaseApiDataSetsPage, + protectionAction: permissions => permissions.isBauUser, +}; + +export const releaseApiDataSetDetailsRoute: ReleaseRouteProps = { + path: '/publication/:publicationId/release/:releaseId/api-data-sets/:dataSetId', + title: 'API data set details', + protectionAction: permissions => permissions.isBauUser, +}; diff --git a/src/explore-education-statistics-admin/src/services/apiDataSetCandidateService.ts b/src/explore-education-statistics-admin/src/services/apiDataSetCandidateService.ts new file mode 100644 index 00000000000..a750dcab370 --- /dev/null +++ b/src/explore-education-statistics-admin/src/services/apiDataSetCandidateService.ts @@ -0,0 +1,18 @@ +import client from '@admin/services/utils/service'; + +export interface ApiDataSetCandidate { + releaseFileId: string; + title: string; +} + +const apiDataSetCandidateService = { + listCandidates(releaseVersionId: string): Promise { + return client.get(`/public-data/data-set-candidates`, { + params: { + releaseVersionId, + }, + }); + }, +}; + +export default apiDataSetCandidateService; diff --git a/src/explore-education-statistics-admin/src/services/apiDataSetService.ts b/src/explore-education-statistics-admin/src/services/apiDataSetService.ts new file mode 100644 index 00000000000..940f24a0eff --- /dev/null +++ b/src/explore-education-statistics-admin/src/services/apiDataSetService.ts @@ -0,0 +1,109 @@ +import { IdTitlePair } from '@admin/services/types/common'; +import client from '@admin/services/utils/service'; +import { PaginatedList } from '@common/services/types/pagination'; + +export interface ApiDataSetSummary { + id: string; + title: string; + summary: string; + status: DataSetStatus; + supersedingDataSetId?: string; + draftVersion?: ApiDataSetDraftVersionSummary; + latestLiveVersion?: ApiDataSetLiveVersionSummary; +} + +export interface ApiDataSetVersionSummary { + id: string; + version: string; + type: DataSetVersionType; +} + +export interface ApiDataSetDraftVersionSummary + extends ApiDataSetVersionSummary { + status: DataSetDraftVersionStatus; +} + +export interface ApiDataSetLiveVersionSummary extends ApiDataSetVersionSummary { + published: string; + status: DataSetLiveVersionStatus; +} + +export interface ApiDataSet { + id: string; + title: string; + summary: string; + status: DataSetStatus; + supersedingDataSetId?: string; + draftVersion?: ApiDataSetDraftVersion; + latestLiveVersion?: ApiDataSetLiveVersion; +} + +export interface ApiDataSetVersion { + id: string; + version: string; + status: DataSetVersionStatus; + type: DataSetVersionType; + dataSetFileId: string; + releaseVersion: IdTitlePair; + totalResults: number; +} + +export interface ApiDataSetDraftVersion extends ApiDataSetVersion { + status: DataSetDraftVersionStatus; + timePeriods?: TimePeriodRange; + geographicLevels?: string[]; + filters?: string[]; + indicators?: string[]; +} + +export interface ApiDataSetLiveVersion extends ApiDataSetVersion { + status: DataSetLiveVersionStatus; + published: string; + timePeriods: TimePeriodRange; + geographicLevels: string[]; + filters: string[]; + indicators: string[]; +} + +export interface TimePeriodRange { + start: string; + end: string; +} + +export type DataSetStatus = 'Draft' | 'Published' | 'Deprecated' | 'Withdrawn'; + +export type DataSetDraftVersionStatus = + | 'Processing' + | 'Failed' + | 'Mapping' + | 'Draft' + | 'Cancelled'; + +export type DataSetLiveVersionStatus = 'Published' | 'Deprecated' | 'Withdrawn'; + +export type DataSetVersionStatus = + | DataSetDraftVersionStatus + | DataSetLiveVersionStatus; + +export type DataSetVersionType = 'Major' | 'Minor'; + +const apiDataSetService = { + async listDataSets(publicationId: string): Promise { + const { results } = await client.get>( + '/public-data/data-sets', + { + params: { + publicationId, + pageSize: 100, + }, + }, + ); + + return results; + }, + createDataSet(data: { releaseFileId: string }): Promise { + return client.post('/public-data/data-sets', data); + }, +}; + +export default apiDataSetService; diff --git a/src/explore-education-statistics-common/test/render.tsx b/src/explore-education-statistics-common/test/render.tsx index 0b6ca4c9f41..d5749496597 100644 --- a/src/explore-education-statistics-common/test/render.tsx +++ b/src/explore-education-statistics-common/test/render.tsx @@ -8,7 +8,7 @@ import userEvent, { UserEvent } from '@testing-library/user-event'; import noop from 'lodash/noop'; import React, { FC, ReactElement, ReactNode } from 'react'; -interface CustomRenderResult extends RenderResult { +export interface CustomRenderResult extends RenderResult { user: UserEvent; } From 532fc3d2d13bc4977733dc2e8a33c06c442479e1 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Wed, 22 May 2024 19:05:36 +0100 Subject: [PATCH 66/73] EES-4366 Add common `TaskList` component --- .../src/components/TaskList.tsx | 23 ++++++++ .../src/components/TaskListItem.module.scss | 25 +++++++++ .../src/components/TaskListItem.tsx | 52 +++++++++++++++++++ .../__tests__/TaskListItem.test.tsx | 42 +++++++++++++++ 4 files changed, 142 insertions(+) create mode 100644 src/explore-education-statistics-common/src/components/TaskList.tsx create mode 100644 src/explore-education-statistics-common/src/components/TaskListItem.module.scss create mode 100644 src/explore-education-statistics-common/src/components/TaskListItem.tsx create mode 100644 src/explore-education-statistics-common/src/components/__tests__/TaskListItem.test.tsx diff --git a/src/explore-education-statistics-common/src/components/TaskList.tsx b/src/explore-education-statistics-common/src/components/TaskList.tsx new file mode 100644 index 00000000000..ddae31f1fea --- /dev/null +++ b/src/explore-education-statistics-common/src/components/TaskList.tsx @@ -0,0 +1,23 @@ +import classNames from 'classnames'; +import React, { ReactNode } from 'react'; + +export interface TaskListProps { + children: ReactNode; + className?: string; + testId?: string; +} + +export default function TaskList({ + children, + className, + testId, +}: TaskListProps) { + return ( +
    + {children} +
+ ); +} diff --git a/src/explore-education-statistics-common/src/components/TaskListItem.module.scss b/src/explore-education-statistics-common/src/components/TaskListItem.module.scss new file mode 100644 index 00000000000..7c3b49beeac --- /dev/null +++ b/src/explore-education-statistics-common/src/components/TaskListItem.module.scss @@ -0,0 +1,25 @@ +@import '~govuk-frontend/dist/govuk/base'; + +.item { + // Can automatically figure this out with CSS instead of having to + // manually apply `govuk-task-list__item--with-link`. + &:has(a, button):hover { + background: govuk-colour('light-grey'); + } +} + +.nameAndHint { + // Copy of `govuk-task-list__link` class that adds a + // transparent box pseudo-element in the child element + // allowing the entire row to be clickable. + a::after, + button::after { + content: ''; + display: block; + position: absolute; + top: 0; + right: 0; + bottom: 0; + left: 0; + } +} diff --git a/src/explore-education-statistics-common/src/components/TaskListItem.tsx b/src/explore-education-statistics-common/src/components/TaskListItem.tsx new file mode 100644 index 00000000000..337910b2b6b --- /dev/null +++ b/src/explore-education-statistics-common/src/components/TaskListItem.tsx @@ -0,0 +1,52 @@ +import classNames from 'classnames'; +import React, { ReactNode } from 'react'; +import styles from './TaskListItem.module.scss'; + +export interface TaskListItemRenderProps { + 'aria-describedby': string; +} + +export interface TaskListItemProps { + children: (props: TaskListItemRenderProps) => ReactNode; + className?: string; + hint?: string; + id: string; + status: ReactNode; +} + +export default function TaskListItem({ + children, + className, + hint, + id, + status, +}: TaskListItemProps) { + const statusId = `${id}-status`; + const hintId = `${id}-hint`; + + return ( +
  • +
    + {children({ + 'aria-describedby': classNames(statusId, { + [hintId]: !!hint, + }), + })} + + {hint && ( +
    + {hint} +
    + )} +
    +
    + {status} +
    +
    • + ); +} diff --git a/src/explore-education-statistics-common/src/components/__tests__/TaskListItem.test.tsx b/src/explore-education-statistics-common/src/components/__tests__/TaskListItem.test.tsx new file mode 100644 index 00000000000..f9cfc096276 --- /dev/null +++ b/src/explore-education-statistics-common/src/components/__tests__/TaskListItem.test.tsx @@ -0,0 +1,42 @@ +import { getAllDescribedBy, getDescribedBy } from '@common-test/queries'; +import TaskListItem from '@common/components/TaskListItem'; +import { render, screen } from '@testing-library/react'; +import React from 'react'; + +describe('TaskListItem', () => { + test('provides correct `aria-describedby` render prop when `hint` is set', () => { + render( + + {props => ( + // eslint-disable-next-line react/jsx-props-no-spreading + + Test + + )} + , + ); + + const descriptors = getAllDescribedBy(screen.getByRole('link')); + + expect(descriptors).toHaveLength(2); + expect(descriptors[0]).toHaveTextContent('Completed'); + expect(descriptors[1]).toHaveTextContent('Test hint'); + }); + + test('provides correct `aria-describedby` render prop when `hint` is not set', () => { + render( + + {props => ( + // eslint-disable-next-line react/jsx-props-no-spreading + + Test + + )} + , + ); + + expect(getDescribedBy(screen.getByRole('link'))).toHaveTextContent( + 'Completed', + ); + }); +}); From 220bc00598ed490c974b23286c0417f64ad55d8b Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Wed, 22 May 2024 23:41:56 +0100 Subject: [PATCH 67/73] EES-4366 Add data set file title to `DataSetVersionViewModel` --- .../Public.Data/DataSetsControllerTests.cs | 20 ++++++++++++------- .../Services/Public.Data/DataSetService.cs | 13 ++++++++++-- .../Public.Data/DataSetVersionViewModels.cs | 2 +- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetsControllerTests.cs index 8d0a4d4083d..f3c73ee4bad 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetsControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetsControllerTests.cs @@ -546,7 +546,8 @@ await TestApp.AddTestData(context => Assert.Equal(liveDataSetVersion.VersionType, viewModel.LatestLiveVersion.Type); Assert.Equal(liveDataSetVersion.TotalResults, viewModel.LatestLiveVersion.TotalResults); Assert.Equal(liveDataSetVersion.Published.TruncateNanoseconds(), viewModel.LatestLiveVersion.Published); - Assert.Equal(liveReleaseFile.File.DataSetFileId, viewModel.LatestLiveVersion.DataSetFileId); + Assert.Equal(liveReleaseFile.File.DataSetFileId, viewModel.LatestLiveVersion.File.Id); + Assert.Equal(liveReleaseFile.Name, viewModel.LatestLiveVersion.File.Title); Assert.Equal( liveDataSetVersion.MetaSummary!.GeographicLevels.Select(l => l.GetEnumLabel()), @@ -569,7 +570,8 @@ await TestApp.AddTestData(context => Assert.Equal(draftDataSetVersion.Status, viewModel.DraftVersion.Status); Assert.Equal(draftDataSetVersion.VersionType, viewModel.DraftVersion.Type); Assert.Equal(draftDataSetVersion.TotalResults, viewModel.DraftVersion.TotalResults); - Assert.Equal(draftReleaseFile.File.DataSetFileId, viewModel.DraftVersion.DataSetFileId); + Assert.Equal(draftReleaseFile.File.DataSetFileId, viewModel.DraftVersion.File.Id); + Assert.Equal(draftReleaseFile.Name, viewModel.DraftVersion.File.Title); Assert.Equal(draftReleaseVersion.Id, viewModel.DraftVersion.ReleaseVersion.Id); Assert.Equal(draftReleaseVersion.Title, viewModel.DraftVersion.ReleaseVersion.Title); @@ -654,8 +656,9 @@ await TestApp.AddTestData(context => Assert.Equal(dataSet.Id, viewModel.Id); - Assert.Equal(file.DataSetFileId, viewModel.LatestLiveVersion!.DataSetFileId); - Assert.Equal(file.DataSetFileId, viewModel.DraftVersion!.DataSetFileId); + Assert.Equal(viewModel.LatestLiveVersion!.File.Id, viewModel.DraftVersion!.File.Id); + Assert.Equal(liveReleaseFile.File.DataSetFileId, viewModel.LatestLiveVersion!.File.Id); + Assert.Equal(draftReleaseFile.File.DataSetFileId, viewModel.DraftVersion!.File.Id); } [Fact] @@ -715,7 +718,8 @@ await TestApp.AddTestData(context => Assert.Equal(dataSetVersion.Version, viewModel.LatestLiveVersion.Version); Assert.Equal(dataSetVersion.Status, viewModel.LatestLiveVersion.Status); Assert.Equal(dataSetVersion.VersionType, viewModel.LatestLiveVersion.Type); - Assert.Equal(releaseFile.File.DataSetFileId, viewModel.LatestLiveVersion.DataSetFileId); + Assert.Equal(releaseFile.File.DataSetFileId, viewModel.LatestLiveVersion!.File.Id); + Assert.Equal(releaseFile.Name, viewModel.LatestLiveVersion!.File.Title); Assert.Null(viewModel.DraftVersion); } @@ -777,7 +781,8 @@ await TestApp.AddTestData(context => Assert.Equal(dataSetVersion.Version, viewModel.DraftVersion.Version); Assert.Equal(dataSetVersion.Status, viewModel.DraftVersion.Status); Assert.Equal(dataSetVersion.VersionType, viewModel.DraftVersion.Type); - Assert.Equal(releaseFile.File.DataSetFileId, viewModel.DraftVersion.DataSetFileId); + Assert.Equal(releaseFile.File.DataSetFileId, viewModel.DraftVersion!.File.Id); + Assert.Equal(releaseFile.Name, viewModel.DraftVersion!.File.Title); Assert.Null(viewModel.LatestLiveVersion); } @@ -992,7 +997,8 @@ await TestApp.AddTestData(context => Assert.Equal(dataSetVersion.Version, content.DraftVersion!.Version); Assert.Equal(dataSetVersion.Status, content.DraftVersion!.Status); Assert.Equal(dataSetVersion.VersionType, content.DraftVersion!.Type); - Assert.Equal(draftReleaseFile.File.DataSetFileId!.Value, content.DraftVersion!.DataSetFileId); + Assert.Equal(draftReleaseFile.File.DataSetFileId, content.DraftVersion!.File.Id); + Assert.Equal(draftReleaseFile.Name, content.DraftVersion!.File.Title); Assert.Equal(draftReleaseVersion.Id, content.DraftVersion!.ReleaseVersion.Id); Assert.Equal(draftReleaseVersion.Title, content.DraftVersion!.ReleaseVersion.Title); Assert.Null(content.DraftVersion!.GeographicLevels); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetService.cs index 26e5f6fcf0c..71074a45602 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetService.cs @@ -204,7 +204,7 @@ private static DataSetVersionViewModel MapDraftVersion( Version = dataSetVersion.Version, Status = dataSetVersion.Status, Type = dataSetVersion.VersionType, - DataSetFileId = releaseFile.File.DataSetFileId!.Value, + File = MapVersionFile(releaseFile), ReleaseVersion = MapReleaseVersion(releaseFile.ReleaseVersion), TotalResults = dataSetVersion.TotalResults, GeographicLevels = dataSetVersion.MetaSummary?.GeographicLevels @@ -228,7 +228,7 @@ private static DataSetLiveVersionViewModel MapLiveVersion( Version = dataSetVersion.Version, Status = dataSetVersion.Status, Type = dataSetVersion.VersionType, - DataSetFileId = releaseFile.File.DataSetFileId!.Value, + File = MapVersionFile(releaseFile), Published = dataSetVersion.Published!.Value, TotalResults = dataSetVersion.TotalResults, ReleaseVersion = MapReleaseVersion(releaseFile.ReleaseVersion), @@ -250,6 +250,15 @@ private static IdTitleViewModel MapReleaseVersion(ReleaseVersion releaseVersion) }; } + private static IdTitleViewModel MapVersionFile(ReleaseFile releaseFile) + { + return new IdTitleViewModel + { + Id = releaseFile.File.DataSetFileId!.Value, + Title = releaseFile.Name ?? string.Empty, + }; + } + private async Task> CheckPublicationExists(Guid publicationId, CancellationToken cancellationToken) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/Public.Data/DataSetVersionViewModels.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/Public.Data/DataSetVersionViewModels.cs index ddb8074bdb0..061783c65e8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/Public.Data/DataSetVersionViewModels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/Public.Data/DataSetVersionViewModels.cs @@ -20,7 +20,7 @@ public record DataSetVersionViewModel [JsonConverter(typeof(StringEnumConverter))] public required DataSetVersionType Type { get; init; } - public required Guid DataSetFileId { get; init; } + public required IdTitleViewModel File { get; init; } public required IdTitleViewModel ReleaseVersion { get; init; } From e34679387857da3fa6676cd9d1d37f85b2173fdd Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 24 May 2024 00:27:43 +0100 Subject: [PATCH 68/73] EES-4366 Add API data set details admin page --- .../pages/release/ReleasePageContainer.tsx | 2 + .../ReleaseApiDataSetDetailsPage.tsx | 197 +++++++++++ .../ReleaseApiDataSetDetailsPage.test.tsx | 299 +++++++++++++++++ .../components/ApiDataSetCreateModal.tsx | 18 +- .../ApiDataSetVersionSummaryList.tsx | 114 +++++++ .../__tests__/ApiDataSetCreateModal.test.tsx | 29 +- .../ApiDataSetVersionSummaryList.test.tsx | 317 ++++++++++++++++++ .../src/queries/apiDataSetQueries.ts | 6 + .../src/routes/releaseRoutes.ts | 2 + .../src/services/apiDataSetService.ts | 5 +- .../src/components/SummaryCard.tsx | 28 ++ .../src/components/SummaryCardAction.tsx | 18 + .../src/components/SummaryList.tsx | 3 + 13 files changed, 1031 insertions(+), 7 deletions(-) create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetDetailsPage.test.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetVersionSummaryList.test.tsx create mode 100644 src/explore-education-statistics-common/src/components/SummaryCard.tsx create mode 100644 src/explore-education-statistics-common/src/components/SummaryCardAction.tsx diff --git a/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx b/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx index 07d0a95c184..18829ac0e6a 100644 --- a/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx @@ -27,6 +27,7 @@ import { releaseSummaryRoute, releaseTableToolRoute, releaseApiDataSetsRoute, + releaseApiDataSetDetailsRoute, } from '@admin/routes/releaseRoutes'; import releaseService from '@admin/services/releaseService'; import LoadingSpinner from '@common/components/LoadingSpinner'; @@ -61,6 +62,7 @@ const routes = [ releaseTableToolRoute, releaseDataBlockCreateRoute, releaseDataBlockEditRoute, + releaseApiDataSetDetailsRoute, ]; interface MatchProps { diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage.tsx new file mode 100644 index 00000000000..9bf3dd0a3d1 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage.tsx @@ -0,0 +1,197 @@ +import Link from '@admin/components/Link'; +import { useConfig } from '@admin/contexts/ConfigContext'; +import ApiDataSetVersionSummaryList from '@admin/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList'; +import { useReleaseContext } from '@admin/pages/release/contexts/ReleaseContext'; +import apiDataSetQueries from '@admin/queries/apiDataSetQueries'; +import { + releaseApiDataSetsRoute, + ReleaseDataSetRouteParams, + ReleaseRouteParams, +} from '@admin/routes/releaseRoutes'; +import ButtonText from '@common/components/ButtonText'; +import LoadingSpinner from '@common/components/LoadingSpinner'; +import SummaryCard from '@common/components/SummaryCard'; +import Tag from '@common/components/Tag'; +import TaskList from '@common/components/TaskList'; +import TaskListItem from '@common/components/TaskListItem'; +import { useQuery } from '@tanstack/react-query'; +import React from 'react'; +import { generatePath, useParams } from 'react-router-dom'; + +// TODO: EES-4370 +const showRemoveDraft = false; +// TODO: Version mapping +const showDraftVersionTasks = false; + +// TODO: EES-4367 +const showChangelog = false; +// TODO: EES-4382 +const showVersionHistory = false; + +export default function ReleaseApiDataSetDetailsPage() { + const { dataSetId } = useParams(); + const { publicAppUrl } = useConfig(); + const { release } = useReleaseContext(); + + const { data: dataSet, isLoading } = useQuery({ + ...apiDataSetQueries.get(dataSetId), + refetchInterval: data => { + return data?.draftVersion?.status === 'Processing' ? 3000 : false; + }, + }); + + const columnSizeClassName = + dataSet?.latestLiveVersion && dataSet?.draftVersion + ? 'govuk-grid-column-one-half-from-desktop' + : 'govuk-grid-column-two-thirds-from-desktop'; + + const draftVersionSummary = dataSet?.draftVersion ? ( + +
  • + Remove draft version +
    • + + ) : undefined + } + /> + ) : null; + + const liveVersionSummary = dataSet?.latestLiveVersion ? ( + +
  • + + View live data set (opens in new tab) + +
    • + {showChangelog && ( +
  • + View changelog and guidance notes +
    • + )} + {showVersionHistory && ( +
  • + View version history +
    • + )} + + } + /> + ) : null; + + return ( + <> + (releaseApiDataSetsRoute.path, { + releaseId: release.id, + publicationId: release.publicationId, + })} + > + Back to API data sets + + + + {dataSet && ( + <> + API data set details +

    {dataSet.title}

    + + {dataSet.draftVersion?.status === 'Mapping' && + showDraftVersionTasks && ( +
    +
    +

    Draft version tasks

    + +

    + To publish the draft version of the API data set, the + following tasks need to be completed: +

    + + + Incomplete} + hint="Define the changes to locations in this version." + > + {props => ( + + Map locations + + )} + + Complete} + hint="Define the changes to filters in this version." + > + {props => ( + + Map filters + + )} + + +
    +
    + )} + +
    + {draftVersionSummary && ( +
    + {dataSet.latestLiveVersion ? ( + + {draftVersionSummary} + + ) : ( + <> +

    Draft version details

    + {draftVersionSummary} + + )} +
    + )} + {liveVersionSummary && ( +
    + {dataSet.draftVersion ? ( + + {liveVersionSummary} + + ) : ( + <> +

    Latest live version details

    + {liveVersionSummary} + + )} +
    + )} +
    + + )} +     + + ); +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetDetailsPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetDetailsPage.test.tsx new file mode 100644 index 00000000000..e16a37c4fd6 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetDetailsPage.test.tsx @@ -0,0 +1,299 @@ +import { TestConfigContextProvider } from '@admin/contexts/ConfigContext'; +import { testRelease } from '@admin/pages/release/__data__/testRelease'; +import ReleaseApiDataSetDetailsPage from '@admin/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage'; +import { ReleaseContextProvider } from '@admin/pages/release/contexts/ReleaseContext'; +import { + releaseApiDataSetDetailsRoute, + ReleaseDataSetRouteParams, +} from '@admin/routes/releaseRoutes'; +import _apiDataSetService, { + ApiDataSet, + ApiDataSetDraftVersion, + ApiDataSetLiveVersion, +} from '@admin/services/apiDataSetService'; +import { Release } from '@admin/services/releaseService'; +import render from '@common-test/render'; +import { screen, within } from '@testing-library/react'; +import React from 'react'; +import { generatePath, MemoryRouter, Route } from 'react-router-dom'; + +jest.mock('@admin/services/apiDataSetService'); + +const apiDataSetService = jest.mocked(_apiDataSetService); + +describe('ReleaseApiDataSetDetailsPage', () => { + const testDataSet: ApiDataSet = { + id: 'data-set-id', + title: 'Data set title', + summary: 'Data set summary', + status: 'Published', + }; + + const testLiveVersion: ApiDataSetLiveVersion = { + id: 'live-version-id', + version: '1.0', + type: 'Minor', + status: 'Published', + totalResults: 10_000, + published: '2024-03-01T09:30:00+00:00', + releaseVersion: { + id: 'release-1-id', + title: 'Test release 1', + }, + file: { + id: 'live-file-id', + title: 'Test live file', + }, + filters: ['Test live filter'], + geographicLevels: ['National', 'Local authority'], + indicators: ['Test live indicator'], + timePeriods: { + start: '2018', + end: '2023', + }, + }; + + const testProcessingVersion: ApiDataSetDraftVersion = { + id: 'processing-version-id', + version: '2.0', + status: 'Processing', + type: 'Minor', + totalResults: 0, + releaseVersion: { + id: 'release-2-id', + title: 'Test release 2', + }, + file: { + id: 'processing-file-id', + title: 'Test processing file', + }, + }; + + const testDraftVersion: ApiDataSetDraftVersion = { + id: 'draft-version-id', + version: '2.0', + status: 'Draft', + type: 'Major', + totalResults: 20_000, + releaseVersion: { + id: 'release-2-id', + title: 'Test release 2', + }, + file: { + id: 'draft-file-id', + title: 'Test draft file', + }, + filters: ['Test draft filter'], + geographicLevels: ['National', 'Local authority'], + indicators: ['Test draft indicator'], + timePeriods: { + start: '2018', + end: '2024', + }, + }; + + test('renders correctly with processing version only', async () => { + apiDataSetService.getDataSet.mockResolvedValue({ + ...testDataSet, + draftVersion: testProcessingVersion, + }); + + renderPage(); + + expect( + await screen.findByText('Draft version details'), + ).toBeInTheDocument(); + + expect( + screen.getByRole('heading', { name: 'Draft version details' }), + ).toBeInTheDocument(); + + const summary = within(screen.getByTestId('draft-version-summary')); + + expect(summary.getByTestId('Version')).toHaveTextContent('v2.0'); + expect(summary.getByTestId('Status')).toHaveTextContent('Processing'); + + expect(summary.queryByTestId('Time periods')).not.toBeInTheDocument(); + expect(summary.queryByTestId('Geographic levels')).not.toBeInTheDocument(); + expect(summary.queryByTestId('Indicators')).not.toBeInTheDocument(); + expect(summary.queryByTestId('Filters')).not.toBeInTheDocument(); + + expect( + screen.queryByRole('heading', { name: 'Latest live version details' }), + ).not.toBeInTheDocument(); + }); + + test('renders correctly with draft version only', async () => { + apiDataSetService.getDataSet.mockResolvedValue({ + ...testDataSet, + draftVersion: testDraftVersion, + }); + + renderPage(); + + expect( + await screen.findByText('Draft version details'), + ).toBeInTheDocument(); + + expect( + screen.getByRole('heading', { name: 'Draft version details' }), + ).toBeInTheDocument(); + + const summary = within(screen.getByTestId('draft-version-summary')); + + expect(summary.getByTestId('Version')).toHaveTextContent('v2.0'); + expect(summary.getByTestId('Status')).toHaveTextContent('Ready'); + expect(summary.getByTestId('Time periods')).toHaveTextContent( + '2018 to 2024', + ); + expect(summary.getByTestId('Geographic levels')).toHaveTextContent( + 'National, Local authority', + ); + expect(summary.getByTestId('Indicators')).toHaveTextContent( + 'Test draft indicator', + ); + expect(summary.getByTestId('Filters')).toHaveTextContent( + 'Test draft filter', + ); + + expect( + screen.queryByRole('heading', { name: 'Latest live version details' }), + ).not.toBeInTheDocument(); + }); + + test('renders correctly with latest live version only', async () => { + apiDataSetService.getDataSet.mockResolvedValue({ + ...testDataSet, + latestLiveVersion: testLiveVersion, + }); + + renderPage(); + + expect( + await screen.findByText('Latest live version details'), + ).toBeInTheDocument(); + + expect( + screen.getByRole('heading', { name: 'Latest live version details' }), + ).toBeInTheDocument(); + + const summary = within(screen.getByTestId('live-version-summary')); + + expect(summary.getByTestId('Version')).toHaveTextContent('v1.0'); + expect(summary.getByTestId('Status')).toHaveTextContent('Published'); + expect(summary.getByTestId('Time periods')).toHaveTextContent( + '2018 to 2023', + ); + expect(summary.getByTestId('Indicators')).toHaveTextContent( + 'Test live indicator', + ); + expect(summary.getByTestId('Filters')).toHaveTextContent( + 'Test live filter', + ); + expect( + within(summary.getByTestId('Actions')).getByRole('link', { + name: /View live data set/, + }), + ).toHaveAttribute( + 'href', + 'http://localhost/data-catalogue/data-set/live-file-id', + ); + + expect( + screen.queryByRole('heading', { name: 'Draft version details' }), + ).not.toBeInTheDocument(); + }); + + test('renders correctly with draft and latest live version', async () => { + apiDataSetService.getDataSet.mockResolvedValue({ + ...testDataSet, + draftVersion: testDraftVersion, + latestLiveVersion: testLiveVersion, + }); + + renderPage(); + + expect( + await screen.findByText('Draft version details'), + ).toBeInTheDocument(); + + // Draft version + + expect( + screen.getByRole('heading', { name: 'Draft version details' }), + ).toBeInTheDocument(); + + const draftSummary = within(screen.getByTestId('draft-version-summary')); + + expect(draftSummary.getByTestId('Version')).toHaveTextContent('v2.0'); + expect(draftSummary.getByTestId('Status')).toHaveTextContent('Ready'); + expect(draftSummary.getByTestId('Time periods')).toHaveTextContent( + '2018 to 2024', + ); + expect(draftSummary.getByTestId('Geographic levels')).toHaveTextContent( + 'National, Local authority', + ); + expect(draftSummary.getByTestId('Indicators')).toHaveTextContent( + 'Test draft indicator', + ); + expect(draftSummary.getByTestId('Filters')).toHaveTextContent( + 'Test draft filter', + ); + + // Latest live version + + expect( + screen.getByRole('heading', { name: 'Latest live version details' }), + ).toBeInTheDocument(); + + const liveSummary = within(screen.getByTestId('live-version-summary')); + + expect(liveSummary.getByTestId('Version')).toHaveTextContent('v1.0'); + expect(liveSummary.getByTestId('Status')).toHaveTextContent('Published'); + expect(liveSummary.getByTestId('Time periods')).toHaveTextContent( + '2018 to 2023', + ); + expect(liveSummary.getByTestId('Indicators')).toHaveTextContent( + 'Test live indicator', + ); + expect(liveSummary.getByTestId('Filters')).toHaveTextContent( + 'Test live filter', + ); + expect( + within(liveSummary.getByTestId('Actions')).getByRole('link', { + name: /View live data set/, + }), + ).toHaveAttribute( + 'href', + 'http://localhost/data-catalogue/data-set/live-file-id', + ); + }); + + function renderPage(options?: { release?: Release; dataSetId?: string }) { + const { release = testRelease, dataSetId = 'data-set-id' } = options ?? {}; + + render( + + + ( + releaseApiDataSetDetailsRoute.path, + { + publicationId: release.publicationId, + releaseId: release.id, + dataSetId, + }, + ), + ]} + > + + + + , + ); + } +}); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx index 09fd9eade9c..ab35b982e58 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx @@ -4,7 +4,9 @@ import ApiDataSetCreateForm, { } from '@admin/pages/release/api-data-sets/components/ApiDataSetCreateForm'; import apiDataSetCandidateQueries from '@admin/queries/apiDataSetCandidateQueries'; import { + releaseApiDataSetDetailsRoute, releaseDataRoute, + ReleaseDataSetRouteParams, ReleaseRouteParams, } from '@admin/routes/releaseRoutes'; import apiDataSetService from '@admin/services/apiDataSetService'; @@ -14,7 +16,7 @@ import WarningMessage from '@common/components/WarningMessage'; import useToggle from '@common/hooks/useToggle'; import { useQuery } from '@tanstack/react-query'; import React from 'react'; -import { generatePath } from 'react-router-dom'; +import { generatePath, useHistory } from 'react-router-dom'; interface Props { publicationId: string; @@ -25,6 +27,7 @@ export default function ApiDataSetCreateModal({ publicationId, releaseId, }: Props) { + const history = useHistory(); const [isOpen, toggleOpen] = useToggle(false); const { @@ -41,9 +44,20 @@ export default function ApiDataSetCreateModal({ const handleSubmit = async ({ releaseFileId, }: ApiDataSetCreateFormValues) => { - await apiDataSetService.createDataSet({ + const dataSet = await apiDataSetService.createDataSet({ releaseFileId, }); + + history.push( + generatePath( + releaseApiDataSetDetailsRoute.path, + { + publicationId, + releaseId, + dataSetId: dataSet.id, + }, + ), + ); }; if (isLoading) { diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList.tsx new file mode 100644 index 00000000000..12692b319cd --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList.tsx @@ -0,0 +1,114 @@ +import Link from '@admin/components/Link'; +import getVersionStatusTagColour from '@admin/pages/release/api-data-sets/utils/getVersionStatusColour'; +import getVersionStatusText from '@admin/pages/release/api-data-sets/utils/getVersionStatusText'; +import { + ReleaseRouteParams, + releaseSummaryRoute, +} from '@admin/routes/releaseRoutes'; +import { + ApiDataSetDraftVersion, + ApiDataSetLiveVersion, +} from '@admin/services/apiDataSetService'; +import CollapsibleList from '@common/components/CollapsibleList'; +import SummaryList from '@common/components/SummaryList'; +import SummaryListItem from '@common/components/SummaryListItem'; +import Tag from '@common/components/Tag'; +import getTimePeriodString from '@common/modules/table-tool/utils/getTimePeriodString'; +import React, { ReactNode } from 'react'; +import { generatePath } from 'react-router'; + +interface ApiDataSetVersionSummaryListProps { + actions?: ReactNode; + collapsibleButtonHiddenText?: string; + dataSetVersion: ApiDataSetDraftVersion | ApiDataSetLiveVersion; + id: string; + publicationId: string; + testId?: string; +} + +export default function ApiDataSetVersionSummaryList({ + actions, + collapsibleButtonHiddenText, + dataSetVersion, + id, + publicationId, + testId = id, +}: ApiDataSetVersionSummaryListProps) { + return ( + + + {`v${dataSetVersion.version}`} + + + + {getVersionStatusText(dataSetVersion.status)} + + + + (releaseSummaryRoute.path, { + releaseId: dataSetVersion.releaseVersion.id, + publicationId, + })} + > + {dataSetVersion.releaseVersion.title} + + + + {dataSetVersion.file.title} + + {dataSetVersion.geographicLevels && ( + + {dataSetVersion.geographicLevels.join(', ')} + + )} + {dataSetVersion.timePeriods && ( + + {getTimePeriodString({ + from: dataSetVersion.timePeriods.start, + to: dataSetVersion.timePeriods.end, + })} + + )} + {dataSetVersion.indicators && dataSetVersion.indicators.length > 0 && ( + + + {dataSetVersion.indicators.map((indicator, index) => ( +
  • {indicator}
    • + ))} +     +     + )} + {dataSetVersion.filters && dataSetVersion.filters.length > 0 && ( + + + {dataSetVersion.filters.map((filter, index) => ( +
  • {filter}
    • + ))} +     +     + )} + {actions ? ( + {actions} + ) : null} +     + ); +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx index 0b289950f33..db67af381bb 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx @@ -5,8 +5,9 @@ import _apiDataSetCandidateService, { import _apiDataSetService from '@admin/services/apiDataSetService'; import baseRender from '@common-test/render'; import { screen, waitFor } from '@testing-library/react'; +import { createMemoryHistory, History } from 'history'; import { ReactElement } from 'react'; -import { MemoryRouter } from 'react-router-dom'; +import { Router } from 'react-router-dom'; jest.mock('@admin/services/apiDataSetService'); jest.mock('@admin/services/apiDataSetCandidateService'); @@ -84,14 +85,23 @@ describe('ApiDataSetCreateModal', () => { ).not.toBeInTheDocument(); }); - test('submitting the form calls correct service', async () => { + test('submitting the form calls correct service and redirects to next page', async () => { apiDataSetCandidateService.listCandidates.mockResolvedValue(testCandidates); + apiDataSetService.createDataSet.mockResolvedValue({ + id: 'data-set-id', + title: 'Test title', + summary: 'Test summary', + status: 'Draft', + }); + + const history = createMemoryHistory(); const { user } = render( , + { history }, ); expect(await screen.findByText('Create API data set')).toBeInTheDocument(); @@ -119,9 +129,20 @@ describe('ApiDataSetCreateModal', () => { releaseFileId: testCandidates[0].releaseFileId, }); }); + + expect(history.location.pathname).toBe( + '/publication/publication-id/release/release-id/api-data-sets/data-set-id', + ); }); - function render(ui: ReactElement) { - return baseRender({ui}); + function render( + ui: ReactElement, + options?: { + history: History; + }, + ) { + const { history = createMemoryHistory() } = options ?? {}; + + return baseRender({ui}); } }); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetVersionSummaryList.test.tsx b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetVersionSummaryList.test.tsx new file mode 100644 index 00000000000..fc10dcfe73f --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetVersionSummaryList.test.tsx @@ -0,0 +1,317 @@ +import ApiDataSetVersionSummaryList from '@admin/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList'; +import { ApiDataSetDraftVersion } from '@admin/services/apiDataSetService'; +import { render as baseRender, screen, within } from '@testing-library/react'; +import { ReactElement } from 'react'; +import { MemoryRouter } from 'react-router-dom'; + +describe('ApiDataSetVersionSummaryList', () => { + const testBaseVersion: ApiDataSetDraftVersion = { + id: 'version-id', + status: 'Draft', + version: '1.0', + type: 'Major', + file: { + id: 'file-id', + title: 'Test data set file', + }, + releaseVersion: { + id: 'release-id', + title: 'Test release', + }, + totalResults: 0, + }; + + test('renders correctly when data set version is missing facets', () => { + render( + , + ); + + expect(screen.getByTestId('Version')).toHaveTextContent('v1.0'); + expect(screen.getByTestId('Status')).toHaveTextContent('Ready'); + expect( + within(screen.getByTestId('Release')).getByRole('link', { + name: 'Test release', + }), + ).toHaveAttribute( + 'href', + '/publication/publication-id/release/release-id/summary', + ); + expect(screen.getByTestId('Data set file')).toHaveTextContent( + 'Test data set file', + ); + + expect(screen.queryByTestId('Geographic levels')).not.toBeInTheDocument(); + expect(screen.queryByTestId('Time periods')).not.toBeInTheDocument(); + expect(screen.queryByTestId('Indicators')).not.toBeInTheDocument(); + expect(screen.queryByTestId('Filters')).not.toBeInTheDocument(); + + expect(screen.queryByTestId('Actions')).not.toBeInTheDocument(); + }); + + test('renders correctly when data set version has facets', () => { + render( + , + ); + + expect(screen.getByTestId('Version')).toHaveTextContent('v1.0'); + expect(screen.getByTestId('Status')).toHaveTextContent('Ready'); + expect( + within(screen.getByTestId('Release')).getByRole('link', { + name: 'Test release', + }), + ).toHaveAttribute( + 'href', + '/publication/publication-id/release/release-id/summary', + ); + expect(screen.getByTestId('Data set file')).toHaveTextContent( + 'Test data set file', + ); + + expect(screen.getByTestId('Geographic levels')).toHaveTextContent( + 'National, Local authority', + ); + expect(screen.getByTestId('Time periods')).toHaveTextContent( + '2018 to 2024', + ); + + const indicators = within(screen.getByTestId('Indicators')).getAllByRole( + 'listitem', + ); + + expect(indicators).toHaveLength(2); + expect(indicators[0]).toHaveTextContent('Test indicator 1'); + expect(indicators[1]).toHaveTextContent('Test indicator 2'); + + const filters = within(screen.getByTestId('Filters')).getAllByRole( + 'listitem', + ); + + expect(filters).toHaveLength(2); + expect(filters[0]).toHaveTextContent('Test filter 1'); + expect(filters[1]).toHaveTextContent('Test filter 2'); + + expect(screen.queryByTestId('Actions')).not.toBeInTheDocument(); + }); + + test('renders indicators as collapsible list', () => { + render( + , + ); + + const indicators = within(screen.getByTestId('Indicators')).getAllByRole( + 'listitem', + ); + + expect(indicators).toHaveLength(3); + expect(indicators[0]).toHaveTextContent('Test indicator 1'); + expect(indicators[1]).toHaveTextContent('Test indicator 2'); + expect(indicators[2]).toHaveTextContent('Test indicator 3'); + + expect( + within(screen.getByTestId('Indicators')).getByRole('button', { + name: 'Show 1 more indicator', + }), + ).toBeInTheDocument(); + }); + + test('renders indicators as pluralised collapsible list', () => { + render( + , + ); + + const indicators = within(screen.getByTestId('Indicators')).getAllByRole( + 'listitem', + ); + + expect(indicators).toHaveLength(3); + expect(indicators[0]).toHaveTextContent('Test indicator 1'); + expect(indicators[1]).toHaveTextContent('Test indicator 2'); + expect(indicators[2]).toHaveTextContent('Test indicator 3'); + + expect( + within(screen.getByTestId('Indicators')).getByRole('button', { + name: 'Show 2 more indicators', + }), + ).toBeInTheDocument(); + }); + + test('renders collapsible indicators list button with hidden text', () => { + render( + , + ); + + expect( + within(screen.getByTestId('Indicators')).getByRole('button', { + name: 'Show 1 more indicator for draft version', + }), + ).toBeInTheDocument(); + }); + + test('renders filters as collapsible list', () => { + render( + , + ); + + const filters = within(screen.getByTestId('Filters')).getAllByRole( + 'listitem', + ); + + expect(filters).toHaveLength(3); + expect(filters[0]).toHaveTextContent('Test filter 1'); + expect(filters[1]).toHaveTextContent('Test filter 2'); + expect(filters[2]).toHaveTextContent('Test filter 3'); + + expect( + within(screen.getByTestId('Filters')).getByRole('button', { + name: 'Show 1 more filter', + }), + ).toBeInTheDocument(); + }); + + test('renders filters as pluralised collapsible list', () => { + render( + , + ); + + const filters = within(screen.getByTestId('Filters')).getAllByRole( + 'listitem', + ); + + expect(filters).toHaveLength(3); + expect(filters[0]).toHaveTextContent('Test filter 1'); + expect(filters[1]).toHaveTextContent('Test filter 2'); + expect(filters[2]).toHaveTextContent('Test filter 3'); + + expect( + within(screen.getByTestId('Filters')).getByRole('button', { + name: 'Show 2 more filters', + }), + ).toBeInTheDocument(); + }); + + test('renders collapsible filters list button with hidden text', () => { + render( + , + ); + + expect( + within(screen.getByTestId('Filters')).getByRole('button', { + name: 'Show 1 more filter for draft version', + }), + ).toBeInTheDocument(); + }); + + test('renders additional actions', () => { + render( + Some action} + />, + ); + + expect( + within(screen.getByTestId('Actions')).getByRole('button', { + name: 'Some action', + }), + ).toBeInTheDocument(); + }); + + function render(ui: ReactElement) { + return baseRender({ui}); + } +}); diff --git a/src/explore-education-statistics-admin/src/queries/apiDataSetQueries.ts b/src/explore-education-statistics-admin/src/queries/apiDataSetQueries.ts index d02e85d1111..72e46741b64 100644 --- a/src/explore-education-statistics-admin/src/queries/apiDataSetQueries.ts +++ b/src/explore-education-statistics-admin/src/queries/apiDataSetQueries.ts @@ -8,6 +8,12 @@ const apiDataSetQueries = createQueryKeys('apiDataSetQueries', { queryFn: () => apiDataSetService.listDataSets(publicationId), }; }, + get(dataSetId: string) { + return { + queryKey: [dataSetId], + queryFn: () => apiDataSetService.getDataSet(dataSetId), + }; + }, }); export default apiDataSetQueries; diff --git a/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts b/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts index 0964e00055e..10ea3f77f86 100644 --- a/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts +++ b/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts @@ -1,4 +1,5 @@ import { ProtectedRouteProps } from '@admin/components/ProtectedRoute'; +import ReleaseApiDataSetDetailsPage from '@admin/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage'; import ReleaseApiDataSetsPage from '@admin/pages/release/api-data-sets/ReleaseApiDataSetsPage'; import ReleaseContentPage from '@admin/pages/release/content/ReleaseContentPage'; import ReleaseDataFilePage from '@admin/pages/release/data/ReleaseDataFilePage'; @@ -170,5 +171,6 @@ export const releaseApiDataSetsRoute: ReleaseRouteProps = { export const releaseApiDataSetDetailsRoute: ReleaseRouteProps = { path: '/publication/:publicationId/release/:releaseId/api-data-sets/:dataSetId', title: 'API data set details', + component: ReleaseApiDataSetDetailsPage, protectionAction: permissions => permissions.isBauUser, }; diff --git a/src/explore-education-statistics-admin/src/services/apiDataSetService.ts b/src/explore-education-statistics-admin/src/services/apiDataSetService.ts index 940f24a0eff..47646aaa87d 100644 --- a/src/explore-education-statistics-admin/src/services/apiDataSetService.ts +++ b/src/explore-education-statistics-admin/src/services/apiDataSetService.ts @@ -43,7 +43,7 @@ export interface ApiDataSetVersion { version: string; status: DataSetVersionStatus; type: DataSetVersionType; - dataSetFileId: string; + file: IdTitlePair; releaseVersion: IdTitlePair; totalResults: number; } @@ -104,6 +104,9 @@ const apiDataSetService = { createDataSet(data: { releaseFileId: string }): Promise { return client.post('/public-data/data-sets', data); }, + getDataSet(dataSetId: string): Promise { + return client.get(`/public-data/data-sets/${dataSetId}`); + }, }; export default apiDataSetService; diff --git a/src/explore-education-statistics-common/src/components/SummaryCard.tsx b/src/explore-education-statistics-common/src/components/SummaryCard.tsx new file mode 100644 index 00000000000..6d328ac803a --- /dev/null +++ b/src/explore-education-statistics-common/src/components/SummaryCard.tsx @@ -0,0 +1,28 @@ +import classNames from 'classnames'; +import React, { ReactNode } from 'react'; + +export interface SummaryCardProps { + actions?: ReactNode; + children: ReactNode; + className?: string; + heading: ReactNode; + headingTag: 'h2' | 'h3' | 'h4'; +} + +export default function SummaryCard({ + actions, + children, + className, + heading, + headingTag: Heading = 'h2', +}: SummaryCardProps) { + return ( +
    +
    + {heading} + {actions &&
      {actions}
    } +
    +
    {children}
    +
    + ); +} diff --git a/src/explore-education-statistics-common/src/components/SummaryCardAction.tsx b/src/explore-education-statistics-common/src/components/SummaryCardAction.tsx new file mode 100644 index 00000000000..48969603bb2 --- /dev/null +++ b/src/explore-education-statistics-common/src/components/SummaryCardAction.tsx @@ -0,0 +1,18 @@ +import classNames from 'classnames'; +import React, { ReactNode } from 'react'; + +interface SummaryCardActionProps { + children: ReactNode; + className?: string; +} + +export default function SummaryCardAction({ + children, + className, +}: SummaryCardActionProps) { + return ( +
  • + {children} +
    • + ); +} diff --git a/src/explore-education-statistics-common/src/components/SummaryList.tsx b/src/explore-education-statistics-common/src/components/SummaryList.tsx index 29d6c07a6ef..71c4b7263f2 100644 --- a/src/explore-education-statistics-common/src/components/SummaryList.tsx +++ b/src/explore-education-statistics-common/src/components/SummaryList.tsx @@ -10,6 +10,7 @@ interface Props { id?: string; noBorder?: boolean; smallKey?: boolean; + testId?: string; } const SummaryList = ({ @@ -20,6 +21,7 @@ const SummaryList = ({ id, noBorder, smallKey = false, + testId, }: Props) => { return (
    {children} From e733599ee4c8369a9f9dabcc1858f7ecd96b6b3f Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Wed, 29 May 2024 18:35:42 +0100 Subject: [PATCH 69/73] EES-4366 Update release context state when publishing completes This updates `ReleasePublishingStatus` so that it can trigger a refetch of the release in the surrounding `ReleaseContext`. --- .../pages/release/ReleasePageContainer.tsx | 20 +-- .../src/pages/release/ReleaseStatusPage.tsx | 20 ++- .../pages/release/ReleaseSummaryEditPage.tsx | 12 +- .../components/ReleasePublishingStages.tsx | 12 +- .../components/ReleasePublishingStatus.tsx | 16 +- .../components/ReleasePublishingStatusTag.tsx | 12 +- .../ReleasePublishingStatus.test.tsx | 138 ++++++++++++++++++ .../pages/release/contexts/ReleaseContext.tsx | 5 +- .../hooks/useReleasePublishingStatus.ts | 51 ++++--- .../src/services/releaseService.ts | 6 +- 10 files changed, 221 insertions(+), 71 deletions(-) create mode 100644 src/explore-education-statistics-admin/src/pages/release/components/__tests__/ReleasePublishingStatus.test.tsx diff --git a/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx b/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx index 18829ac0e6a..04e9e851b65 100644 --- a/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx @@ -6,6 +6,7 @@ import ProtectedRoute from '@admin/components/ProtectedRoute'; import { useAuthContext } from '@admin/contexts/AuthContext'; import { ReleaseContextProvider } from '@admin/pages/release/contexts/ReleaseContext'; import { getReleaseApprovalStatusLabel } from '@admin/pages/release/utils/releaseSummaryUtil'; +import releaseQueries from '@admin/queries/releaseQueries'; import { releaseContentRoute, releaseDataBlockCreateRoute, @@ -29,10 +30,9 @@ import { releaseApiDataSetsRoute, releaseApiDataSetDetailsRoute, } from '@admin/routes/releaseRoutes'; -import releaseService from '@admin/services/releaseService'; import LoadingSpinner from '@common/components/LoadingSpinner'; import Tag from '@common/components/Tag'; -import useAsyncHandledRetry from '@common/hooks/useAsyncHandledRetry'; +import { useQuery } from '@tanstack/react-query'; import React, { useMemo } from 'react'; import { generatePath, RouteComponentProps, Switch } from 'react-router'; import { publicationReleasesRoute } from '@admin/routes/publicationRoutes'; @@ -79,13 +79,10 @@ const ReleasePageContainer = ({ const { user } = useAuthContext(); const { - value: release, - setState: setRelease, + data: release, isLoading: loadingRelease, - } = useAsyncHandledRetry( - () => releaseService.getRelease(releaseId), - [releaseId], - ); + refetch, + } = useQuery(releaseQueries.get(releaseId)); const navRoutes = useMemo(() => { return allNavRoutes.filter(route => { @@ -193,12 +190,7 @@ const ReleasePageContainer = ({ label="Release" /> - { - setRelease({ value: nextRelease }); - }} - > + {routes.map(route => ( diff --git a/src/explore-education-statistics-admin/src/pages/release/ReleaseStatusPage.tsx b/src/explore-education-statistics-admin/src/pages/release/ReleaseStatusPage.tsx index e4f0711fd75..3f89e7639e2 100644 --- a/src/explore-education-statistics-admin/src/pages/release/ReleaseStatusPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/ReleaseStatusPage.tsx @@ -7,7 +7,10 @@ import ReleaseStatusEditPage from '@admin/pages/release/ReleaseStatusEditPage'; import permissionService, { ReleaseStatusPermissions, } from '@admin/services/permissionService'; -import releaseService, { ReleaseStatus } from '@admin/services/releaseService'; +import releaseService, { + ReleaseStageStatus, + ReleaseStatus, +} from '@admin/services/releaseService'; import Button from '@common/components/Button'; import FormattedDate from '@common/components/FormattedDate'; import LoadingSpinner from '@common/components/LoadingSpinner'; @@ -33,7 +36,7 @@ const statusMap: { Approved: 'Approved', }; -const ReleaseStatusPage = () => { +export default function ReleaseStatusPage() { const [isEditing, toggleEditing] = useToggle(false); const location = useLocation(); @@ -64,6 +67,12 @@ const ReleaseStatusPage = () => { const { publicAppUrl } = useConfig(); + const handlePublishingStatusChange = (status: ReleaseStageStatus) => { + if (status.overallStage === 'Complete') { + onReleaseChange(); + } + }; + if (!release) { return ; } @@ -80,7 +89,7 @@ const ReleaseStatusPage = () => { onCancel={toggleEditing.off} onUpdate={nextRelease => { setRelease({ value: nextRelease }); - onReleaseChange(nextRelease); + onReleaseChange(); toggleEditing.off(); }} /> @@ -112,6 +121,7 @@ const ReleaseStatusPage = () => { )} @@ -188,6 +198,4 @@ const ReleaseStatusPage = () => { )} ); -}; - -export default ReleaseStatusPage; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/ReleaseSummaryEditPage.tsx b/src/explore-education-statistics-admin/src/pages/release/ReleaseSummaryEditPage.tsx index 980750b445f..9d63bcd5b85 100644 --- a/src/explore-education-statistics-admin/src/pages/release/ReleaseSummaryEditPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/ReleaseSummaryEditPage.tsx @@ -13,7 +13,9 @@ import useAsyncRetry from '@common/hooks/useAsyncRetry'; import React from 'react'; import { generatePath, RouteComponentProps, useLocation } from 'react-router'; -const ReleaseSummaryEditPage = ({ history }: RouteComponentProps) => { +export default function ReleaseSummaryEditPage({ + history, +}: RouteComponentProps) { const location = useLocation(); const lastLocation = useLastLocation(); @@ -36,7 +38,7 @@ const ReleaseSummaryEditPage = ({ history }: RouteComponentProps) => { throw new Error('Could not update missing release'); } - const nextRelease = await releaseService.updateRelease(releaseId, { + await releaseService.updateRelease(releaseId, { year: Number(values.timePeriodCoverageStartYear), timePeriodCoverage: { value: values.timePeriodCoverageCode, @@ -45,7 +47,7 @@ const ReleaseSummaryEditPage = ({ history }: RouteComponentProps) => { preReleaseAccessList: release.preReleaseAccessList, }); - onReleaseChange(nextRelease); + onReleaseChange(); history.push( generatePath(releaseSummaryRoute.path, { @@ -88,6 +90,4 @@ const ReleaseSummaryEditPage = ({ history }: RouteComponentProps) => { )} ); -}; - -export default ReleaseSummaryEditPage; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStages.tsx b/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStages.tsx index 3bfd5769d74..9400ff84bc0 100644 --- a/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStages.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStages.tsx @@ -1,6 +1,6 @@ import StatusBlock from '@admin/components/StatusBlock'; import getStatusDetail from '@admin/pages/release/utils/getStatusDetail'; -import { ReleaseStageStatuses } from '@admin/services/releaseService'; +import { ReleaseStageStatus } from '@admin/services/releaseService'; import Details from '@common/components/Details'; import React from 'react'; @@ -8,15 +8,15 @@ const notStartedStatuses = ['Validating', 'Invalid']; interface Props { checklistStyle?: boolean; - currentStatus?: ReleaseStageStatuses; + currentStatus?: ReleaseStageStatus; includeScheduled?: boolean; } -const ReleasePublishingStages = ({ +export default function ReleasePublishingStages({ checklistStyle = false, currentStatus, includeScheduled = false, -}: Props) => { +}: Props) { if ( !currentStatus || (includeScheduled @@ -79,6 +79,4 @@ const ReleasePublishingStages = ({ )} ); -}; - -export default ReleasePublishingStages; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStatus.tsx b/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStatus.tsx index a2bcd5db293..3a55e3b072b 100644 --- a/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStatus.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStatus.tsx @@ -1,33 +1,33 @@ import useReleasePublishingStatus from '@admin/pages/release/hooks/useReleasePublishingStatus'; import ReleasePublishingStatusTag from '@admin/pages/release/components/ReleasePublishingStatusTag'; import ReleasePublishingStages from '@admin/pages/release/components/ReleasePublishingStages'; +import { ReleaseStageStatus } from '@admin/services/releaseService'; import React from 'react'; interface ReleasePublishingStatusProps { - isApproved?: boolean; releaseId: string; refreshPeriod?: number; + onChange?: (status: ReleaseStageStatus) => void; } -const ReleasePublishingStatus = ({ - isApproved = false, +export default function ReleasePublishingStatus({ releaseId, refreshPeriod, -}: ReleasePublishingStatusProps) => { + onChange, +}: ReleasePublishingStatusProps) { const { currentStatus, currentStatusDetail } = useReleasePublishingStatus({ releaseId, refreshPeriod, + onChange, }); + return ( <> ); -}; - -export default ReleasePublishingStatus; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStatusTag.tsx b/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStatusTag.tsx index ed178500687..b61660e6388 100644 --- a/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStatusTag.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/components/ReleasePublishingStatusTag.tsx @@ -1,5 +1,5 @@ import StatusBlock, { StatusBlockColors } from '@admin/components/StatusBlock'; -import { ReleaseStageStatuses } from '@admin/services/releaseService'; +import { ReleaseStageStatus } from '@admin/services/releaseService'; import React from 'react'; import Tag from '@common/components/Tag'; @@ -7,15 +7,15 @@ const approvedStatuses = ['Complete', 'Scheduled']; interface Props { color?: StatusBlockColors; - currentStatus?: ReleaseStageStatuses; + currentStatus?: ReleaseStageStatus; isApproved?: boolean; } -const ReleasePublishingStatusTag = ({ +export default function ReleasePublishingStatusTag({ color, currentStatus, isApproved = false, -}: Props) => { +}: Props) { if (!currentStatus) { return null; } @@ -43,6 +43,4 @@ const ReleasePublishingStatusTag = ({ /> ); -}; - -export default ReleasePublishingStatusTag; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/components/__tests__/ReleasePublishingStatus.test.tsx b/src/explore-education-statistics-admin/src/pages/release/components/__tests__/ReleasePublishingStatus.test.tsx new file mode 100644 index 00000000000..2e42c24c07d --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/components/__tests__/ReleasePublishingStatus.test.tsx @@ -0,0 +1,138 @@ +import ReleasePublishingStatus from '@admin/pages/release/components/ReleasePublishingStatus'; +import _releaseService from '@admin/services/releaseService'; +import { render, screen } from '@testing-library/react'; +import React from 'react'; + +jest.mock('@admin/services/releaseService'); + +const releaseService = jest.mocked(_releaseService); + +describe('ReleasePublishingStatus', () => { + test('renders correctly with initial status', async () => { + releaseService.getReleaseStatus.mockResolvedValue({ + overallStage: 'Started', + filesStage: 'Queued', + contentStage: 'Complete', + publishingStage: 'NotStarted', + }); + + render(); + + expect(await screen.findByText('Started')).toBeInTheDocument(); + + expect(screen.getByText('Files - queued')).toBeInTheDocument(); + expect(screen.getByText('Content - complete')).toBeInTheDocument(); + expect(screen.getByText('Publishing - not started')).toBeInTheDocument(); + }); + + test('re-renders correctly when status changes to complete', async () => { + jest.useFakeTimers(); + + releaseService.getReleaseStatus.mockResolvedValue({ + overallStage: 'Started', + }); + + render(); + + expect(await screen.findByText('Started')).toBeInTheDocument(); + + releaseService.getReleaseStatus.mockResolvedValue({ + overallStage: 'Complete', + contentStage: 'Complete', + publishingStage: 'Complete', + filesStage: 'Complete', + }); + + jest.runOnlyPendingTimers(); + + expect(await screen.findByText('Complete')).toBeInTheDocument(); + + expect(screen.getByText('Content - complete')).toBeInTheDocument(); + expect(screen.getByText('Files - complete')).toBeInTheDocument(); + expect(screen.getByText('Publishing - complete')).toBeInTheDocument(); + + jest.useRealTimers(); + }); + + test('calls `onChange` only when status changes', async () => { + jest.useFakeTimers(); + + releaseService.getReleaseStatus.mockResolvedValue({ + overallStage: 'Started', + }); + + const handleChange = jest.fn(); + + render( + , + ); + + expect(await screen.findByText('Started')).toBeInTheDocument(); + + expect(handleChange).toHaveBeenCalledTimes(1); + + jest.runOnlyPendingTimers(); + + expect(await screen.findByText('Started')).toBeInTheDocument(); + + // Not called again because the status hasn't changed + expect(handleChange).toHaveBeenCalledTimes(1); + + releaseService.getReleaseStatus.mockResolvedValue({ + overallStage: 'Complete', + contentStage: 'Complete', + publishingStage: 'Complete', + filesStage: 'Complete', + }); + + jest.runOnlyPendingTimers(); + + expect(await screen.findByText('Complete')).toBeInTheDocument(); + + expect(handleChange).toHaveBeenCalledTimes(2); + + jest.useRealTimers(); + }); + + test('does not re-render or call service after overall status is no longer `Started`', async () => { + jest.useFakeTimers(); + + releaseService.getReleaseStatus.mockResolvedValue({ + overallStage: 'Started', + }); + + const handleChange = jest.fn(); + + render( + , + ); + + expect(await screen.findByText('Started')).toBeInTheDocument(); + + expect(releaseService.getReleaseStatus).toHaveBeenCalledTimes(1); + + releaseService.getReleaseStatus.mockResolvedValue({ + overallStage: 'Complete', + }); + + jest.runOnlyPendingTimers(); + + expect(await screen.findByText('Complete')).toBeInTheDocument(); + + expect(releaseService.getReleaseStatus).toHaveBeenCalledTimes(2); + + // Can't really transition from Complete to Failed, but + // just simulating a change for the test. + releaseService.getReleaseStatus.mockResolvedValue({ + overallStage: 'Failed', + }); + + jest.runOnlyPendingTimers(); + + expect(await screen.findByText('Complete')).toBeInTheDocument(); + + expect(releaseService.getReleaseStatus).toHaveBeenCalledTimes(2); + + jest.useRealTimers(); + }); +}); diff --git a/src/explore-education-statistics-admin/src/pages/release/contexts/ReleaseContext.tsx b/src/explore-education-statistics-admin/src/pages/release/contexts/ReleaseContext.tsx index b17a2c99d62..63924907254 100644 --- a/src/explore-education-statistics-admin/src/pages/release/contexts/ReleaseContext.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/contexts/ReleaseContext.tsx @@ -5,7 +5,7 @@ import React, { createContext, ReactNode, useContext, useMemo } from 'react'; export interface ReleaseContextState { release: Release; releaseId: string; - onReleaseChange: (nextRelease: Release) => void; + onReleaseChange: () => void; } const ReleaseContext = createContext( @@ -15,7 +15,7 @@ const ReleaseContext = createContext( interface ReleaseContextProviderProps { children: ReactNode; release: Release; - onReleaseChange?: (nextRelease: Release) => void; + onReleaseChange?: () => void; } export const ReleaseContextProvider = ({ @@ -44,5 +44,6 @@ export function useReleaseContext() { 'useReleaseContext must be used within a ReleaseContextProvider', ); } + return context; } diff --git a/src/explore-education-statistics-admin/src/pages/release/hooks/useReleasePublishingStatus.ts b/src/explore-education-statistics-admin/src/pages/release/hooks/useReleasePublishingStatus.ts index c8ad27faf46..1957732d2e5 100644 --- a/src/explore-education-statistics-admin/src/pages/release/hooks/useReleasePublishingStatus.ts +++ b/src/explore-education-statistics-admin/src/pages/release/hooks/useReleasePublishingStatus.ts @@ -1,8 +1,9 @@ import { StatusBlockProps } from '@admin/components/StatusBlock'; import getStatusDetail from '@admin/pages/release/utils/getStatusDetail'; import releaseService, { - ReleaseStageStatuses, + ReleaseStageStatus, } from '@admin/services/releaseService'; +import isEqual from 'lodash/isEqual'; import { useCallback, useEffect, useRef, useState } from 'react'; import { forceCheck } from 'react-lazyload'; @@ -14,6 +15,7 @@ export interface StatusDetail { interface Options { refreshPeriod?: number; releaseId: string; + onChange?: (status: ReleaseStageStatus) => void; } /** @@ -22,11 +24,12 @@ interface Options { export default function useReleasePublishingStatus({ refreshPeriod = 10000, releaseId, + onChange, }: Options): { - currentStatus?: ReleaseStageStatuses; + currentStatus?: ReleaseStageStatus; currentStatusDetail: StatusDetail; } { - const [currentStatus, setCurrentStatus] = useState(); + const [currentStatus, setCurrentStatus] = useState(); const [currentStatusDetail, setStatusDetail] = useState({ color: 'blue', text: '', @@ -34,45 +37,57 @@ export default function useReleasePublishingStatus({ const timeoutRef = useRef(); - const fetchReleasePublishingStatus = useCallback(async () => { + const fetchNextStatus = useCallback(async () => { const status = await releaseService.getReleaseStatus(releaseId); + + const setNextStatus = (nextStatus: ReleaseStageStatus) => { + setCurrentStatus(prevStatus => { + if (onChange && !isEqual(prevStatus, nextStatus)) { + onChange(nextStatus); + } + + return nextStatus; + }); + }; + if (!status) { // 204 response waiting for status - setCurrentStatus({ overallStage: 'Validating' }); - timeoutRef.current = setTimeout( - fetchReleasePublishingStatus, - refreshPeriod, - ); + setNextStatus({ overallStage: 'Validating' }); + + timeoutRef.current = setTimeout(fetchNextStatus, refreshPeriod); } else { - setCurrentStatus(status); + setNextStatus(status); + if (status && status.overallStage === 'Started') { - timeoutRef.current = setTimeout( - fetchReleasePublishingStatus, - refreshPeriod, - ); + timeoutRef.current = setTimeout(fetchNextStatus, refreshPeriod); } } forceCheck(); - }, [releaseId, refreshPeriod]); + }, [releaseId, onChange, refreshPeriod]); function cancelTimer() { - if (timeoutRef.current) clearInterval(timeoutRef.current); + if (timeoutRef.current) { + clearInterval(timeoutRef.current); + } } useEffect(() => { - fetchReleasePublishingStatus(); + fetchNextStatus(); + return () => { // cleans up the timeout cancelTimer(); }; - }, [fetchReleasePublishingStatus]); + }, [fetchNextStatus]); useEffect(() => { if (currentStatus && currentStatus.overallStage) { const status = getStatusDetail(currentStatus.overallStage); + if (status.color === 'red' || status.color === 'green') { cancelTimer(); } + setStatusDetail(status); } }, [currentStatus]); diff --git a/src/explore-education-statistics-admin/src/services/releaseService.ts b/src/explore-education-statistics-admin/src/services/releaseService.ts index b8122f551e7..10afbd1ebd1 100644 --- a/src/explore-education-statistics-admin/src/services/releaseService.ts +++ b/src/explore-education-statistics-admin/src/services/releaseService.ts @@ -129,7 +129,7 @@ type TaskStage = | 'Started' | 'Scheduled'; -export interface ReleaseStageStatuses { +export interface ReleaseStageStatus { releaseId?: string; contentStage?: TaskStage; filesStage?: TaskStage; @@ -261,8 +261,8 @@ const releaseService = { ); }, - getReleaseStatus(releaseId: string): Promise { - return client.get( + getReleaseStatus(releaseId: string): Promise { + return client.get( `/releases/${releaseId}/stage-status`, ); }, From 0a4db51bbe38ed498203354925dfa5f2d87da9bc Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 30 May 2024 11:29:05 +0100 Subject: [PATCH 70/73] EES-5174 fix data catalogue download all button --- .../data-catalogue/DataCataloguePageNew.tsx | 47 +++++++++---------- .../__tests__/DataCataloguePage.test.tsx | 4 +- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePageNew.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePageNew.tsx index 5baf58f797e..3d92aa739ba 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePageNew.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePageNew.tsx @@ -8,10 +8,9 @@ import WarningMessage from '@common/components/WarningMessage'; import VisuallyHidden from '@common/components/VisuallyHidden'; import ButtonText from '@common/components/ButtonText'; import Tag from '@common/components/Tag'; -import Button from '@common/components/Button'; +import ButtonLink from '@frontend/components/ButtonLink'; import NotificationBanner from '@common/components/NotificationBanner'; import { releaseTypes } from '@common/services/types/releaseType'; -import downloadService from '@common/services/downloadService'; import { Theme } from '@common/services/publicationService'; import { useMobileMedia } from '@common/hooks/useMedia'; import { SortDirection } from '@common/services/types/sort'; @@ -229,20 +228,6 @@ export default function DataCataloguePageNew({ showTypeFilter }: Props) { }); }; - const handleDownload = async () => { - if (!selectedPublication || !selectedRelease) { - return; - } - const fileIds = dataSets.map(dataSet => dataSet.fileId); - await downloadService.downloadFiles(selectedRelease.id, fileIds); - - logEvent({ - category: 'Data catalogue', - action: 'Data set file download - all', - label: `Publication: ${selectedPublication.title}, Release: ${selectedRelease.title}`, - }); - }; - const handleSortByChange = async (nextSortBy: DataSetFileSortOption) => { await updateQueryParams({ ...omit(router.query, 'page'), @@ -516,14 +501,28 @@ export default function DataCataloguePageNew({ showTypeFilter }: Props) {

    - + { + logEvent({ + category: 'Data catalogue', + action: 'Data set file download - all', + label: `Publication: ${selectedPublication.title}, Release: ${selectedRelease.title}`, + }); + }} + > + {`Download ${ + totalResults === 1 + ? '1 data set' + : `all ${totalResults} data sets` + } (ZIP)`} + +
    + + Download includes data guidance and supporting + files. +

    )} diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataCataloguePage.test.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataCataloguePage.test.tsx index e5306d5a3bd..03d1ad9e539 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataCataloguePage.test.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataCataloguePage.test.tsx @@ -828,7 +828,7 @@ describe('DataCataloguePage', () => { releaseInfo.getByText('This is the latest data'), ).toBeInTheDocument(); expect( - releaseInfo.getByRole('button', { + releaseInfo.getByRole('link', { name: 'Download 1 data set (ZIP)', }), ).toBeInTheDocument(); @@ -937,7 +937,7 @@ describe('DataCataloguePage', () => { releaseInfo.getByText('This is not the latest data'), ).toBeInTheDocument(); expect( - releaseInfo.getByRole('button', { + releaseInfo.getByRole('link', { name: 'Download all 2 data sets (ZIP)', }), ).toBeInTheDocument(); From c9336d4e5cb11b8d7fac57b6d8a35b38a7776d45 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Thu, 30 May 2024 15:09:15 +0100 Subject: [PATCH 71/73] EES-5183 Use UTC for ReleaseFiles.Published --- .../Database/ContentDbContext.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs index d5322bdf433..9bd1ecb0965 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs @@ -402,6 +402,11 @@ private static void ConfigureReleaseFile(ModelBuilder modelBuilder) .HasConversion( v => JsonConvert.SerializeObject(v), v => JsonConvert.DeserializeObject>(v)); + entity.Property(rf => rf.Published) + .HasConversion(v => v, + v => v.HasValue + ? DateTime.SpecifyKind(v.Value, DateTimeKind.Utc) + : null); }); } From 649d5feaa30801b2fb478d4e33b152225d6031bb Mon Sep 17 00:00:00 2001 From: Sam Biram <63285990+sambiramairelogic@users.noreply.github.com> Date: Fri, 31 May 2024 11:10:02 +0100 Subject: [PATCH 72/73] EES-5167 Upload new redirects and use an object instead of an array (#4916) --- .../redirects.js | 1807 ++++++++++------- .../server.js | 9 +- .../tests/public/redirects.spec.ts | 8 +- 3 files changed, 1052 insertions(+), 772 deletions(-) diff --git a/src/explore-education-statistics-frontend/redirects.js b/src/explore-education-statistics-frontend/redirects.js index 6ecf21afaa1..478ec3cc559 100644 --- a/src/explore-education-statistics-frontend/redirects.js +++ b/src/explore-education-statistics-frontend/redirects.js @@ -1,763 +1,1048 @@ -const seoRedirects = [ - { - from: '/nbsp;andnbsp;https://www.gov.uk/government/collections/abortion-statistics-for-england-and-wales', - to: '/', - }, - { from: '/!nd-statistics/permanent-and-', to: '/' }, - { - from: '/api/methodologies/%7BmethodologyId%7D/images/7763f949-eff9-4571-93b4-08db2bd4eec9', - to: '/', - }, - { - from: '/api/methodologies/%7BmethodologyId%7D/images/9348e5bc-c09d-43b4-de90-08d90ae3d820', - to: '/', - }, - { - from: '/api/methodologies/%7BmethodologyId%7D/images/a095bcaa-7c7c-4a99-de75-08d90ae3d820', - to: '/', - }, - { - from: '/api/methodologies/%7BmethodologyId%7D/images/cd50d521-0d75-498e-143b-08dac7b96540', - to: '/', - }, - { - from: '/api/methodologies/%7BmethodologyId%7D/images/d8ecf739-f299-413f-126f-08dacbba9031', - to: '/', - }, - { - from: '/api/methodologies/%7BmethodologyId%7D/images/ef57d23a-f281-4b6b-a55e-08da6007d7ab', - to: '/', - }, - { - from: '/api/methodologies/%7BmethodologyId%7D/images/fcfa509c-8572-4b61-ca3a-08dbcefb484c', - to: '/', - }, - { - from: '/api/methodologies/%7BmethodologyId%7D/images/feeffb4e-f022-469a-6e23-08dacc6823fb', - to: '/', - }, - { - from: '/api/releases/%7BreleaseId%7D/images/0eb3fc80-77ad-4d66-8090-a6c2509d50c5', - to: '/', - }, - { - from: '/api/releases/%7BreleaseId%7D/images/26f6ac12-25d0-4831-5293-08db35b5456a', - to: '/', - }, - { - from: '/api/releases/%7BreleaseId%7D/images/fdb52a67-704d-44f3-9721-752000ff9ab9', - to: '/', - }, - { from: '/data', to: '/' }, - { from: '/data-', to: '/' }, - { from: '/data-table', to: '/' }, - { from: '/data-tablesnbsp;-nbsp;springnbsp;2022', to: '/' }, - { from: '/data-tables/fast-track/%5BdataBlockId%5D', to: '/' }, - { from: '/data-tables/fast-track/%5BdataBlockParentId%5D', to: '/' }, - { from: '/data-tables/permalink/%5Bpermalink%5D', to: '/data-tables' }, - { - from: '/data-tables/permalink/09fca49f-7bd8-475f-dd03-08db8e99978c.', - to: '/data-tables', - }, - { from: '/data-tables/permalink/167d18fb-', to: '/data-tables' }, - { - from: '/data-tables/permalink/203a8889-429b-469f-abe1-', - to: '/data-tables', - }, - { - from: '/data-tables/permalink/2e97cf7e-3fe6-4e97-8525-', - to: '/data-tables', - }, - { - from: '/data-tables/permalink/36526263-f93b-4bb1-9d1f-', - to: '/data-tables', - }, - { - from: '/data-tables/permalink/47223cca-6ffb-42da-be50-', - to: '/data-tables', - }, - { - from: '/data-tables/permalink/55ca2487-7a4d-43c4-487d-08db8e99c26c.', - to: '/data-tables', - }, - { - from: '/data-tables/permalink/59fc4e59-160c-4755-c612-08', - to: '/data-tables', - }, - { - from: '/data-tables/permalink/5ab5dbf2-393b-4a43-b7a1-', - to: '/data-tables', - }, - { - from: '/data-tables/permalink/67961e79-5204-410e-b521-', - to: '/data-tables', - }, - { - from: '/data-tables/permalink/85ba124e-1489-4648-bfde-', - to: '/data-tables', - }, - { - from: '/data-tables/permalink/85ba124e-1489-4648-bfde-3eff6fdf76a2;', - to: '/data-tables', - }, - { from: '/data-tables/permalink/8f50fce8-', to: '/data-tables' }, - { from: '/data-tables/permalink/970e9bb8-d185-', to: '/data-tables' }, - { - from: '/data-tables/permalink/9795440a-015e-47eb-86e2-', - to: '/data-tables', - }, - { from: '/data-tables/permalink/9e420c30-', to: '/data-tables' }, - { - from: '/data-tables/permalink/c0319f82-e1e3-4adf-9db1-', - to: '/data-tables', - }, - { from: '/data-tables/permalink/c4cb6884-', to: '/data-tables' }, - { from: '/data-tables/permalink/e0980465-', to: '/data-tables' }, - { from: '/data-tables/permalink/e8942369-b2a3-', to: '/data-tables' }, - { from: '/datatables/apprenticeships-and-traineeships', to: '/data-tables' }, - { from: '/datatables/permalink/48f9a035-9123-477e-', to: '/data-tables' }, - { from: '/datatables/permalink/b169759c-', to: '/data-tables' }, - { from: '/find-st%3C/body%3E%3C/html%3E', to: '/find-statistics' }, - { from: '/find-sta', to: '/find-statistics' }, - { from: '/find-stati', to: '/find-statistics' }, - { from: '/find-statis-', to: '/find-statistics' }, - { from: '/find-statistic', to: '/find-statistics' }, - { from: '/find-statistics/[publication]', to: '/find-statistics' }, - { from: '/find-statistics/[publication]/[release]', to: '/find-statistics' }, - { from: '/find-statistics/%3ca%20href=', to: '/find-statistics' }, - { from: '/find-statistics/16-18-', to: '/find-statistics' }, - { - from: '/find-statistics/16-18-destination-measure', - to: '/find-statistics', - }, - { from: '/find-statistics/a-', to: '/find-statistics' }, - { from: '/find-statistics/a-level', to: '/find-statistics' }, - { from: '/find-statistics/a-level-and-other-16-to-', to: '/find-statistics' }, - { - from: '/find-statistics/a-level-and-other-16-to-18-', - to: '/find-statistics', - }, - { - from: '/find-statistics/a-level-and-other-16-to-18-results/2019-20 %e2%80%93', - to: '/find-statistics', - }, - { - from: '/find-statistics/a-level-and-other-level-3-results-in-england-academic-year-2019-to-2020', - to: '/find-statistics', - }, - { from: '/find-statistics/admission-', to: '/find-statistics' }, - { from: '/find-statistics/apprenticeships-', to: '/find-statistics' }, - { from: '/find-statistics/apprenticeships-and', to: '/find-statistics' }, - { - from: '/find-statistics/apprenticeships-and-traineeships/2020-21 - apprenticeship starts down 0.3% in 21-22 vs 19-20', - to: '/find-statistics', - }, - { - from: '/find-statistics/apprenticeships-and-traineeshipsThe', - to: '/find-statistics', - }, - { from: '/find-statistics/at-', to: '/find-statistics' }, - { - from: '/find-statistics/attainment-8-score-averages-by-previous-attainment/2019-to-2020-revised', - to: '/find-statistics', - }, - { from: '/find-statistics/attendance-', to: '/find-statistics' }, - { from: '/find-statistics/attendance-in-', to: '/find-statistics' }, - { from: '/find-statistics/attendance-in-edu', to: '/find-statistics' }, - { from: '/find-statistics/attendance-in-educa-', to: '/find-statistics' }, - { from: '/find-statistics/attendance-in-education', to: '/find-statistics' }, - { from: '/find-statistics/attendance-in-education-', to: '/find-statistics' }, - { - from: '/find-statistics/attendance-in-education-and-', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-earl', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-early-', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-early-years', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-early-years-', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-early-years-settings-during-the-', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-covid-19-', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-covid-19-outbreak 2', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-covid-19-outbreak/2020-week-', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-covid-19-outbreak/2021-week-', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-covid-19-outbreak/2022-week-', - to: '/find-statistics', - }, - { - from: '/find-statistics/attendance-in-education-andearly-years-settings-during-the-coronavirus-covid-19-outbreak', - to: '/find-statistics', - }, - { from: '/find-statistics/attendance-ineducation-', to: '/find-statistics' }, - { from: '/find-statistics/characteristics-of-', to: '/find-statistics' }, - { - from: '/find-statistics/characteristics-of-children-in-', - to: '/find-statistics', - }, - { - from: '/find-statistics/characteristics-of-children-in-n', - to: '/find-statistics', - }, - { - from: '/find-statistics/characteristics-of-children-in-need/2022 ', - to: '/find-statistics', - }, - { - from: '/find-statistics/characteristics-of-children-in-need ', - to: '/find-statistics', - }, - { - from: '/find-statistics/characteristics-of-children-inneed/2020', - to: '/find-statistics', - }, - { from: '/find-statistics/childcare', to: '/find-statistics' }, - { from: '/find-statistics/childcare-and-', to: '/find-statistics' }, - { from: '/find-statistics/childcare-and-early-', to: '/find-statistics' }, - - { from: '/find-statistics/children', to: '/find-statistics' }, - { from: '/find-statistics/children-', to: '/find-statistics' }, - - { from: '/find-statistics/children-loo', to: '/find-statistics' }, - { from: '/find-statistics/children-looked', to: '/find-statistics' }, - { from: '/find-statistics/children-looked-after-in', to: '/find-statistics' }, - { - from: '/find-statistics/children-looked-after-in-england', - to: '/find-statistics', - }, - { - from: '/find-statistics/children-looked-after-in-england-includ', - to: '/find-statistics', - }, - { - from: '/find-statistics/children-looked-after-in-england-including', - to: '/find-statistics', - }, - { - from: '/find-statistics/children-looked-after-in-england-including-', - to: '/find-statistics', - }, - { - from: '/find-statistics/children-looked-after-in-england-including-adopti', - to: '/find-statistics', - }, - { - from: '/find-statistics/children-looked-after-in-england-including-adoptions/2020 - dataDownloads-1', - to: '/find-statistics', - }, - { - from: '/find-statistics/children-looked-after-in-england-including-adoptions/2020developmentofattachmentsbetweenolder', - to: '/find-statistics', - }, - { - from: '/Find-Statistics/Children-Looked-after-in-England-Including-Adoptions/2021', - to: '/find-statistics', - }, - { - from: '/find-statistics/children-looked-after-in-england-including-adoptions/2021 /l dataDownloads-1', - to: '/find-statistics', - }, - { - from: '/find-statistics/children-looked-after-in-england-including-adoptions/2022.', - to: '/find-statistics', - }, - { from: '/find-statistics/children-s-social-work', to: '/find-statistics' }, - { - from: '/find-statistics/children-s-social-work-workforce-attrition-', - to: '/find-statistics', - }, - { - from: '/find-statistics/children-s-social-work-workforce/2022 ', - to: '/find-statistics', - }, - { from: '/find-statistics/delivery-of-air-', to: '/find-statistics' }, - { - from: '/find-statistics/early-years-foundation-stage-', - to: '/find-statistics', - }, - { from: '/find-statistics/education-', to: '/find-statistics' }, - { from: '/find-statistics/education-and-training-', to: '/find-statistics' }, - { - from: '/find-statistics/education-and-training-statistics-for-the-', - to: '/find-statistics', - }, - { - from: '/find-statistics/education-and-training-statistics-for-the-uk/2022;', - to: '/find-statistics', - }, - { from: '/find-statistics/education-health-', to: '/find-statistics' }, - { - from: '/find-statistics/education-health-and-care', - to: '/find-statistics', - }, - { - from: '/find-statistics/education-health-and-care-plan', - to: '/find-statistics', - }, - { - from: '/find-statistics/education-health-and-care-plans (opens in external window)', - to: '/find-statistics', - }, - { - from: '/find-statistics/education-health-and-care-plans/www.gov.uk/courts-tribunals/first-tier-tribunal-specialeducational-needs-and-disability', - to: '/find-statistics', - }, - { - from: '/find-statistics/education-provision-children-under-5/2020 (universal three and four-year old offer and disadvantaged two year old offer) and https://www.gov.uk/government/statistics/childcare-and-early-years-survey-of-parents-2019', - to: '/find-statistics', - }, - { - from: '/find-statistics/elective-home-education/2022-', - to: '/find-statistics', - }, - { from: '/find-statistics/free-school-meals-', to: '/find-statistics' }, - { from: '/find-statistics/further-', to: '/find-statistics' }, - { from: '/find-statistics/further-education-and-', to: '/find-statistics' }, - { - from: '/find-statistics/further-education-outcome-', - to: '/find-statistics', - }, - { - from: '/find-statistics/further-education-outcome-based-success-mesures', - to: '/find-statistics', - }, - { from: '/find-statistics/graduate-', to: '/find-statistics' }, - { from: '/find-statistics/graduate-labour-', to: '/find-statistics' }, - { from: '/find-statistics/graduate-outcomes-', to: '/find-statistics' }, - { - from: '/find-statistics/he.modelling@education.gov.uk', - to: '/find-statistics', - }, - { from: '/find-statistics/higher-', to: '/find-statistics' }, - { - from: '/find-statistics/higher-level-learners-in-england/2018-19.', - to: '/find-statistics', - }, - { from: '/find-statistics/initial-', to: '/find-statistics' }, - { from: '/find-statistics/initial-teacher-train-', to: '/find-statistics' }, - { - from: '/find-statistics/initial-teacher-training-', - to: '/find-statistics', - }, - { - from: '/find-statistics/initial-teacher-training-census/2022-', - to: '/find-statistics', - }, - { - from: '/find-statistics/initial-teacher-training-performance-', - to: '/find-statistics', - }, - { - from: '/find-statistics/initial-teacher-training-performance-profiles/2019-20.', - to: '/find-statistics', - }, - { - from: '/find-statistics/initial-teacher-trainingcensus/2020-21', - to: '/find-statistics', - }, - { from: '/find-statistics/key-stage-', to: '/find-statistics' }, - { from: '/find-statistics/key-stage-1-and-phonics-', to: '/find-statistics' }, - { - from: '/find-statistics/key-stage-1-and-phonics-screening-check-', - to: '/find-statistics', - }, - { from: '/find-statistics/key-stage-2-', to: '/find-statistics' }, - { - from: '/find-statistics/key-stage-2-attainment-national-headlines/2021', - to: '/find-statistics', - }, - { - from: '/find-statistics/key-stage-2-attainment-national-headlines/2021-22 Accessed 31st July 2022', - to: '/find-statistics', - }, - { - from: '/find-statistics/key-stage-2-attainment/2021-', - to: '/find-statistics', - }, - { from: '/find-statistics/key-stage-4', to: '/find-statistics' }, - { from: '/find-statistics/key-stage-4-', to: '/find-statistics' }, - { - from: '/find-statistics/key-stage-4-destination-measures/2018-', - to: '/find-statistics', - }, - { from: '/find-statistics/key-stage-4-performance-', to: '/find-statistics' }, - { - from: '/find-statistics/key-stage-4-performance-revised/2020-', - to: '/find-statistics', - }, - { - from: '/find-statistics/key-stage-4-performance-revised/2021%e2%80%9322', - to: '/find-statistics', - }, - { - from: '/find-statistics/la-and-school-expenditure/2019-2', - to: '/find-statistics', - }, - { - from: '/find-statistics/la-and-school-expenditure/2020-21;', - to: '/find-statistics', - }, - { from: '/find-statistics/laptops-and-', to: '/find-statistics' }, - { - from: '/find-statistics/level-2-and-3-attainment-by-youn', - to: '/find-statistics', - }, - { - from: '/find-statistics/level-2-and-3-attainment-by-young-people-', - to: '/find-statistics', - }, - { - from: '/find-statistics/level-2-and-3-attainment-by-young-people-aged-19/2020-', - to: '/find-statistics', - }, - { - from: '/find-statistics/looked-after-children-aged-16-to-17-in-independent-or-semi-', - to: '/find-statistics', - }, - { - from: '/find-statistics/multiplication-tables-check-', - to: '/find-statistics', - }, - { from: '/find-statistics/national-tutoring-', to: '/find-statistics' }, - { from: '/find-statistics/neet-statistics-', to: '/find-statistics' }, - { - from: '/find-statistics/outcomes-for-children-in-need-', - to: '/find-statistics', - }, - { - from: '/find-statistics/outcomes-for-children-in-need-including-c', - to: '/find-statistics', - }, - { - from: '/find-statistics/outcomes-for-children-in-needincluding-children-', - to: '/find-statistics', - }, - { - from: '/find-statistics/outcomes-for-children-in-needincluding-children-looked-after-by-local-authorities-in-england', - to: '/find-statistics', - }, - { from: '/find-statistics/parental', to: '/find-statistics' }, - { from: '/find-statistics/participation', to: '/find-statistics' }, - { - from: '/find-statistics/participation-in-education-and-', - to: '/find-statistics', - }, - { - from: '/find-statistics/participation-in-education-training-and-', - to: '/find-statistics', - }, - { - from: '/find-statistics/participation-measures-in-higher-', - to: '/find-statistics', - }, - { from: '/find-statistics/permanent-', to: '/find-statistics' }, - { from: '/find-statistics/permanent-and', to: '/find-statistics' }, - { from: '/find-statistics/permanent-and-fixed', to: '/find-statistics' }, - { from: '/find-statistics/permanent-and-fixed-', to: '/find-statistics' }, - { from: '/find-statistics/permanent-and-fixed-p', to: '/find-statistics' }, - { - from: '/find-statistics/permanent-and-fixed-period-', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanent-and-fixed-period-exclu', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanent-and-fixed-period-exclusions-in-', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanent-and-fixed-period-exclusions-in-eng', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanent-and-fixed-period-exclusions-in-eng-', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanent-and-fixed-period-exclusions-in-england/2018-', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanent-and-fixed-period-exclusions-in-england/2018-19yougov.uk', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanent-and-fixed-period-exclusions-in-englandFigures', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanent-and-fixed-period-exclusions-inengland/2018-19', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanent-and-fixed-periodexclusions-in-england', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanent-and-fixed-term-exclusions-in-england', - to: '/find-statistics', - }, - { - from: '/find-statistics/permanentand-fixed-period-exclusions-in-england', - to: '/find-statistics', - }, - { - from: '/find-statistics/postgraduate-initial-teacher-', - to: '/find-statistics', - }, - { - from: '/find-statistics/progression-to-%3ca href==', - to: '/find-statistics', - }, - { from: '/find-statistics/progression-to-higher-ed', to: '/find-statistics' }, - { from: '/find-statistics/pupil', to: '/find-statistics' }, - { from: '/find-statistics/pupil-absence-', to: '/find-statistics' }, - { from: '/find-statistics/pupil-absence-in-', to: '/find-statistics' }, - { - from: '/find-statistics/pupil-absence-in-schools-in-engl', - to: '/find-statistics', - }, - { - from: '/find-statistics/pupil-absence-in-schools-in-england-autumn-and-', - to: '/find-statistics', - }, - { - from: '/find-statistics/pupil-absence-in-schools-in-england-autumn-term/data', - to: '/find-statistics', - }, - { - from: '/find-statistics/pupil-absence-in-schools-in-england)', - to: '/find-statistics', - }, - { - from: '/find-statistics/pupil-absence-inschools-in-england-', - to: '/find-statistics', - }, - { from: '/find-statistics/pupil-attendance', to: '/find-statistics' }, - { from: '/find-statistics/pupil-attendance-in-', to: '/find-statistics' }, - { - from: '/find-statistics/pupil-attendance-in-schools/2023-7week-29', - to: '/find-statistics', - }, - { from: '/find-statistics/school-funding-', to: '/find-statistics' }, - { - from: '/find-statistics/school-funding-statistics/2021-', - to: '/find-statistics', - }, - { from: '/find-statistics/school-placements-', to: '/find-statistics' }, - { from: '/find-statistics/school-pup', to: '/find-statistics' }, - { from: '/find-statistics/school-pupils-', to: '/find-statistics' }, - { from: '/find-statistics/school-pupils-and-', to: '/find-statistics' }, - { from: '/find-statistics/school-pupils-and-thei', to: '/find-statistics' }, - { from: '/find-statistics/school-pupils-and-their-', to: '/find-statistics' }, - { - from: '/find-statistics/school-pupils-and-their-ch', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-pupils-and-their-cha', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-pupils-and-their-characteristics;', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-pupils-and-their-characteristics/2019-20.', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-pupils-and-their-characteristicsa', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-pupils-and-their-characteristicshttps:/explore-education-statistics.service.gov.uk/find-statistics/school-pupils-and-their-characteristics', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-pupils-and-their-characteristicsm', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-pupils-and-theircharacteristics', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-pupils-andtheir-characteristics', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-pupilsand-their-characteristics', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-workforce-in-englan%3c/p%3e %3cp%3e for more detailed information about the cookies we use, please see our %3ca href=', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-workforce-in-england,', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-workforce-in-englandhttps:/explore-education-statistics.service.gov.uk/find-statistics/school-workforce-in-england', - to: '/find-statistics', - }, - { - from: '/find-statistics/school-workforce-inengland', - to: '/find-statistics', - }, - { - from: '/find-statistics/schoolworkforce-in-england', - to: '/find-statistics', - }, - { from: '/find-statistics/serious-', to: '/find-statistics' }, - { - from: '/find-statistics/serious-incident-notifications ', - to: '/find-statistics', - }, - { from: '/find-statistics/skills-bootcamps-', to: '/find-statistics' }, - { from: '/find-statistics/special-', to: '/find-statistics' }, - { - from: '/find-statistics/special-educational-needs-', - to: '/find-statistics', - }, - { - from: '/find-statistics/special-educational-needs-in-england/2021-', - to: '/find-statistics', - }, - { - from: '/find-statistics/special-educational-needs-in-england/2021-2', - to: '/find-statistics', - }, - { - from: '/find-statistics/special-educational-needs-in-england/221-22', - to: '/find-statistics', - }, - { - from: '/find-statistics/special-educationalneeds-in-england', - to: '/find-statistics', - }, - { from: '/find-statistics/student-', to: '/find-statistics' }, - { - from: '/find-statistics/student-loan-forecasts-for', - to: '/find-statistics', - }, - { - from: '/find-statistics/student-loan-forecasts-for-', - to: '/find-statistics', - }, - { - from: '/find-statistics/the-link-between-absence-and-attainment-', - to: '/find-statistics', - }, - { - from: '/find-statistics/the-link-between-absence-and-attainment-at-ks2-', - to: '/find-statistics', - }, - { - from: '/find-statistics/uk-revenue-from-education-related-exports-', - to: '/find-statistics', - }, - { - from: '/find-statistics/uk-revenue-from-education-related-exports-and-', - to: '/find-statistics', - }, - { from: '/find-statistics/widen-', to: '/find-statistics' }, - { from: '/find-statistics/widening', to: '/find-statistics' }, - { from: '/find-statistics/widening-', to: '/find-statistics' }, - { - from: '/find-statistics/widening-participation-in-higher-', - to: '/find-statistics', - }, - { - from: '/find-statistics/www.gov.uk/courts-tribunals/first-tier-tribunal-specialeducational-needs-and-disability', - to: '/find-statistics', - }, - { - from: '/findstatistics/apprenticeships-and-traineeships/2021-22', - to: '/find-statistics', - }, - { - from: '/findstatistics/apprenticeships-in-england-by-industry-characteristics', - to: '/find-statistics', - }, - { from: '/findstatistics/attendance-in-', to: '/find-statistics' }, - { - from: '/findstatistics/attendance-in-education-and-early-years-', - to: '/find-statistics', - }, - { - from: '/findstatistics/children-s-social-work-workforce', - to: '/find-statistics', - }, - { - from: '/findstatistics/education-and-training-statistics-for-the-uk/2020', - to: '/find-statistics', - }, - { from: '/findstatistics/leo-graduate-and', to: '/find-statistics' }, - { - from: '/findstatistics/special-educational-needs-in-england', - to: '/find-statistics', - }, - { from: '/methodol', to: '/methodology' }, - { from: '/methodology/16-18-destination-', to: '/methodology' }, - { from: '/methodology/attendance-in-', to: '/methodology' }, - { - from: '/methodology/attendance-ineducation-and-early-years-settings-during-the-coronavirus-covid-19-outbreakmethodology', - to: '/methodology', - }, - { from: '/methodology/children-looked-after-', to: '/methodology' }, - { - from: '/methodology/permanent-and-fixed-period-exclusions-in-england', - to: '/methodology', - }, - { from: '/methodology/secondary-and-primary-school-', to: '/methodology' }, - { - from: '/methodology/student-loan-forecasts-for-england-', - to: '/methodology', - }, - { from: '/methodology/widening-', to: '/methodology' }, - { - from: '/methodology/widening-participation-in-higher-', - to: '/methodology', - }, - { from: '/service', to: '/' }, - { - from: '/statistics/pupil-absence-in-schools-in-england', - to: '/find-statistics', - }, -]; +const seoRedirects = { + '/%20and%20%0ahttps%3a/www.gov.uk/government/collections/abortion-statistics-for-england-and-wales': + '/', + '/%21nd-statistics/permanent-and-': '/', + '/api/methodologies/%7bmethodologyid%7d/images/7763f949-eff9-4571-93b4-08db2bd4eec9': + '/', + '/api/methodologies/%7bmethodologyid%7d/images/9348e5bc-c09d-43b4-de90-08d90ae3d820': + '/', + '/api/methodologies/%7bmethodologyid%7d/images/a095bcaa-7c7c-4a99-de75-08d90ae3d820': + '/', + '/api/methodologies/%7bmethodologyid%7d/images/cd50d521-0d75-498e-143b-08dac7b96540': + '/', + '/api/methodologies/%7bmethodologyid%7d/images/d8ecf739-f299-413f-126f-08dacbba9031': + '/', + '/api/methodologies/%7bmethodologyid%7d/images/ef57d23a-f281-4b6b-a55e-08da6007d7ab': + '/', + '/api/methodologies/%7bmethodologyid%7d/images/fcfa509c-8572-4b61-ca3a-08dbcefb484c': + '/', + '/api/methodologies/%7bmethodologyid%7d/images/feeffb4e-f022-469a-6e23-08dacc6823fb': + '/', + '/api/releases/%7breleaseid%7d/images/0eb3fc80-77ad-4d66-8090-a6c2509d50c5': + '/', + '/api/releases/%7breleaseid%7d/images/26f6ac12-25d0-4831-5293-08db35b5456a': + '/', + '/api/releases/%7breleaseid%7d/images/fdb52a67-704d-44f3-9721-752000ff9ab9': + '/', + '/data': '/', + '/data-': '/', + '/data-table': '/', + '/data-tables%20-%20spring%20%0a2022': '/', + '/data-tables/fast-track/%5bdatablockid%5d': '/', + '/data-tables/fast-track/%5bdatablockparentid%5d': '/', + '/data-tables/fast-track/0164bb4a-6777-4d8f-a1a0-34ce69f75153': + '/data-tables', + '/data-tables/fast-track/02b8c449-017b-4cd3-8db9-f96fba507c1c': + '/data-tables', + '/data-tables/fast-track/02f116d1-72e5-4024-be0c-f3920918c7c0': + '/data-tables', + '/data-tables/fast-track/03f531e7-d182-416a-9ceb-dce13c9896f4': + '/data-tables', + '/data-tables/fast-track/043db729-7788-4155-a191-ac44aafbd810': + '/data-tables', + '/data-tables/fast-track/04a0798b-a8a1-411e-819f-329a056b1693': + '/data-tables', + '/data-tables/fast-track/04e92348-e4ae-4b3b-8e41-77c80b3ce875': + '/data-tables', + '/data-tables/fast-track/05d00ec0-c666-4615-a41f-a0526c474af3': + '/data-tables', + '/data-tables/fast-track/064c8fdc-f45f-4cad-c109-08daa787e284': + '/data-tables', + '/data-tables/fast-track/089eb3b5-d646-473e-bfd7-4d14a681b491': + '/data-tables', + '/data-tables/fast-track/0ae5c9ca-2068-44a1-b271-dbb9a7cff76e': + '/data-tables', + '/data-tables/fast-track/0bb752c7-678f-4c1a-968b-9df1dc40310a': + '/data-tables', + '/data-tables/fast-track/0f6dcc5d-eaa1-4815-c0b4-08daa787e284': + '/data-tables', + '/data-tables/fast-track/0fb43712-f795-4141-359e-08daa20fa891': + '/data-tables', + '/data-tables/fast-track/1031cbca-36e4-4773-e008-08dbe03b87c8/data-tables/fast-track/1031cbca-36e4-4773-e008-08dbe03b87c8': + '/data-tables', + '/data-tables/fast-track/10afd5a5-6fb8-4823-a373-cb4f6f566df4': + '/data-tables', + '/data-tables/fast-track/121310d5-6398-4b9f-7758-08dab100bfc2': + '/data-tables', + '/data-tables/fast-track/1237e676-3142-41bf-35a8-08daa20fa891': + '/data-tables', + '/data-tables/fast-track/12e52f19-45bf-41e3-d76f-08db513a05d2': + '/data-tables', + '/data-tables/fast-track/12f1cb60-31bd-4cb0-f7f1-08da8c214b78': + '/data-tables', + '/data-tables/fast-track/13217d2f-c1e9-4743-42d2-08da3eebca15': + '/data-tables', + '/data-tables/fast-track/13bd481b-e92b-428b-8969-f6aedced3ce3': + '/data-tables', + '/data-tables/fast-track/142bc228-0ca7-47c5-358d-08daa20fa891': + '/data-tables', + '/data-tables/fast-track/14a38393-361d-445c-80d9-7222fab822cf': + '/data-tables', + '/data-tables/fast-track/14ca58d9-6747-405c-5f81-08db3a70d65c': + '/data-tables', + '/data-tables/fast-track/15192eda-4efb-47a0-ac7d-765029c1989a': + '/data-tables', + '/data-tables/fast-track/1624d963-b636-4952-8264-6595225a344a': + '/data-tables', + '/data-tables/fast-track/165dedb9-d2e4-4377-957b-db92a041a6f5': + '/data-tables', + '/data-tables/fast-track/1773b6af-9d37-4274-a26b-27b30a88a098': + '/data-tables', + '/data-tables/fast-track/18ad21e0-efd6-480b-8d72-cbd55ec57811': + '/data-tables', + '/data-tables/fast-track/19abadb2-8a3f-4394-b017-25fc34686ddc': + '/data-tables', + '/data-tables/fast-track/1a9aa06b-02ae-4791-8c59-08d884b70554': + '/data-tables', + '/data-tables/fast-track/1baac6fd-656d-49b1-9b06-2f6b1c7f235d': + '/data-tables', + '/data-tables/fast-track/1c15f472-6e49-42e4-9b1d-6812050d6e8b': + '/data-tables', + '/data-tables/fast-track/1c334d03-b7b5-4413-b7a8-c51d123ff3c3': + '/data-tables', + '/data-tables/fast-track/1c4f7d08-7455-4236-776e-08dab100bfc2': + '/data-tables', + '/data-tables/fast-track/1c50c543-e9cf-49cd-965e-a55fd8bb6e5b': + '/data-tables', + '/data-tables/fast-track/1cf335a1-1591-4941-a5e5-b7d78cf694d5': + '/data-tables', + '/data-tables/fast-track/1d5beeb2-9ef9-4262-b70d-6ef635c04ee8': + '/data-tables', + '/data-tables/fast-track/1e235c86-4dcd-4777-9afa-ba5f0cfb3b17': + '/data-tables', + '/data-tables/fast-track/1e7c1415-2200-421e-ad74-67510e97096d': + '/data-tables', + '/data-tables/fast-track/1ecc6995-3bd1-437e-88dd-08f3824c114f': + '/data-tables', + '/data-tables/fast-track/1ef9174c-bbfd-4f1b-b0c4-1266a089863b': + '/data-tables', + '/data-tables/fast-track/2111b824-eb8c-4a5b-8479-e8e26f5ec987': + '/data-tables', + '/data-tables/fast-track/217b0881-4784-4a0f-b55d-f0ba95d1bc06': + '/data-tables', + '/data-tables/fast-track/228441a3-c132-47f2-8002-921db4696bfc': + '/data-tables', + '/data-tables/fast-track/22922154-4afb-48fe-835e-71ceac3bab3b': + '/data-tables', + '/data-tables/fast-track/22f54370-e54d-4dbd-a932-c9f16fb63e54': + '/data-tables', + '/data-tables/fast-track/22f9e886-6e9b-43ec-86bc-08da7eccd8d3': + '/data-tables', + '/data-tables/fast-track/2300c602-baef-48de-a1aa-97025213bf54': + '/data-tables', + '/data-tables/fast-track/234cab35-71ee-4acb-a88d-db86f20868e4': + '/data-tables', + '/data-tables/fast-track/25336f4d-a1a2-4f13-89b7-3388efc9d6a8': + '/data-tables', + '/data-tables/fast-track/258cee77-825e-432f-af12-246e30dcd8cf': + '/data-tables', + '/data-tables/fast-track/26ec1527-61b9-4d39-def5-08dacc6b24db': + '/data-tables', + '/data-tables/fast-track/27ef17ce-5f53-48dc-a769-efbe2eb0f5db': + '/data-tables', + '/data-tables/fast-track/284556f5-36e6-4ce0-8562-9acc7c3137e3': + '/data-tables', + '/data-tables/fast-track/28f77e89-3c0c-46bb-8288-ed7ec1a490a5': + '/data-tables', + '/data-tables/fast-track/2a395d39-8f77-46ae-aed3-d2c694c84381': + '/data-tables', + '/data-tables/fast-track/2c16da26-d63d-4d9e-8add-409ac19317a8': + '/data-tables', + '/data-tables/fast-track/2c4ad26b-fd81-4580-bc4e-27ba6ca6ef6a': + '/data-tables', + '/data-tables/fast-track/2cdefe2c-6027-4f58-9a39-9eba029fc168': + '/data-tables', + '/data-tables/fast-track/2d2e4f14-a48a-4859-bef1-ac45b9a1b83e': + '/data-tables', + '/data-tables/fast-track/2ed7b34f-8ad6-4516-31fc-08d90ed7dffe': + '/data-tables', + '/data-tables/fast-track/32d01225-09b9-4629-bc4a-a90fd62c3e20': + '/data-tables', + '/data-tables/fast-track/32f9af5e-1cae-4a40-b384-08da9bbbb7c1': + '/data-tables', + '/data-tables/fast-track/3303ab40-23f1-4089-9f19-b6a19f399197': + '/data-tables', + '/data-tables/fast-track/336e9909-3c43-4413-b06a-1642c7968ae3': + '/data-tables', + '/data-tables/fast-track/34401325-bf6a-48c1-ac5e-73963b3466b1': + '/data-tables', + '/data-tables/fast-track/34494aa3-e6c1-48a1-acdb-82d5da0021d7': + '/data-tables', + '/data-tables/fast-track/35a09351-7129-43b9-8fb0-83a052453471': + '/data-tables', + '/data-tables/fast-track/35cfe914-9590-40cb-c105-08daa787e284': + '/data-tables', + '/data-tables/fast-track/361ea940-2a1b-42c9-a87f-01f85cfcdc19': + '/data-tables', + '/data-tables/fast-track/3683eb72-2ec5-403b-878f-0c9814265fd3': + '/data-tables', + '/data-tables/fast-track/37176a2a-e4d6-436b-a0c1-5': '/data-tables', + '/data-tables/fast-track/386dc2c6-d953-473a-8947-1f081c3094b6': + '/data-tables', + '/data-tables/fast-track/389db54b-ab55-4f38-8345-f3967fc0d131': + '/data-tables', + '/data-tables/fast-track/38f63f82-e005-48ec-bf9a-97b03f57a48c': + '/data-tables', + '/data-tables/fast-track/3aa9d070-0c6a-4b38-a749-f758bbe74b9e': + '/data-tables', + '/data-tables/fast-track/3c928d23-70c0-4203-a1ad-9f4c8d185d2f': + '/data-tables', + '/data-tables/fast-track/3e0a5bf2-b2a3-4c18-88c9-6d8039b3007e': + '/data-tables', + '/data-tables/fast-track/3e94ab36-bceb-4a5f-9c6d-64a6f714dcf0': + '/data-tables', + '/data-tables/fast-track/3f7f9488-e896-4622-9472-4b01fefba33a': + '/data-tables', + '/data-tables/fast-track/3f8f4e21-c0f2-4378-b3c3-114cf81465cd': + '/data-tables', + '/data-tables/fast-track/3f93fe61-36f5-4844-ba9e-6aad692d91dd': + '/data-tables', + '/data-tables/fast-track/403a75f6-222f-435a-8947-f12d37b77ee5': + '/data-tables', + '/data-tables/fast-track/40aea90b-3d2c-420c-a013-5d02393dd102': + '/data-tables', + '/data-tables/fast-track/423c4ef5-c2ec-4f07-8e75-0a81630d3757': + '/data-tables', + '/data-tables/fast-track/42f62bef-bfe3-4a1e-a27b-3ac8e2928b45': + '/data-tables', + '/data-tables/fast-track/43401fa9-3cee-4ebe-b7de-34ad4eb525eb': + '/data-tables', + '/data-tables/fast-track/44433f05-d1e1-4ced-86be-08da7eccd8d3': + '/data-tables', + '/data-tables/fast-track/45a93b5d-861d-4cf1-b2cb-e2c738a8c99f': + '/data-tables', + '/data-tables/fast-track/46a36a97-a5e0-4324-a575-31d229942124': + '/data-tables', + '/data-tables/fast-track/46bfea21-74cb-473b-961a-1480bda0e904': + '/data-tables', + '/data-tables/fast-track/4760154a-9a9c-439a-d44f-08dabceba8ca': + '/data-tables', + '/data-tables/fast-track/4891da30-91d7-44b5-75ef-08db786e904b': + '/data-tables', + '/data-tables/fast-track/48a27133-7247-4b7d-5f7c-08db3a70d65c': + '/data-tables', + '/data-tables/fast-track/490b6032-3d52-4be7-9223-04da13b956dd': + '/data-tables', + '/data-tables/fast-track/4a716316-3b92-4c19-3593-08daa20fa891': + '/data-tables', + '/data-tables/fast-track/4b48701a-9025-475a-c0c5-08daa787e284': + '/data-tables', + '/data-tables/fast-track/4c98c839-dc1e-4421-b2ea-25e7f4208af8': + '/data-tables', + '/data-tables/fast-track/4ccb033a-d706-4eac-bee3-9695e512d76e': + '/data-tables', + '/data-tables/fast-track/4ea0eaa3-36ca-43d2-42e4-08da3eebca15': + '/data-tables', + '/data-tables/fast-track/4ef5a904-ecd6-4da1-820e-ff218f0ef17c': + '/data-tables', + '/data-tables/fast-track/4f0fb5cb-b591-45fb-3552-08daa20fa891': + '/data-tables', + '/data-tables/fast-track/4f4da0c4-6b09-48a8-a731-126d3de01e20': + '/data-tables', + '/data-tables/fast-track/4fa4f6f7-57fb-4220-b0b4-32008be6c43b': + '/data-tables', + '/data-tables/fast-track/4fede41b-deb6-4781-b018-e799723ff1a4': + '/data-tables', + '/data-tables/fast-track/50208d7d-3c3c-4609-b98d-67435406676b': + '/data-tables', + '/data-tables/fast-track/50747948-ee8c-495c-8d54-6318c1d37d27': + '/data-tables', + '/data-tables/fast-track/508bfccc-4a82-4fdf-a3c9-79dd3086c658': + '/data-tables', + '/data-tables/fast-track/51051d54-1b04-4797-8d92-c71893dab2aa': + '/data-tables', + '/data-tables/fast-track/52da011b-c5a0-483b-89f4-000b6bd38ea7': + '/data-tables', + '/data-tables/fast-track/5384d9eb-749b-4457-92d4-063699f79f69': + '/data-tables', + '/data-tables/fast-track/53c9de7d-536f-45c8-8e31-1fcd7f1bb9f2': + '/data-tables', + '/data-tables/fast-track/54d83471-6d8c-4c83-adee-08d8fff8a85b': + '/data-tables', + '/data-tables/fast-track/5593f530-5816-4342-bdea-80c738d583b1': + '/data-tables', + '/data-tables/fast-track/569c3a6b-8328-4e96-8865-00fd965687b1': + '/data-tables', + '/data-tables/fast-track/5832704e-405b-41d3-8697-08da7eccd8d3': + '/data-tables', + '/data-tables/fast-track/583c1413-605d-488a-9870-42df33a5f83a': + '/data-tables', + '/data-tables/fast-track/5942c462-dd79-4cca-a038-be465e8d535e': + '/data-tables', + '/data-tables/fast-track/5bf308dd-22b5-4ae2-89a4-000c6073b24b': + '/data-tables', + '/data-tables/fast-track/5c994200-1736-42ed-acc6-12dbcf89466d': + '/data-tables', + '/data-tables/fast-track/5cb00347-4735-45fd-b23b-451942c1278c': + '/data-tables', + '/data-tables/fast-track/5f78743f-cdf3-4725-abad-ff4ed7a3a552': + '/data-tables', + '/data-tables/fast-track/5f9b466e-8dec-4f68-c15e-08daa787e284': + '/data-tables', + '/data-tables/fast-track/6012c61b-e007-4c35-8959-78eda64429ad': + '/data-tables', + '/data-tables/fast-track/61424f5b-e434-47bd-8401-33cbd44f3bba': + '/data-tables', + '/data-tables/fast-track/62452bd5-31df-': '/data-tables', + '/data-tables/fast-track/62452bd5-31df-449a-931e-516e8fe7705b': + '/data-tables', + '/data-tables/fast-track/6356f6c1-2cb9-4ffb-a8aa-08da8f1f9388': + '/data-tables', + '/data-tables/fast-track/63fdd805-85d9-422b-a54c-c94591b17319': + '/data-tables', + '/data-tables/fast-track/646bd6fb-7f44-4224-ba29-2f49b076be80': + '/data-tables', + '/data-tables/fast-track/6657ce22-abac-4508-5b6a-08d98e357d76': + '/data-tables', + '/data-tables/fast-track/66b09df2-dfcd-4055-849d-9e68a37cfd21': + '/data-tables', + '/data-tables/fast-track/6858cd82-d149-49bd-bf41-27d659d5c428': + '/data-tables', + '/data-tables/fast-track/6989b06f-c55d-40a4-8ea6-4eb90b8e5eaf': + '/data-tables', + '/data-tables/fast-track/69e72292-508e-4558-a15d-fd5bf85af7b4': + '/data-tables', + '/data-tables/fast-track/6a846ce2-0f94-47d1-a9c3-6d11ffecd170': + '/data-tables', + '/data-tables/fast-track/6b4bc912-b742-4109-a78b-6093cba746e8': + '/data-tables', + '/data-tables/fast-track/6be2d494-6a00-48db-defa-08dacc6b24db': + '/data-tables', + '/data-tables/fast-track/717daf19-e125-413f-bba3-482dc9fab22b': + '/data-tables', + '/data-tables/fast-track/719569fe-cb80-43f3-8bf7-eb23345d0f49': + '/data-tables', + '/data-tables/fast-track/71c6ed96-25d2-4985-f248-08db998c77c8': + '/data-tables', + '/data-tables/fast-track/7427255d-9f04-4869-afba-dc8c57ef6c51': + '/data-tables', + '/data-tables/fast-track/74fca436-5e3b-4d7e-8ccc-83a78086caee': + '/data-tables', + '/data-tables/fast-track/7578e83c-3a69-4bac-928a-74194081aed9': + '/data-tables', + '/data-tables/fast-track/764a0fa2-5722-4a3d-b006-e5b044cc520a': + '/data-tables', + '/data-tables/fast-track/775cc77f-420e-4ecc-8304-8dc472a57c78': + '/data-tables', + '/data-tables/fast-track/776af074-bc69-4482-b84f-a12ea30f1c2c': + '/data-tables', + '/data-tables/fast-track/77bf7a68-beff-4a39-ab32-16376aedf127': + '/data-tables', + '/data-tables/fast-track/7b87da46-9624-4edf-96e4-3b8a322f5598': + '/data-tables', + '/data-tables/fast-track/7ce027d2-7cbb-46da-': '/data-tables', + '/data-tables/fast-track/7e7a2ca3-309b-44f1-b4ea-55f39e3bb239': + '/data-tables', + '/data-tables/fast-track/7eea0c3c-b1f7-4703-35bd-08daa20fa891': + '/data-tables', + '/data-tables/fast-track/7f0de236-f5a1-433d-ad3b-1a75cd7e09cb': + '/data-tables', + '/data-tables/fast-track/809209ed-4edf-4c2e-7761-08dab100bfc2': + '/data-tables', + '/data-tables/fast-track/80c23194-68b9-4c2e-b7bd-a164f6163c06': + '/data-tables', + '/data-tables/fast-track/814bd896-1d3b-475e-b21d-d0c5faf6ee8f': + '/data-tables', + '/data-tables/fast-track/81ab3729-6cfb-4738-a502-10d2859604b2': + '/data-tables', + '/data-tables/fast-track/81d1c4f7-e9da-48d9-9729-6a9e34f9f578': + '/data-tables', + '/data-tables/fast-track/8288f4d3-2e4e-4135-a74f-638a1d849714': + '/data-tables', + '/data-tables/fast-track/8321bebd-a884-42e0-86d4-408e82ed5f48': + '/data-tables', + '/data-tables/fast-track/83c1c8fd-ce0c-42a0-c46a-08d9bbe39aeb': + '/data-tables', + '/data-tables/fast-track/8413bd02-26f0-': '/data-tables', + '/data-tables/fast-track/84a9b3f5-ca6f-446b-bcf9-f208b79b6a0e': + '/data-tables', + '/data-tables/fast-track/84b992d0-9abf-427c-a3b2-393a673795be': + '/data-tables', + '/data-tables/fast-track/84e08b66-a434-4a3c-903d-83537af066cb': + '/data-tables', + '/data-tables/fast-track/8560a280-ceb8-40e9-8c24-a760035a008e': + '/data-tables', + '/data-tables/fast-track/85b4b7ac-f715-4eab-acb8-c872c8ae5281': + '/data-tables', + '/data-tables/fast-track/8706180c-2bdb-481e-20da-': '/data-tables', + '/data-tables/fast-track/8706180c-2bdb-481e-20da-08d99c9bf8eb': + '/data-tables', + '/data-tables/fast-track/8836e106-600d-4126-b324-c07a9ea970ce': + '/data-tables', + '/data-tables/fast-track/896ddead-213c-43aa-a080-aaa185af2cd9': + '/data-tables', + '/data-tables/fast-track/89e05fef-0b22-4d68-9260-9e23039e0dc7': + '/data-tables', + '/data-tables/fast-track/89f774e6-088e-4345-b3ce-3a62b4e7e32b': + '/data-tables', + '/data-tables/fast-track/8a41e161-2b53-44a7-a9c6-08da8befc70c': + '/data-tables', + '/data-tables/fast-track/8a804f47-9581-40cf-8f18-0479b35f2dab': + '/data-tables', + '/data-tables/fast-track/8b83d7ec-1ba4-44dc-b21c-adb5fa5a4bc8': + '/data-tables', + '/data-tables/fast-track/8ba8c85a-f067-48b2-81ed-2efa4040f3f9': + '/data-tables', + '/data-tables/fast-track/8c5674ec-4e2c-448f-80f0-a4dca6be5df8': + '/data-tables', + '/data-tables/fast-track/8c693d87-28d7-4ff1-9374-a5896c326662': + '/data-tables', + '/data-tables/fast-track/8c6ae42d-b89e-4e88-bd26-519da67832e1': + '/data-tables', + '/data-tables/fast-track/8d979f9b-ba2a-42a8-a9f9-a6d43413b89d': + '/data-tables', + '/data-tables/fast-track/8e066026-ce43-40b1-a534-f32df5dd5adb': + '/data-tables', + '/data-tables/fast-track/8fa4f92b-980a-46dc-982b-2b35b89b11f9': + '/data-tables', + '/data-tables/fast-track/90691076-386a-4ba9-9524-08d9986262b5': + '/data-tables', + '/data-tables/fast-track/908beb34-e6c3-4dd9-88af-00c13d597b04': + '/data-tables', + '/data-tables/fast-track/9125b59f-95be-4600-85ea-6d583b7566e1': + '/data-tables', + '/data-tables/fast-track/91805ad0-1ba2-4cdf-bb44-83aad652fe69': + '/data-tables', + '/data-tables/fast-track/91a730e7-e3d5-46d6-aae5-2eb481c9fb0f': + '/data-tables', + '/data-tables/fast-track/91e9e185-2da3-47fa-93a9-773249a3c7d6': + '/data-tables', + '/data-tables/fast-track/9201b32b-cc91-4365-3574-08daa20fa891': + '/data-tables', + '/data-tables/fast-track/92031aef-7efe-4c70-9204-da82ac02124d': + '/data-tables', + '/data-tables/fast-track/92d88d1f-1d56-43e2-912d-c13f17916a52': + '/data-tables', + '/data-tables/fast-track/94385483-4d6a-4d6d-c05f-08db465931c0': + '/data-tables', + '/data-tables/fast-track/948a5007-b090-4e43-8e9d-8c0b7cfa0fef': + '/data-tables', + '/data-tables/fast-track/952dc5bd-d0a0-4f22-bf5b-7a597bfc6f68': + '/data-tables', + '/data-tables/fast-track/97a699fd-1d72-473e-d48c-08dabceba8ca': + '/data-tables', + '/data-tables/fast-track/9a2cb3b7-6908-415a-8604-ae9eee5265a9': + '/data-tables', + '/data-tables/fast-track/9a360800-506b-4550-f597-08d8e93777ac': + '/data-tables', + '/data-tables/fast-track/9af514a7-00d2-4090-80d9-7e5af1677970': + '/data-tables', + '/data-tables/fast-track/9b1e4a4e-02b1-4795-8c74-b65acfd4d20f': + '/data-tables', + '/data-tables/fast-track/9b25fded-9844-477d-8694-08da7eccd8d3': + '/data-tables', + '/data-tables/fast-track/9b4a435d-7a43-4f74-3bed-08db5b843ba5': + '/data-tables', + '/data-tables/fast-track/9cee6f89-d584-43e1-f595-08d8e93777ac': + '/data-tables', + '/data-tables/fast-track/9dffde6f-0a27-49ee-a2a5-5fbeae8d9f3a': + '/data-tables', + '/data-tables/fast-track/9f2d0cdf-84dc-4c08-aa5f-29dcdfc6fe54': + '/data-tables', + '/data-tables/fast-track/a001d78f-6edf-45e0-af48-f130003c2040': + '/data-tables', + '/data-tables/fast-track/a011ef71-e150-4c87-8d5d-035b86dd7cd1': + '/data-tables', + '/data-tables/fast-track/a03ee004-1fd7-44bc-c145-08daa787e284': + '/data-tables', + '/data-tables/fast-track/a0f9ca30-f651-4876-ad2b-878ce32919cc': + '/data-tables', + '/data-tables/fast-track/a1413c9c-1b78-48c4-afed-91eb47b66c1b': + '/data-tables', + '/data-tables/fast-track/a2698ca3-0950-496b-8096-0b4125e3f65f': + '/data-tables', + '/data-tables/fast-track/a397dea6-5c24-4700-9a02-2ea7957593ef': + '/data-tables', + '/data-tables/fast-track/a4042cfc-53d4-4349-9a1f-e153999374ac': + '/data-tables', + '/data-tables/fast-track/a494ffaa-a1cd-428d-9084-ee918506fae8': + '/data-tables', + '/data-tables/fast-track/a55bef42-f8af-4ca6-9093-3b743d13f2b8': + '/data-tables', + '/data-tables/fast-track/a6b4d538-b919-490e-c217-08dba88b1a38': + '/data-tables', + '/data-tables/fast-track/a89db6b7-815e-44cd-a32c-8ce718a9fd39': + '/data-tables', + '/data-tables/fast-track/ab4809de-fe59-4dc4-a30b-b3b064db9216': + '/data-tables', + '/data-tables/fast-track/aca1d4a5-88eb-4a96-a8e3-08da8f1f9388': + '/data-tables', + '/data-tables/fast-track/acbd1b5c-b775-447b-993e-b7ef479a7139': + '/data-tables', + '/data-tables/fast-track/acff0bb0-4d7a-4971-84da-79a0cd9fe36b': + '/data-tables', + '/data-tables/fast-track/ae12ebc0-73b6-4a40-d771-08db513a05d2': + '/data-tables', + '/data-tables/fast-track/afa0571f-a9af-413e-ae52-8e9467c5e30f': + '/data-tables', + '/data-tables/fast-track/b048b06b-23a4-467f-986b-b471ff54da05': + '/data-tables', + '/data-tables/fast-track/b071bf10-58a5-41bf-bc4b-08da8695004d': + '/data-tables', + '/data-tables/fast-track/b0af25f4-84c8-4bb4-b498-de8b3abb5312': + '/data-tables', + '/data-tables/fast-track/b2b15e44-f8a9-41ff-a8af-b2baa28d60bc': + '/data-tables', + '/data-tables/fast-track/b2bd9f8c-768e-4d9f-bce0-e9e39a564e22': + '/data-tables', + '/data-tables/fast-track/b2d0cfe2-e605-4fc6-3beb-08db5b843ba5': + '/data-tables', + '/data-tables/fast-track/b31d8b95-f7bc-4e11-c064-08db465931c0': + '/data-tables', + '/data-tables/fast-track/b343e11c-f175-45fa-8541-45c370599f5f': + '/data-tables', + '/data-tables/fast-track/b4793cfd-5991-4cd0-870d-e41cfb5a65af': + '/data-tables', + '/data-tables/fast-track/b5d1a528-8b18-44da-aeb6-b47bd3481b18': + '/data-tables', + '/data-tables/fast-track/b6dc17c3-7d42-4155-9f69-f884ec61bd75': + '/data-tables', + '/data-tables/fast-track/b82a3833-8d45-4332-aeb2-a51be6c15146': + '/data-tables', + '/data-tables/fast-track/b84ee31e-4ba5-4cea-5692-08d9924a81c2': + '/data-tables', + '/data-tables/fast-track/b88bae97-d789-40b5-81aa-508713e7ccc8': + '/data-tables', + '/data-tables/fast-track/b91ccb4d-7d8b-4ced-969f-a050aecbc77f': + '/data-tables', + '/data-tables/fast-track/b97bf61a-89cf-4818-b32c-3f83482ad63d': + '/data-tables', + '/data-tables/fast-track/bba7bc5f-fd01-43c9-35a4-08daa20fa891': + '/data-tables', + '/data-tables/fast-track/bbbdd826-8df9-45b1-a3d0-b54e18fbd46f': + '/data-tables', + '/data-tables/fast-track/bbcaf878-50f3-4755-a987-36b0e9a752cd': + '/data-tables', + '/data-tables/fast-track/bbf9ee63-096a-472a-a50f-0b3039805ebe': + '/data-tables', + '/data-tables/fast-track/bd25d934-c0b5-4de2-81df-75eec0e9bbbb': + '/data-tables', + '/data-tables/fast-track/bf12cd2d-83d7-4ed8-a439-00f1ca0ccf06': + '/data-tables', + '/data-tables/fast-track/bf1beb50-1dd5-4426-b9fa-9db0df20ea7e': + '/data-tables', + '/data-tables/fast-track/bfa15bab-a5e0-4dee-86ea-4bc2aa1bf3b3': + '/data-tables', + '/data-tables/fast-track/bfc4c50e-734f-42c3-a4e2-7090e3c11641': + '/data-tables', + '/data-tables/fast-track/c0057c9f-ca9a-4360-8ae4-10a03d4d7f29': + '/data-tables', + '/data-tables/fast-track/c0a53e6e-2c9a-4417-bcf6-1e357dede040': + '/data-tables', + '/data-tables/fast-track/c15f4679-2b6a-4bdb-86b9-08da7eccd8d3': + '/data-tables', + '/data-tables/fast-track/c18ea9c7-b699-4b46-adfc-9718206e1dd4': + '/data-tables', + '/data-tables/fast-track/c298fa6e-b23f-49da-9a42-46f313c47983': + '/data-tables', + '/data-tables/fast-track/c2dbb675-1aea-4c79-9938-7406e44e9bea': + '/data-tables', + '/data-tables/fast-track/c2e09977-1a41-4554-8fcc-ab1a147f50f6': + '/data-tables', + '/data-tables/fast-track/c34c7ebc-30e8-4a77-b9a1-bbcf40c624ec': + '/data-tables', + '/data-tables/fast-track/c3ed3a98-edc1-47e1-917d-98fc0eabbb77': + '/data-tables', + '/data-tables/fast-track/c4061205-6cd7-4ec2-b2e7-9e5a9cb31b17': + '/data-tables', + '/data-tables/fast-track/c4bf6323-e7a8-4267-94fd-71084d159296': + '/data-tables', + '/data-tables/fast-track/c69292a7-d49e-4b43-8fc8-bdd0e46f4e04': + '/data-tables', + '/data-tables/fast-track/c6ae8b83-29b3-4d68-bac6-8690ea94917d': + '/data-tables', + '/data-tables/fast-track/c7997707-e733-4591-b993-3faa4ac3668f': + '/data-tables', + '/data-tables/fast-track/c7ead7a2-5660-4192-7732-08dab100bfc2': + '/data-tables', + '/data-tables/fast-track/c82b1eab-10d8-4475-872c-207ff32688e3': + '/data-tables', + '/data-tables/fast-track/c88edc91-fde0-4f08-8f82-bdb3dce8555c': + '/data-tables', + '/data-tables/fast-track/c991e329-49ec-480a-bfe4-2383cd6f5170': + '/data-tables', + '/data-tables/fast-track/c99814ff-dbaa-49ac-8a4c-40a0d2d829c1': + '/data-tables', + '/data-tables/fast-track/c9da44a5-527e-4f75-9812-12fe0490403e': + '/data-tables', + '/data-tables/fast-track/ca08e074-2ce1-4448-ba3b-08a1db9a57a3': + '/data-tables', + '/data-tables/fast-track/caca8a1b-9658-4fc2-bfbc-7e5e957a4d2b': + '/data-tables', + '/data-tables/fast-track/cbb62019-fb9f-48f1-c06a-08db465931c0': + '/data-tables', + '/data-tables/fast-track/cc345cf1-82b4-4155-b2ba-ffae201f226e': + '/data-tables', + '/data-tables/fast-track/ccb3ab9a-89a8-49e9-ae93-cfd5634685b2': + '/data-tables', + '/data-tables/fast-track/ccf319fc-08f4-4720-bd0f-b873127346c7': + '/data-tables', + '/data-tables/fast-track/cd305910-93d7-419d-9404-0ba98bb43de1': + '/data-tables', + '/data-tables/fast-track/cd386772-c34a-43e9-a1e6-bef4686d4346': + '/data-tables', + '/data-tables/fast-track/ce0359d5-a5d0-4e75-aaa1-34e75b017d23': + '/data-tables', + '/data-tables/fast-track/ce103d9b-f827-43dd-a8ea-f8b71028732f': + '/data-tables', + '/data-tables/fast-track/cf653512-96c0-4420-b570-b680a32f4e90': + '/data-tables', + '/data-tables/fast-track/cf753eaf-c214-410d-a832-5dac9980453b': + '/data-tables', + '/data-tables/fast-track/cf964186-58f0-47ac-bb30-1515940251ce': + '/data-tables', + '/data-tables/fast-track/d0df05d5-3038-': '/data-tables', + '/data-tables/fast-track/d40df679-2092-4d7f-89f7-91e4da6134ae': + '/data-tables', + '/data-tables/fast-track/d55b6ea8-6ae4-49e8-8749-216e706f5f17': + '/data-tables', + '/data-tables/fast-track/d596f5e0-3aff-4638-a553-af0515d9e755': + '/data-tables', + '/data-tables/fast-track/d5aa5ea1-8111-42af-b63e-c1158f00c96c': + '/data-tables', + '/data-tables/fast-track/d662e080-a7ee-4310-b798-8788c516e8dc': + '/data-tables', + '/data-tables/fast-track/d77da3cd-db3b-4b87-f58c-08d8e93777ac': + '/data-tables', + '/data-tables/fast-track/d7ee63f3-f398-4003-9062-250161008878': + '/data-tables', + '/data-tables/fast-track/da9e690a-16dc-4211-8114-04c3abfd5c4d': + '/data-tables', + '/data-tables/fast-track/dcdc0385-ef22-477f-8ff4-0f478c04f75d': + '/data-tables', + '/data-tables/fast-track/dd6a66de-2636-453f-9dca-8ad9f1497c32': + '/data-tables', + '/data-tables/fast-track/de51d27f-dbf3-42aa-9d1a-6c099919de96': + '/data-tables', + '/data-tables/fast-track/df2f55d9-3076-4f3c-87a7-d33258f7868a': + '/data-tables', + '/data-tables/fast-track/e0418641-27c9-4b5a-8804-ac070a8fad2c': + '/data-tables', + '/data-tables/fast-track/e07a81a3-8a6d-4185-b7ee-03bfe245340c': + '/data-tables', + '/data-tables/fast-track/e1b81c76-d59e-4ba2-8d0d-e756d48063bd': + '/data-tables', + '/data-tables/fast-track/e45f43af-b7da-46dd-3575-08daa20fa891': + '/data-tables', + '/data-tables/fast-track/e4def50d-5224-45bf-baac-473b88f1a0fb': + '/data-tables', + '/data-tables/fast-track/e50aa2ec-4725-4087-a184-e612db99de03': + '/data-tables', + '/data-tables/fast-track/e71dd1d5-86d3-': '/data-tables', + '/data-tables/fast-track/e79cf591-5f85-45b1-7755-08dab100bfc2': + '/data-tables', + '/data-tables/fast-track/e8b0b479-d39e-42ef-b4d2-877bbe678c39': + '/data-tables', + '/data-tables/fast-track/e912c09e-cda2-4829-af3b-bc5027f4f528': + '/data-tables', + '/data-tables/fast-track/ea1384a0-03d0-42b5-9b6d-3e81ead18f95': + '/data-tables', + '/data-tables/fast-track/ea917bb0-97d5-4cb5-bd11-9f5d80a5cd17': + '/data-tables', + '/data-tables/fast-track/eb1178b4-d357-4a6d-d716-08db513a05d2': + '/data-tables', + '/data-tables/fast-track/ebcd4c02-fdb7-4b8d-9fff-c80cf62bed40': + '/data-tables', + '/data-tables/fast-track/ec5f9b52-75e0-4029-16e3-08da47b0392d': + '/data-tables', + '/data-tables/fast-track/ed3120a8-e235-40df-9ce3-3adc103db4aa': + '/data-tables', + '/data-tables/fast-track/edf5dc3e-cd41-4a9e-3554-08daa20fa891': + '/data-tables', + '/data-tables/fast-track/ee086150-1e90-4e3e-b4d3-cc11d87a6fc7': + '/data-tables', + '/data-tables/fast-track/eeb158cd-46c2-40c0-8ca3-1e0ab067f59d': + '/data-tables', + '/data-tables/fast-track/eee26b50-6367-4e73-86c0-08da7eccd8d3': + '/data-tables', + '/data-tables/fast-track/ef95998d-f96a-4c4f-9d2a-e00ef17684e7': + '/data-tables', + '/data-tables/fast-track/f0423305-94d6-4552-86ba-08da7eccd8d3': + '/data-tables', + '/data-tables/fast-track/f05583c6-4ee5-4c26-bb98-515cc90885c3': + '/data-tables', + '/data-tables/fast-track/f171d6ff-6e7d-4bb0-b90e-cca1eb1f8767': + '/data-tables', + '/data-tables/fast-track/f1c0e0bd-af8e-4644-9b25-5490fabd8acc': + '/data-tables', + '/data-tables/fast-track/f2ed11d3-7320-46c9-b2a4-938fa0a0c2af': + '/data-tables', + '/data-tables/fast-track/f360a303-3d24-4516-f7f3-08da8c214b78': + '/data-tables', + '/data-tables/fast-track/f43f768d-df1b-4f48-1d7e-08db8eaeaf03': + '/data-tables', + '/data-tables/fast-track/f4d42479-': '/data-tables', + '/data-tables/fast-track/f4e2ad47-fc89-4020-a3fd-8536a7e7667d': + '/data-tables', + '/data-tables/fast-track/f5c96542-0736-493b-90f4-917b53717579': + '/data-tables', + '/data-tables/fast-track/f5e19431-a6c5-443a-bf3d-9b497aa41bc4': + '/data-tables', + '/data-tables/fast-track/f61793f8-e54e-477a-a6ae-0a856bac4be5': + '/data-tables', + '/data-tables/fast-track/f6272eee-e3a0-44ac-82bf-f8c7f517e778': + '/data-tables', + '/data-tables/fast-track/f85ae336-f947-4fd2-829c-94a3c4f613d7': + '/data-tables', + '/data-tables/fast-track/f88c7e10-7af0-4987-b4ae-cb02a6e82119': + '/data-tables', + '/data-tables/fast-track/f8d4dcd7-f5a0-4625-9369-bd40bd43c7eb': + '/data-tables', + '/data-tables/fast-track/f9ae0266-dbeb-4fb6-a0cd-ca0bc0d67ac3': + '/data-tables', + '/data-tables/fast-track/fbd1ee4f-f072-48c4-abe5-ac15f1c1dd57': + '/data-tables', + '/data-tables/fast-track/fc23c1d7-6cee-4d9d-8acb-f47879a6f7db': + '/data-tables', + '/data-tables/fast-track/fd88636c-c6fc-4802-98f6-260e6670cd58': + '/data-tables', + '/data-tables/fast-track/fd94f5a9-7eac-4017-9d20-a90fe89a627c': + '/data-tables', + '/data-tables/fast-track/fef5a785-caa5-4de9-a8d8-08da8f1f9388': + '/data-tables', + '/data-tables/fast-track/ff7bbf92-c452-4805-bd1d-282aec2c9772': + '/data-tables', + '/data-tables/permalink/%5bpermalink%5d': '/data-tables', + '/data-tables/permalink/09fca49f-7bd8-475f-dd03-08db8e99978c.': + '/data-tables', + '/data-tables/permalink/167d18fb-': '/data-tables', + '/data-tables/permalink/203a8889-429b-469f-abe1-': '/data-tables', + '/data-tables/permalink/2e97cf7e-3fe6-4e97-8525-': '/data-tables', + '/data-tables/permalink/36526263-f93b-4bb1-9d1f-': '/data-tables', + '/data-tables/permalink/47223cca-6ffb-42da-be50-': '/data-tables', + '/data-tables/permalink/55ca2487-7a4d-43c4-487d-08db8e99c26c.': + '/data-tables', + '/data-tables/permalink/59fc4e59-160c-4755-c612-08': '/data-tables', + '/data-tables/permalink/5ab5dbf2-393b-4a43-b7a1-': '/data-tables', + '/data-tables/permalink/67961e79-5204-410e-b521-': '/data-tables', + '/data-tables/permalink/85ba124e-1489-4648-bfde-': '/data-tables', + '/data-tables/permalink/85ba124e-1489-4648-bfde-3eff6fdf76a2%3b': + '/data-tables', + '/data-tables/permalink/8f50fce8-': '/data-tables', + '/data-tables/permalink/970e9bb8-d185-': '/data-tables', + '/data-tables/permalink/9795440a-015e-47eb-86e2-': '/data-tables', + '/data-tables/permalink/9e420c30-': '/data-tables', + '/data-tables/permalink/c0319f82-e1e3-4adf-9db1-': '/data-tables', + '/data-tables/permalink/c4cb6884-': '/data-tables', + '/data-tables/permalink/e0980465-': '/data-tables', + '/data-tables/permalink/e8942369-b2a3-': '/data-tables', + '/datatables/apprenticeships-and-traineeships': '/data-tables', + '/datatables/permalink/48f9a035-9123-477e-': '/data-tables', + '/datatables/permalink/b169759c-': '/data-tables', + '/find-st%3c/body%3e%3c/html%3e': '/find-statistics', + '/find-sta': '/find-statistics', + '/find-stati': '/find-statistics', + '/find-statis-': '/find-statistics', + '/find-statistic': '/find-statistics', + '/find-statistics/%5bpublication%5d': '/find-statistics', + '/find-statistics/%5bpublication%5d/%5brelease%5d': '/find-statistics', + '/find-statistics/%3ca%20href%3d': '/find-statistics', + '/find-statistics/16-18-': '/find-statistics', + '/find-statistics/16-18-destination-measure': '/find-statistics', + '/find-statistics/a-': '/find-statistics', + '/find-statistics/a-level': '/find-statistics', + '/find-statistics/a-level-and-other-16-to-': '/find-statistics', + '/find-statistics/a-level-and-other-16-to-18-': '/find-statistics', + '/find-statistics/a-level-and-other-16-to-18-results/2019-20%20%e2%80%93': + '/find-statistics', + '/find-statistics/a-level-and-other-level-3-results-in-england-academic-year-2019-to-2020': + '/find-statistics', + '/find-statistics/admission-': '/find-statistics', + '/find-statistics/apprenticeships-': '/find-statistics', + '/find-statistics/apprenticeships-and': '/find-statistics', + '/find-statistics/apprenticeships-and-traineeships/2020-21%20%0a-%20apprenticeship%20starts%20down%200.3%25%20in%2021-22%20vs%2019-20': + '/find-statistics', + '/find-statistics/apprenticeships-and-traineeshipsthe': '/find-statistics', + '/find-statistics/at-': '/find-statistics', + '/find-statistics/attainment-8-score-averages-by-previous-attainment/2019-to-2020-revised': + '/find-statistics', + '/find-statistics/attendance-': '/find-statistics', + '/find-statistics/attendance-in-': '/find-statistics', + '/find-statistics/attendance-in-edu': '/find-statistics', + '/find-statistics/attendance-in-educa-': '/find-statistics', + '/find-statistics/attendance-in-education': '/find-statistics', + '/find-statistics/attendance-in-education-': '/find-statistics', + '/find-statistics/attendance-in-education-and-': '/find-statistics', + '/find-statistics/attendance-in-education-and-earl': '/find-statistics', + '/find-statistics/attendance-in-education-and-early-': '/find-statistics', + '/find-statistics/attendance-in-education-and-early-years': + '/find-statistics', + '/find-statistics/attendance-in-education-and-early-years-': + '/find-statistics', + '/find-statistics/attendance-in-education-and-early-years-settings-during-the-': + '/find-statistics', + '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-': + '/find-statistics', + '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-covid-19-': + '/find-statistics', + '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-covid-19-outbreak%20%0a2': + '/find-statistics', + '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-covid-19-outbreak/2020-week-': + '/find-statistics', + '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-covid-19-outbreak/2021-week-': + '/find-statistics', + '/find-statistics/attendance-in-education-and-early-years-settings-during-the-coronavirus-covid-19-outbreak/2022-week-': + '/find-statistics', + '/find-statistics/attendance-in-education-andearly-years-settings-during-the-coronavirus-covid-19-outbreak': + '/find-statistics', + '/find-statistics/attendance-ineducation-': '/find-statistics', + '/find-statistics/characteristics-of-': '/find-statistics', + '/find-statistics/characteristics-of-children-in-': '/find-statistics', + '/find-statistics/characteristics-of-children-in-n': '/find-statistics', + '/find-statistics/characteristics-of-children-in-need/2022%26nbsp': + '/find-statistics', + '/find-statistics/characteristics-of-children-in-need%26nbsp': + '/find-statistics', + '/find-statistics/characteristics-of-children-inneed/2020': + '/find-statistics', + '/find-statistics/childcare': '/find-statistics', + '/find-statistics/childcare-and-': '/find-statistics', + '/find-statistics/childcare-and-early-': '/find-statistics', + '/find-statistics/children': '/find-statistics', + '/find-statistics/children-': '/find-statistics', + '/find-statistics/children-loo': '/find-statistics', + '/find-statistics/children-looked': '/find-statistics', + '/find-statistics/children-looked-after-in': '/find-statistics', + '/find-statistics/children-looked-after-in-england': '/find-statistics', + '/find-statistics/children-looked-after-in-england-includ': + '/find-statistics', + '/find-statistics/children-looked-after-in-england-including': + '/find-statistics', + '/find-statistics/children-looked-after-in-england-including-': + '/find-statistics', + '/find-statistics/children-looked-after-in-england-including-adopti': + '/find-statistics', + '/find-statistics/children-looked-after-in-england-including-adoptions/2020%20%0a-%20datadownloads-1': + '/find-statistics', + '/find-statistics/children-looked-after-in-england-including-adoptions/2020developmentofattachmentsbetweenolder': + '/find-statistics', + '/find-statistics/children-looked-after-in-england-including-adoptions/2021%22%20%0a/l%20%22datadownloads-1': + '/find-statistics', + '/find-statistics/children-s-social-work': '/find-statistics', + '/find-statistics/children-s-social-work-workforce-attrition-': + '/find-statistics', + '/find-statistics/children-s-social-work-workforce/2022%26nbsp': + '/find-statistics', + '/find-statistics/delivery-of-air-': '/find-statistics', + '/find-statistics/early-years-foundation-stage-': '/find-statistics', + '/find-statistics/education-': '/find-statistics', + '/find-statistics/education-and-training-': '/find-statistics', + '/find-statistics/education-and-training-statistics-for-the-': + '/find-statistics', + '/find-statistics/education-and-training-statistics-for-the-uk/2022%3b': + '/find-statistics', + '/find-statistics/education-health-': '/find-statistics', + '/find-statistics/education-health-and-care': '/find-statistics', + '/find-statistics/education-health-and-care-plan': '/find-statistics', + '/find-statistics/education-health-and-care-plans%20%0a%28opens%20in%20external%20window%29': + '/find-statistics', + '/find-statistics/education-health-and-care-plans/www.gov.uk/courts-tribunals/first-tier-tribunal-specialeducational-needs-and-disability': + '/find-statistics', + '/find-statistics/education-provision-children-under-5/2020%20%0a%28universal%20three%20and%20four-year%20old%20offer%20and%20disadvantaged%20two%20year%20old%20%0aoffer%29%20and%20%0ahttps%3a/www.gov.uk/government/statistics/childcare-and-early-years-survey-of-parents-2019': + '/find-statistics', + '/find-statistics/elective-home-education/2022-': '/find-statistics', + '/find-statistics/free-school-meals-': '/find-statistics', + '/find-statistics/further-': '/find-statistics', + '/find-statistics/further-education-and-': '/find-statistics', + '/find-statistics/further-education-outcome-': '/find-statistics', + '/find-statistics/further-education-outcome-based-success-mesures': + '/find-statistics', + '/find-statistics/graduate-': '/find-statistics', + '/find-statistics/graduate-labour-': '/find-statistics', + '/find-statistics/graduate-outcomes-': '/find-statistics', + '/find-statistics/he.modelling%40education.gov.uk': '/find-statistics', + '/find-statistics/higher-': '/find-statistics', + '/find-statistics/higher-level-learners-in-england/2018-19.': + '/find-statistics', + '/find-statistics/higher-level-learners-in-england%23datadownloads-1': + '/find-statistics', + '/find-statistics/initial-': '/find-statistics', + '/find-statistics/initial-teacher-train-': '/find-statistics', + '/find-statistics/initial-teacher-training-': '/find-statistics', + '/find-statistics/initial-teacher-training-census/2022-': '/find-statistics', + '/find-statistics/initial-teacher-training-performance-': '/find-statistics', + '/find-statistics/initial-teacher-training-performance-profiles/2019-20.': + '/find-statistics', + '/find-statistics/initial-teacher-trainingcensus/2020-21': '/find-statistics', + '/find-statistics/key-stage-': '/find-statistics', + '/find-statistics/key-stage-1-and-phonics-': '/find-statistics', + '/find-statistics/key-stage-1-and-phonics-screening-check-': + '/find-statistics', + '/find-statistics/key-stage-2-': '/find-statistics', + '/find-statistics/key-stage-2-attainment-national-headlines/2021': + '/find-statistics', + '/find-statistics/key-stage-2-attainment-national-headlines/2021-22%20%0aaccessed%2031st%20july%202022': + '/find-statistics', + '/find-statistics/key-stage-2-attainment/2021-': '/find-statistics', + '/find-statistics/key-stage-4': '/find-statistics', + '/find-statistics/key-stage-4-': '/find-statistics', + '/find-statistics/key-stage-4-destination-measures/2018-': '/find-statistics', + '/find-statistics/key-stage-4-performance': '/find-statistics', + '/find-statistics/key-stage-4-performance-': '/find-statistics', + '/find-statistics/key-stage-4-performance-revised/2020-': '/find-statistics', + '/find-statistics/key-stage-4-performance-revised/2021%e2%80%9322': + '/find-statistics', + '/find-statistics/la-and-school-expenditure/2019-2': '/find-statistics', + '/find-statistics/la-and-school-expenditure/2020-21%3b': '/find-statistics', + '/find-statistics/laptops-and-': '/find-statistics', + '/find-statistics/level-2-and-3-attainment-by-youn': '/find-statistics', + '/find-statistics/level-2-and-3-attainment-by-young-people-': + '/find-statistics', + '/find-statistics/level-2-and-3-attainment-by-young-people-aged-19/2020-': + '/find-statistics', + '/find-statistics/looked-after-children-aged-16-to-17-in-independent-or-semi-': + '/find-statistics', + '/find-statistics/multiplication-tables-check-': '/find-statistics', + '/find-statistics/national-tutoring-': '/find-statistics', + '/find-statistics/neet-statistics-': '/find-statistics', + '/find-statistics/outcomes-for-children-in-need-': '/find-statistics', + '/find-statistics/outcomes-for-children-in-need-including-c': + '/find-statistics', + '/find-statistics/outcomes-for-children-in-needincluding-children-': + '/find-statistics', + '/find-statistics/outcomes-for-children-in-needincluding-children-looked-after-by-local-authorities-in-england': + '/find-statistics', + '/find-statistics/parental': '/find-statistics', + '/find-statistics/participation': '/find-statistics', + '/find-statistics/participation-in-education-and-': '/find-statistics', + '/find-statistics/participation-in-education-training-and-': + '/find-statistics', + '/find-statistics/participation-measures-in-higher-': '/find-statistics', + '/find-statistics/permanent-': '/find-statistics', + '/find-statistics/permanent-and': '/find-statistics', + '/find-statistics/permanent-and-fixed': '/find-statistics', + '/find-statistics/permanent-and-fixed-': '/find-statistics', + '/find-statistics/permanent-and-fixed-p': '/find-statistics', + '/find-statistics/permanent-and-fixed-period-': '/find-statistics', + '/find-statistics/permanent-and-fixed-period-exclu': '/find-statistics', + '/find-statistics/permanent-and-fixed-period-exclusions-in-': + '/find-statistics', + '/find-statistics/permanent-and-fixed-period-exclusions-in-eng': + '/find-statistics', + '/find-statistics/permanent-and-fixed-period-exclusions-in-eng-': + '/find-statistics', + '/find-statistics/permanent-and-fixed-period-exclusions-in-england/2018-': + '/find-statistics', + '/find-statistics/permanent-and-fixed-period-exclusions-in-england/2018-19yougov.uk': + '/find-statistics', + '/find-statistics/permanent-and-fixed-period-exclusions-in-englandfigures': + '/find-statistics', + '/find-statistics/permanent-and-fixed-period-exclusions-inengland/2018-19': + '/find-statistics', + '/find-statistics/permanent-and-fixed-periodexclusions-in-england': + '/find-statistics', + '/find-statistics/permanent-and-fixed-term-exclusions-in-england': + '/find-statistics', + '/find-statistics/permanentand-fixed-period-exclusions-in-england': + '/find-statistics', + '/find-statistics/postgraduate-initial-teacher-': '/find-statistics', + '/find-statistics/progression-to-%3ca%20%0ahref%3d': '/find-statistics', + '/find-statistics/progression-to-higher-ed': '/find-statistics', + '/find-statistics/pupil': '/find-statistics', + '/find-statistics/pupil-absence-': '/find-statistics', + '/find-statistics/pupil-absence-in-': '/find-statistics', + '/find-statistics/pupil-absence-in-schools-in-engl': '/find-statistics', + '/find-statistics/pupil-absence-in-schools-in-england-autumn-and-': + '/find-statistics', + '/find-statistics/pupil-absence-in-schools-in-england-autumn-term/data': + '/find-statistics', + '/find-statistics/pupil-absence-in-schools-in-england%29': '/find-statistics', + '/find-statistics/pupil-absence-inschools-in-england-': '/find-statistics', + '/find-statistics/pupil-attendance': '/find-statistics', + '/find-statistics/pupil-attendance-in-': '/find-statistics', + '/find-statistics/pupil-attendance-in-schools/2023-7week-29': + '/find-statistics', + '/find-statistics/school-funding-': '/find-statistics', + '/find-statistics/school-funding-statistics/2021-': '/find-statistics', + '/find-statistics/school-placements-': '/find-statistics', + '/find-statistics/school-pup': '/find-statistics', + '/find-statistics/school-pupils-': '/find-statistics', + '/find-statistics/school-pupils-and-': '/find-statistics', + '/find-statistics/school-pupils-and-thei': '/find-statistics', + '/find-statistics/school-pupils-and-their-': '/find-statistics', + '/find-statistics/school-pupils-and-their-ch': '/find-statistics', + '/find-statistics/school-pupils-and-their-cha': '/find-statistics', + '/find-statistics/school-pupils-and-their-characteristics%3b': + '/find-statistics', + '/find-statistics/school-pupils-and-their-characteristics%22': + '/find-statistics', + '/find-statistics/school-pupils-and-their-characteristics/2019-20.': + '/find-statistics', + '/find-statistics/school-pupils-and-their-characteristicsa': + '/find-statistics', + '/find-statistics/school-pupils-and-their-characteristicshttps%3a/explore-education-statistics.service.gov.uk/find-statistics/school-pupils-and-their-characteristics': + '/find-statistics', + '/find-statistics/school-pupils-and-their-characteristicsm': + '/find-statistics', + '/find-statistics/school-pupils-and-theircharacteristics': '/find-statistics', + '/find-statistics/school-pupils-andtheir-characteristics': '/find-statistics', + '/find-statistics/school-pupilsand-their-characteristics': '/find-statistics', + '/find-statistics/school-workforce-in-englan%3c/p%3e%20%0a%3cp%3e%20for%20more%20detailed%20information%20about%20the%20cookies%20we%20use%2c%20please%20see%20our%20%0a%3ca%20href%3d': + '/find-statistics', + '/find-statistics/school-workforce-in-england%2c': '/find-statistics', + '/find-statistics/school-workforce-in-england%23data': '/find-statistics', + '/find-statistics/school-workforce-in-england%23datablock-f47ed575-f6eb-4bd0-926a-02830e272452-charts': + '/find-statistics', + '/find-statistics/school-workforce-in-englandhttps%3a/explore-education-statistics.service.gov.uk/find-statistics/school-workforce-in-england': + '/find-statistics', + '/find-statistics/school-workforce-inengland': '/find-statistics', + '/find-statistics/schoolworkforce-in-england': '/find-statistics', + '/find-statistics/serious-': '/find-statistics', + '/find-statistics/serious-incident-notifications%26nbsp': '/find-statistics', + '/find-statistics/skills-bootcamps-': '/find-statistics', + '/find-statistics/special-': '/find-statistics', + '/find-statistics/special-educational-needs-': '/find-statistics', + '/find-statistics/special-educational-needs-in-england/2021-': + '/find-statistics', + '/find-statistics/special-educational-needs-in-england/2021-2': + '/find-statistics', + '/find-statistics/special-educational-needs-in-england/221-22': + '/find-statistics', + '/find-statistics/special-educationalneeds-in-england': '/find-statistics', + '/find-statistics/student-': '/find-statistics', + '/find-statistics/student-loan-forecasts-for': '/find-statistics', + '/find-statistics/student-loan-forecasts-for-': '/find-statistics', + '/find-statistics/the-link-between-absence-and-attainment-': + '/find-statistics', + '/find-statistics/the-link-between-absence-and-attainment-at-ks2-': + '/find-statistics', + '/find-statistics/uk-revenue-from-education-related-exports-': + '/find-statistics', + '/find-statistics/uk-revenue-from-education-related-exports-and-': + '/find-statistics', + '/find-statistics/widen-': '/find-statistics', + '/find-statistics/widening': '/find-statistics', + '/find-statistics/widening-': '/find-statistics', + '/find-statistics/widening-participation-in-higher-': '/find-statistics', + '/find-statistics/www.gov.uk/courts-tribunals/first-tier-tribunal-specialeducational-needs-and-disability': + '/find-statistics', + '/findstatistics/apprenticeships-and-traineeships/2021-22': + '/find-statistics', + '/findstatistics/apprenticeships-in-england-by-industry-characteristics': + '/find-statistics', + '/findstatistics/attendance-in-': '/find-statistics', + '/findstatistics/attendance-in-education-and-early-years-': + '/find-statistics', + '/findstatistics/children-s-social-work-workforce': '/find-statistics', + '/findstatistics/education-and-training-statistics-for-the-uk/2020': + '/find-statistics', + '/findstatistics/leo-graduate-and': '/find-statistics', + '/findstatistics/special-educational-needs-in-england': '/find-statistics', + '/methodol': '/methodology', + '/methodology/16-18-destination-': '/methodology', + '/methodology/attendance-in-': '/methodology', + '/methodology/attendance-ineducation-and-early-years-settings-during-the-coronavirus-covid-19-outbreakmethodology': + '/methodology', + '/methodology/children-looked-after-': '/methodology', + '/methodology/permanent-and-fixed-period-exclusions-in-england': + '/methodology', + '/methodology/secondary-and-primary-school-': '/methodology', + '/methodology/student-loan-forecasts-for-england-': '/methodology', + '/methodology/widening-': '/methodology', + '/methodology/widening-participation-in-higher-': '/methodology', + '/service': '/', + '/statistics/pupil-absence-in-schools-in-england': '/find-statistics', + '/subscriptions': '/', +}; module.exports = seoRedirects; diff --git a/src/explore-education-statistics-frontend/server.js b/src/explore-education-statistics-frontend/server.js index ae29ca2cd23..741f93e6446 100644 --- a/src/explore-education-statistics-frontend/server.js +++ b/src/explore-education-statistics-frontend/server.js @@ -83,7 +83,7 @@ async function startServer() { * @returns An absolute URL if redirection is required; undefined otherwise. */ function getRedirectUrl(request) { - let redirectPath = request.path; + let redirectPath = request.path.toLowerCase(); // Redirect URLs with trailing slash to equivalent without slash with 301 redirectPath = replaceLastOccurrence(redirectPath, '/', ''); @@ -94,12 +94,7 @@ async function startServer() { // Search wrongly indexed pages from Google Search Console for matches // Redirect away if found to (eventually) clear routes from index - const seoRedirect = seoRedirects.find( - seoRedirectPath => seoRedirectPath.from === request.url, - ); - if (seoRedirect) { - redirectPath = seoRedirect.to; - } + redirectPath = seoRedirects[redirectPath] ?? redirectPath; const redirectionRequired = request.hostname.startsWith('www') || redirectPath !== request.path; diff --git a/tests/playwright-tests/tests/public/redirects.spec.ts b/tests/playwright-tests/tests/public/redirects.spec.ts index 6e65f8335bd..4210b5add03 100644 --- a/tests/playwright-tests/tests/public/redirects.spec.ts +++ b/tests/playwright-tests/tests/public/redirects.spec.ts @@ -50,10 +50,10 @@ test.describe('Redirect behaviour', () => { // Not ideal, I'd rather it.each like Jest has. But from the docs: // https://playwright.dev/docs/test-parameterize - for (const redirect of seoRedirects) { - test.skip(`Redirects from ${redirect.from}`, async ({ page }) => { - await page.goto(`${PUBLIC_URL}${redirect.from}`); - await expect(page).toHaveURL(`${PUBLIC_URL}${redirect.to}`); + for (const redirect of Object.keys(seoRedirects)) { + test.skip(`Redirects from ${redirect}`, async ({ page }) => { + await page.goto(`${PUBLIC_URL}${redirect}`); + await expect(page).toHaveURL(`${PUBLIC_URL}${seoRedirects[redirect]}`); }); } }); From 684ac2d876c01c158973108e6cd409b539c66ecd Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Fri, 31 May 2024 19:32:22 +0100 Subject: [PATCH 73/73] EES-5097 Add Processor functions to create data set meta, import to DuckDb and export to Parquet data files (#4880) * EES-5097 Rename ParquetFilesOptions to DataFilesOptions to reflect that the files directory holds more than just parquet files. Change base path from data/public-api-parquet to data/public-api-data * EES-4948 Convert backslash in absolute version directory path in Windows before replacing lines in load.sql * EES-5097 Add DuckDbConnection extension method to allow creating an SqlBuilder with a (non-interpolated) string command * EES-5097 Convert DuckDb queries in SeedDataCommand.Seeder to use SqlBuilder API * EES-5097 Replace default data source string with DuckDBConnectionStringBuilder.InMemoryConnectionString constant * EES-5097 Create data set version metadata, import metadata and data to DuckDb and export parquet data files * EES-5097 Convert DuckDb queries to use SqlBuilder API * EES-5097 Separate out DuckDb repository classes * EES-5097 Change methods to use Guid dataSetVersionId parameters * EES-5162 Enable prepared transactions in PostgreSQL container --- .gitignore | 2 +- docker-compose.yml | 1 + .../{fileShares.bicep => fileShare.bicep} | 0 .../public-api/deploy-stage-template.yml | 2 +- .../templates/public-api/main.bicep | 30 +- .../AbsenceSchool/data.parquet | Bin .../AbsenceSchool/filter_options.parquet | Bin .../AbsenceSchool/indicators.parquet | Bin .../AbsenceSchool/load.sql | 0 .../AbsenceSchool/location_options.parquet | Bin .../AbsenceSchool/schema.sql | 0 .../AbsenceSchool/source.csv | 0 .../AbsenceSchool/source.meta.csv | 0 .../AbsenceSchool/time_periods.parquet | Bin .../appsettings.IntegrationTest.json | 4 +- .../Startup.cs | 4 +- .../appsettings.Development.json | 4 +- .../DataSetFilenames.cs | 10 + .../DataSetVersionImportStage.cs | 20 + .../DuckDb/DuckDbConnection.cs | 7 +- .../DuckDb/DuckDbConnectionExtensions.cs | 8 + .../DuckDb/DuckDbDapperSqlBuilder.cs | 9 + .../DuckDb/DuckDbSqlBuilder.cs | 7 + .../DuckDb/IDuckDbConnection.cs | 6 +- .../DuckDb/ProfiledDuckDbConnection.cs | 15 +- ...askOrchestrationEntityFeatureExtensions.cs | 23 ++ .../Functions/CopyCsvFilesFunctionTests.cs | 133 ++++--- ...ocessInitialDataSetVersionFunctionTests.cs | 366 +++++++++++++----- ...tistics.Public.Data.Processor.Tests.csproj | 6 + .../DataFiles/AbsenceSchool/data.csv.gz | Bin 0 -> 5352 bytes .../DataFiles/AbsenceSchool/metadata.csv.gz | Bin 0 -> 310 bytes .../DataFiles/AbsenceSchool/source.csv | 217 +++++++++++ .../DataFiles/AbsenceSchool/source.meta.csv | 9 + .../Entities/ActivityLock.cs | 8 + .../TaskOrchestrationContextExtensions.cs | 70 ++++ .../BaseProcessDataSetVersionFunction.cs | 26 ++ .../Functions/CopyCsvFilesFunction.cs | 39 +- .../Functions/HealthCheckFunctions.cs | 12 +- .../ProcessInitialDataSetVersionFunction.cs | 116 +++--- ...ionStatistics.Public.Data.Processor.csproj | 1 + .../Models/MetaFileRow.cs | 34 ++ .../ProcessorHostBuilder.cs | 22 +- .../Properties/AssemblyInfo.cs | 4 + .../Repository/DataDuckDbRepository.cs | 168 ++++++++ .../Repository/FilterMetaRepository.cs | 131 +++++++ .../FilterOptionsDuckDbRepository.cs | 55 +++ .../GeographicLevelMetaRepository.cs | 42 ++ .../Repository/IndicatorMetaRepository.cs | 46 +++ .../Repository/IndicatorsDuckDbRepository.cs | 47 +++ .../Interfaces/IDataDuckDbRepository.cs | 8 + .../Interfaces/IFilterMetaRepository.cs | 13 + .../IFilterOptionsDuckDbRepository.cs | 12 + .../IGeographicLevelMetaRepository.cs | 12 + .../Interfaces/IIndicatorMetaRepository.cs | 13 + .../Interfaces/IIndicatorsDuckDbRepository.cs | 12 + .../Interfaces/ILocationMetaRepository.cs | 13 + .../Interfaces/ILocationsDuckDbRepository.cs | 12 + .../Interfaces/ITimePeriodMetaRepository.cs | 12 + .../ITimePeriodsDuckDbRepository.cs | 12 + .../Repository/LocationMetaRepository.cs | 209 ++++++++++ .../Repository/LocationsDuckDbRepository.cs | 95 +++++ .../Repository/TimePeriodMetaRepository.cs | 45 +++ .../Repository/TimePeriodsDuckDbRepository.cs | 52 +++ .../Services/DataSetMetaService.cs | 155 ++++++++ .../Services/DataSetService.cs | 3 +- .../Interfaces/IDataSetMetaService.cs | 8 + .../Services/Interfaces/IParquetService.cs | 8 + .../Services/ParquetService.cs | 48 +++ .../appsettings.json | 4 +- .../host.json | 2 + .../Commands/SeedDataCommand.cs | 273 +++++++------ .../Models/MetaFileRow.cs | 4 +- .../DataSetVersionPathResolverTests.cs | 48 ++- .../TestDataSetVersionPathResolver.cs | 2 +- .../DataSetVersionPathResolver.cs | 6 +- .../Interfaces/IDataSetVersionPathResolver.cs | 13 +- ...uetFilesOptions.cs => DataFilesOptions.cs} | 6 +- 77 files changed, 2355 insertions(+), 449 deletions(-) rename infrastructure/templates/public-api/components/{fileShares.bicep => fileShare.bicep} (100%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/{ParquetFiles => DataFiles}/AbsenceSchool/data.parquet (100%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/{ParquetFiles => DataFiles}/AbsenceSchool/filter_options.parquet (100%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/{ParquetFiles => DataFiles}/AbsenceSchool/indicators.parquet (100%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/{ParquetFiles => DataFiles}/AbsenceSchool/load.sql (100%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/{ParquetFiles => DataFiles}/AbsenceSchool/location_options.parquet (100%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/{ParquetFiles => DataFiles}/AbsenceSchool/schema.sql (100%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/{ParquetFiles => DataFiles}/AbsenceSchool/source.csv (100%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/{ParquetFiles => DataFiles}/AbsenceSchool/source.meta.csv (100%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/{ParquetFiles => DataFiles}/AbsenceSchool/time_periods.parquet (100%) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetFilenames.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Extensions/MockTaskOrchestrationEntityFeatureExtensions.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/data.csv.gz create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/metadata.csv.gz create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/source.csv create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/source.meta.csv create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Entities/ActivityLock.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Extensions/TaskOrchestrationContextExtensions.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/BaseProcessDataSetVersionFunction.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/MetaFileRow.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Properties/AssemblyInfo.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/DataDuckDbRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterOptionsDuckDbRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorsDuckDbRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IDataDuckDbRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterMetaRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterOptionsDuckDbRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IGeographicLevelMetaRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorMetaRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorsDuckDbRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationMetaRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationsDuckDbRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodMetaRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodsDuckDbRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationMetaRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationsDuckDbRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodMetaRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodsDuckDbRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IParquetService.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/ParquetService.cs rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Options/{ParquetFilesOptions.cs => DataFilesOptions.cs} (63%) diff --git a/.gitignore b/.gitignore index 72d70dd789b..f0c2ca2da00 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ bin obj /data/ees-mssql /data/public-api-db -/data/public-api-parquet +/data/public-api-data dfe-meta.db ## CSharp diff --git a/docker-compose.yml b/docker-compose.yml index 44f4937f4d8..7001bcd9b7e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,6 +43,7 @@ services: public-api-db: image: postgres:16.1-alpine + command: postgres -c max_prepared_transactions=100 ports: - "5432:5432" volumes: diff --git a/infrastructure/templates/public-api/components/fileShares.bicep b/infrastructure/templates/public-api/components/fileShare.bicep similarity index 100% rename from infrastructure/templates/public-api/components/fileShares.bicep rename to infrastructure/templates/public-api/components/fileShare.bicep diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index feeb60bf176..a7677dcdf91 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -99,7 +99,7 @@ stages: --settings \ "CoreStorage=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ - "ParquetFiles__BasePath=$(parquetFileShareMountPath)" + "DataFiles__BasePath=$(dataFilesFileShareMountPath)" az webapp config connection-string set \ --name $(dataProcessorFunctionAppName) \ diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index c1c13c6b955..cf0b884343f 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -91,8 +91,8 @@ var keyVaultName = '${subscription}-kv-ees-01' var acrName = 'eesacr' var vNetName = '${subscription}-vnet-ees' var containerAppEnvironmentNameSuffix = '01' -var parquetFileShareMountName = 'parquet-fileshare-mount' -var parquetFileShareMountPath = '/data/public-api-parquet' +var dataFilesFileShareMountName = 'data-files-fileshare-mount' +var dataFilesFileShareMountPath = '/data/public-api-data' var publicApiStorageAccountName = '${subscription}eespapisa' var tagValues = union(resourceTags ?? {}, { @@ -158,7 +158,7 @@ resource publicApiStorageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' var publicApiStorageAccountAccessKey = publicApiStorageAccount.listKeys().keys[0].value // Deploy File Share. -module parquetFileShareModule 'components/fileShares.bicep' = { +module fileShareModule 'components/fileShare.bicep' = { name: 'fileShareDeploy' params: { resourcePrefix: resourcePrefix @@ -237,10 +237,10 @@ module containerAppEnvironmentModule 'components/containerAppEnvironment.bicep' tagValues: tagValues azureFileStorages: [ { - storageName: parquetFileShareModule.outputs.fileShareName + storageName: fileShareModule.outputs.fileShareName storageAccountName: publicApiStorageAccountName storageAccountKey: publicApiStorageAccountAccessKey - fileShareName: parquetFileShareModule.outputs.fileShareName + fileShareName: fileShareModule.outputs.fileShareName accessMode: 'ReadWrite' } ] @@ -260,15 +260,15 @@ module apiContainerAppModule 'components/containerApp.bicep' = if (deployContain managedEnvironmentId: containerAppEnvironmentModule.outputs.containerAppEnvironmentId volumeMounts: [ { - volumeName: parquetFileShareMountName - mountPath: parquetFileShareMountPath + volumeName: dataFilesFileShareMountName + mountPath: dataFilesFileShareMountPath } ] volumes: [ { - name: parquetFileShareMountName + name: dataFilesFileShareMountName storageType: 'AzureFile' - storageName: parquetFileShareModule.outputs.fileShareName + storageName: fileShareModule.outputs.fileShareName } ] appSettings: [ @@ -295,8 +295,8 @@ module apiContainerAppModule 'components/containerApp.bicep' = if (deployContain value: 'true' } { - name: 'ParquetFiles__BasePath' - value: parquetFileShareMountPath + name: 'DataFiles__BasePath' + value: dataFilesFileShareMountPath } { // This property informs the Container App of the name of the Admin's system-assigned identity. @@ -358,11 +358,11 @@ module dataProcessorFunctionAppModule 'components/functionApp.bicep' = { } preWarmedInstanceCount: 1 azureFileShares: [{ - storageName: parquetFileShareModule.outputs.fileShareName + storageName: fileShareModule.outputs.fileShareName storageAccountKey: publicApiStorageAccountAccessKey storageAccountName: publicApiStorageAccountName - fileShareName: parquetFileShareModule.outputs.fileShareName - mountPath: parquetFileShareMountPath + fileShareName: fileShareModule.outputs.fileShareName + mountPath: dataFilesFileShareMountPath }] storageFirewallRules: storageFirewallRules tagValues: tagValues @@ -428,4 +428,4 @@ output dataProcessorFunctionAppManagedIdentityClientId string = dataProcessorFun output coreStorageConnectionStringSecretKey string = coreStorageConnectionStringSecretKey output keyVaultName string = keyVaultName -output parquetFileShareMountPath string = parquetFileShareMountPath +output dataFilesFileShareMountPath string = dataFilesFileShareMountPath diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/data.parquet b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/data.parquet similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/data.parquet rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/data.parquet diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/filter_options.parquet b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/filter_options.parquet similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/filter_options.parquet rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/filter_options.parquet diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/indicators.parquet b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/indicators.parquet similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/indicators.parquet rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/indicators.parquet diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/load.sql b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/load.sql similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/load.sql rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/load.sql diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/location_options.parquet b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/location_options.parquet similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/location_options.parquet rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/location_options.parquet diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/schema.sql b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/schema.sql similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/schema.sql rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/schema.sql diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/source.csv b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/source.csv similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/source.csv rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/source.csv diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/source.meta.csv b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/source.meta.csv similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/source.meta.csv rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/source.meta.csv diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/time_periods.parquet b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/time_periods.parquet similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/ParquetFiles/AbsenceSchool/time_periods.parquet rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Resources/DataFiles/AbsenceSchool/time_periods.parquet diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/appsettings.IntegrationTest.json b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/appsettings.IntegrationTest.json index 7ba5992e60e..9a889dc0c34 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/appsettings.IntegrationTest.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/appsettings.IntegrationTest.json @@ -5,7 +5,7 @@ "MiniProfiler": { "Enabled": false }, - "ParquetFiles": { - "BasePath": "Resources/ParquetFiles" + "DataFiles": { + "BasePath": "Resources/DataFiles" } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs index f5a5d06d76c..7249eaf48fa 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs @@ -162,8 +162,8 @@ public void ConfigureServices(IServiceCollection services) services.AddOptions() .Bind(configuration.GetRequiredSection(ContentApiOptions.Section)); - services.AddOptions() - .Bind(configuration.GetRequiredSection(ParquetFilesOptions.Section)); + services.AddOptions() + .Bind(configuration.GetRequiredSection(DataFilesOptions.Section)); services.AddOptions() .Bind(_miniProfilerConfig); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/appsettings.Development.json b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/appsettings.Development.json index dd73fc11f00..0eff5e94180 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/appsettings.Development.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/appsettings.Development.json @@ -16,7 +16,7 @@ "MiniProfiler": { "Enabled": true }, - "ParquetFiles": { - "BasePath": "data/public-api-parquet" + "DataFiles": { + "BasePath": "data/public-api-data" } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetFilenames.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetFilenames.cs new file mode 100644 index 00000000000..4cc38d0c2a0 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetFilenames.cs @@ -0,0 +1,10 @@ +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model; + +public static class DataSetFilenames +{ + public const string CsvDataFile = "data.csv.gz"; + public const string CsvMetadataFile = "metadata.csv.gz"; + public const string DuckDbDatabaseFile = "data.db"; + public const string DuckDbLoadSqlFile = "load.sql"; + public const string DuckDbSchemaSqlFile = "schema.sql"; +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs index ae2e99f0992..4884f9356b5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model; @@ -7,5 +8,24 @@ public enum DataSetVersionImportStage { Pending, CopyingCsvFiles, + ImportingMetadata, + ImportingData, + WritingDataFiles, Completing } + +public static class DataSetVersionImportStageExtensions +{ + public static DataSetVersionImportStage PreviousStage(this DataSetVersionImportStage stage) + { + var stages = EnumUtil.GetEnums(); + var prevIndex = stages.IndexOf(stage) - 1; + + return prevIndex == -1 + ? throw new ArgumentOutOfRangeException( + nameof(stage), + stage, + $"No previous stage has been defined for '{stage}'") + : stages[prevIndex]; + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnection.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnection.cs index 68750b3891c..f1224f99e86 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnection.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnection.cs @@ -2,9 +2,14 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; -public class DuckDbConnection(string connectionString = "DataSource=:memory:") +public class DuckDbConnection(string connectionString = DuckDBConnectionStringBuilder.InMemoryConnectionString) : DuckDBConnection(connectionString), IDuckDbConnection { + public static DuckDbConnection CreateFileConnection(string filename) + { + return new DuckDbConnection($"DataSource={filename}"); + } + public override DuckDbCommand CreateCommand() { // Bit rubbish to do this but we don't have access to the diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnectionExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnectionExtensions.cs index bf1f07c8aa0..0c6e187a3ac 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnectionExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnectionExtensions.cs @@ -18,6 +18,14 @@ public static DuckDbDapperSqlBuilder SqlBuilder( return new DuckDbDapperSqlBuilder(connection, command, options); } + public static DuckDbDapperSqlBuilder SqlBuilder( + this IDuckDbConnection connection, + string command, + InterpolatedSqlBuilderOptions? options = null) + { + return new DuckDbDapperSqlBuilder(connection, command, options); + } + public static DuckDbDapperSqlBuilder SqlBuilder( this IDuckDbConnection connection, InterpolatedSqlBuilderOptions options) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbDapperSqlBuilder.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbDapperSqlBuilder.cs index dce4be80212..53c9121cd2b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbDapperSqlBuilder.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbDapperSqlBuilder.cs @@ -30,6 +30,15 @@ public DuckDbDapperSqlBuilder( DbConnection = connection; } + public DuckDbDapperSqlBuilder( + IDbConnection connection, + string value, + InterpolatedSqlBuilderOptions? options = null) + : base(value, options) + { + DbConnection = connection; + } + protected internal DuckDbDapperSqlBuilder( IDbConnection connection, InterpolatedSqlBuilderOptions? options, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbSqlBuilder.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbSqlBuilder.cs index 12dd6ebc69d..43fcf7bbe94 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbSqlBuilder.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbSqlBuilder.cs @@ -34,6 +34,13 @@ public DuckDbSqlBuilder(IInterpolatedSql value, InterpolatedSqlBuilderOptions? o ResetAutoSpacing(); } + public DuckDbSqlBuilder(string value, InterpolatedSqlBuilderOptions? options = null) + : this(options: options, format: null, arguments: null) + { + AppendLiteral(value); + ResetAutoSpacing(); + } + protected DuckDbSqlBuilder( InterpolatedSqlBuilderOptions? options = null, StringBuilder? format = null, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/IDuckDbConnection.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/IDuckDbConnection.cs index 4eee864838c..165a64f7576 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/IDuckDbConnection.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/IDuckDbConnection.cs @@ -1,5 +1,9 @@ using System.Data; +using DuckDB.NET.Data; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; -public interface IDuckDbConnection : IDbConnection; +public interface IDuckDbConnection : IDbConnection +{ + DuckDBAppender CreateAppender(string table); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/ProfiledDuckDbConnection.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/ProfiledDuckDbConnection.cs index eae1309ff11..2eda1056439 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/ProfiledDuckDbConnection.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/ProfiledDuckDbConnection.cs @@ -1,3 +1,4 @@ +using DuckDB.NET.Data; using StackExchange.Profiling; using StackExchange.Profiling.Data; @@ -6,5 +7,15 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; /// /// DuckDB connection profiled using MiniProfiler. /// -public class ProfiledDuckDbConnection(string connectionString = "DataSource=:memory:", IDbProfiler? profiler = null) - : ProfiledDbConnection(new DuckDbConnection(connectionString), profiler ?? MiniProfiler.Current), IDuckDbConnection; +public class ProfiledDuckDbConnection( + string connectionString = DuckDBConnectionStringBuilder.InMemoryConnectionString, + IDbProfiler? profiler = null) + : ProfiledDbConnection(new DuckDbConnection(connectionString), profiler ?? MiniProfiler.Current), IDuckDbConnection +{ + private DuckDbConnection WrappedDuckDbConnection => (DuckDbConnection)WrappedConnection; + + public DuckDBAppender CreateAppender(string table) + { + return WrappedDuckDbConnection.CreateAppender(table); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Extensions/MockTaskOrchestrationEntityFeatureExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Extensions/MockTaskOrchestrationEntityFeatureExtensions.cs new file mode 100644 index 00000000000..e542f095fc9 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Extensions/MockTaskOrchestrationEntityFeatureExtensions.cs @@ -0,0 +1,23 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Entities; +using Microsoft.DurableTask.Entities; +using Moq; +using Moq.Language.Flow; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; + +internal static class MockTaskOrchestrationEntityFeatureExtensions +{ + internal static IReturnsResult SetupLockForActivity( + this Mock mock, + string name) + { + return mock.Setup(entityFeature => entityFeature.LockEntitiesAsync( + new EntityInstanceId(nameof(ActivityLock), name))) + .ReturnsAsync(new NoopAsyncDisposable()); + } + + private sealed class NoopAsyncDisposable : IAsyncDisposable + { + ValueTask IAsyncDisposable.DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CopyCsvFilesFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CopyCsvFilesFunctionTests.cs index 997466c6a05..25f64507a0d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CopyCsvFilesFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CopyCsvFilesFunctionTests.cs @@ -1,3 +1,4 @@ +using System.Reflection; using GovUk.Education.ExploreEducationStatistics.Common; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -23,17 +24,11 @@ public abstract class CopyCsvFilesFunctionTests(ProcessorFunctionsIntegrationTes public class CopyCsvFilesTests(ProcessorFunctionsIntegrationTestFixture fixture) : CopyCsvFilesFunctionTests(fixture) { - private const string DataFileContent = - "time_period,time_identifier,geographic_level,country_code,country_name,enrolments"; - - private const string MetadataFileContent = - "col_name,col_type,label,indicator_grouping,indicator_unit,indicator_dp,filter_hint,filter_grouping_column"; + private const DataSetVersionImportStage Stage = DataSetVersionImportStage.CopyingCsvFiles; [Fact] public async Task Success() { - var instanceId = Guid.NewGuid(); - var subjectId = Guid.NewGuid(); ReleaseVersion releaseVersion = DataFixture.DefaultReleaseVersion() @@ -55,67 +50,74 @@ await AddTestData(context => context.ReleaseFiles.AddRange(releaseDataFile, releaseMetaFile); }); - DataSet dataSet = DataFixture.DefaultDataSet(); - - await AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion dataSetVersion = DataFixture - .DefaultDataSetVersion() - .WithDataSet(dataSet) - .WithStatus(DataSetVersionStatus.Processing) - .WithReleaseFileId(releaseDataFile.Id) - .WithImports(() => DataFixture - .DefaultDataSetVersionImport() - .WithInstanceId(instanceId) - .WithStage(DataSetVersionImportStage.Pending) - .Generate(1)) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); - - await AddTestData(context => - { - context.DataSetVersions.Add(dataSetVersion); - context.DataSets.Update(dataSet); - }); + var (dataSetVersion, instanceId) = await CreateDataSetVersion(releaseDataFile.Id, Stage.PreviousStage()); var blobStorageService = GetRequiredService(); - await blobStorageService.UploadStream( - BlobContainers.PrivateReleaseFiles, - releaseDataFile.Path(), - DataFileContent.ToStream(), - ContentTypes.Csv); + var testDataDirectoryPath = Path.Combine( + Assembly.GetExecutingAssembly().GetDirectoryPath(), + "Resources", + "DataFiles", + "AbsenceSchool" + ); - await blobStorageService.UploadStream( - BlobContainers.PrivateReleaseFiles, - releaseMetaFile.Path(), - MetadataFileContent.ToStream(), - ContentTypes.Csv); + var sourceDataFileContent = await File.ReadAllTextAsync(Path.Combine( + testDataDirectoryPath, + "source.csv" + )); + + await using (var contentStream = sourceDataFileContent.ToStream()) + { + await blobStorageService.UploadStream( + BlobContainers.PrivateReleaseFiles, + releaseDataFile.Path(), + contentStream, + ContentTypes.Csv); + } + + var sourceMetadataFileContent = await File.ReadAllTextAsync(Path.Combine( + testDataDirectoryPath, + "source.meta.csv" + )); + + await using (var contentStream = sourceMetadataFileContent.ToStream()) + { + await blobStorageService.UploadStream( + BlobContainers.PrivateReleaseFiles, + releaseMetaFile.Path(), + contentStream, + ContentTypes.Csv); + } var function = GetRequiredService(); - await function.CopyCsvFiles( - dataSetVersionId: dataSetVersion.Id, - instanceId: instanceId, - CancellationToken.None); + await function.CopyCsvFiles(instanceId, CancellationToken.None); await using var publicDataDbContext = GetDbContext(); - var savedDataSetVersion = await publicDataDbContext.DataSetVersions - .Include(dsv => dsv.Imports) - .FirstAsync(dsv => dsv.Id == dataSetVersion.Id); + var savedImport = await publicDataDbContext.DataSetVersionImports + .Include(dataSetVersionImport => dataSetVersionImport.DataSetVersion) + .SingleAsync(i => i.InstanceId == instanceId); - var savedImport = savedDataSetVersion.Imports.Single(); - Assert.Equal(DataSetVersionImportStage.CopyingCsvFiles, savedImport.Stage); + Assert.Equal(DataSetVersionStatus.Processing, savedImport.DataSetVersion.Status); + Assert.Equal(Stage, savedImport.Stage); var dataSetVersionPathResolver = GetRequiredService(); + Assert.True(Directory.Exists(dataSetVersionPathResolver.DirectoryPath(dataSetVersion))); + var actualDataSetVersionFiles = Directory.GetFiles(dataSetVersionPathResolver.DirectoryPath(dataSetVersion)) + .Select(Path.GetFullPath) + .ToArray(); + + Assert.Equal(2, actualDataSetVersionFiles.Length); + var expectedCsvDataPath = dataSetVersionPathResolver.CsvDataPath(dataSetVersion); - Assert.True(File.Exists(expectedCsvDataPath)); - Assert.Equal(DataFileContent, await DecompressFileToString(expectedCsvDataPath)); + Assert.Contains(expectedCsvDataPath, actualDataSetVersionFiles); + Assert.Equal(sourceDataFileContent, await DecompressFileToString(expectedCsvDataPath)); var expectedCsvMetadataPath = dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion); - Assert.True(File.Exists(expectedCsvMetadataPath)); - Assert.Equal(MetadataFileContent, await DecompressFileToString(expectedCsvMetadataPath)); + Assert.Contains(expectedCsvMetadataPath, actualDataSetVersionFiles); + Assert.Equal(sourceMetadataFileContent, await DecompressFileToString(expectedCsvMetadataPath)); } private static async Task DecompressFileToString(string path) @@ -123,5 +125,34 @@ private static async Task DecompressFileToString(string path) var bytes = await File.ReadAllBytesAsync(path); return await CompressionUtils.DecompressToString(bytes, ContentEncodings.Gzip); } + + private async Task<(DataSetVersion dataSetVersion, Guid instanceId)> CreateDataSetVersion( + Guid releaseFileId, + DataSetVersionImportStage importStage) + { + DataSet dataSet = DataFixture.DefaultDataSet(); + + await AddTestData(context => context.DataSets.Add(dataSet)); + + DataSetVersionImport dataSetVersionImport = DataFixture + .DefaultDataSetVersionImport() + .WithStage(importStage); + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithDataSet(dataSet) + .WithReleaseFileId(releaseFileId) + .WithStatus(DataSetVersionStatus.Processing) + .WithImports(() => [dataSetVersionImport]) + .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + + await AddTestData(context => + { + context.DataSetVersions.Add(dataSetVersion); + context.DataSets.Update(dataSet); + }); + + return (dataSetVersion, dataSetVersionImport.InstanceId); + } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs index 366b1d971d9..44665617214 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs @@ -1,10 +1,17 @@ +using System.Reflection; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.DurableTask; +using Microsoft.DurableTask.Entities; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -17,46 +24,70 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests public abstract class ProcessInitialDataSetVersionFunctionTests(ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { + private readonly string _testDataDirectoryPath = Path.Combine( + Assembly.GetExecutingAssembly().GetDirectoryPath(), + "Resources", + "DataFiles", + "AbsenceSchool" + ); + + private readonly string[] _allDataSetVersionFiles = + [ + DataSetFilenames.CsvDataFile, + DataSetFilenames.CsvMetadataFile, + DataSetFilenames.DuckDbDatabaseFile, + DataSetFilenames.DuckDbLoadSqlFile, + DataSetFilenames.DuckDbSchemaSqlFile, + DataTable.ParquetFile, + FilterOptionsTable.ParquetFile, + IndicatorsTable.ParquetFile, + LocationOptionsTable.ParquetFile, + TimePeriodsTable.ParquetFile + ]; + public class ProcessInitialDataSetVersionTests(ProcessorFunctionsIntegrationTestFixture fixture) : ProcessInitialDataSetVersionFunctionTests(fixture) { [Fact] public async Task Success() { - var dataSetVersionId = Guid.NewGuid(); - var mockOrchestrationContext = DefaultMockOrchestrationContext(); + // Expect an entity lock to be acquired for calling the ImportMetadata activity + var mockEntityFeature = new Mock(MockBehavior.Strict); + mockEntityFeature.SetupLockForActivity(nameof(ProcessInitialDataSetVersionFunction.ImportMetadata)); + mockOrchestrationContext.SetupGet(context => context.Entities) + .Returns(mockEntityFeature.Object); + var activitySequence = new MockSequence(); - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync( - nameof(CopyCsvFilesFunction.CopyCsvFiles), - dataSetVersionId, - null)) - .Returns(Task.CompletedTask); + string[] expectedActivitySequence = + [ + nameof(CopyCsvFilesFunction.CopyCsvFiles), + nameof(ProcessInitialDataSetVersionFunction.ImportMetadata), + nameof(ProcessInitialDataSetVersionFunction.ImportData), + nameof(ProcessInitialDataSetVersionFunction.WriteDataFiles), + nameof(ProcessInitialDataSetVersionFunction.CompleteProcessing) + ]; - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync( - nameof(ProcessInitialDataSetVersionFunction.CompleteProcessing), - dataSetVersionId, + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, null)) - .Returns(Task.CompletedTask); + .Returns(Task.CompletedTask); + } - await ProcessInitialDataSetVersion(mockOrchestrationContext.Object, dataSetVersionId); + await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); - VerifyAllMocks(mockOrchestrationContext); + VerifyAllMocks(mockOrchestrationContext, mockEntityFeature); } [Fact] public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() { - var dataSetVersionId = Guid.NewGuid(); - var mockOrchestrationContext = DefaultMockOrchestrationContext(); var activitySequence = new MockSequence(); @@ -64,36 +95,32 @@ public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() mockOrchestrationContext .InSequence(activitySequence) .Setup(context => - context.CallActivityAsync( - nameof(CopyCsvFilesFunction.CopyCsvFiles), - dataSetVersionId, + context.CallActivityAsync(nameof(CopyCsvFilesFunction.CopyCsvFiles), + mockOrchestrationContext.Object.InstanceId, null)) .Throws(); mockOrchestrationContext .InSequence(activitySequence) .Setup(context => - context.CallActivityAsync( - nameof(ProcessInitialDataSetVersionFunction.HandleProcessingFailure), - dataSetVersionId, + context.CallActivityAsync(nameof(ProcessInitialDataSetVersionFunction.HandleProcessingFailure), + null, null)) .Returns(Task.CompletedTask); - await ProcessInitialDataSetVersion(mockOrchestrationContext.Object, dataSetVersionId); + await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); VerifyAllMocks(mockOrchestrationContext); } - private async Task ProcessInitialDataSetVersion( - TaskOrchestrationContext orchestrationContext, - Guid dataSetVersionId) + private async Task ProcessInitialDataSetVersion(TaskOrchestrationContext orchestrationContext) { var function = GetRequiredService(); await function.ProcessInitialDataSetVersion( orchestrationContext, new ProcessInitialDataSetVersionContext { - DataSetVersionId = dataSetVersionId + DataSetVersionId = Guid.NewGuid() }); } @@ -120,107 +147,242 @@ private static Mock DefaultMockOrchestrationContext( } } - public class HandleProcessingFailureTests(ProcessorFunctionsIntegrationTestFixture fixture) + public class ImportMetadataTests(ProcessorFunctionsIntegrationTestFixture fixture) : ProcessInitialDataSetVersionFunctionTests(fixture) { + private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ImportingMetadata; + [Fact] public async Task Success() { - var instanceId = Guid.NewGuid(); - - DataSet dataSet = DataFixture.DefaultDataSet(); - - await AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion dataSetVersion = DataFixture - .DefaultDataSetVersion() - .WithDataSet(dataSet) - .WithStatus(DataSetVersionStatus.Processing) - .WithImports( - () => DataFixture - .DefaultDataSetVersionImport() - .WithInstanceId(instanceId) - .WithStage(DataSetVersionImportStage.Pending) - .Generate(1) - ) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); - - await AddTestData( - context => - { - context.DataSetVersions.Add(dataSetVersion); - context.DataSets.Update(dataSet); - } - ); + var (dataSetVersion, instanceId) = await CreateDataSetVersion(Stage.PreviousStage()); + + // Prepare the data set version directory with compressed data and metadata CSV files + SetupCsvDataFiles(dataSetVersion); var function = GetRequiredService(); - await function.HandleProcessingFailure( - dataSetVersion.Id, - instanceId, - CancellationToken.None - ); + await function.ImportMetadata(instanceId, CancellationToken.None); await using var publicDataDbContext = GetDbContext(); - var savedDataSetVersion = await publicDataDbContext.DataSetVersions - .Include(dsv => dsv.Imports) - .FirstAsync(dsv => dsv.Id == dataSetVersion.Id); + var savedImport = await publicDataDbContext.DataSetVersionImports + .Include(dataSetVersionImport => dataSetVersionImport.DataSetVersion) + .SingleAsync(i => i.InstanceId == instanceId); - Assert.Equal(DataSetVersionStatus.Failed, savedDataSetVersion.Status); + Assert.Equal(DataSetVersionStatus.Processing, savedImport.DataSetVersion.Status); + Assert.Equal(Stage, savedImport.Stage); - var savedImport = savedDataSetVersion.Imports.Single(); - Assert.Equal(DataSetVersionImportStage.Pending, savedImport.Stage); - savedImport.Completed.AssertUtcNow(); + AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, + [ + DataSetFilenames.CsvDataFile, + DataSetFilenames.CsvMetadataFile, + DataSetFilenames.DuckDbDatabaseFile + ]); } } - public class CompleteProcessingTests(ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessorFunctionsIntegrationTest(fixture) + public class ImportDataTests(ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessInitialDataSetVersionFunctionTests(fixture) { + private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ImportingData; + [Fact] public async Task Success() { - var instanceId = Guid.NewGuid(); + var (dataSetVersion, instanceId) = await CreateDataSetVersion(Stage.PreviousStage()); - DataSet dataSet = DataFixture - .DefaultDataSet(); + // Prepare the data set version directory with compressed data and metadata CSV files + SetupCsvDataFiles(dataSetVersion); - await AddTestData(context => context.DataSets.Add(dataSet)); + var function = GetRequiredService(); - DataSetVersion dataSetVersion = DataFixture - .DefaultDataSetVersion() - .WithDataSet(dataSet) - .WithStatus(DataSetVersionStatus.Processing) - .WithImports(() => DataFixture - .DefaultDataSetVersionImport() - .WithInstanceId(instanceId) - .WithStage(DataSetVersionImportStage.Pending) - .Generate(1)) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + // Prepare the metadata before calling the ImportData function + await function.ImportMetadata(instanceId, CancellationToken.None); - await AddTestData(context => - { - context.DataSetVersions.Add(dataSetVersion); - context.DataSets.Update(dataSet); - }); + await function.ImportData(instanceId, CancellationToken.None); + + await using var publicDataDbContext = GetDbContext(); + + var savedImport = await publicDataDbContext.DataSetVersionImports + .Include(dataSetVersionImport => dataSetVersionImport.DataSetVersion) + .SingleAsync(i => i.InstanceId == instanceId); + + Assert.Equal(DataSetVersionStatus.Processing, savedImport.DataSetVersion.Status); + Assert.Equal(Stage, savedImport.Stage); + + AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, + [ + DataSetFilenames.CsvDataFile, + DataSetFilenames.CsvMetadataFile, + DataSetFilenames.DuckDbDatabaseFile + ]); + } + } + + public class WriteDataFilesTests(ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessInitialDataSetVersionFunctionTests(fixture) + { + private const DataSetVersionImportStage Stage = DataSetVersionImportStage.WritingDataFiles; + + [Fact] + public async Task Success() + { + var (dataSetVersion, instanceId) = await CreateDataSetVersion(Stage.PreviousStage()); + + // Prepare the data set version directory with compressed data and metadata CSV files + SetupCsvDataFiles(dataSetVersion); + + var function = GetRequiredService(); + + // Prepare the metadata and data before calling the WriteDataFiles function + await function.ImportMetadata(instanceId, CancellationToken.None); + await function.ImportData(instanceId, CancellationToken.None); + + await function.WriteDataFiles(instanceId, CancellationToken.None); + + await using var publicDataDbContext = GetDbContext(); + + var savedImport = await publicDataDbContext.DataSetVersionImports + .Include(dataSetVersionImport => dataSetVersionImport.DataSetVersion) + .SingleAsync(i => i.InstanceId == instanceId); + + Assert.Equal(DataSetVersionStatus.Processing, savedImport.DataSetVersion.Status); + Assert.Equal(Stage, savedImport.Stage); + + AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, _allDataSetVersionFiles); + } + } + + public class HandleProcessingFailureTests(ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessInitialDataSetVersionFunctionTests(fixture) + { + [Fact] + public async Task Success() + { + // The stage which the failure occured in - This should not be altered by the handler + const DataSetVersionImportStage failedStage = DataSetVersionImportStage.CopyingCsvFiles; + + var (_, instanceId) = await CreateDataSetVersion(failedStage); var function = GetRequiredService(); - await function.CompleteProcessing( - dataSetVersion.Id, - instanceId, - CancellationToken.None); + await function.HandleProcessingFailure(instanceId, CancellationToken.None); await using var publicDataDbContext = GetDbContext(); - var savedDataSetVersion = await publicDataDbContext.DataSetVersions - .Include(dsv => dsv.Imports) - .FirstAsync(dsv => dsv.Id == dataSetVersion.Id); + var savedImport = await publicDataDbContext.DataSetVersionImports + .Include(i => i.DataSetVersion) + .SingleAsync(i => i.InstanceId == instanceId); + + Assert.Equal(DataSetVersionStatus.Failed, savedImport.DataSetVersion.Status); + Assert.Equal(failedStage, savedImport.Stage); + savedImport.Completed.AssertUtcNow(); + } + } + + public class CompleteProcessingTests(ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessInitialDataSetVersionFunctionTests(fixture) + { + private const DataSetVersionImportStage Stage = DataSetVersionImportStage.Completing; + + [Fact] + public async Task Success() + { + var (dataSetVersion, instanceId) = await CreateDataSetVersion(Stage.PreviousStage()); - Assert.Equal(DataSetVersionStatus.Draft, savedDataSetVersion.Status); + var dataSetVersionPathResolver = GetRequiredService(); + Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(dataSetVersion)); - var savedImport = savedDataSetVersion.Imports.Single(); - Assert.Equal(DataSetVersionImportStage.Completing, savedImport.Stage); + await CompleteProcessing(instanceId); + + await using var publicDataDbContext = GetDbContext(); + + var savedImport = await publicDataDbContext.DataSetVersionImports + .Include(i => i.DataSetVersion) + .SingleAsync(i => i.InstanceId == instanceId); + + Assert.Equal(DataSetVersionStatus.Draft, savedImport.DataSetVersion.Status); + Assert.Equal(Stage, savedImport.Stage); savedImport.Completed.AssertUtcNow(); } + + [Fact] + public async Task DuckDbFileIsDeleted() + { + var (dataSetVersion, instanceId) = await CreateDataSetVersion(Stage.PreviousStage()); + + // Create empty data set version files for all file paths + var dataSetVersionPathResolver = GetRequiredService(); + var directoryPath = dataSetVersionPathResolver.DirectoryPath(dataSetVersion); + Directory.CreateDirectory(directoryPath); + foreach (var filename in _allDataSetVersionFiles) + { + await File.Create(Path.Combine(directoryPath, filename)).DisposeAsync(); + } + + await CompleteProcessing(instanceId); + + // Ensure the duck db database file is the only file that was deleted + AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, + _allDataSetVersionFiles + .Where(file => file != DataSetFilenames.DuckDbDatabaseFile) + .ToArray()); + } + + private async Task CompleteProcessing(Guid instanceId) + { + var function = GetRequiredService(); + await function.CompleteProcessing(instanceId, CancellationToken.None); + } + } + + private async Task<(DataSetVersion dataSetVersion, Guid instanceId)> CreateDataSetVersion( + DataSetVersionImportStage importStage) + { + DataSet dataSet = DataFixture.DefaultDataSet(); + + await AddTestData(context => context.DataSets.Add(dataSet)); + + DataSetVersionImport dataSetVersionImport = DataFixture + .DefaultDataSetVersionImport() + .WithStage(importStage); + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithDataSet(dataSet) + .WithStatus(DataSetVersionStatus.Processing) + .WithImports(() => [dataSetVersionImport]) + .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + + await AddTestData(context => + { + context.DataSetVersions.Add(dataSetVersion); + context.DataSets.Update(dataSet); + }); + + return (dataSetVersion, dataSetVersionImport.InstanceId); + } + + private void SetupCsvDataFiles(DataSetVersion dataSetVersion) + { + var dataSetVersionPathResolver = GetRequiredService(); + Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(dataSetVersion)); + File.Copy(Path.Combine(_testDataDirectoryPath, DataSetFilenames.CsvDataFile), + dataSetVersionPathResolver.CsvDataPath(dataSetVersion)); + File.Copy(Path.Combine(_testDataDirectoryPath, DataSetFilenames.CsvMetadataFile), + dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion)); + } + + private void AssertDataSetVersionDirectoryContainsOnlyFiles( + DataSetVersion dataSetVersion, + params string[] expectedFiles) + { + var dataSetVersionPathResolver = GetRequiredService(); + var actualFiles = Directory.GetFiles(dataSetVersionPathResolver.DirectoryPath(dataSetVersion)) + .Select(Path.GetFileName) + .ToArray(); + + // Assert that the directory contains the expected files and no others + Assert.Equal(expectedFiles.Length, actualFiles.Length); + Assert.True(ComparerUtils.SequencesAreEqualIgnoringOrder(expectedFiles, actualFiles)); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.csproj b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.csproj index 5ac867bbba0..6c6b5e3b42b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.csproj +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.csproj @@ -26,4 +26,10 @@ + + + + Always + + diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/data.csv.gz b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/data.csv.gz new file mode 100644 index 0000000000000000000000000000000000000000..0c7874f40a66b1d56ab405e1c962868f61b0ea70 GIT binary patch literal 5352 zcmV)@+?dO5Bt;NK_k4vv;8Kbk?^pstut0!- z4J*{x1!K|Ejk-M`|2=;26ayA$>^9UYGa{?+iR`S%jJ&#hxqkKidbr$Q;I4gnvF@%e zzqwow_;%fYd)R#c;`02f?fS#I#rgi#?&@&;)%pHn{k5~(yj*cuzrEb=?j85-d;5NS z@zr*7Z{Mvy{@UJd?)}H}7yJG8t5=8JFU{>{Jzj0T{-yi-QCHXBuh^V#F4mXV_nqDO z^}0D=-5vJZmp9x;9M|LVtMB*61%7$H*k9fF@#W3%Gsl=%2HIL&0`3%X zg{Ap`vWUgJ5vhcYs2Q{;h5{|~(15eU8`P!E{f%ffCQ~e6nL|-{WmPw*A%$6-(QibF zg6t}Bbhiv0=9tFvnHR}?1W*QBw zaSsL4+XJ;st3HNuA}HC4l-4eocv&a3GR9F$OTHQ1<9fc|UHqopAU#q*VXG4^#qI

    Gkzfymn_ZGvK`@?s~7ng_i?8bif z#f$aq!&fiAUJtMRa?5)%P})oqHXbR^aGD|)weY7wNi@er;ZdX-bac{C!wk@uHqEC& zY0OMaL$kZeF=4I>73~h_5~cMtC=_%WClSt)34BRNUFl!S_@ptwI2!X;oRfX*@9xRNtn`<1)7@%d|{@a z2#$i1v~q-NO%2q$Sqb$nF~~CVS9{(^`}58A?B^rK)%CxFv=Y28efM7C-OXWl+^*Mu zPFv|_GSF&k9k^?7U@4}9S;TctPML5?YQsh~mf}_UZk?Is5#sc76S#`en@V z`~B{?Kb&oEd_UK>zt$j=7K*a$Z~;Y>($J1ng)P&S|J(2rs@m;HX(bJok}Oax(vXX$ zog#Y4jJxPvwD9S=yF6a)ck9^)ht12Ih<*wBflo4%=>)4rVn7Se9#F%*VN1Kse>6A> zRI->bwb2oHdP_8{CtX2ZOwAuPJo`1g!RIg5Z*K0iwimCtp_%j{T11--PYbJr(x?h9 zIgHbDCfd_VpbQ-qXlpkWX6>S=B{TZ;oDHf@5o#Wq1y5RLKOww;38g?eNkMpM z1z6g21$86`v_)O(1fUNcoXpxR(W2ohu$ZAW)Tf5%MOE_IVJE1;fQe8-V8(C;)TYHT zi;Vg+pngqN8ZJt-Xo$l?MVYWNwS+AspCL8{-Z}-Z^6Ca7nwsd&4EQ2G_ImF7{QhQf z>0#yfljg(+hjrb(xZG|ZI41mY-E(ZLVXazj$Qd6(IV0erI`y9lDB&t?jr8<(m}=?- zG_<6kON97SMkKAgb-PI8yhI=4OR3u=d8QtLAT6K}K z{$E8*0juc_T2EHOvxXI5Fvbvzd&R%F)UrE^;*(cb>*4I98<73(wb$5bIXN(=XBte~ zj1FTf9bD8!A5TBQ3KfKpw>T`b%b=N(idwwNA5Xo}ETh1KEGksZ+yZVgm(tOxLx-DCKv~-64MzXq!J)=&9GtW;MdV#~L{rcU$LGml5-P$OFf^qF+8i%~ zOPLGE#YFqf@;toJrxtpGCJsi7`@pnJ0=`&HJRuMTW9FLBN_Gx7YFeJEhITELeK=_detFm<_wNx-ru zqo8c_Hn38fW$&NiS$uiX|qy1-<0ce6J7Zx^X5Q97}4$luAuRnJMUt zcsvJC3MMBd5k0As2+c5qT3X&0yY?ic6l%_`Plb;6_S?%F(CzxqDi|>x1hahtuAw;s zrfN5QnHK$z#PSpOX0szl5d`5^*#`&}85#L|o4Ee!H{ zXV$mTs7v`pR(Ez*bwy###s>1?m?MlcON9Bf!H-2g&%`Lx8Zxi7!uP-Z`1;{7-d_*j z{QUZdSC>uAPg=87rXo8T-$_-)~RSY)NcRaR%AL zF*wT_Ix6-{&{2r)>cke!HzBUYG_&V~LomF3?YL@+C;2 zAza&zqPgG^wjv~-9n8t}yPcDNe>-KAh+IXdp;_UAm&yb?GwUx()|a48-tklQ@zbDVy=b%9UWC;~Lt{0CIy=-| zN~3x9LJT+g+o2C%|MBX_UoPj`U;nRuaXr|Mbl+hkitB`DS{7)hI`8w5e5w=_#;0u2 zMJ*lj8p{T2xPv^Lyw9eD!6}CrVMe^!kA64&5%fQ_Yd>ASD?B{9AKvX!O$Hb8jd4L# zRU>RB9n@*feO9y2lMos>l*86-5f<`3R}t=p{F*8=yn+x$7B3+L@soClhCS$M5ntOm@N(% zhxP8Jd3ryGwL_|;JCv1S4*6^lc6zj*Co&CT!wF39mO0!b2v?;I>L_$g_h+|q6iqU8 zh3z`jZfpmutq1gDl%5BsFl*6+kK`&n?s@WK}C03LE+Y$*pL=`EVph#R@jbNLf-Bc zZ^>d7k?maYblY6fp54tcwj5&0DlMVd?rivkvk_-RzY!b_UOko2b~guM*7Sw<<_e5s z+PLD~(9WUGULiL;L~v?kphtRPPdfePl!caCJGjjo13ZxN2?A5qjEt% zs+oJkE;w0f$n_~}u$Hodrcwp!gqzLu?u-TMgJV zu>?J4N(ifP*h#y#!e5@9A>hj@Lu7hcf)N}U!bu;HXPWWF>MztXEifFD5sDm>gIpE* zu-+~HUYg@!Fs)cr*<=VDu|;Si2Y!62UZ^x0q|6n9k!wWAVg=`x0y<;eS8=zra#WTV zW`mrCcTcY)Y&=0fv6HvQEJT!?=Q{a9Aq0r&d_TJCK^=T1%`dOXshNFT&u=RM^61J!q|s|VdJIy`DKg;;Tj zI<71C%P>9!G+Ub>S7Qy$tda0_p8@S;#=a;`e{*jxH&6vJ5KO4`U#lVWG=Q=p5{XJ8U6ynu&R0GXE=K z2Bs8osC7gp)XHmQb!#!t1z*O0B;~mDD!`ubCd|zm!aa({Jc{{q;~NZG7^=Ms_FT4L zE{~v}SoWWrzp#CisAg0Zd_BcN)+rD7Bz1poJ_B1dBKG!VLUZmUY|jw+?5ut(z^`u6 z@__1c73{KYP!pM`=e+)FG!1bS1(R+QD+dYBWD)a3>Kn~psyW6niQA5=hRsedtiBeQ z&sG0LAO>>VgUu;vF%e87ol}W;Jf|1W-)%YA zELg3%!;PmvJs#-|U<YhGZEU173e1a*hQ-K%pBc@I^W8Z|DoiNKwBT!A z9K78s?89jO3#1Gv!fouN=!D{~qQHuqf*!l|LjBhx2*cgYg)`Y}D6_`}k8_NCl;VfT zG>SWkN$z#Raola>YIcy%`TGO)$)Up{(HzY2WI_d2J;*_j2qrfu-Hy8;wr7X8%p=bW3bb2=__6qw}o99A0&=6w(&rsd0Y~dbP>XWfzngGeT+4UC3CkQP1ft*An&tBHKVKW~G#dK{B7 z!N^q_y}M^5ce!9bqTWt9sB-y`?&CEk(kXC9LFh3j-%e>v*LbkSiU>$Ib>O5*AfN5` z_ww5yoC=(}E*BX(t#A(d`)}NS=lnE6L_wUGLV0UCk*d8|6el(4$9w*4)B~|f1F3x5 zu*Vgc92f4nxq3hIm-5XV2W}t+f*)!K>}K=&|Hlj;iflOY?P$ZT8c`ZSnEJ*$U3zB0 zoz)a{ojnOr8wCA0@()<-Z&e(($3UMM6L5@Ng082Gc{bi#@ZZZA44tcuE>E=Jaa9?r z*@d6(sDIYJK_$Z&UGN0g?3v*84C-OdYyY>E4vW@i6eiZIL@r4sT-jIH$7}zkatw4` z*iffM!!f~I+&BvH$n00{cV-+}la=7)MFeb>YM`yPKs{AazaO+Q^4@~sR19*@L3lFH zpdZfZ_j4}9CI`db-Gs%`24Nh^)(5B5ke~+-3^Yv)2ARfQ2V! z7v7?-z{$LAWXD#Zp30xO01RAYWn(ir+u(GJ0%Bzh7|&+uCHMt3$EG_As%1q(S=u#N zc~xLO(Ko-0r?KTA!Odt`(VGj35dy#3Vt;ON97tt2_qH7?|MS;($>lgAca9ur# z?9Au`r$ZZXL<)Si+Aq*A?K!loT-v; z!dnrbHdJuO9TCqd`TH>&rzM8W z_0Hy>bx`o@@4JyRfu7JFJl)X?Z_@o~>JIlrp8f_A1{iVO$V*lWj<)ZR!Hxy<@%_2e_5IUJ6;%V~U3Wnf!-V2= zjeJhMf1y`{o{~}NGqf{%1Vdiv$xi+iR^K=@rHMt&RktCjsut{;4#ZRQmzpuqij2T! z7a6!*<$;mM0sU<6pE-*^uH)z{cjAggLAzu&B!>t-bARa7{c#ybjb+T0TVIML3QSiU zhwVmXa1-4V89Y{K$q_z-c1WG?GE<2%KZS2iN$Py GmH+?%XM7_7 literal 0 HcmV?d00001 diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/metadata.csv.gz b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/metadata.csv.gz new file mode 100644 index 0000000000000000000000000000000000000000..e6666058dc2893cf00e8def5a594428bd6c7cf01 GIT binary patch literal 310 zcmV-60m=R!iwFo%N?2wB0CR73a${vKZDn*}E@N|c0Hu=6O2kkMh4+1mFyJl=>l=tD zx*0_uprp5H8@Nr%{Zab%PFw0!5OgrB+>@N|EOMRsN@-x6n>3A#o@(|e+K8iMq3VL= zQMB7hStaeB&4{h|#G&d%)7Edk>O{_i7OcViz?vdHZC1QLi;ql=7iFM2vO0!V*f$aJ z8nK8RV>McgQhVO{k9nv$fVHpdV=99D*U^NOy z1WP@K4~YzF@J>GS0Sk2#Q`nG4N@7YL?s5p?vd;r7^HaT&b{wG%wrUYdNCstzpN;;? IvCRSi04-ONk^lez literal 0 HcmV?d00001 diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/source.csv b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/source.csv new file mode 100644 index 00000000000..af6d16b86e2 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/source.csv @@ -0,0 +1,217 @@ +time_period,time_identifier,geographic_level,country_code,country_name,region_code,region_name,old_la_code,new_la_code,la_name,school_urn,school_laestab,school_name,school_type,academy_type,ncyear,enrolments,sess_possible,sess_authorised,sess_unauthorised,sess_unauthorised_percent +202021,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 4,930365,8380405,4410042,36349,10.0359 +202021,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 6,390233,2895910,2734525,418548,12.8344 +202021,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 8,966035,3669102,4767788,12507,9.4158 +202021,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 10,687704,8880914,3426082,16844,10.9951 +202021,Academic year,National,E92000001,England,,,,,,,,,State-funded primary,,Year 4,233870,4666313,1794452,164845,7.9822 +202021,Academic year,National,E92000001,England,,,,,,,,,State-funded primary,,Year 6,510682,608287,3047386,276594,7.806 +202021,Academic year,National,E92000001,England,,,,,,,,,State-funded secondary,,Year 8,114560,1018241,4071886,105461,3.6011 +202021,Academic year,National,E92000001,England,,,,,,,,,State-funded secondary,,Year 10,496128,8738376,3932498,276495,10.7961 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 4,748965,7281611,394560,254132,6.2354 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 6,819402,4571123,292963,79165,14.8326 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 8,999598,2688774,953422,202885,2.5727 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 10,863196,5417065,516529,70706,7.8286 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded primary,,Year 4,960185,6838156,460125,452731,2.9259 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded primary,,Year 6,415706,2949928,2711122,443827,1.7428 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded secondary,,Year 8,498680,7171811,3213217,134325,4.1664 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded secondary,,Year 10,706374,5427979,2488176,248703,0.9058 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 4,643309,2783829,442519,360329,9.0429 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 6,406128,3922516,529884,228791,9.5496 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 8,954008,2510684,1058785,159578,10.3886 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 10,890581,4446119,4096651,6031,5.4532 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded primary,,Year 4,863407,6514957,4870020,470418,2.8281 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded primary,,Year 6,134998,1099421,351548,206551,12.3929 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded secondary,,Year 8,881018,9926047,4934983,417201,5.3926 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded secondary,,Year 10,939893,502110,2088017,401671,14.2018 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,State-funded primary,Primary sponsor led academy,Year 4,296352,7368540,1002108,285622,3.9876 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,State-funded primary,Primary sponsor led academy,Year 6,222884,6082113,360617,103563,6.2636 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,State-funded secondary,,Year 8,661641,1779059,1354689,125448,3.8439 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,State-funded secondary,,Year 10,206716,3239290,4648058,389642,12.3234 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 4,996491,8002818,4635376,213122,1.6915 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 6,846370,2152920,2828812,338402,7.6658 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 8,122971,2345267,4613665,214274,12.1115 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 10,219610,545275,1728142,129069,13.2427 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded primary,,Year 4,338336,8025420,3599138,249791,8.2206 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded primary,,Year 6,12515,3001050,3315415,239804,9.0924 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded secondary,,Year 8,454021,802504,3520216,373236,3.111 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded secondary,,Year 10,48297,4774788,1558126,88665,4.0493 +202021,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,State-funded primary,,Year 4,193938,3822743,699491,21990,1.2297 +202021,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,State-funded primary,,Year 6,420838,646484,3226295,252707,2.5504 +202021,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,State-funded secondary,Secondary sponsor led academy,Year 8,425445,6702005,3926588,19445,2.0738 +202021,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,State-funded secondary,Secondary sponsor led academy,Year 10,178144,1389676,4679175,151593,5.4475 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 4,636969,3414663,3283314,318214,1.2402 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 6,17520,5494804,366873,266062,7.4774 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 8,817310,4511712,2233435,43576,7.9209 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 10,46096,4687214,862914,43010,10.8394 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded primary,,Year 4,794394,1963532,564394,90260,8.1949 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded primary,,Year 6,200199,1526080,2819473,407340,11.3028 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded secondary,,Year 8,454627,8951304,1559449,327305,4.3795 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded secondary,,Year 10,933426,2761869,1918148,185997,5.1204 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 4,991280,5127375,1163515,388622,2.175 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 6,436370,4420177,1343905,412913,7.4523 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 8,224856,47538,1958476,337417,11.5369 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 10,906144,3803424,864910,168235,4.6508 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded primary,,Year 4,435305,3679657,2183483,176088,3.8776 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded primary,,Year 6,742448,3740647,2876036,96880,8.604 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded secondary,,Year 8,941614,1936128,3212126,57332,14.045 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded secondary,,Year 10,969606,5896296,526533,236361,8.2235 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,State-funded primary,,Year 4,10329,5168495,4252690,162288,4.9802 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,State-funded primary,,Year 6,900808,3281802,4018891,402870,2.0455 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,State-funded secondary,Secondary sponsor led academy,Year 8,655344,6237122,3417277,99042,3.7976 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,State-funded secondary,Secondary sponsor led academy,Year 10,789368,5583863,4304947,456541,5.7804 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 4,500911,9586188,578180,21607,3.3617 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 6,477675,9610295,3190487,392393,12.0798 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 8,216141,5952523,4784805,486676,12.6185 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 10,423972,61023,2440931,332898,8.5528 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded primary,,Year 4,154631,5934750,716423,263603,0.8892 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded primary,,Year 6,48296,3079505,4666548,373958,10.2297 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded secondary,,Year 8,708802,8694392,3370782,109291,13.5185 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded secondary,,Year 10,498079,7008979,406947,380686,6.9813 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,State-funded primary,,Year 4,924172,5229817,461921,255056,10.6752 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,State-funded primary,,Year 6,828609,7613912,3268057,168198,5.1854 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,State-funded secondary,Secondary free school,Year 8,485419,3294823,313667,408852,13.7387 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,State-funded secondary,Secondary free school,Year 10,948391,8779565,2607975,135041,12.8666 +202122,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 4,263424,8721586,4167742,383478,5.0415 +202122,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 6,198408,5264285,2710311,494993,5.0129 +202122,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 8,484230,1614115,3534379,87050,6.8986 +202122,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 10,155301,4190557,3231686,207486,5.8223 +202122,Academic year,National,E92000001,England,,,,,,,,,State-funded primary,,Year 4,611553,9635683,786941,477230,14.2482 +202122,Academic year,National,E92000001,England,,,,,,,,,State-funded primary,,Year 6,752711,3936811,752220,157318,14.1615 +202122,Academic year,National,E92000001,England,,,,,,,,,State-funded secondary,,Year 8,1072,3365122,3565402,328941,0.7839 +202122,Academic year,National,E92000001,England,,,,,,,,,State-funded secondary,,Year 10,408184,8799602,22441,236508,6.3432 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 4,610330,7640992,3720172,167078,12.0133 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 6,890478,5184289,26656,197176,6.2351 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 8,649107,4218939,3217805,343102,13.4956 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 10,63988,3357225,4901965,332298,8.693 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded primary,,Year 4,343528,6477506,3420795,94339,9.0596 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded primary,,Year 6,974891,3272204,3420489,457219,1.7708 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded secondary,,Year 8,809181,8606505,3497757,284217,4.8312 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded secondary,,Year 10,267865,3977718,4007129,279593,14.0394 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 4,375793,5942431,186830,124526,9.5212 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 6,732072,1971603,4584099,297972,7.3678 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 8,262655,7628392,370082,207129,5.1999 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 10,683791,569658,2478482,281011,4.9364 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded primary,,Year 4,977317,9595365,1839374,414659,12.3688 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded primary,,Year 6,211650,5928607,1691306,386536,13.4202 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded secondary,,Year 8,163986,9675857,661047,250482,1.0812 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded secondary,,Year 10,130938,1788491,1248863,40642,0.505 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,State-funded primary,Primary sponsor led academy,Year 4,291660,4552605,1441830,421128,11.8714 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,State-funded primary,Primary sponsor led academy,Year 6,691504,8180301,3458032,11936,7.1623 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,State-funded secondary,,Year 8,87349,6608543,1831731,38458,2.2866 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,State-funded secondary,,Year 10,335593,4145901,430493,219634,3.8077 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 4,65626,8941189,2778792,40825,12.4985 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 6,446000,8180991,4241359,160601,4.396 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 8,672736,2069839,1789330,25253,13.106 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 10,710863,4131441,412250,234342,12.7331 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded primary,,Year 4,735373,1481670,3320679,249061,7.5083 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded primary,,Year 6,315922,1657744,1998333,182112,12.0564 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded secondary,,Year 8,322703,7598586,4782452,34528,4.1753 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded secondary,,Year 10,807678,777312,2469553,493181,6.233 +202122,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,State-funded primary,,Year 4,389695,1193807,3889011,292288,10.5549 +202122,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,State-funded primary,,Year 6,298947,3650745,4370595,53048,14.3211 +202122,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,State-funded secondary,Secondary sponsor led academy,Year 8,29393,2940163,460442,337613,7.0733 +202122,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,State-funded secondary,Secondary sponsor led academy,Year 10,752009,5840033,262396,356628,7.901 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 4,934820,1670483,4183571,467206,7.207 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 6,10142,7126488,1679362,418191,14.0816 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 8,444973,5610555,4967515,359239,9.3129 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 10,855721,8989241,2794631,444771,3.8782 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded primary,,Year 4,38547,1265455,4454444,127033,5.9484 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded primary,,Year 6,217205,4114641,824011,477351,7.0324 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded secondary,,Year 8,474073,9860956,4416348,426962,13.956 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded secondary,,Year 10,38647,8375228,700134,343753,0.4792 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 4,165477,3982999,2881117,494074,5.1466 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 6,322047,7661435,4262799,97475,12.4722 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 8,18416,6851799,2025239,270083,7.1532 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 10,940546,4035799,1299367,17693,14.8835 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded primary,,Year 4,27697,9793497,4056200,22348,11.8263 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded primary,,Year 6,967464,8034305,1390806,421037,4.7727 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded secondary,,Year 8,932041,2054142,4961909,486625,8.1061 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded secondary,,Year 10,200464,8732143,2099505,150975,2.4462 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,State-funded primary,,Year 4,466618,4331684,2336745,474374,3.2321 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,State-funded primary,,Year 6,51873,7318963,4091886,382787,3.9492 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,State-funded secondary,Secondary sponsor led academy,Year 8,324513,9629567,2090853,165020,13.2842 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,State-funded secondary,Secondary sponsor led academy,Year 10,12135,418821,3617819,471557,1.9472 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 4,429828,2541098,4189699,191836,4.6809 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 6,800267,821476,4743561,380527,5.9484 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 8,277672,1725609,1984888,476621,3.1818 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 10,938510,8382254,4570224,373427,5.9204 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded primary,,Year 4,563379,9886857,211791,138796,8.1423 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded primary,,Year 6,262520,5231953,1946565,190296,4.8397 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded secondary,,Year 8,319358,3860000,2182522,407747,11.8051 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded secondary,,Year 10,958386,9724817,3819718,327783,11.8484 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,State-funded primary,,Year 4,825224,8846588,1975540,131067,5.4658 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,State-funded primary,,Year 6,922630,6219569,737030,39992,3.7797 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,State-funded secondary,Secondary free school,Year 8,887815,8905360,511940,110254,8.985 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,State-funded secondary,Secondary free school,Year 10,826025,3168076,1954888,14999,3.7619 +202223,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 4,219863,1211324,2461223,227036,6.8479 +202223,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 6,326899,9400873,3596596,84154,6.4159 +202223,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 8,714610,561356,2907605,433541,5.2496 +202223,Academic year,National,E92000001,England,,,,,,,,,Total,,Year 10,383602,1803285,287864,2883,0.4685 +202223,Academic year,National,E92000001,England,,,,,,,,,State-funded primary,,Year 4,654884,3220663,4379854,207878,4.939 +202223,Academic year,National,E92000001,England,,,,,,,,,State-funded primary,,Year 6,235647,9247378,3142734,451761,3.1564 +202223,Academic year,National,E92000001,England,,,,,,,,,State-funded secondary,,Year 8,625982,1024199,387768,48184,1.1214 +202223,Academic year,National,E92000001,England,,,,,,,,,State-funded secondary,,Year 10,703403,7341429,3350111,90245,6.689 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 4,356747,6400821,178815,353079,7.2406 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 6,478301,6941924,3385453,137969,14.2775 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 8,634983,9628460,3931328,65536,3.7371 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,Total,,Year 10,540587,6663635,2179562,323178,1.3373 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded primary,,Year 4,783767,9416803,3342067,61772,10.9315 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded primary,,Year 6,818474,6521510,3326180,148818,10.1909 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded secondary,,Year 8,935461,7564846,874297,88073,9.912 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,State-funded secondary,,Year 10,340526,2628965,2973134,435451,2.0839 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 4,362381,1027328,577798,217717,12.313 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 6,448489,2859195,602823,176301,3.8162 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 8,654686,7414000,339649,162343,11.753 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,Total,,Year 10,540640,9354927,2039373,98018,14.8837 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded primary,,Year 4,316064,4874652,2347907,416236,5.8798 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded primary,,Year 6,314436,3348975,886840,62684,9.9694 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded secondary,,Year 8,634778,1878275,2963673,212801,14.4038 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,State-funded secondary,,Year 10,198910,6941070,4072452,296754,7.803 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,State-funded primary,Primary sponsor led academy,Year 4,294564,6547062,4745554,25788,4.6605 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,State-funded primary,Primary sponsor led academy,Year 6,123104,7290192,4578941,19677,5.1375 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,State-funded secondary,,Year 8,628305,7084823,3811123,453695,8.4844 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,State-funded secondary,,Year 10,659044,2634214,4070198,97405,0.2416 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 4,357353,1739000,4425257,285842,12.5505 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 6,724898,1043207,389401,473394,14.1792 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 8,216117,7168883,4215280,78439,13.679 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,Total,,Year 10,957960,9934276,1638073,147204,10.8101 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded primary,,Year 4,563389,9121298,3303786,403465,10.987 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded primary,,Year 6,624827,2354255,3052199,48720,12.3966 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded secondary,,Year 8,228238,7828108,2950344,452062,3.2874 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,State-funded secondary,,Year 10,467375,9278504,422419,244619,11.617 +202223,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,State-funded primary,,Year 4,296027,2759449,4467649,78337,7.2387 +202223,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,State-funded primary,,Year 6,98921,9008990,2602662,484586,6.1155 +202223,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,State-funded secondary,Secondary sponsor led academy,Year 8,929132,182167,4080556,483888,8.1479 +202223,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,State-funded secondary,Secondary sponsor led academy,Year 10,751028,1449807,175843,263574,9.6395 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 4,577746,7493602,1377020,315021,7.8505 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 6,720401,3593781,3441828,186132,12.8428 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 8,470788,4851295,1733986,360850,5.5305 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,Total,,Year 10,903536,8331786,1760720,215307,10.8352 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded primary,,Year 4,834635,2581506,2352884,153955,13.0697 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded primary,,Year 6,854011,18306,4206838,121002,13.2874 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded secondary,,Year 8,304325,2900642,690657,217905,1.9265 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,State-funded secondary,,Year 10,965673,3633069,1760280,40032,1.8564 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 4,288238,9847925,4084552,444552,11.8302 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 6,775782,7953739,787260,180127,8.7798 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 8,783460,233298,1339828,148935,7.5016 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,Total,,Year 10,524941,7991277,1172325,363994,10.5637 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded primary,,Year 4,930007,9048500,128025,460991,0.9231 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded primary,,Year 6,929516,9022992,4420240,350476,7.6263 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded secondary,,Year 8,535524,5970083,3697539,328489,4.6321 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,State-funded secondary,,Year 10,435263,4919212,1055600,422276,12.6506 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,State-funded primary,,Year 4,519041,7906994,4405937,89041,4.744 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,State-funded primary,,Year 6,674329,776260,1342817,241868,9.1741 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,State-funded secondary,Secondary sponsor led academy,Year 8,455256,921184,4608260,20074,8.3842 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,State-funded secondary,Secondary sponsor led academy,Year 10,135692,7302541,324393,244258,7.7954 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 4,303770,670289,4766068,486141,13.9153 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 6,37346,4344143,1516845,387614,3.5746 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 8,903110,9011174,417857,433299,9.0392 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,Total,,Year 10,100966,3931562,4064499,301608,13.4953 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded primary,,Year 4,750557,1399665,270713,209296,3.9268 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded primary,,Year 6,147524,53494,2877891,10248,2.3631 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded secondary,,Year 8,92708,5795664,2225558,165760,0.5731 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,State-funded secondary,,Year 10,620300,8365110,938239,103917,14.8688 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,State-funded primary,,Year 4,695463,951512,2881136,224021,1.7931 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,State-funded primary,,Year 6,57424,7954628,3860012,134564,11.4722 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,State-funded secondary,Secondary free school,Year 8,304011,8070409,1297358,26349,2.1772 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,State-funded secondary,Secondary free school,Year 10,422137,145499,2199762,452887,8.9538 \ No newline at end of file diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/source.meta.csv b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/source.meta.csv new file mode 100644 index 00000000000..74426ec2c16 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Resources/DataFiles/AbsenceSchool/source.meta.csv @@ -0,0 +1,9 @@ +col_name,col_type,label,indicator_grouping,indicator_unit,indicator_dp,filter_hint,filter_grouping_column +enrolments,Indicator,Enrolments,Headline absence fields,,0,, +sess_possible,Indicator,Number of possible sessions,Headline absence fields,,0,, +sess_authorised,Indicator,Number of authorised sessions,Headline absence fields,,0,, +sess_unauthorised,Indicator,Number of unauthorised sessions,Headline absence fields,,0,, +sess_unauthorised_percent,Indicator,Percentage of unauthorised sessions,Headline absence fields,%,2,, +school_type,Filter,School type,,,,, +academy_type,Filter,Academy type,,,,"Only applicable for academies, otherwise no value", +ncyear,Filter,National Curriculum year,,,,Ranges from years 1 to 11, \ No newline at end of file diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Entities/ActivityLock.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Entities/ActivityLock.cs new file mode 100644 index 00000000000..22b858a7d32 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Entities/ActivityLock.cs @@ -0,0 +1,8 @@ +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Entities; + +///

    +/// Durable entity which has no state or operations defined except for the built-in lock and unlock operations. +/// Obtaining a lock on this entity can be used to create a critical section during orchestration. +/// +// ReSharper disable once ClassNeverInstantiated.Global +internal class ActivityLock; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Extensions/TaskOrchestrationContextExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Extensions/TaskOrchestrationContextExtensions.cs new file mode 100644 index 00000000000..3dba57676f1 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Extensions/TaskOrchestrationContextExtensions.cs @@ -0,0 +1,70 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Entities; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; + +public static class TaskOrchestrationContextExtensions +{ + public static async Task CallActivity( + this TaskOrchestrationContext context, + TaskName name, + ILogger logger, + object? input = null, + TaskOptions? options = null) + { + logger.LogInformation( + "Calling activity '{ActivityName}' (InstanceId={InstanceId})", + name, + context.InstanceId); + + await context.CallActivityAsync(name, input, options); + + logger.LogInformation( + "Activity '{ActivityName}' completed (InstanceId={InstanceId})", + name, + context.InstanceId); + } + + public static async Task CallActivityExclusively( + this TaskOrchestrationContext context, + TaskName name, + ILogger logger, + object? input = null, + TaskOptions? options = null) + { + // Create an entity id which represents the activity name + var activityLockId = new EntityInstanceId(nameof(ActivityLock), name); + + logger.LogInformation("Attempting to acquire lock '{EntityId}' (InstanceId={InstanceId})", + activityLockId, + context.InstanceId); + + // Acquire a lock on the entity instance ensuring that no other orchestrations can execute the same activity + // concurrently provided that they also acquire a lock on the same entity. + // The lock is automatically released at the end of the using statement. + await using (await context.Entities.LockEntitiesAsync(activityLockId)) + { + logger.LogInformation("Acquired lock '{EntityId}' (InstanceId={InstanceId})", + activityLockId, + context.InstanceId); + + logger.LogInformation( + "Calling activity '{ActivityName}' (InstanceId={InstanceId})", + name, + context.InstanceId); + + await context.CallActivityAsync(name, input, options); + + logger.LogInformation( + "Activity '{ActivityName}' completed (InstanceId={InstanceId})", + name, + context.InstanceId); + + logger.LogInformation("Releasing lock '{EntityId}' (InstanceId={InstanceId})", + activityLockId, + context.InstanceId); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/BaseProcessDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/BaseProcessDataSetVersionFunction.cs new file mode 100644 index 00000000000..f76d38cb5e5 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/BaseProcessDataSetVersionFunction.cs @@ -0,0 +1,26 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public abstract class BaseProcessDataSetVersionFunction(PublicDataDbContext publicDataDbContext) +{ + protected async Task GetDataSetVersionImport( + Guid instanceId, + CancellationToken cancellationToken) + { + return await publicDataDbContext.DataSetVersionImports + .Include(i => i.DataSetVersion) + .SingleAsync(i => i.InstanceId == instanceId, cancellationToken: cancellationToken); + } + + protected async Task UpdateImportStage( + DataSetVersionImport dataSetVersionImport, + DataSetVersionImportStage stage, + CancellationToken cancellationToken) + { + dataSetVersionImport.Stage = stage; + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CopyCsvFilesFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CopyCsvFilesFunction.cs index bda49ff4c75..69c7fde45a8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CopyCsvFilesFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CopyCsvFilesFunction.cs @@ -4,12 +4,12 @@ using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.Azure.Functions.Worker; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using static GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionImportStage; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; @@ -18,20 +18,17 @@ public class CopyCsvFilesFunction( ContentDbContext contentDbContext, PublicDataDbContext publicDataDbContext, IDataSetVersionPathResolver dataSetVersionPathResolver, - IPrivateBlobStorageService privateBlobStorageService) + IPrivateBlobStorageService privateBlobStorageService) : BaseProcessDataSetVersionFunction(publicDataDbContext) { [Function(nameof(CopyCsvFiles))] public async Task CopyCsvFiles( - [ActivityTrigger] Guid dataSetVersionId, - Guid instanceId, + [ActivityTrigger] Guid instanceId, CancellationToken cancellationToken) { - var dataSetVersion = await GetDataSetVersion( - dataSetVersionId: dataSetVersionId, - instanceId: instanceId, - cancellationToken); + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + var dataSetVersion = dataSetVersionImport.DataSetVersion; - await UpdateStage(dataSetVersion, instanceId, DataSetVersionImportStage.CopyingCsvFiles, cancellationToken); + await UpdateImportStage(dataSetVersionImport, CopyingCsvFiles, cancellationToken); var csvDataFile = await contentDbContext.ReleaseFiles .Where(rf => rf.Id == dataSetVersion.ReleaseFileId) @@ -59,7 +56,7 @@ private async Task CopyCsvFile( if (File.Exists(destinationPath)) { - logger.LogWarning("Destination csv file '{destination}' already exists and will be overwritten.", + logger.LogWarning("Destination csv file '{DestinationPath}' already exists and will be overwritten.", destinationPath); } @@ -71,26 +68,4 @@ private async Task CopyCsvFile( await using var fileStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write); await CompressionUtils.CompressToStream(blobStream, fileStream, ContentEncodings.Gzip, cancellationToken); } - - private async Task GetDataSetVersion( - Guid dataSetVersionId, - Guid instanceId, - CancellationToken cancellationToken) - { - return await publicDataDbContext.DataSetVersions - .Include(dsv => dsv.Imports.Where(i => i.InstanceId == instanceId)) - .SingleAsync(dsv => dsv.Id == dataSetVersionId, cancellationToken: cancellationToken); - } - - private async Task UpdateStage( - DataSetVersion dataSetVersion, - Guid instanceId, - DataSetVersionImportStage stage, - CancellationToken cancellationToken) - { - var dataSetVersionImport = dataSetVersion.Imports.Single(i => i.InstanceId == instanceId); - - dataSetVersionImport.Stage = stage; - await publicDataDbContext.SaveChangesAsync(cancellationToken); - } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs index af246cfffd7..d5175f58db3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HealthCheckFunctions.cs @@ -12,7 +12,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Funct public class HealthCheckFunctions( ILogger logger, PublicDataDbContext publicDataDbContext, - IOptions parquetFileOptions) + IOptions dataFilesOptions) { [Function(nameof(CountDataSets))] public async Task CountDataSets( @@ -22,7 +22,7 @@ public async Task CountDataSets( { try { - var message = $"Found {await publicDataDbContext.DataSets.CountAsync()} datasets."; + var message = $"Found {await publicDataDbContext.DataSets.CountAsync()} data sets."; logger.LogInformation(message); return message; } @@ -32,7 +32,7 @@ public async Task CountDataSets( throw; } } - + [Function(nameof(CheckForFileShareMount))] public Task CheckForFileShareMount( #pragma warning disable IDE0060 @@ -40,10 +40,10 @@ public Task CheckForFileShareMount( #pragma warning restore IDE0060 { logger.LogInformation("Attempting to read from file share"); - + try { - if (Directory.Exists(parquetFileOptions.Value.BasePath)) + if (Directory.Exists(dataFilesOptions.Value.BasePath)) { logger.LogInformation("Successfully found the file share mount"); } @@ -57,7 +57,7 @@ public Task CheckForFileShareMount( logger.LogError(e, "Error encountered when attempting to find the file share mount"); throw; } - + return Task.CompletedTask; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs index 2313728c3f5..7a158a5cf9f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs @@ -1,15 +1,25 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.Azure.Functions.Worker; using Microsoft.DurableTask; -using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class ProcessInitialDataSetVersionFunction(PublicDataDbContext publicDataDbContext) +public class ProcessInitialDataSetVersionFunction( + PublicDataDbContext publicDataDbContext, + IDataSetMetaService dataSetMetaService, + IDataDuckDbRepository dataDuckDbRepository, + IParquetService parquetService, + IDataSetVersionPathResolver dataSetVersionPathResolver) : BaseProcessDataSetVersionFunction(publicDataDbContext) { + private readonly PublicDataDbContext _publicDataDbContext = publicDataDbContext; + [Function(nameof(ProcessInitialDataSetVersion))] public async Task ProcessInitialDataSetVersion( [OrchestrationTrigger] TaskOrchestrationContext context, @@ -24,28 +34,11 @@ public async Task ProcessInitialDataSetVersion( try { - await context.CallActivityAsync(nameof(CopyCsvFilesFunction.CopyCsvFiles), - input.DataSetVersionId); - - logger.LogInformation( - "Activity '{ActivityName}' completed (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - nameof(CopyCsvFilesFunction.CopyCsvFiles), - context.InstanceId, - input.DataSetVersionId); - - // Other activity function calls to be added here to cover the following stages: - // Create meta summary for the DataSetVersion - // Import metadata to DuckDb - // Import data to DuckDb - // Export to Parquet files - - await context.CallActivityAsync(nameof(CompleteProcessing), input.DataSetVersionId); - - logger.LogInformation( - "Activity '{ActivityName}' completed (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - nameof(CompleteProcessing), - context.InstanceId, - input.DataSetVersionId); + await context.CallActivity(nameof(CopyCsvFilesFunction.CopyCsvFiles), logger, context.InstanceId); + await context.CallActivityExclusively(nameof(ImportMetadata), logger, context.InstanceId); + await context.CallActivity(nameof(ImportData), logger, context.InstanceId); + await context.CallActivity(nameof(WriteDataFiles), logger, context.InstanceId); + await context.CallActivity(nameof(CompleteProcessing), logger, context.InstanceId); } catch (Exception e) { @@ -54,59 +47,68 @@ await context.CallActivityAsync(nameof(CopyCsvFilesFunction.CopyCsvFiles), context.InstanceId, input.DataSetVersionId); - await context.CallActivityAsync(nameof(HandleProcessingFailure), input.DataSetVersionId); + await context.CallActivity(nameof(HandleProcessingFailure), logger); } } + [Function(nameof(ImportMetadata))] + public async Task ImportMetadata( + [ActivityTrigger] Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.ImportingMetadata, cancellationToken); + await dataSetMetaService.CreateDataSetVersionMeta(dataSetVersionImport.DataSetVersionId, cancellationToken); + } + + [Function(nameof(ImportData))] + public async Task ImportData( + [ActivityTrigger] Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.ImportingData, cancellationToken); + await dataDuckDbRepository.CreateDataTable(dataSetVersionImport.DataSetVersionId, cancellationToken); + } + + [Function(nameof(WriteDataFiles))] + public async Task WriteDataFiles( + [ActivityTrigger] Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.WritingDataFiles, cancellationToken); + await parquetService.WriteDataFiles(dataSetVersionImport.DataSetVersionId, cancellationToken); + } + [Function(nameof(CompleteProcessing))] public async Task CompleteProcessing( - [ActivityTrigger] Guid dataSetVersionId, - Guid instanceId, + [ActivityTrigger] Guid instanceId, CancellationToken cancellationToken) { - var dataSetVersion = await GetDataSetVersion( - dataSetVersionId: dataSetVersionId, - instanceId: instanceId, - cancellationToken); + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.Completing, cancellationToken); - var dataSetVersionImport = dataSetVersion.Imports.Single(i => i.InstanceId == instanceId); + var dataSetVersion = dataSetVersionImport.DataSetVersion; - dataSetVersionImport.Stage = DataSetVersionImportStage.Completing; - await publicDataDbContext.SaveChangesAsync(cancellationToken); - - // Any additional logic to tidy up after importing will be added here + // Delete the DuckDb database file as it is no longer needed + File.Delete(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); dataSetVersion.Status = DataSetVersionStatus.Draft; dataSetVersionImport.Completed = DateTimeOffset.UtcNow; - await publicDataDbContext.SaveChangesAsync(cancellationToken); + await _publicDataDbContext.SaveChangesAsync(cancellationToken); } [Function(nameof(HandleProcessingFailure))] public async Task HandleProcessingFailure( - [ActivityTrigger] Guid dataSetVersionId, - Guid instanceId, + [ActivityTrigger] Guid instanceId, CancellationToken cancellationToken) { - var dataSetVersion = await GetDataSetVersion( - dataSetVersionId: dataSetVersionId, - instanceId: instanceId, - cancellationToken); + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + var dataSetVersion = dataSetVersionImport.DataSetVersion; dataSetVersion.Status = DataSetVersionStatus.Failed; - - var dataSetVersionImport = dataSetVersion.Imports.Single(i => i.InstanceId == instanceId); dataSetVersionImport.Completed = DateTimeOffset.UtcNow; - - await publicDataDbContext.SaveChangesAsync(cancellationToken); - } - - private async Task GetDataSetVersion( - Guid dataSetVersionId, - Guid instanceId, - CancellationToken cancellationToken) - { - return await publicDataDbContext.DataSetVersions - .Include(dsv => dsv.Imports.Where(i => i.InstanceId == instanceId)) - .SingleAsync(dsv => dsv.Id == dataSetVersionId, cancellationToken: cancellationToken); + await _publicDataDbContext.SaveChangesAsync(cancellationToken); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.csproj b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.csproj index 232c599aaf8..b8d5be4188f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.csproj +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.csproj @@ -34,6 +34,7 @@ + diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/MetaFileRow.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/MetaFileRow.cs new file mode 100644 index 00000000000..fc1672fbbc9 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/MetaFileRow.cs @@ -0,0 +1,34 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Models; + +public record MetaFileRow +{ + public required string ColName { get; init; } + + public ColumnType ColType { get; init; } + + public required string Label { get; init; } + + public string? IndicatorGrouping { get; init; } + + private string? _indicatorUnit { get; init; } + + public IndicatorUnit? IndicatorUnit => + _indicatorUnit is not null + ? EnumUtil.GetFromEnumValue(_indicatorUnit) + : null; + + public byte? IndicatorDp { get; init; } + + public string? FilterGroupingColumn { get; init; } + + public string? FilterHint { get; init; } + + public enum ColumnType + { + Indicator, + Filter, + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs index 6324fe3f6fa..49a1f20c3a9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs @@ -1,5 +1,6 @@ using Azure.Core; using Azure.Identity; +using Dapper; using FluentValidation; using GovUk.Education.ExploreEducationStatistics.Common.Database; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; @@ -8,6 +9,8 @@ using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; @@ -97,6 +100,9 @@ public static IHostBuilder ConfigureProcessorHostBuilder(this IHostBuilder hostB } } + // Configure Dapper to match CSV columns with underscores + DefaultTypeMap.MatchNamesWithUnderscores = true; + services .AddApplicationInsightsTelemetryWorkerService() .ConfigureFunctionsApplicationInsights() @@ -107,12 +113,24 @@ public static IHostBuilder ConfigureProcessorHostBuilder(this IHostBuilder hostB .EnableSensitiveDataLogging(hostEnvironment.IsDevelopment())) .AddFluentValidation() .AddScoped() + .AddScoped() .AddScoped() + .AddScoped() + .AddScoped() + .AddScoped() + .AddScoped() + .AddScoped() + .AddScoped() + .AddScoped() + .AddScoped() + .AddScoped() + .AddScoped() + .AddScoped() .AddScoped() .AddScoped, InitialDataSetVersionCreateRequest.Validator>() - .Configure( - hostBuilderContext.Configuration.GetSection(ParquetFilesOptions.Section)); + .Configure( + hostBuilderContext.Configuration.GetSection(DataFilesOptions.Section)); }); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Properties/AssemblyInfo.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..eec38e8e98e --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Properties/AssemblyInfo.cs @@ -0,0 +1,4 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests"), + InternalsVisibleTo("DynamicProxyGenAssembly2")] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/DataDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/DataDuckDbRepository.cs new file mode 100644 index 00000000000..2f8cca6e565 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/DataDuckDbRepository.cs @@ -0,0 +1,168 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using InterpolatedSql.Dapper; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; + +public class DataDuckDbRepository( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver) : IDataDuckDbRepository +{ + public async Task CreateDataTable( + Guid dataSetVersionId, + CancellationToken cancellationToken = default) + { + var dataSetVersion = await publicDataDbContext.DataSetVersions + .AsNoTracking() + .Include(dsv => dsv.FilterMetas) + .Include(dsv => dsv.IndicatorMetas) + .Include(dsv => dsv.LocationMetas) + .Include(dsv => dsv.TimePeriodMetas) + .AsSplitQuery() + .SingleAsync(dsv => dsv.Id == dataSetVersionId, cancellationToken: cancellationToken); + + await using var duckDbConnection = + DuckDbConnection.CreateFileConnection(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); + duckDbConnection.Open(); + + await duckDbConnection.SqlBuilder("CREATE SEQUENCE data_seq START 1") + .ExecuteAsync(cancellationToken: cancellationToken); + + string[] columns = + [ + $"{DataTable.Cols.Id} UINTEGER NOT NULL PRIMARY KEY", + $"{DataTable.Cols.TimePeriodId} INTEGER NOT NULL", + $"{DataTable.Cols.GeographicLevel} VARCHAR NOT NULL", + ..dataSetVersion.LocationMetas.Select(location => + $"{DataTable.Cols.LocationId(location)} INTEGER NOT NULL"), + ..dataSetVersion.FilterMetas.Select(filter => $"{DataTable.Cols.Filter(filter)} INTEGER NOT NULL"), + ..dataSetVersion.IndicatorMetas.Select(indicator => + $"{DataTable.Cols.Indicator(indicator)} VARCHAR NOT NULL"), + ]; + + await duckDbConnection.SqlBuilder( + $""" + CREATE TABLE {DataTable.TableName:raw} + ({columns.JoinToString(",\n"):raw}) + """) + .ExecuteAsync(cancellationToken: cancellationToken); + + string[] insertColumns = + [ + "nextval('data_seq') AS id", + $"{TimePeriodsTable.Ref().Id} AS {DataTable.Cols.TimePeriodId}", + DataSourceTable.Ref.GeographicLevel, + ..dataSetVersion.LocationMetas.Select(location => + $"COALESCE({LocationOptionsTable.Ref(location).Id}, 0) AS {DataTable.Cols.LocationId(location)}"), + ..dataSetVersion.FilterMetas.Select(filter => + $"COALESCE({FilterOptionsTable.Ref(filter).Id}, 0) AS {DataTable.Cols.Filter(filter)}"), + ..dataSetVersion.IndicatorMetas.Select(DataTable.Cols.Indicator), + ]; + + string[] insertJoins = + [ + ..dataSetVersion.LocationMetas.Select( + location => + { + var codeColumns = GetLocationCodeColumns(location.Level); + + string[] conditions = + [ + ..codeColumns.Select(col => + $"{LocationOptionsTable.Ref(location).Col(col.Name)} = {DataSourceTable.Ref.Col(col.CsvName)}"), + $"{LocationOptionsTable.Ref(location).Label} = {DataSourceTable.Ref.Col(location.Level.CsvNameColumn())}" + ]; + + return $""" + LEFT JOIN {LocationOptionsTable.TableName} AS {LocationOptionsTable.Alias(location)} + ON {conditions.JoinToString(" AND ")} + """; + } + ), + ..dataSetVersion.FilterMetas.Select( + filter => $""" + LEFT JOIN {FilterOptionsTable.TableName} AS {FilterOptionsTable.Alias(filter)} + ON {FilterOptionsTable.Ref(filter).FilterId} = '{filter.PublicId}' + AND {FilterOptionsTable.Ref(filter).Label} = {DataSourceTable.Ref.Col(filter.PublicId)} + """ + ), + $""" + JOIN {TimePeriodsTable.TableName} + ON {TimePeriodsTable.Ref().Period} = {DataSourceTable.Ref.TimePeriod} + AND {TimePeriodsTable.Ref().Identifier} = {DataSourceTable.Ref.TimeIdentifier} + """ + ]; + + var dataFilePath = dataSetVersionPathResolver.CsvDataPath(dataSetVersion); + + await duckDbConnection.SqlBuilder( + $""" + INSERT INTO {DataTable.TableName:raw} + SELECT + {insertColumns.JoinToString(",\n"):raw} + FROM read_csv('{dataFilePath:raw}', ALL_VARCHAR = true) AS {DataSourceTable.TableName:raw} + {insertJoins.JoinToString('\n'):raw} + ORDER BY + {DataSourceTable.Ref.GeographicLevel:raw} ASC, + {DataSourceTable.Ref.TimePeriod:raw} DESC + """ + ).ExecuteAsync(cancellationToken: cancellationToken); + } + + private static LocationColumn[] GetLocationCodeColumns(GeographicLevel geographicLevel) + { + return geographicLevel switch + { + GeographicLevel.LocalAuthority => + [ + new LocationColumn(Name: LocationOptionsTable.Cols.Code, + CsvName: GeographicLevelUtils.LocalAuthorityCsvColumns.NewCode), + new LocationColumn(Name: LocationOptionsTable.Cols.OldCode, + CsvName: GeographicLevelUtils.LocalAuthorityCsvColumns.OldCode) + ], + GeographicLevel.Provider => + [ + new LocationColumn(Name: LocationOptionsTable.Cols.Ukprn, + CsvName: GeographicLevelUtils.ProviderCsvColumns.Ukprn) + ], + GeographicLevel.RscRegion => [], + GeographicLevel.School => + [ + new LocationColumn(Name: LocationOptionsTable.Cols.Urn, + CsvName: GeographicLevelUtils.SchoolCsvColumns.Urn), + new LocationColumn(Name: LocationOptionsTable.Cols.LaEstab, + CsvName: GeographicLevelUtils.SchoolCsvColumns.LaEstab) + ], + _ => + [ + new LocationColumn(Name: LocationOptionsTable.Cols.Code, + CsvName: geographicLevel.CsvCodeColumns().First()) + ], + }; + } + + private record LocationColumn(string Name, string CsvName); + + private static class DataSourceTable + { + public const string TableName = "data_source"; + + public static readonly TableRef Ref = new(TableName); + + public class TableRef(string table) + { + public readonly string TimePeriod = $"{table}.time_period"; + public readonly string TimeIdentifier = $"{table}.time_identifier"; + public readonly string GeographicLevel = $"{table}.geographic_level"; + + public string Col(string column) => $"{table}.\"{column}\""; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs new file mode 100644 index 00000000000..2e0c23e53c0 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs @@ -0,0 +1,131 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Models; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; +using InterpolatedSql.Dapper; +using LinqToDB; +using LinqToDB.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; + +public class FilterMetaRepository( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver) : IFilterMetaRepository +{ + public async Task CreateFilterMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default) + { + var metas = (await duckDbConnection.SqlBuilder( + $""" + SELECT * + FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}' + WHERE "col_type" = {MetaFileRow.ColumnType.Filter.ToString()} + AND "col_name" IN ({allowedColumns}) + """) + .QueryAsync(cancellationToken: cancellationToken) + ) + .OrderBy(row => row.Label) + .Select( + row => new FilterMeta + { + PublicId = row.ColName, + DataSetVersionId = dataSetVersion.Id, + Label = row.Label, + Hint = row.FilterHint ?? string.Empty, + } + ) + .ToList(); + + publicDataDbContext.FilterMetas.AddRange(metas); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + foreach (var meta in metas) + { + var options = (await duckDbConnection.SqlBuilder( + $""" + SELECT DISTINCT "{meta.PublicId:raw}" + FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', ALL_VARCHAR = true) AS data + WHERE "{meta.PublicId:raw}" != '' + ORDER BY "{meta.PublicId:raw}" + """ + ).QueryAsync(cancellationToken: cancellationToken)) + .Select( + label => new FilterOptionMeta + { + Label = label, + IsAggregate = label == "Total" ? true : null + } + ) + .ToList(); + + var optionTable = publicDataDbContext.GetTable(); + + // Merge to only inserting new filter options + // that don't already exist in the table. + await optionTable + .Merge() + .Using(options) + .On( + o => new { o.Label, o.IsAggregate }, + o => new { o.Label, o.IsAggregate } + ) + .InsertWhenNotMatched() + .MergeAsync(cancellationToken); + + var startIndex = await publicDataDbContext.FilterOptionMetaLinks.CountAsync(token: cancellationToken); + + var current = 0; + const int batchSize = 1000; + + while (current < options.Count) + { + var batchStartIndex = startIndex + current; + var batch = options + .Skip(current) + .Take(batchSize) + .ToList(); + + // Although not necessary for filter options, we've adopted the 'row key' + // technique that was used for the location meta. This is more for + // future-proofing if we ever add more columns to the filter options table. + var batchRowKeys = batch + .Select(o => o.Label + ',' + (o.IsAggregate == true ? "True" : "")) + .ToHashSet(); + + var links = await optionTable + .Where(o => + batchRowKeys.Contains(o.Label + ',' + (o.IsAggregate == true ? "True" : ""))) + .Select((option, index) => new FilterOptionMetaLink + { + PublicId = SqidEncoder.Encode(batchStartIndex + index), + MetaId = meta.Id, + OptionId = option.Id + }) + .ToListAsync(token: cancellationToken); + + publicDataDbContext.FilterOptionMetaLinks.AddRange(links); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + current += batchSize; + } + + var insertedLinks = await publicDataDbContext.FilterOptionMetaLinks + .CountAsync(l => l.MetaId == meta.Id, + cancellationToken: cancellationToken); + + if (insertedLinks != options.Count) + { + throw new InvalidOperationException( + $"Inserted incorrect number of filter option meta links for {meta.PublicId}. " + + $"Inserted: {insertedLinks}, expected: {options.Count}"); + } + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterOptionsDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterOptionsDuckDbRepository.cs new file mode 100644 index 00000000000..7a315a1b65d --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterOptionsDuckDbRepository.cs @@ -0,0 +1,55 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using InterpolatedSql.Dapper; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; + +public class FilterOptionsDuckDbRepository(PublicDataDbContext publicDataDbContext) : IFilterOptionsDuckDbRepository +{ + public async Task CreateFilterOptionsTable( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default) + { + await publicDataDbContext + .Entry(dataSetVersion) + .Collection(dsv => dsv.FilterMetas) + .Query() + .Include(m => m.Options) + .LoadAsync(cancellationToken); + + await duckDbConnection.SqlBuilder( + $""" + CREATE TABLE {FilterOptionsTable.TableName:raw}( + {FilterOptionsTable.Cols.Id:raw} INTEGER PRIMARY KEY, + {FilterOptionsTable.Cols.Label:raw} VARCHAR, + {FilterOptionsTable.Cols.PublicId:raw} VARCHAR, + {FilterOptionsTable.Cols.FilterId:raw} VARCHAR + ) + """ + ).ExecuteAsync(cancellationToken: cancellationToken); + + var id = 1; + + foreach (var filter in dataSetVersion.FilterMetas) + { + using var appender = duckDbConnection.CreateAppender(table: FilterOptionsTable.TableName); + + foreach (var link in filter.OptionLinks.OrderBy(l => l.Option.Label)) + { + var insertRow = appender.CreateRow(); + + insertRow.AppendValue(id++); + insertRow.AppendValue(link.Option.Label); + insertRow.AppendValue(link.PublicId); + insertRow.AppendValue(filter.PublicId); + + insertRow.EndRow(); + } + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs new file mode 100644 index 00000000000..e2a61829b78 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs @@ -0,0 +1,42 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Converters; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using InterpolatedSql.Dapper; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; + +public class GeographicLevelMetaRepository( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver) : IGeographicLevelMetaRepository +{ + public async Task CreateGeographicLevelMeta( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default) + { + var geographicLevels = + (await duckDbConnection.SqlBuilder( + $""" + SELECT DISTINCT geographic_level + FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', ALL_VARCHAR = true) + """ + ).QueryAsync(cancellationToken: cancellationToken)) + .Select(EnumToEnumLabelConverter.FromProvider) + .ToList(); + + var meta = new GeographicLevelMeta + { + DataSetVersionId = dataSetVersion.Id, + Levels = geographicLevels + }; + + publicDataDbContext.GeographicLevelMetas.Add(meta); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + return meta; + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs new file mode 100644 index 00000000000..0a9875b4128 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs @@ -0,0 +1,46 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Models; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using InterpolatedSql.Dapper; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; + +public class IndicatorMetaRepository( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver) : IIndicatorMetaRepository +{ + public async Task CreateIndicatorMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default) + { + var metas = (await duckDbConnection.SqlBuilder( + $""" + SELECT * + FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}' + WHERE "col_type" = {MetaFileRow.ColumnType.Indicator.ToString()} + AND "col_name" IN ({allowedColumns}) + """) + .QueryAsync(cancellationToken: cancellationToken) + ) + .OrderBy(row => row.Label) + .Select( + row => new IndicatorMeta + { + DataSetVersionId = dataSetVersion.Id, + PublicId = row.ColName, + Label = row.Label, + Unit = row.IndicatorUnit, + DecimalPlaces = row.IndicatorDp + } + ) + .ToList(); + + publicDataDbContext.IndicatorMetas.AddRange(metas); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorsDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorsDuckDbRepository.cs new file mode 100644 index 00000000000..00aee5959ac --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorsDuckDbRepository.cs @@ -0,0 +1,47 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using InterpolatedSql.Dapper; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; + +public class IndicatorsDuckDbRepository(PublicDataDbContext publicDataDbContext) : IIndicatorsDuckDbRepository +{ + public async Task CreateIndicatorsTable( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default) + { + await publicDataDbContext + .Entry(dataSetVersion) + .Collection(dsv => dsv.IndicatorMetas) + .LoadAsync(cancellationToken); + + await duckDbConnection.SqlBuilder( + $""" + CREATE TABLE {IndicatorsTable.TableName:raw}( + {IndicatorsTable.Cols.Id:raw} VARCHAR PRIMARY KEY, + {IndicatorsTable.Cols.Label:raw} VARCHAR, + {IndicatorsTable.Cols.Unit:raw} VARCHAR, + {IndicatorsTable.Cols.DecimalPlaces:raw} TINYINT, + ) + """ + ).ExecuteAsync(cancellationToken: cancellationToken); + + using var appender = duckDbConnection.CreateAppender(table: IndicatorsTable.TableName); + + foreach (var meta in dataSetVersion.IndicatorMetas) + { + var insertRow = appender.CreateRow(); + + insertRow.AppendValue(meta.PublicId); + insertRow.AppendValue(meta.Label); + insertRow.AppendValue(meta.Unit?.GetEnumLabel() ?? string.Empty); + insertRow.AppendValue(meta.DecimalPlaces); + insertRow.EndRow(); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IDataDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IDataDuckDbRepository.cs new file mode 100644 index 00000000000..96b59ac0769 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IDataDuckDbRepository.cs @@ -0,0 +1,8 @@ +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; + +public interface IDataDuckDbRepository +{ + Task CreateDataTable( + Guid dataSetVersionId, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterMetaRepository.cs new file mode 100644 index 00000000000..19a7200adc6 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterMetaRepository.cs @@ -0,0 +1,13 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; + +public interface IFilterMetaRepository +{ + Task CreateFilterMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterOptionsDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterOptionsDuckDbRepository.cs new file mode 100644 index 00000000000..442bd85b8a8 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterOptionsDuckDbRepository.cs @@ -0,0 +1,12 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; + +public interface IFilterOptionsDuckDbRepository +{ + Task CreateFilterOptionsTable( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IGeographicLevelMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IGeographicLevelMetaRepository.cs new file mode 100644 index 00000000000..f1a73550a84 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IGeographicLevelMetaRepository.cs @@ -0,0 +1,12 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; + +public interface IGeographicLevelMetaRepository +{ + Task CreateGeographicLevelMeta( + IDuckDbConnection duckDb, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorMetaRepository.cs new file mode 100644 index 00000000000..c0d6643fb75 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorMetaRepository.cs @@ -0,0 +1,13 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; + +public interface IIndicatorMetaRepository +{ + Task CreateIndicatorMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorsDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorsDuckDbRepository.cs new file mode 100644 index 00000000000..94f0b20eb44 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorsDuckDbRepository.cs @@ -0,0 +1,12 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; + +public interface IIndicatorsDuckDbRepository +{ + Task CreateIndicatorsTable( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationMetaRepository.cs new file mode 100644 index 00000000000..3f0fc89a868 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationMetaRepository.cs @@ -0,0 +1,13 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; + +public interface ILocationMetaRepository +{ + Task CreateLocationMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationsDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationsDuckDbRepository.cs new file mode 100644 index 00000000000..9d0439466d1 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationsDuckDbRepository.cs @@ -0,0 +1,12 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; + +public interface ILocationsDuckDbRepository +{ + Task CreateLocationsTable( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodMetaRepository.cs new file mode 100644 index 00000000000..7e631d7a573 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodMetaRepository.cs @@ -0,0 +1,12 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; + +public interface ITimePeriodMetaRepository +{ + Task> CreateTimePeriodMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodsDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodsDuckDbRepository.cs new file mode 100644 index 00000000000..a33d33634fc --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodsDuckDbRepository.cs @@ -0,0 +1,12 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; + +public interface ITimePeriodsDuckDbRepository +{ + Task CreateTimePeriodsTable( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationMetaRepository.cs new file mode 100644 index 00000000000..17a3784d4d7 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationMetaRepository.cs @@ -0,0 +1,209 @@ +using System.Linq.Expressions; +using GovUk.Education.ExploreEducationStatistics.Common.Converters; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; +using InterpolatedSql.Dapper; +using LinqToDB; +using LinqToDB.Data; +using LinqToDB.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; +using static GovUk.Education.ExploreEducationStatistics.Common.Utils.GeographicLevelUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; + +public class LocationMetaRepository( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver) : ILocationMetaRepository +{ + public async Task CreateLocationMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default) + { + var levels = ListLocationLevels(allowedColumns); + + var metas = levels + .Select(level => new LocationMeta + { + DataSetVersionId = dataSetVersion.Id, + Level = level, + }) + .ToList(); + + publicDataDbContext.LocationMetas.AddRange(metas); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + foreach (var meta in metas) + { + var nameCol = meta.Level.CsvNameColumn(); + var codeCols = meta.Level.CsvCodeColumns(); + string[] cols = [..codeCols, nameCol]; + + var options = (await duckDbConnection.SqlBuilder( + $""" + SELECT {cols.JoinToString(", "):raw} + FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', ALL_VARCHAR = true) + WHERE {cols.Select(col => $"{col} != ''").JoinToString(" AND "):raw} + GROUP BY {cols.JoinToString(", "):raw} + ORDER BY {cols.JoinToString(", "):raw} + """ + ).QueryAsync(cancellationToken: cancellationToken) + ) + .Cast>() + .Select(row => row.ToDictionary( + kv => kv.Key, + kv => (string)kv.Value! + )) + .Select(row => MapLocationOptionMeta(row, meta.Level).ToRow()) + .ToList(); + + var optionTable = publicDataDbContext + .GetTable() + .TableName(nameof(PublicDataDbContext.LocationOptionMetas)); + + var current = 0; + + const int batchSize = 1000; + + while (current < options.Count) + { + var batch = options + .Skip(current) + .Take(batchSize) + .ToList(); + + // We create a 'row key' for each option that allows us to quickly + // find the option rows that exist in the database. It's typically + // much slower to have multiple WHERE clauses for each row that check + // against every other row. Out of several such attempts, the 'row key' + // technique was the fastest and simplest way to create the links. + var batchRowKeys = batch + .Select(o => o.GetRowKey()) + .ToHashSet(); + + Expression> hasBatchRowKey = + o => o.Type == batch[0].Type && + batchRowKeys.Contains( + o.Type + ',' + + o.Label + ',' + + (o.Code ?? "null") + ',' + + (o.OldCode ?? "null") + ',' + + (o.Urn ?? "null") + ',' + + (o.LaEstab ?? "null") + ',' + + (o.Ukprn ?? "null") + ); + + var existingRowKeys = (await optionTable + .Where(hasBatchRowKey) + .ToListAsync(token: cancellationToken)) + .Select(o => o.GetRowKey()) + .ToHashSet(); + + if (existingRowKeys.Count != batch.Count) + { + var newOptions = batch + .Where(o => !existingRowKeys.Contains(o.GetRowKey())) + .ToList(); + + var startIndex = await publicDataDbContext.LocationOptionMetas.CountAsync(token: cancellationToken); + + foreach (var option in newOptions) + { + option.Id = startIndex++; + option.PublicId = SqidEncoder.Encode(option.Id); + } + + await optionTable.BulkCopyAsync(newOptions, cancellationToken); + } + + var links = await optionTable + .Where(hasBatchRowKey) + .Select((option, index) => new LocationOptionMetaLink + { + MetaId = meta.Id, + OptionId = option.Id + }) + .ToListAsync(token: cancellationToken); + + publicDataDbContext.LocationOptionMetaLinks.AddRange(links); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + current += batchSize; + } + + var insertedLinks = await publicDataDbContext.LocationOptionMetaLinks + .CountAsync(l => l.MetaId == meta.Id, cancellationToken: cancellationToken); + + if (insertedLinks != options.Count) + { + throw new InvalidOperationException( + $"Inserted incorrect number of location option meta links for {meta.Level}. " + + $"Inserted: {insertedLinks}, expected: {options.Count}" + ); + } + } + } + + private static List ListLocationLevels(IReadOnlySet allowedColumns) + { + return + [ + .. allowedColumns + .Where(CsvColumnsToGeographicLevel.ContainsKey) + .Select(col => CsvColumnsToGeographicLevel[col]) + .Distinct() + .OrderBy(EnumToEnumLabelConverter.ToProvider) + ]; + } + + private static LocationOptionMeta MapLocationOptionMeta( + IDictionary row, + GeographicLevel level) + { + var cols = level.CsvColumns(); + var label = row[level.CsvNameColumn()]; + + return level switch + { + GeographicLevel.LocalAuthority => new LocationLocalAuthorityOptionMeta + { + PublicId = string.Empty, + Label = label, + Code = row[LocalAuthorityCsvColumns.NewCode], + OldCode = row[LocalAuthorityCsvColumns.OldCode] + }, + GeographicLevel.School => new LocationSchoolOptionMeta + { + PublicId = string.Empty, + Label = label, + Urn = row[SchoolCsvColumns.Urn], + LaEstab = row[SchoolCsvColumns.LaEstab] + }, + GeographicLevel.Provider => new LocationProviderOptionMeta + { + PublicId = string.Empty, + Label = label, + Ukprn = row[ProviderCsvColumns.Ukprn] + }, + GeographicLevel.RscRegion => new LocationRscRegionOptionMeta + { + PublicId = string.Empty, + Label = label + }, + _ => new LocationCodedOptionMeta + { + PublicId = string.Empty, + Label = label, + Code = row[cols.Codes.First()] + } + }; + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationsDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationsDuckDbRepository.cs new file mode 100644 index 00000000000..85b16819220 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationsDuckDbRepository.cs @@ -0,0 +1,95 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using InterpolatedSql.Dapper; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; + +public class LocationsDuckDbRepository(PublicDataDbContext publicDataDbContext) : ILocationsDuckDbRepository +{ + public async Task CreateLocationsTable( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default) + { + await publicDataDbContext + .Entry(dataSetVersion) + .Collection(dsv => dsv.LocationMetas) + .Query() + .Include(m => m.Options) + .LoadAsync(cancellationToken); + + await duckDbConnection.SqlBuilder( + $""" + CREATE TABLE {LocationOptionsTable.TableName:raw}( + {LocationOptionsTable.Cols.Id:raw} INTEGER PRIMARY KEY, + {LocationOptionsTable.Cols.Label:raw} VARCHAR, + {LocationOptionsTable.Cols.Level:raw} VARCHAR, + {LocationOptionsTable.Cols.PublicId:raw} VARCHAR, + {LocationOptionsTable.Cols.Code:raw} VARCHAR, + {LocationOptionsTable.Cols.OldCode:raw} VARCHAR, + {LocationOptionsTable.Cols.Urn:raw} VARCHAR, + {LocationOptionsTable.Cols.LaEstab:raw} VARCHAR, + {LocationOptionsTable.Cols.Ukprn:raw} VARCHAR + ) + """ + ).ExecuteAsync(cancellationToken: cancellationToken); + + var id = 1; + + foreach (var location in dataSetVersion.LocationMetas) + { + using var appender = duckDbConnection.CreateAppender(table: LocationOptionsTable.TableName); + + var insertRow = appender.CreateRow(); + + foreach (var link in location.OptionLinks.OrderBy(l => l.Option.Label)) + { + var option = link.Option; + + insertRow.AppendValue(id++); + insertRow.AppendValue(option.Label); + insertRow.AppendValue(location.Level.GetEnumValue()); + insertRow.AppendValue(option.PublicId); + + switch (option) + { + case LocationLocalAuthorityOptionMeta laOption: + insertRow.AppendValue(laOption.Code); + insertRow.AppendValue(laOption.OldCode); + insertRow.AppendNullValue(); + insertRow.AppendNullValue(); + insertRow.AppendNullValue(); + break; + case LocationCodedOptionMeta codedOption: + insertRow.AppendValue(codedOption.Code); + insertRow.AppendNullValue(); + insertRow.AppendNullValue(); + insertRow.AppendNullValue(); + insertRow.AppendNullValue(); + break; + case LocationProviderOptionMeta providerOption: + insertRow.AppendNullValue(); + insertRow.AppendNullValue(); + insertRow.AppendNullValue(); + insertRow.AppendNullValue(); + insertRow.AppendValue(providerOption.Ukprn); + break; + case LocationSchoolOptionMeta schoolOption: + insertRow.AppendNullValue(); + insertRow.AppendNullValue(); + insertRow.AppendValue(schoolOption.Urn); + insertRow.AppendValue(schoolOption.LaEstab); + insertRow.AppendNullValue(); + break; + } + + insertRow.EndRow(); + } + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodMetaRepository.cs new file mode 100644 index 00000000000..c3553534203 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodMetaRepository.cs @@ -0,0 +1,45 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Converters; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; +using InterpolatedSql.Dapper; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; + +public class TimePeriodMetaRepository( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver) : ITimePeriodMetaRepository +{ + public async Task> CreateTimePeriodMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default) + { + var metas = (await duckDbConnection.SqlBuilder( + $""" + SELECT DISTINCT time_period, time_identifier + FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', ALL_VARCHAR = true) + ORDER BY time_period + """ + ).QueryAsync<(string TimePeriod, string TimeIdentifier)>(cancellationToken: cancellationToken)) + .Select(tuple => new TimePeriodMeta + { + DataSetVersionId = dataSetVersion.Id, + Period = TimePeriodFormatter.FormatFromCsv(tuple.TimePeriod), + Code = EnumToEnumLabelConverter.FromProvider(tuple.TimeIdentifier) + } + ) + .OrderBy(meta => meta.Period) + .ThenBy(meta => meta.Code) + .ToList(); + + publicDataDbContext.TimePeriodMetas.AddRange(metas); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + return metas; + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodsDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodsDuckDbRepository.cs new file mode 100644 index 00000000000..b9d2a4e9325 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodsDuckDbRepository.cs @@ -0,0 +1,52 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; +using InterpolatedSql.Dapper; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; + +public class TimePeriodsDuckDbRepository(PublicDataDbContext publicDataDbContext) : ITimePeriodsDuckDbRepository +{ + public async Task CreateTimePeriodsTable( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default) + { + await publicDataDbContext + .Entry(dataSetVersion) + .Collection(dsv => dsv.TimePeriodMetas) + .LoadAsync(cancellationToken); + + await duckDbConnection.SqlBuilder( + $""" + CREATE TABLE {TimePeriodsTable.TableName:raw}( + {TimePeriodsTable.Cols.Id:raw} INTEGER PRIMARY KEY, + {TimePeriodsTable.Cols.Period:raw} VARCHAR, + {TimePeriodsTable.Cols.Identifier:raw} VARCHAR + ) + """ + ).ExecuteAsync(cancellationToken: cancellationToken); + + using var appender = duckDbConnection.CreateAppender(table: TimePeriodsTable.TableName); + + var timePeriods = dataSetVersion.TimePeriodMetas + .OrderBy(tp => tp.Period) + .ThenBy(tp => tp.Code); + + var id = 1; + + foreach (var timePeriod in timePeriods) + { + var insertRow = appender.CreateRow(); + + insertRow.AppendValue(id++); + insertRow.AppendValue(TimePeriodFormatter.FormatToCsv(timePeriod.Period)); + insertRow.AppendValue(timePeriod.Code.GetEnumLabel()); + insertRow.EndRow(); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs new file mode 100644 index 00000000000..f895d59e269 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs @@ -0,0 +1,155 @@ +using Dapper; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Models; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using InterpolatedSql.Dapper; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services; + +public class DataSetMetaService( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver, + IFilterMetaRepository filterMetaRepository, + IIndicatorMetaRepository indicatorMetaRepository, + IGeographicLevelMetaRepository geographicLevelMetaRepository, + ILocationMetaRepository locationMetaRepository, + ITimePeriodMetaRepository timePeriodMetaRepository, + IFilterOptionsDuckDbRepository filterOptionsDuckDbRepository, + IIndicatorsDuckDbRepository indicatorsDuckDbRepository, + ILocationsDuckDbRepository locationsDuckDbRepository, + ITimePeriodsDuckDbRepository timePeriodsDuckDbRepository +) : IDataSetMetaService +{ + public async Task CreateDataSetVersionMeta( + Guid dataSetVersionId, + CancellationToken cancellationToken = default) + { + var dataSetVersion = await publicDataDbContext.DataSetVersions + .SingleAsync(dsv => dsv.Id == dataSetVersionId, cancellationToken: cancellationToken); + + await using var duckDbConnection = + DuckDbConnection.CreateFileConnection(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); + duckDbConnection.Open(); + + var columns = (await duckDbConnection.SqlBuilder( + $"DESCRIBE SELECT * FROM '{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}'") + .QueryAsync<(string ColumnName, string ColumnType)>(cancellationToken: cancellationToken)) + .Select(row => row.ColumnName) + .ToList(); + + var allowedColumns = columns.ToHashSet(); + + var metaFileRows = (await duckDbConnection.SqlBuilder( + $"SELECT * FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}'") + .QueryAsync(cancellationToken: cancellationToken)).AsList(); + + await filterMetaRepository.CreateFilterMetas( + duckDbConnection, + dataSetVersion, + allowedColumns, + cancellationToken); + + await indicatorMetaRepository.CreateIndicatorMetas( + duckDbConnection, + dataSetVersion, + allowedColumns, + cancellationToken); + + var geographicLevelMeta = await geographicLevelMetaRepository.CreateGeographicLevelMeta( + duckDbConnection, + dataSetVersion, + cancellationToken); + + await locationMetaRepository.CreateLocationMetas( + duckDbConnection, + dataSetVersion, + allowedColumns, + cancellationToken); + + var timePeriodMetas = await timePeriodMetaRepository.CreateTimePeriodMetas( + duckDbConnection, + dataSetVersion, + cancellationToken); + + dataSetVersion.MetaSummary = + BuildMetaSummary(timePeriodMetas, metaFileRows, allowedColumns, geographicLevelMeta); + dataSetVersion.TotalResults = await CountCsvRows(duckDbConnection, dataSetVersion, cancellationToken); + + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + await indicatorsDuckDbRepository.CreateIndicatorsTable( + duckDbConnection, + dataSetVersion, + cancellationToken); + + await locationsDuckDbRepository.CreateLocationsTable( + duckDbConnection, + dataSetVersion, + cancellationToken); + + await filterOptionsDuckDbRepository.CreateFilterOptionsTable( + duckDbConnection, + dataSetVersion, + cancellationToken); + + await timePeriodsDuckDbRepository.CreateTimePeriodsTable( + duckDbConnection, + dataSetVersion, + cancellationToken); + } + + private static DataSetVersionMetaSummary BuildMetaSummary( + IList timePeriodMetas, + IList metaFileRows, + HashSet allowedColumns, + GeographicLevelMeta geographicLevelMeta) => + new() + { + TimePeriodRange = new TimePeriodRange + { + Start = new TimePeriodRangeBound + { + Period = timePeriodMetas[0].Period, + Code = timePeriodMetas[0].Code + }, + End = new TimePeriodRangeBound + { + Period = timePeriodMetas[^1].Period, + Code = timePeriodMetas[^1].Code + }, + }, + Filters = metaFileRows + .Where(row => row.ColType == MetaFileRow.ColumnType.Filter + && allowedColumns.Contains(row.ColName)) + .OrderBy(row => row.Label) + .Select(row => row.Label) + .ToList(), + Indicators = metaFileRows + .Where( + row => row.ColType == MetaFileRow.ColumnType.Indicator + && allowedColumns.Contains(row.ColName) + ) + .OrderBy(row => row.Label) + .Select(row => row.Label) + .ToList(), + GeographicLevels = geographicLevelMeta.Levels + }; + + private async Task CountCsvRows( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken) + { + return await duckDbConnection.SqlBuilder( + $""" + SELECT COUNT(*) + FROM '{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}' + """ + ).QuerySingleAsync(cancellationToken: cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs index 79ab483d39c..79b5f27b768 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs @@ -49,7 +49,8 @@ await CreateDataSetVersionImport(dataSetVersion, instanceId, cancellationToken)) .OnSuccessDo(async dataSetVersion => await UpdateFilePublicDataSetVersionId(releaseFile, dataSetVersion, cancellationToken)) .OnSuccessDo(transactionScope.Complete) - .OnSuccess(dataSetVersion => (dataSetId: dataSetVersion.DataSetId, dataSetVersionId: dataSetVersion.Id))); + .OnSuccess(dataSetVersion => + (dataSetId: dataSetVersion.DataSetId, dataSetVersionId: dataSetVersion.Id))); }); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs new file mode 100644 index 00000000000..7c92c998cc6 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs @@ -0,0 +1,8 @@ +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; + +public interface IDataSetMetaService +{ + Task CreateDataSetVersionMeta( + Guid dataSetVersionId, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IParquetService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IParquetService.cs new file mode 100644 index 00000000000..7b9dfb7766c --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IParquetService.cs @@ -0,0 +1,8 @@ +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; + +public interface IParquetService +{ + Task WriteDataFiles( + Guid dataSetVersionId, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/ParquetService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/ParquetService.cs new file mode 100644 index 00000000000..4b738b58e22 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/ParquetService.cs @@ -0,0 +1,48 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using InterpolatedSql.Dapper; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services; + +public class ParquetService( + ILogger logger, + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver +) : IParquetService +{ + public async Task WriteDataFiles( + Guid dataSetVersionId, + CancellationToken cancellationToken = default) + { + var dataSetVersion = await publicDataDbContext.DataSetVersions + .SingleAsync(dsv => dsv.Id == dataSetVersionId, cancellationToken: cancellationToken); + + var versionDir = dataSetVersionPathResolver.DirectoryPath(dataSetVersion); + + logger.LogDebug("Writing data files to data set version directory '{VersionDir}'", versionDir); + + await using var duckDbConnection = + DuckDbConnection.CreateFileConnection(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); + duckDbConnection.Open(); + + await duckDbConnection.SqlBuilder( + $"EXPORT DATABASE '{versionDir:raw}' (FORMAT PARQUET, CODEC ZSTD)") + .ExecuteAsync(cancellationToken: cancellationToken); + + // Convert absolute paths in load.sql to relative paths otherwise + // these refer to the machine that the script was run on. + + var loadSqlFilePath = dataSetVersionPathResolver.DuckDbLoadSqlPath(dataSetVersion); + + var absolutePathToReplace = $"{versionDir.Replace('\\', '/')}/"; + + var newLines = (await File.ReadAllLinesAsync(loadSqlFilePath, cancellationToken)) + .Select(line => line.Replace(absolutePathToReplace, "")); + + await File.WriteAllLinesAsync(loadSqlFilePath, newLines, cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/appsettings.json b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/appsettings.json index 44bdb53a280..de89935dc15 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/appsettings.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/appsettings.json @@ -6,7 +6,7 @@ } }, "CoreStorage": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://data-storage:10000/devstoreaccount1;QueueEndpoint=http://data-storage:10001/devstoreaccount1;TableEndpoint=http://data-storage:10002/devstoreaccount1", - "ParquetFiles": { - "BasePath": "data/public-api-parquet" + "DataFiles": { + "BasePath": "data/public-api-data" } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json index ac2d37bfc79..bb436797e8d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/host.json @@ -19,6 +19,8 @@ }, "logLevel": { "default": "Error", + "DurableTask.AzureStorage": "Warning", + "DurableTask.Core": "Warning", "Function": "Information" } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs index e4dfe2b2d99..eff794aa535 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs @@ -6,7 +6,6 @@ using CliFx.Infrastructure; using CliWrap; using Dapper; -using DuckDB.NET.Data; using GovUk.Education.ExploreEducationStatistics.Common.Converters; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -14,10 +13,12 @@ using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts.Models; using GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts.Seeds; using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; +using InterpolatedSql.Dapper; using LinqToDB; using LinqToDB.Data; using LinqToDB.EntityFrameworkCore; @@ -66,13 +67,13 @@ public async ValueTask ExecuteAsync(IConsole console) foreach (var seed in dataSetSeeds) { - await using var duckDb = new DuckDBConnection("DataSource=:memory:"); + await using var duckDb = new DuckDbConnection(); await duckDb.OpenAsync(cancellationToken); var seeder = new Seeder( seed: seed, dbContext: dbContext, - duckDb: duckDb, + duckDbConnection: duckDb, console: console, cancellationToken: cancellationToken ); @@ -174,7 +175,7 @@ private class Seeder { private readonly DataSetSeed _seed; private readonly PublicDataDbContext _dbContext; - private readonly DuckDBConnection _duckDb; + private readonly IDuckDbConnection _duckDbConnection; private readonly IConsole _console; private readonly CancellationToken _cancellationToken; @@ -184,13 +185,13 @@ private class Seeder public Seeder( DataSetSeed seed, PublicDataDbContext dbContext, - DuckDBConnection duckDb, + IDuckDbConnection duckDbConnection, IConsole console, CancellationToken cancellationToken) { _seed = seed; _dbContext = dbContext; - _duckDb = duckDb; + _duckDbConnection = duckDbConnection; _console = console; _cancellationToken = cancellationToken; @@ -218,7 +219,7 @@ public async Task Generate() await _console.Output.WriteLineAsync("=> Started seeding meta"); - var columns = _duckDb.Query<(string ColumnName, string ColumnType)>( + var columns = _duckDbConnection.Query<(string ColumnName, string ColumnType)>( $"DESCRIBE SELECT * FROM '{_dataFilePath}'" ) .Select(row => row.ColumnName) @@ -226,13 +227,9 @@ public async Task Generate() var allowedColumns = columns.ToHashSet(); - var metaFileRows = (await _duckDb.QueryAsync( - new CommandDefinition( - $"SELECT * FROM '{_metaFilePath}'", - cancellationToken: _cancellationToken - ) - )) - .ToList(); + var metaFileRows = (await _duckDbConnection.SqlBuilder( + $"SELECT * FROM '{_metaFilePath:raw}'") + .QueryAsync(cancellationToken: _cancellationToken)).AsList(); await using var transaction = await _dbContext.Database.BeginTransactionAsync(_cancellationToken); @@ -270,30 +267,30 @@ private async Task CreateDataSetVersion( IList metaFileRows, HashSet allowedColumns) { - var totalResults = await _duckDb.QuerySingleAsync($"SELECT COUNT(*) FROM '{_dataFilePath}'"); + var totalResults = await _duckDbConnection.SqlBuilder( + $""" + SELECT COUNT(*) + FROM '{_dataFilePath:raw}' + """ + ).QuerySingleAsync(cancellationToken: _cancellationToken); - var geographicLevels = (await _duckDb.QueryAsync( - new CommandDefinition( - $""" - SELECT DISTINCT geographic_level - FROM read_csv_auto('{_dataFilePath}', ALL_VARCHAR = true) - """, - cancellationToken: _cancellationToken - ) - )) + var geographicLevels = + (await _duckDbConnection.SqlBuilder( + $""" + SELECT DISTINCT geographic_level + FROM read_csv('{_dataFilePath:raw}', ALL_VARCHAR = true) + """ + ).QueryAsync(cancellationToken: _cancellationToken)) .Select(EnumToEnumLabelConverter.FromProvider) .ToList(); - var timePeriods = (await _duckDb.QueryAsync<(string TimePeriod, string TimeIdentifier)>( - new CommandDefinition( + var timePeriods = (await _duckDbConnection.SqlBuilder( $""" SELECT DISTINCT time_period, time_identifier - FROM read_csv_auto('{_dataFilePath}', ALL_VARCHAR = true) + FROM read_csv('{_dataFilePath:raw}', ALL_VARCHAR = true) ORDER BY time_period - """, - cancellationToken: _cancellationToken - ) - )) + """ + ).QueryAsync<(string TimePeriod, string TimeIdentifier)>(cancellationToken: _cancellationToken)) .Select( row => ( Period: TimePeriodFormatter.FormatFromCsv(row.TimePeriod), @@ -420,17 +417,15 @@ private async Task CreateFilterMetas( foreach (var meta in metas) { - var options = (await _duckDb.QueryAsync( - new CommandDefinition( + var options = (await _duckDbConnection.SqlBuilder( $""" - SELECT DISTINCT "{meta.PublicId}" - FROM read_csv_auto('{_dataFilePath}', ALL_VARCHAR = true) AS data - WHERE "{meta.PublicId}" != '' - ORDER BY "{meta.PublicId}" - """, - cancellationToken: _cancellationToken - ) - )) + SELECT DISTINCT "{meta.PublicId:raw}" + FROM read_csv('{_dataFilePath:raw}', ALL_VARCHAR = true) AS data + WHERE "{meta.PublicId:raw}" != '' + ORDER BY "{meta.PublicId:raw}" + """ + ).QueryAsync(cancellationToken: _cancellationToken) + ) .Select( label => new FilterOptionMeta { @@ -476,7 +471,7 @@ await optionTable var links = await optionTable .Where(o => - batchRowKeys.Contains(o.Label + ',' + (o.IsAggregate == true ? "True" : "" ))) + batchRowKeys.Contains(o.Label + ',' + (o.IsAggregate == true ? "True" : ""))) .Select((option, index) => new FilterOptionMetaLink { PublicId = SqidEncoder.Encode(batchStartIndex + index), @@ -525,18 +520,15 @@ private async Task CreateLocationMetas(IReadOnlySet allowedColumns) var codeCols = meta.Level.CsvCodeColumns(); string[] cols = [..codeCols, nameCol]; - var options = (await _duckDb.QueryAsync( - new CommandDefinition( + var options = (await _duckDbConnection.SqlBuilder( $""" - SELECT {cols.JoinToString(", ")} - FROM read_csv_auto('{_dataFilePath}', ALL_VARCHAR = true) - WHERE {cols.Select(col => $"{col} != ''").JoinToString(" AND ")} - GROUP BY {cols.JoinToString(", ")} - ORDER BY {cols.JoinToString(", ")} - """, - cancellationToken: _cancellationToken - ) - )) + SELECT {cols.JoinToString(", "):raw} + FROM read_csv('{_dataFilePath:raw}', ALL_VARCHAR = true) + WHERE {cols.Select(col => $"{col} != ''").JoinToString(" AND "):raw} + GROUP BY {cols.JoinToString(", "):raw} + ORDER BY {cols.JoinToString(", "):raw} + """ + ).QueryAsync(cancellationToken: _cancellationToken)) .Cast>() .Select(row => row.ToDictionary( kv => kv.Key, @@ -569,7 +561,7 @@ FROM read_csv_auto('{_dataFilePath}', ALL_VARCHAR = true) .Select(o => o.GetRowKey()) .ToHashSet(); - Expression> hasBatchRowKey = + Expression> hasBatchRowKey = o => o.Type == batch[0].Type && batchRowKeys.Contains( o.Type + ',' + @@ -582,8 +574,8 @@ FROM read_csv_auto('{_dataFilePath}', ALL_VARCHAR = true) ); var existingRowKeys = (await optionTable - .Where(hasBatchRowKey) - .ToListAsync(token: _cancellationToken)) + .Where(hasBatchRowKey) + .ToListAsync(token: _cancellationToken)) .Select(o => o.GetRowKey()) .ToHashSet(); @@ -621,7 +613,7 @@ FROM read_csv_auto('{_dataFilePath}', ALL_VARCHAR = true) var insertedLinks = await _dbContext.LocationOptionMetaLinks .CountAsync( - l => l.MetaId == meta.Id, + l => l.MetaId == meta.Id, cancellationToken: _cancellationToken); if (insertedLinks != options.Count) @@ -689,16 +681,13 @@ private static List ListLocationLevels(IReadOnlySet all private async Task CreateTimePeriodMeta() { - var metas = (await _duckDb.QueryAsync<(string TimePeriod, string TimeIdentifier)>( - new CommandDefinition( - $""" - SELECT DISTINCT time_period, time_identifier - FROM read_csv_auto('{_dataFilePath}', ALL_VARCHAR = true) - ORDER BY time_period - """, - cancellationToken: _cancellationToken - ) - )) + var metas = (await _duckDbConnection.SqlBuilder( + $""" + SELECT DISTINCT time_period, time_identifier + FROM read_csv('{_dataFilePath:raw}', ALL_VARCHAR = true) + ORDER BY time_period + """ + ).QueryAsync<(string TimePeriod, string TimeIdentifier)>(cancellationToken: _cancellationToken)) .Select( tuple => new TimePeriodMeta { @@ -735,7 +724,8 @@ private async Task SeedParquetData() // and seems to regularly cause DuckDB crashes for larger data sets. await CreateParquetMetaTables(version); - await _duckDb.ExecuteAsync("CREATE SEQUENCE data_seq START 1"); + await _duckDbConnection.SqlBuilder("CREATE SEQUENCE data_seq START 1") + .ExecuteAsync(cancellationToken: _cancellationToken); string[] columns = [ @@ -747,7 +737,12 @@ private async Task SeedParquetData() ..version.IndicatorMetas.Select(indicator => $"{DataTable.Cols.Indicator(indicator)} VARCHAR NOT NULL"), ]; - await _duckDb.ExecuteAsync($"CREATE TABLE {DataTable.TableName}({columns.JoinToString(",\n")})"); + await _duckDbConnection.SqlBuilder( + $""" + CREATE TABLE {DataTable.TableName:raw} + ({columns.JoinToString(",\n"):raw}) + """) + .ExecuteAsync(cancellationToken: _cancellationToken); string[] insertColumns = [ @@ -756,7 +751,7 @@ private async Task SeedParquetData() DataSourceTable.Ref.GeographicLevel, ..version.LocationMetas.Select(location => $"COALESCE({LocationOptionsTable.Ref(location).Id}, 0) AS {DataTable.Cols.LocationId(location)}"), - ..version.FilterMetas.Select(filter => + ..version.FilterMetas.Select(filter => $"COALESCE({FilterOptionsTable.Ref(filter).Id}, 0) AS {DataTable.Cols.Filter(filter)}"), ..version.IndicatorMetas.Select(DataTable.Cols.Indicator), ]; @@ -789,27 +784,24 @@ private async Task SeedParquetData() """ ), $""" - JOIN {TimePeriodsTable.TableName} - ON {TimePeriodsTable.Ref().Period} = {DataSourceTable.Ref.TimePeriod} - AND {TimePeriodsTable.Ref().Identifier} = {DataSourceTable.Ref.TimeIdentifier} - """ + JOIN {TimePeriodsTable.TableName} + ON {TimePeriodsTable.Ref().Period} = {DataSourceTable.Ref.TimePeriod} + AND {TimePeriodsTable.Ref().Identifier} = {DataSourceTable.Ref.TimeIdentifier} + """ ]; - await _duckDb.ExecuteAsync( - new CommandDefinition( - $""" - INSERT INTO {DataTable.TableName} - SELECT - {insertColumns.JoinToString(",\n")} - FROM read_csv_auto('{_dataFilePath}', ALL_VARCHAR = true) AS {DataSourceTable.TableName} - {insertJoins.JoinToString('\n')} - ORDER BY - {DataSourceTable.Ref.GeographicLevel} ASC, - {DataSourceTable.Ref.TimePeriod} DESC - """, - cancellationToken: _cancellationToken - ) - ); + await _duckDbConnection.SqlBuilder( + $""" + INSERT INTO {DataTable.TableName:raw} + SELECT + {insertColumns.JoinToString(",\n"):raw} + FROM read_csv('{_dataFilePath:raw}', ALL_VARCHAR = true) AS {DataSourceTable.TableName:raw} + {insertJoins.JoinToString('\n'):raw} + ORDER BY + {DataSourceTable.Ref.GeographicLevel:raw} ASC, + {DataSourceTable.Ref.TimePeriod:raw} DESC + """ + ).ExecuteAsync(cancellationToken: _cancellationToken); await OutputParquetFiles(); } @@ -817,14 +809,14 @@ ORDER BY private async Task OutputParquetFiles() { var projectRootPath = PathUtils.ProjectRootPath; - var parquetDir = Path.Combine(projectRootPath, "data", "public-api-parquet"); + var dataDir = Path.Combine(projectRootPath, "data", "public-api-data"); - if (!Path.Exists(parquetDir)) + if (!Path.Exists(dataDir)) { - Directory.CreateDirectory(parquetDir); + Directory.CreateDirectory(dataDir); } - var dataSetDir = Path.Combine(parquetDir, _seed.DataSet.Id.ToString()); + var dataSetDir = Path.Combine(dataDir, _seed.DataSet.Id.ToString()); if (Path.Exists(dataSetDir)) { @@ -846,20 +838,19 @@ private async Task OutputParquetFiles() Directory.CreateDirectory(versionDir); - await _duckDb.ExecuteAsync( - new CommandDefinition( - $"EXPORT DATABASE '{versionDir}' (FORMAT PARQUET, CODEC ZSTD)", - cancellationToken: _cancellationToken - ) - ); + await _duckDbConnection.SqlBuilder( + $"EXPORT DATABASE '{versionDir:raw}' (FORMAT PARQUET, CODEC ZSTD)") + .ExecuteAsync(cancellationToken: _cancellationToken); // Convert absolute paths in load.sql to relative paths otherwise - // these refer to the machine that the script was ran on. + // these refer to the machine that the script was run on. + + var loadSqlFilePath = Path.Combine(versionDir, DataSetFilenames.DuckDbLoadSqlFile); - var loadSqlFilePath = Path.Combine(versionDir, "load.sql"); + var absolutePathToReplace = $"{versionDir.Replace('\\', '/')}/"; var newLines = (await File.ReadAllLinesAsync(loadSqlFilePath, _cancellationToken)) - .Select(line => line.Replace($"{versionDir}{Path.DirectorySeparatorChar}", "")); + .Select(line => line.Replace(absolutePathToReplace, "")); await File.WriteAllLinesAsync(loadSqlFilePath, newLines, _cancellationToken); } @@ -874,18 +865,18 @@ private async Task CreateParquetMetaTables(DataSetVersion version) private async Task CreateParquetIndicatorTable(DataSetVersion version) { - await _duckDb.ExecuteAsync( + await _duckDbConnection.SqlBuilder( $""" - CREATE TABLE {IndicatorsTable.TableName}( - {IndicatorsTable.Cols.Id} VARCHAR PRIMARY KEY, - {IndicatorsTable.Cols.Label} VARCHAR, - {IndicatorsTable.Cols.Unit} VARCHAR, - {IndicatorsTable.Cols.DecimalPlaces} TINYINT, + CREATE TABLE {IndicatorsTable.TableName:raw}( + {IndicatorsTable.Cols.Id:raw} VARCHAR PRIMARY KEY, + {IndicatorsTable.Cols.Label:raw} VARCHAR, + {IndicatorsTable.Cols.Unit:raw} VARCHAR, + {IndicatorsTable.Cols.DecimalPlaces:raw} TINYINT, ) """ - ); + ).ExecuteAsync(cancellationToken: _cancellationToken); - using var appender = _duckDb.CreateAppender(table: IndicatorsTable.TableName); + using var appender = _duckDbConnection.CreateAppender(table: IndicatorsTable.TableName); foreach (var meta in version.IndicatorMetas) { @@ -901,27 +892,27 @@ await _duckDb.ExecuteAsync( private async Task CreateParquetLocationMetaTable(DataSetVersion version) { - await _duckDb.ExecuteAsync( + await _duckDbConnection.SqlBuilder( $""" - CREATE TABLE {LocationOptionsTable.TableName}( - {LocationOptionsTable.Cols.Id} INTEGER PRIMARY KEY, - {LocationOptionsTable.Cols.Label} VARCHAR, - {LocationOptionsTable.Cols.Level} VARCHAR, - {LocationOptionsTable.Cols.PublicId} VARCHAR, - {LocationOptionsTable.Cols.Code} VARCHAR, - {LocationOptionsTable.Cols.OldCode} VARCHAR, - {LocationOptionsTable.Cols.Urn} VARCHAR, - {LocationOptionsTable.Cols.LaEstab} VARCHAR, - {LocationOptionsTable.Cols.Ukprn} VARCHAR + CREATE TABLE {LocationOptionsTable.TableName:raw}( + {LocationOptionsTable.Cols.Id:raw} INTEGER PRIMARY KEY, + {LocationOptionsTable.Cols.Label:raw} VARCHAR, + {LocationOptionsTable.Cols.Level:raw} VARCHAR, + {LocationOptionsTable.Cols.PublicId:raw} VARCHAR, + {LocationOptionsTable.Cols.Code:raw} VARCHAR, + {LocationOptionsTable.Cols.OldCode:raw} VARCHAR, + {LocationOptionsTable.Cols.Urn:raw} VARCHAR, + {LocationOptionsTable.Cols.LaEstab:raw} VARCHAR, + {LocationOptionsTable.Cols.Ukprn:raw} VARCHAR ) """ - ); + ).ExecuteAsync(cancellationToken: _cancellationToken); var id = 1; foreach (var location in version.LocationMetas) { - using var appender = _duckDb.CreateAppender(table: LocationOptionsTable.TableName); + using var appender = _duckDbConnection.CreateAppender(table: LocationOptionsTable.TableName); var insertRow = appender.CreateRow(); @@ -973,22 +964,22 @@ await _duckDb.ExecuteAsync( private async Task CreateParquetFilterMetaTable(DataSetVersion version) { - await _duckDb.ExecuteAsync( + await _duckDbConnection.SqlBuilder( $""" - CREATE TABLE {FilterOptionsTable.TableName}( - {FilterOptionsTable.Cols.Id} INTEGER PRIMARY KEY, - {FilterOptionsTable.Cols.Label} VARCHAR, - {FilterOptionsTable.Cols.PublicId} VARCHAR, - {FilterOptionsTable.Cols.FilterId} VARCHAR + CREATE TABLE {FilterOptionsTable.TableName:raw}( + {FilterOptionsTable.Cols.Id:raw} INTEGER PRIMARY KEY, + {FilterOptionsTable.Cols.Label:raw} VARCHAR, + {FilterOptionsTable.Cols.PublicId:raw} VARCHAR, + {FilterOptionsTable.Cols.FilterId:raw} VARCHAR ) """ - ); + ).ExecuteAsync(cancellationToken: _cancellationToken); var id = 1; foreach (var filter in version.FilterMetas) { - using var appender = _duckDb.CreateAppender(table: FilterOptionsTable.TableName); + using var appender = _duckDbConnection.CreateAppender(table: FilterOptionsTable.TableName); foreach (var link in filter.OptionLinks.OrderBy(l => l.Option.Label)) { @@ -1006,17 +997,17 @@ await _duckDb.ExecuteAsync( private async Task CreateParquetTimePeriodMetaTable(DataSetVersion version) { - await _duckDb.ExecuteAsync( + await _duckDbConnection.SqlBuilder( $""" - CREATE TABLE {TimePeriodsTable.TableName}( - {TimePeriodsTable.Cols.Id} INTEGER PRIMARY KEY, - {TimePeriodsTable.Cols.Period} VARCHAR, - {TimePeriodsTable.Cols.Identifier} VARCHAR + CREATE TABLE {TimePeriodsTable.TableName:raw}( + {TimePeriodsTable.Cols.Id:raw} INTEGER PRIMARY KEY, + {TimePeriodsTable.Cols.Period:raw} VARCHAR, + {TimePeriodsTable.Cols.Identifier:raw} VARCHAR ) """ - ); + ).ExecuteAsync(cancellationToken: _cancellationToken); - using var appender = _duckDb.CreateAppender(table: TimePeriodsTable.TableName); + using var appender = _duckDbConnection.CreateAppender(table: TimePeriodsTable.TableName); var timePeriods = version.TimePeriodMetas .OrderBy(tp => tp.Period) @@ -1042,7 +1033,8 @@ private LocationColumn[] GetParquetLocationCodeColumns(GeographicLevel geographi GeographicLevel.LocalAuthority => [ new LocationColumn(Name: LocationOptionsTable.Cols.Code, CsvName: LocalAuthorityCsvColumns.NewCode), - new LocationColumn(Name: LocationOptionsTable.Cols.OldCode, CsvName: LocalAuthorityCsvColumns.OldCode) + new LocationColumn(Name: LocationOptionsTable.Cols.OldCode, + CsvName: LocalAuthorityCsvColumns.OldCode) ], GeographicLevel.Provider => [ @@ -1056,7 +1048,8 @@ private LocationColumn[] GetParquetLocationCodeColumns(GeographicLevel geographi ], _ => [ - new LocationColumn(Name: LocationOptionsTable.Cols.Code, CsvName: geographicLevel.CsvCodeColumns().First()) + new LocationColumn(Name: LocationOptionsTable.Cols.Code, + CsvName: geographicLevel.CsvCodeColumns().First()) ], }; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Models/MetaFileRow.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Models/MetaFileRow.cs index 82e59e7fbd6..3e80025c002 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Models/MetaFileRow.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Models/MetaFileRow.cs @@ -1,5 +1,5 @@ -using GovUk.Education.ExploreEducationStatistics.Common.Converters; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts.Models; @@ -17,7 +17,7 @@ public record MetaFileRow public IndicatorUnit? IndicatorUnit => _indicatorUnit is not null - ? EnumToEnumValueConverter.FromProvider(_indicatorUnit) + ? EnumUtil.GetFromEnumValue(_indicatorUnit) : null; public byte? IndicatorDp { get; init; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs index eabfb494888..a7f357c9d64 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs @@ -28,7 +28,7 @@ public class ConstructorTests : DataSetVersionPathResolverTests public void EmptyBasePath_Throws(string basePath) { Assert.Throws(() => - BuildService(options: new ParquetFilesOptions + BuildService(options: new DataFilesOptions { BasePath = basePath }) @@ -58,16 +58,16 @@ public void DevelopmentEnv_ValidBasePath() .SetupGet(s => s.EnvironmentName) .Returns(Environments.Development); - var resolver = BuildService(options: new ParquetFilesOptions + var resolver = BuildService(options: new DataFilesOptions { - BasePath = Path.Combine("data", "parquet-files") + BasePath = Path.Combine("data", "data-files") }); Assert.Equal( Path.Combine( PathUtils.ProjectRootPath, "data", - "parquet-files" + "data-files" ), resolver.BasePath()); } @@ -79,9 +79,9 @@ public void IntegrationTestEnv_ValidBasePath() .SetupGet(s => s.EnvironmentName) .Returns(HostEnvironmentExtensions.IntegrationTestEnvironment); - var resolver = BuildService(options: new ParquetFilesOptions + var resolver = BuildService(options: new DataFilesOptions { - BasePath = Path.Combine("data", "parquet-files") + BasePath = Path.Combine("data", "data-files") }); var basePath = resolver.BasePath(); @@ -95,7 +95,7 @@ public void IntegrationTestEnv_ValidBasePath() Path.Combine( Assembly.GetExecutingAssembly().GetDirectoryPath(), "data", - "parquet-files", + "data-files", randomTestInstanceDir.ToString() ), basePath @@ -109,15 +109,15 @@ public void ProductionEnv_ValidBasePath() .SetupGet(s => s.EnvironmentName) .Returns(Environments.Production); - var resolver = BuildService(options: new ParquetFilesOptions + var resolver = BuildService(options: new DataFilesOptions { - BasePath = Path.Combine("data", "parquet-files") + BasePath = Path.Combine("data", "data-files") }); Assert.Equal( Path.Combine( "data", - "parquet-files" + "data-files" ), resolver.BasePath()); } @@ -132,9 +132,9 @@ public void ValidDirectoryPath(string environmentName) .SetupGet(s => s.EnvironmentName) .Returns(environmentName); - var resolver = BuildService(options: new ParquetFilesOptions + var resolver = BuildService(options: new DataFilesOptions { - BasePath = Path.Combine("data", "parquet-files") + BasePath = Path.Combine("data", "data-files") }); Assert.Equal( @@ -156,21 +156,33 @@ public void ValidFilePaths(string environmentName) .SetupGet(s => s.EnvironmentName) .Returns(environmentName); - var resolver = BuildService(options: new ParquetFilesOptions + var resolver = BuildService(options: new DataFilesOptions { - BasePath = Path.Combine("data", "parquet-files") + BasePath = Path.Combine("data", "data-files") }); var directoryPath = resolver.DirectoryPath(version); Assert.Equal( - Path.Combine(directoryPath, "data.csv.gz"), + Path.Combine(directoryPath, DataSetFilenames.CsvDataFile), resolver.CsvDataPath(version) ); Assert.Equal( - Path.Combine(directoryPath, "metadata.csv.gz"), + Path.Combine(directoryPath, DataSetFilenames.CsvMetadataFile), resolver.CsvMetadataPath(version) ); + Assert.Equal( + Path.Combine(directoryPath, DataSetFilenames.DuckDbDatabaseFile), + resolver.DuckDbPath(version) + ); + Assert.Equal( + Path.Combine(directoryPath, DataSetFilenames.DuckDbLoadSqlFile), + resolver.DuckDbLoadSqlPath(version) + ); + Assert.Equal( + Path.Combine(directoryPath, DataSetFilenames.DuckDbSchemaSqlFile), + resolver.DuckDbSchemaSqlPath(version) + ); Assert.Equal( Path.Combine(directoryPath, DataTable.ParquetFile), resolver.DataPath(version) @@ -195,11 +207,11 @@ public void ValidFilePaths(string environmentName) } private IDataSetVersionPathResolver BuildService( - ParquetFilesOptions options, + DataFilesOptions options, IWebHostEnvironment? webHostEnvironment = null) { return new DataSetVersionPathResolver( - new OptionsWrapper(options), + new OptionsWrapper(options), webHostEnvironment ?? _webHostEnvironmentMock.Object ); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/TestDataSetVersionPathResolver.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/TestDataSetVersionPathResolver.cs index 6d982104fe5..1a5611eb706 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/TestDataSetVersionPathResolver.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/TestDataSetVersionPathResolver.cs @@ -10,7 +10,7 @@ public class TestDataSetVersionPathResolver : IDataSetVersionPathResolver private readonly string _basePath = Path.Combine( Assembly.GetExecutingAssembly().GetDirectoryPath(), "Resources", - "ParquetFiles" + "DataFiles" ); public string BasePath() => _basePath; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs index bf1c62ff9e9..77f8787caee 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs @@ -12,11 +12,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Services; public class DataSetVersionPathResolver : IDataSetVersionPathResolver { - private readonly IOptions _options; + private readonly IOptions _options; private readonly IWebHostEnvironment _environment; private readonly string _basePath; - public DataSetVersionPathResolver(IOptions options, IWebHostEnvironment environment) + public DataSetVersionPathResolver(IOptions options, IWebHostEnvironment environment) { _options = options; _environment = environment; @@ -24,7 +24,7 @@ public DataSetVersionPathResolver(IOptions options, IWebHos if (_options.Value.BasePath.IsNullOrWhitespace()) { throw new ArgumentException( - message: $"'{nameof(ParquetFilesOptions.BasePath)}' must not be blank", + message: $"'{nameof(DataFilesOptions.BasePath)}' must not be blank", paramName: nameof(options) ); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Interfaces/IDataSetVersionPathResolver.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Interfaces/IDataSetVersionPathResolver.cs index 83bfa5117a9..f2636ec020d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Interfaces/IDataSetVersionPathResolver.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Interfaces/IDataSetVersionPathResolver.cs @@ -10,10 +10,19 @@ public interface IDataSetVersionPathResolver string DirectoryPath(DataSetVersion dataSetVersion); string CsvDataPath(DataSetVersion dataSetVersion) - => Path.Combine(DirectoryPath(dataSetVersion), "data.csv.gz"); + => Path.Combine(DirectoryPath(dataSetVersion), DataSetFilenames.CsvDataFile); string CsvMetadataPath(DataSetVersion dataSetVersion) - => Path.Combine(DirectoryPath(dataSetVersion), "metadata.csv.gz"); + => Path.Combine(DirectoryPath(dataSetVersion), DataSetFilenames.CsvMetadataFile); + + string DuckDbPath(DataSetVersion dataSetVersion) + => Path.Combine(DirectoryPath(dataSetVersion), DataSetFilenames.DuckDbDatabaseFile); + + string DuckDbLoadSqlPath(DataSetVersion dataSetVersion) + => Path.Combine(DirectoryPath(dataSetVersion), DataSetFilenames.DuckDbLoadSqlFile); + + string DuckDbSchemaSqlPath(DataSetVersion dataSetVersion) + => Path.Combine(DirectoryPath(dataSetVersion), DataSetFilenames.DuckDbSchemaSqlFile); string DataPath(DataSetVersion dataSetVersion) => Path.Combine(DirectoryPath(dataSetVersion), DataTable.ParquetFile); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Options/ParquetFilesOptions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Options/DataFilesOptions.cs similarity index 63% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Options/ParquetFilesOptions.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Options/DataFilesOptions.cs index 25acaa2e1a9..8321babd5fb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Options/ParquetFilesOptions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Options/DataFilesOptions.cs @@ -1,11 +1,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Options; -public class ParquetFilesOptions +public class DataFilesOptions { - public static readonly string Section = "ParquetFiles"; + public static readonly string Section = "DataFiles"; /// - /// Base path where Parquet files are stored. This should be an absolute path + /// Base path where data files are stored. This should be an absolute path /// in a non-local environment (i.e. where the File Share has been mounted), /// or a relative path in a local environment. ///