diff --git a/.eslintrc.json b/.eslintrc.json index 6a7f42fa215..2344ee8e8c5 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -118,6 +118,12 @@ } ], "no-shadow": "off", + "no-underscore-dangle": [ + "error", + { + "allow": ["_def"] + } + ], "no-unreachable": "error", "no-use-before-define": "off", "no-useless-constructor": "off" diff --git a/infrastructure/parameters/dev.parameters.json b/infrastructure/parameters/dev.parameters.json index f425e4936f1..ed98169cd41 100644 --- a/infrastructure/parameters/dev.parameters.json +++ b/infrastructure/parameters/dev.parameters.json @@ -50,7 +50,7 @@ "devopsSPN": { "value": "e541f669-7fac-4d33-b480-29b523b9d968", "metadata": { - "comments": "ObjectId for s101d-datahub-spn-ees-dfe-gov-uk" + "comments": "ObjectId for s101d-datahub-spn-ees-dfe-gov-uk" } }, "domain": { @@ -62,6 +62,9 @@ "dataApiUrl": { "value": "data.dev.explore-education-statistics.service.gov.uk" }, + "publicApiUrl": { + "value": "dev.statistics.api.education.gov.uk" + }, "detailedErrors": { "value": true }, @@ -72,7 +75,7 @@ "value": "G-GRPHH2FN0L" }, "enableSwagger": { - "value": true + "value": true }, "enableThemeDeletion": { "value": true @@ -81,7 +84,7 @@ "value": 10 }, "branch": { - "value": "dev" + "value": "dev" }, "skuContentDb": { "value": "Standard" diff --git a/infrastructure/parameters/pre-prod.parameters.json b/infrastructure/parameters/pre-prod.parameters.json index f9081f998ac..33226dadc3a 100644 --- a/infrastructure/parameters/pre-prod.parameters.json +++ b/infrastructure/parameters/pre-prod.parameters.json @@ -41,7 +41,7 @@ "devopsSPN": { "value": "2d5a7bf2-a6b1-4474-b202-ab17dd87c375", "metadata": { - "comments": "ObjectId for s101prep-datahub-spn-ees-dfe-gov-uk" + "comments": "ObjectId for s101prep-datahub-spn-ees-dfe-gov-uk" } }, "domain": { @@ -53,6 +53,9 @@ "dataApiUrl": { "value": "data.pre-production.explore-education-statistics.service.gov.uk" }, + "publicApiUrl": { + "value": "pre-production.statistics.api.education.gov.uk" + }, "publicAppGATrackingId": { "value": "G-8FSLWXTV2W" }, diff --git a/infrastructure/parameters/prod.parameters.json b/infrastructure/parameters/prod.parameters.json index 0565acc742b..8f2c9a54a75 100644 --- a/infrastructure/parameters/prod.parameters.json +++ b/infrastructure/parameters/prod.parameters.json @@ -44,7 +44,7 @@ "devopsSPN": { "value": "911681c3-8cc7-4afc-8355-4cf60359d743", "metadata": { - "comments": "ObjectId for s101p-datahub-spn-ees-dfe-gov-uk" + "comments": "ObjectId for s101p-datahub-spn-ees-dfe-gov-uk" } }, "domain": { @@ -56,6 +56,9 @@ "dataApiUrl": { "value": "data.explore-education-statistics.service.gov.uk" }, + "publicApiUrl": { + "value": "statistics.api.education.gov.uk" + }, "publicAppGATrackingId": { "value": "G-9YG8ESXR5Y" }, diff --git a/infrastructure/parameters/test.parameters.json b/infrastructure/parameters/test.parameters.json index a06eec6b30b..1b5cc5578ef 100644 --- a/infrastructure/parameters/test.parameters.json +++ b/infrastructure/parameters/test.parameters.json @@ -41,7 +41,7 @@ "devopsSPN": { "value": "22888fee-3aa4-411d-8016-bb8a8c3b825a", "metadata": { - "comments": "ObjectId for s101t-datahub-spn-ees-dfe-gov-uk" + "comments": "ObjectId for s101t-datahub-spn-ees-dfe-gov-uk" } }, "domain": { @@ -53,6 +53,9 @@ "dataApiUrl": { "value": "data.test.explore-education-statistics.service.gov.uk" }, + "publicApiUrl": { + "value": "test.statistics.api.education.gov.uk" + }, "detailedErrors": { "value": true }, diff --git a/infrastructure/templates/public-api/components/containerApp.bicep b/infrastructure/templates/public-api/components/containerApp.bicep index 9cb99503774..71308b918c1 100644 --- a/infrastructure/templates/public-api/components/containerApp.bicep +++ b/infrastructure/templates/public-api/components/containerApp.bicep @@ -12,12 +12,19 @@ param containerAppImageName string @minLength(2) @maxLength(32) -@description('Specifies the name of the container app.') +@description('Specifies the name of the Container App.') param containerAppName string @description('Specifies the container port.') param containerAppTargetPort int = 8080 +@description('The CORS policy to use for the Container App.') +param corsPolicy { + allowedHeaders: string[]? + allowedMethods: string[]? + allowedOrigins: string[]? +} + @description('Number of CPU cores the container can use. Can be with a maximum of two decimals.') @allowed([ '1' @@ -88,7 +95,7 @@ param volumeMounts { var containerImageName = '${acrLoginServer}/${containerAppImageName}' var containerApplicationName = toLower('${resourcePrefix}-ca-${containerAppName}') -resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { +resource containerApp 'Microsoft.App/containerApps@2023-11-02-preview' = { name: containerApplicationName location: location identity: { @@ -106,6 +113,7 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { external: true targetPort: containerAppTargetPort allowInsecure: false + corsPolicy: corsPolicy traffic: [ { latestRevision: true @@ -123,7 +131,7 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { template: { containers: [ { - name: containerAppName + name: containerAppName image: containerImageName env: appSettings resources: { diff --git a/infrastructure/templates/public-api/deploy-stage-template.yml b/infrastructure/templates/public-api/deploy-stage-template.yml index 5269cbbf777..5b66cfe4d5b 100644 --- a/infrastructure/templates/public-api/deploy-stage-template.yml +++ b/infrastructure/templates/public-api/deploy-stage-template.yml @@ -20,8 +20,8 @@ stages: - stage: ${{parameters.stageName}} displayName: 'Deploy ${{parameters.environment}} Infrastructure and Applications' # Prevent this stage from running in parallel with the same deploy stage in other ongoing runs of this pipeline. - # Instead, multiple executions of this stage will be queued and run sequentially in the order that their pipelines - # were triggered. + # Instead, multiple executions of this stage will be queued and run sequentially in the order that their pipelines + # were triggered. lockBehavior: sequential condition: ${{parameters.condition}} variables: @@ -68,7 +68,6 @@ stages: --parameters \ subscription='$(subscription)' \ resourceTags='$(resourceTags)' \ - publicUrls='$(publicUrls)' \ postgreSqlAdminName='$(postgreSqlAdminName)' \ postgreSqlAdminPassword='$(postgreSqlAdminPassword)' \ postgreSqlFirewallRules='$(maintenanceFirewallRules)' \ @@ -122,7 +121,7 @@ stages: --settings \ "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" - # TODO EES-5128 - add Private Endpoint to Data Processor Function App into the VMSS VNet to allow DevOps to + # TODO EES-5128 - add Private Endpoint to Data Processor Function App into the VMSS VNet to allow DevOps to # deploy the Data Processor Function App without having to temporarily make it publicly accessible. - task: AzureCLI@2 displayName: 'Deploy Data Processor Function App - temporarily enable public network access before deploy' @@ -142,11 +141,11 @@ stages: publicNetworkAccess=Enabled \ siteConfig.publicNetworkAccess=Enabled - # TODO EES-5128 - we will try several attempts to deploy the Function App in order to allow the staging + # TODO EES-5128 - we will try several attempts to deploy the Function App in order to allow the staging # slot the time to fully restart after appsettings and network visibility settings have been updated prior to - # attempting the deploy. Deploying prematurely results in a 500 from the deployment endpoint until the + # attempting the deploy. Deploying prematurely results in a 500 from the deployment endpoint until the # endpoint is ready to accept the deployment request. In the future it would be preferable to have a health - # check Function that we could call to establish that the site is ready, but this will require adding the + # check Function that we could call to establish that the site is ready, but this will require adding the # Service Principal to allowed Client IDs / Identities that can access the Function App. The Service Principal # that is performing the deploy can be accessed by using the "addSpnToEnvironment" config option in the task # definition and using the $(servicePrincipalId) variable. @@ -165,7 +164,7 @@ stages: --resource-group $(resourceGroupName) \ --slot staging - # TODO EES-5128 - add Private Endpoint to Data Processor Function App into the VMSS VNet to allow DevOps to + # TODO EES-5128 - add Private Endpoint to Data Processor Function App into the VMSS VNet to allow DevOps to # deploy the Data Processor Function App without having to temporarily make it publicly accessible. - task: AzureCLI@2 displayName: 'Deploy Data Processor Function App - disable public network access after deploy' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index bbfa38eda75..cab9e2a1a9a 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -64,7 +64,7 @@ param dockerImagesTag string = '' @description('Can we deploy the Container App yet? This is dependent on the user-assigned Managed Identity for the API Container App being created with the AcrPull role, and the database users added to PSQL.') param deployContainerApp bool = true -// TODO EES-5128 - Note that this has been added temporarily to avoid 10+ minute deploys where it appears that PSQL +// TODO EES-5128 - Note that this has been added temporarily to avoid 10+ minute deploys where it appears that PSQL // will redeploy even if no changes exist in this deploy from the previous one. @description('Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys.') param updatePsqlFlexibleServer bool = false @@ -72,7 +72,8 @@ param updatePsqlFlexibleServer bool = false @description('Public URLs of other components in the service.') param publicUrls { contentApi: string -}? + publicApp: string +} @description('Specifies whether or not the Data Processor Function App already exists.') param dataProcessorFunctionAppExists bool = false @@ -256,11 +257,18 @@ module apiContainerAppModule 'components/containerApp.bicep' = if (deployContain params: { resourcePrefix: resourcePrefix location: location - containerAppName: apiContainerAppName + containerAppName: apiContainerAppName acrLoginServer: containerRegistry.properties.loginServer containerAppImageName: 'ees-public-api/api:${dockerImagesTag}' userAssignedManagedIdentityId: apiContainerAppManagedIdentity.id managedEnvironmentId: containerAppEnvironmentModule.outputs.containerAppEnvironmentId + corsPolicy: { + allowedOrigins: [ + publicUrls.publicApp + 'http://localhost:3000' + 'http://127.0.0.1' + ] + } volumeMounts: [ { volumeName: dataFilesFileShareMountName @@ -291,7 +299,7 @@ module apiContainerAppModule 'components/containerApp.bicep' = if (deployContain } { name: 'ContentApi__Url' - value: publicUrls!.contentApi + value: publicUrls.contentApi } { name: 'MiniProfiler__Enabled' diff --git a/infrastructure/templates/public-api/parameters/main-dev.bicepparam b/infrastructure/templates/public-api/parameters/main-dev.bicepparam index 91bbb5a4a30..9e75e7a3743 100644 --- a/infrastructure/templates/public-api/parameters/main-dev.bicepparam +++ b/infrastructure/templates/public-api/parameters/main-dev.bicepparam @@ -3,6 +3,11 @@ using '../main.bicep' // Environment Params param environmentName = 'Development' +param publicUrls = { + contentApi: 'https://content.dev.explore-education-statistics.service.gov.uk' + publicApp: 'https://dev.explore-education-statistics.service.gov.uk' +} + // PostgreSQL Database Params param postgreSqlSkuName = 'Standard_B1ms' param postgreSqlStorageSizeGB = 32 diff --git a/infrastructure/templates/public-api/parameters/main-preprod.bicepparam b/infrastructure/templates/public-api/parameters/main-preprod.bicepparam index 360d6436c23..050601a36ab 100644 --- a/infrastructure/templates/public-api/parameters/main-preprod.bicepparam +++ b/infrastructure/templates/public-api/parameters/main-preprod.bicepparam @@ -3,6 +3,11 @@ using '../main.bicep' // Environment Params param environmentName = 'Pre-Production' +param publicUrls = { + contentApi: 'https://s101p02-as-ees-content.azurewebsites.net' + publicApp: 'https://pre-production.explore-education-statistics.service.gov.uk' +} + // PostgreSQL Database Params param postgreSqlSkuName = 'Standard_B1ms' param postgreSqlStorageSizeGB = 32 diff --git a/infrastructure/templates/public-api/parameters/main-prod.bicepparam b/infrastructure/templates/public-api/parameters/main-prod.bicepparam index fb0b536f833..5439ef60564 100644 --- a/infrastructure/templates/public-api/parameters/main-prod.bicepparam +++ b/infrastructure/templates/public-api/parameters/main-prod.bicepparam @@ -3,6 +3,11 @@ using '../main.bicep' // Environment Params param environmentName = 'Production' +param publicUrls = { + contentApi: 'https://content.explore-education-statistics.service.gov.uk' + publicApp: 'https://explore-education-statistics.service.gov.uk' +} + // PostgreSQL Database Params param postgreSqlSkuName = 'Standard_B1ms' param postgreSqlStorageSizeGB = 32 diff --git a/infrastructure/templates/public-api/parameters/main-test.bicepparam b/infrastructure/templates/public-api/parameters/main-test.bicepparam index f4029830ec3..8cff51412d3 100644 --- a/infrastructure/templates/public-api/parameters/main-test.bicepparam +++ b/infrastructure/templates/public-api/parameters/main-test.bicepparam @@ -3,6 +3,11 @@ using '../main.bicep' // Environment Params param environmentName = 'Test' +param publicUrls = { + contentApi: 'https://content.test.explore-education-statistics.service.gov.uk' + publicApp: 'https://test.explore-education-statistics.service.gov.uk' +} + // PostgreSQL Database Params param postgreSqlSkuName = 'Standard_B1ms' param postgreSqlStorageSizeGB = 32 diff --git a/infrastructure/templates/public-api/validate-stage-template.yml b/infrastructure/templates/public-api/validate-stage-template.yml index 2a4603a44a2..8cc5b7807da 100644 --- a/infrastructure/templates/public-api/validate-stage-template.yml +++ b/infrastructure/templates/public-api/validate-stage-template.yml @@ -17,8 +17,8 @@ stages: - stage: ${{parameters.stageName}} displayName: 'Validate ${{parameters.environment}} Infrastructure' # Prevent this stage from running in parallel with the same deploy stage in other ongoing runs of this pipeline. - # Instead, multiple executions of this stage will be queued and run sequentially in the order that their pipelines - # were triggered. + # Instead, multiple executions of this stage will be queued and run sequentially in the order that their pipelines + # were triggered. lockBehavior: sequential condition: ${{parameters.condition}} variables: @@ -52,7 +52,6 @@ stages: --parameters \ subscription='$(subscription)' \ resourceTags='$(resourceTags)' \ - publicUrls='$(publicUrls)' \ postgreSqlAdminName='$(postgreSqlAdminName)' \ postgreSqlAdminPassword='$(postgreSqlAdminPassword)' \ postgreSqlFirewallRules='$(maintenanceFirewallRules)' \ diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index 6eadcf63fe6..6ba381a6337 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -11,6 +11,9 @@ "dataApiUrl": { "type": "string" }, + "publicApiUrl": { + "type": "string" + }, "subscription": { "type": "string", "defaultValue": "", @@ -2061,6 +2064,7 @@ "NEXT_CONFIG_MODE": "server", "NODE_ENV": "production", "PUBLIC_URL": "[concat(variables('publicAppUrl'), '/')]", + "PUBLIC_API_BASE_URL": "[concat('https://', parameters('publicApiUrl'),'/api/v1.0')]", "WEBSITE_NODE_DEFAULT_VERSION": "20.14.0", "WEBSITES_PORT": 3000 } diff --git a/renovate.json b/renovate.json index 6b23536053c..f4937f5ad97 100644 --- a/renovate.json +++ b/renovate.json @@ -66,6 +66,12 @@ "matchPackageNames": ["node"], "groupName": "Node version", "enabled": false + }, + { + "matchDatasources": ["azure-bicep-resource"], + "groupName": "Azure Bicep resources", + "description": "Prevent unofficial resource versions being suggested", + "enabled": false } ] } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/DataReplacementControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/DataReplacementControllerTests.cs new file mode 100644 index 00000000000..e2fd5fa1581 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/DataReplacementControllerTests.cs @@ -0,0 +1,165 @@ +#nullable enable +using GovUk.Education.ExploreEducationStatistics.Admin.Controllers.Api; +using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Admin.Validators; +using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using Moq; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Controllers.Api; + +public abstract class DataReplacementControllerTests +{ + public class GetReplacementPlanTests : DataReplacementControllerTests + { + [Fact] + public async Task Success() + { + var replacementService = new Mock(MockBehavior.Strict); + + var releaseVersionId = Guid.NewGuid(); + var originalFileId = Guid.NewGuid(); + var replacementFileId = Guid.NewGuid(); + + var dataReplacementPlan = new DataReplacementPlanViewModel + { + DataBlocks = [ + new DataBlockReplacementPlanViewModel( + id: Guid.NewGuid(), + name: "my data block", + originalFilters: new Dictionary() { + { + Guid.NewGuid(), + new FilterReplacementViewModel( + id: Guid.NewGuid(), + label: "filter replacement lebel", + name: "filter replacement name", + groups: new Dictionary() { + { + Guid.NewGuid(), + new FilterGroupReplacementViewModel( + id: Guid.NewGuid(), + label: "filter group replacement label", + filters: [ + new FilterItemReplacementViewModel( + id: Guid.NewGuid(), + label: "filter item replacement label", + target: Guid.NewGuid()) + ]) + } + }) + } + }) + ], + Footnotes = [], + DeleteApiDataSetVersionPlan = new DeleteApiDataSetVersionPlanViewModel + { + DataSetId = Guid.NewGuid(), + DataSetTitle = "my data set", + Id = Guid.NewGuid(), + Version = "v1.0", + Status = DataSetVersionStatus.Draft, + Valid = false + }, + OriginalSubjectId = Guid.NewGuid(), + ReplacementSubjectId = Guid.NewGuid() + }; + + replacementService + .Setup(s => s.GetReplacementPlan( + releaseVersionId, + originalFileId, + replacementFileId, + It.IsAny())) + .ReturnsAsync(dataReplacementPlan); + + var controller = BuildController(replacementService: replacementService.Object); + + var result = await controller.GetReplacementPlan( + releaseVersionId: releaseVersionId, + fileId: originalFileId, + replacementFileId: replacementFileId); + + MockUtils.VerifyAllMocks(replacementService); + + var returnedPlan = result.AssertOkResult(); + + Assert.Equal( + JsonConvert.SerializeObject(dataReplacementPlan.ToSummary()), + JsonConvert.SerializeObject(returnedPlan)); + } + } + + public class ReplaceTests : DataReplacementControllerTests + { + [Fact] + public async Task Success() + { + var replacementService = new Mock(MockBehavior.Strict); + + var releaseVersionId = Guid.NewGuid(); + var originalFileId = Guid.NewGuid(); + var replacementFileId = Guid.NewGuid(); + + replacementService + .Setup(service => service.Replace( + releaseVersionId, + originalFileId, + replacementFileId)) + .ReturnsAsync(Unit.Instance); + + var controller = BuildController(replacementService: replacementService.Object); + + var result = await controller.Replace( + releaseVersionId: releaseVersionId, + fileId: originalFileId, + replacementFileId: replacementFileId); + + MockUtils.VerifyAllMocks(replacementService); + + result.AssertOkResult(); + } + + [Fact] + public async Task ValidationProblem() + { + var replacementService = new Mock(MockBehavior.Strict); + + var releaseVersionId = Guid.NewGuid(); + var originalFileId = Guid.NewGuid(); + var replacementFileId = Guid.NewGuid(); + + replacementService + .Setup(service => service.Replace( + releaseVersionId, + originalFileId, + replacementFileId)) + .ReturnsAsync(ValidationUtils.ValidationActionResult(ValidationErrorMessages.ReplacementMustBeValid)); + + var controller = BuildController(replacementService: replacementService.Object); + + var result = await controller.Replace( + releaseVersionId: releaseVersionId, + fileId: originalFileId, + replacementFileId: replacementFileId); + + MockUtils.VerifyAllMocks(replacementService); + + result.AssertValidationProblem(ValidationErrorMessages.ReplacementMustBeValid); + } + } + + private static DataReplacementController BuildController(IReplacementService? replacementService = null) + { + return new DataReplacementController( + replacementService ?? Mock.Of(MockBehavior.Strict)); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetCandidatesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetCandidatesControllerTests.cs index 0cf735faffd..814b40a1552 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetCandidatesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetCandidatesControllerTests.cs @@ -7,6 +7,7 @@ using GovUk.Education.ExploreEducationStatistics.Admin.Security; using GovUk.Education.ExploreEducationStatistics.Admin.Tests.Fixture; using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.Public.Data; +using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; @@ -30,21 +31,27 @@ public async Task Success() Release release = DataFixture .DefaultRelease(publishedVersions: 0, draftVersion: true); - var files = DataFixture - .DefaultFile() + var dataImports = DataFixture + .DefaultDataImport() + .WithStatus(DataImportStatus.COMPLETE) .GenerateList(3); var releaseVersion = release.Versions.Single(); - var releaseFiles = DataFixture - .DefaultReleaseFile() - .ForIndex(0, rf => rf.SetFile(files[0])) - .ForIndex(1, rf => rf.SetFile(files[1])) - .ForIndex(2, rf => rf.SetFile(files[2])) - .WithReleaseVersion(releaseVersion) - .GenerateList(3); + var releaseFiles = dataImports + .Select(di => DataFixture + .DefaultReleaseFile() + .WithFile(di.File) + .WithReleaseVersion(releaseVersion) + .Generate() + ) + .ToList(); - await TestApp.AddTestData(context => context.ReleaseFiles.AddRange(releaseFiles)); + await TestApp.AddTestData(context => + { + context.DataImports.AddRange(dataImports); + context.ReleaseFiles.AddRange(releaseFiles); + }); var response = await GetDataSetCandidates(releaseVersion.Id); @@ -94,7 +101,6 @@ public async Task NoReleaseFileExists_ReturnsEmptyList() var candidates = response.AssertOk>(); - Assert.NotNull(candidates); Assert.Empty(candidates); } @@ -105,24 +111,30 @@ public async Task ReleaseFileIsReplacement_NotReturned() Release release = DataFixture .DefaultRelease(publishedVersions: 0, draftVersion: true); - File file = DataFixture - .DefaultFile() - .WithReplacingId(Guid.NewGuid()); + DataImport dataImport = DataFixture + .DefaultDataImport() + .WithFile(DataFixture + .DefaultFile(FileType.Data) + .WithReplacingId(Guid.NewGuid()) + ); var releaseVersion = release.Versions.Single(); ReleaseFile releaseFile = DataFixture .DefaultReleaseFile() - .WithFile(file) + .WithFile(dataImport.File) .WithReleaseVersion(releaseVersion); - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => + { + context.DataImports.Add(dataImport); + context.ReleaseFiles.Add(releaseFile); + }); var response = await GetDataSetCandidates(releaseVersion.Id); var candidates = response.AssertOk>(); - Assert.NotNull(candidates); Assert.Empty(candidates); } @@ -132,24 +144,30 @@ public async Task ReleaseFileIsReplaced_NotReturned() Release release = DataFixture .DefaultRelease(publishedVersions: 0, draftVersion: true); - File file = DataFixture - .DefaultFile() - .WithReplacedById(Guid.NewGuid()); + DataImport dataImport = DataFixture + .DefaultDataImport() + .WithFile(DataFixture + .DefaultFile(FileType.Data) + .WithReplacedById(Guid.NewGuid()) + ); var releaseVersion = release.Versions.Single(); ReleaseFile releaseFile = DataFixture .DefaultReleaseFile() - .WithFile(file) + .WithFile(dataImport.File) .WithReleaseVersion(releaseVersion); - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => + { + context.DataImports.Add(dataImport); + context.ReleaseFiles.Add(releaseFile); + }); var response = await GetDataSetCandidates(releaseVersion.Id); var candidates = response.AssertOk>(); - Assert.NotNull(candidates); Assert.Empty(candidates); } @@ -159,32 +177,74 @@ public async Task ReleaseFileHasAssociatedDataSet_NotReturned() Release release = DataFixture .DefaultRelease(publishedVersions: 0, draftVersion: true); - File file = DataFixture - .DefaultFile() + DataImport dataImport = DataFixture + .DefaultDataImport() + .WithFile(DataFixture.DefaultFile(FileType.Data)); + + var releaseVersion = release.Versions.Single(); + + ReleaseFile releaseFile = DataFixture + .DefaultReleaseFile() + .WithFile(dataImport.File) + .WithReleaseVersion(releaseVersion) .WithPublicApiDataSetId(Guid.NewGuid()); + await TestApp.AddTestData(context => + { + context.DataImports.Add(dataImport); + context.ReleaseFiles.Add(releaseFile); + }); + + var response = await GetDataSetCandidates(releaseVersion.Id); + + var candidates = response.AssertOk>(); + + Assert.Empty(candidates); + } + + [Theory] + [InlineData(DataImportStatus.QUEUED)] + [InlineData(DataImportStatus.STAGE_1)] + [InlineData(DataImportStatus.STAGE_2)] + [InlineData(DataImportStatus.STAGE_3)] + [InlineData(DataImportStatus.FAILED)] + [InlineData(DataImportStatus.NOT_FOUND)] + [InlineData(DataImportStatus.CANCELLED)] + [InlineData(DataImportStatus.CANCELLING)] + public async Task ReleaseFileImportIsNotComplete_NotReturned(DataImportStatus status) + { + Release release = DataFixture + .DefaultRelease(publishedVersions: 0, draftVersion: true); + + DataImport dataImport = DataFixture + .DefaultDataImport() + .WithFile(DataFixture.DefaultFile(FileType.Data)) + .WithStatus(status); + var releaseVersion = release.Versions.Single(); ReleaseFile releaseFile = DataFixture .DefaultReleaseFile() - .WithFile(file) + .WithFile(dataImport.File) .WithReleaseVersion(releaseVersion); - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => + { + context.DataImports.Add(dataImport); + context.ReleaseFiles.Add(releaseFile); + }); var response = await GetDataSetCandidates(releaseVersion.Id); var candidates = response.AssertOk>(); - Assert.NotNull(candidates); Assert.Empty(candidates); } [Fact] public async Task ReleaseVersionDoesNotExist_Returns404() { - var response = await GetDataSetCandidates( - releaseVersionId: Guid.NewGuid()); + var response = await GetDataSetCandidates(releaseVersionId: Guid.NewGuid()); response.AssertNotFound(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionsControllerTests.cs index 55b456b4e13..c566a049cfb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionsControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionsControllerTests.cs @@ -4,14 +4,19 @@ using System.Security.Claims; using System.Threading; using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Admin.Requests.Public.Data; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data; using GovUk.Education.ExploreEducationStatistics.Admin.Tests.Fixture; +using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.Public.Data; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.ViewModels; +using LinqToDB; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Testing; using Moq; @@ -19,47 +24,126 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Controllers.Api.Public.Data; -public abstract class DataSetVersionsControllerTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) +public abstract class DataSetVersionsControllerTests( + TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { private const string BaseUrl = "api/public-data/data-set-versions"; - public class DeleteVersionTests(TestApplicationFactory testApp) : DataSetVersionsControllerTests(testApp) + public class CreateNextVersionTests( + TestApplicationFactory testApp) : DataSetVersionsControllerTests(testApp) { [Fact] public async Task Success() { + var nextReleaseFileId = Guid.NewGuid(); + DataSet dataSet = DataFixture .DefaultDataSet() - .WithStatusDraft(); + .WithStatusPublished(); await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - DataSetVersion dataSetVersion = DataFixture + DataSetVersion currentDataSetVersion = DataFixture .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(1, 0, 0) - .WithStatusDraft() + .WithVersionNumber(major: 1, minor: 0) + .WithStatusPublished() .WithDataSet(dataSet) - .WithImports(() => DataFixture - .DefaultDataSetVersionImport() - .Generate(1)) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); await TestApp.AddTestData(context => { - context.DataSetVersions.Add(dataSetVersion); + context.DataSetVersions.Add(currentDataSetVersion); context.DataSets.Update(dataSet); }); - var processorClient = new Mock(); + DataSetVersion? nextVersion = null; + + var processorClient = new Mock(MockBehavior.Strict); + + processorClient + .Setup(c => c.CreateNextDataSetVersion(dataSet.Id, + nextReleaseFileId, It.IsAny())) + .Returns(async () => + { + var savedDataSet = await TestApp.GetDbContext() + .DataSets + .SingleAsync(ds => ds.Id == dataSet.Id); + + nextVersion = DataFixture + .DefaultDataSetVersion() + .WithStatusMapping() + .WithVersionNumber(major: 1, minor: 1) + .WithReleaseFileId(nextReleaseFileId) + .WithDataSet(savedDataSet) + .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + + await TestApp.AddTestData(context => + { + context.DataSetVersions.Add(nextVersion); + context.DataSets.Update(savedDataSet); + }); + + return new CreateDataSetResponseViewModel + { + DataSetId = dataSet.Id, + DataSetVersionId = nextVersion.Id, + InstanceId = Guid.NewGuid() + }; + }); + + var client = BuildApp(processorClient.Object).CreateClient(); + + var response = await CreateNextVersion( + dataSetId: dataSet.Id, + releaseFileId: nextReleaseFileId, + client); + + var viewModel = response.AssertOk(); + + Assert.NotNull(nextVersion); + Assert.Equal(viewModel.Id, nextVersion.Id); + Assert.Equal(viewModel.Version, nextVersion.Version); + Assert.Equal(viewModel.Status, nextVersion.Status); + Assert.Equal(viewModel.Type, nextVersion.VersionType); + } + + private async Task CreateNextVersion( + Guid dataSetId, + Guid releaseFileId, + HttpClient? client = null) + { + client ??= BuildApp().CreateClient(); + + var uri = new Uri(BaseUrl, UriKind.Relative); + + return await client.PostAsync(uri, + new JsonNetContent(new NextDataSetVersionCreateRequest + { + DataSetId = dataSetId, + ReleaseFileId = releaseFileId + })); + } + } + + public class DeleteVersionTests( + TestApplicationFactory testApp) : DataSetVersionsControllerTests(testApp) + { + [Fact] + public async Task Success() + { + var dataSetVersionId = Guid.NewGuid(); + + var processorClient = new Mock(MockBehavior.Strict); + processorClient .Setup(c => c.DeleteDataSetVersion( - dataSetVersion.Id, + dataSetVersionId, It.IsAny())) .ReturnsAsync(new Either(Unit.Instance)); var client = BuildApp(processorClient.Object).CreateClient(); - var response = await DeleteVersion(dataSetVersion.Id, client); + var response = await DeleteVersion(dataSetVersionId, client); response.AssertNoContent(); } @@ -85,7 +169,7 @@ public async Task ProcessorReturns404_Returns404() DataSetVersion dataSetVersion = DataFixture .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(1, 0, 0) + .WithVersionNumber(1, 0) .WithStatusDraft() .WithDataSet(dataSet) .WithImports(() => DataFixture @@ -99,7 +183,7 @@ await TestApp.AddTestData(context => context.DataSets.Update(dataSet); }); - var processorClient = new Mock(); + var processorClient = new Mock(MockBehavior.Strict); processorClient .Setup(c => c.DeleteDataSetVersion( dataSetVersion.Id, @@ -124,7 +208,7 @@ public async Task ProcessorReturns400_Returns400() DataSetVersion dataSetVersion = DataFixture .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(1, 0, 0) + .WithVersionNumber(1, 0) .WithStatusDraft() .WithDataSet(dataSet) .WithImports(() => DataFixture @@ -138,7 +222,7 @@ await TestApp.AddTestData(context => context.DataSets.Update(dataSet); }); - var processorClient = new Mock(); + var processorClient = new Mock(MockBehavior.Strict); processorClient .Setup(c => c.DeleteDataSetVersion( dataSetVersion.Id, @@ -148,12 +232,13 @@ await TestApp.AddTestData(context => new ValidationProblemViewModel { Errors = new ErrorViewModel[] + { + new() { - new() { - Code = "error code", - Path = "error path" - } + Code = "error code", + Path = "error path" } + } }))); var client = BuildApp(processorClient.Object).CreateClient(); @@ -162,7 +247,7 @@ await TestApp.AddTestData(context => var validationProblem = response.AssertValidationProblem(); - var error = validationProblem.AssertHasError("error path", "error code"); + validationProblem.AssertHasError("error path", "error code"); } [Fact] @@ -176,7 +261,7 @@ public async Task ProcessorFailureStatusCode_Returns500() DataSetVersion dataSetVersion = DataFixture .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(1, 0, 0) + .WithVersionNumber(1, 0) .WithStatusDraft() .WithDataSet(dataSet) .WithImports(() => DataFixture @@ -190,29 +275,21 @@ await TestApp.AddTestData(context => context.DataSets.Update(dataSet); }); - var processorClient = new Mock(); + var processorClient = new Mock(MockBehavior.Strict); processorClient .Setup(c => c.DeleteDataSetVersion( dataSetVersion.Id, It.IsAny())) .ThrowsAsync(new HttpRequestException()); - + var client = BuildApp(processorClient.Object).CreateClient(); - var exception = await Assert.ThrowsAsync(async () => await DeleteVersion(dataSetVersion.Id, client)); + var exception = + await Assert.ThrowsAsync(async () => + await DeleteVersion(dataSetVersion.Id, client)); Assert.IsType(exception.InnerException); } - private WebApplicationFactory BuildApp( - IProcessorClient? processorClient = null, - ClaimsPrincipal? user = null) - { - return TestApp.ConfigureServices( - services => { services.ReplaceService(processorClient ?? Mock.Of()); } - ) - .SetUser(user ?? BauUser()); - } - private async Task DeleteVersion( Guid dataSetVersionId, HttpClient? client = null) @@ -224,4 +301,14 @@ private async Task DeleteVersion( return await client.DeleteAsync(uri); } } + + private WebApplicationFactory BuildApp( + IProcessorClient? processorClient = null, + ClaimsPrincipal? user = null) + { + return TestApp.ConfigureServices( + services => { services.ReplaceService(processorClient ?? Mock.Of()); } + ) + .SetUser(user ?? BauUser()); + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetsControllerTests.cs index 77c16f60c56..468179ea9f3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetsControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetsControllerTests.cs @@ -13,6 +13,7 @@ using GovUk.Education.ExploreEducationStatistics.Admin.Tests.Fixture; using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.Public.Data; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils; using GovUk.Education.ExploreEducationStatistics.Common.Validators; @@ -31,11 +32,13 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Controllers.Api.Public.Data; -public class DataSetsControllerTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) +public abstract class DataSetsControllerTests( + TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { private const string BaseUrl = "api/public-data/data-sets"; - public class ListDataSetsTests(TestApplicationFactory testApp) : DataSetsControllerTests(testApp) + public class ListDataSetsTests( + TestApplicationFactory testApp) : DataSetsControllerTests(testApp) { [Fact] public async Task PublicationHasSingleDataSet_Success_CorrectViewModel() @@ -450,15 +453,9 @@ private async Task ListPublicationDataSets( var queryParams = new Dictionary { - { - "page", page?.ToString() - }, - { - "pageSize", pageSize?.ToString() - }, - { - "publicationId", publicationId.ToString() - }, + { "page", page?.ToString() }, + { "pageSize", pageSize?.ToString() }, + { "publicationId", publicationId.ToString() }, }; var uri = QueryHelpers.AddQueryString(BaseUrl, queryParams); @@ -467,7 +464,8 @@ private async Task ListPublicationDataSets( } } - public class GetDataSetTests(TestApplicationFactory testApp) : DataSetsControllerTests(testApp) + public class GetDataSetTests( + TestApplicationFactory testApp) : DataSetsControllerTests(testApp) { [Fact] public async Task Success() @@ -486,12 +484,12 @@ public async Task Success() ReleaseFile liveReleaseFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture.DefaultFile()) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(liveReleaseVersion); ReleaseFile draftReleaseFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture.DefaultFile()) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(draftReleaseVersion); await TestApp.AddTestData(context => @@ -599,7 +597,7 @@ public async Task ReleaseFilesWithSameFileId_Returns200_SameDataSetFileId() .Generate(1) ); - File file = DataFixture.DefaultFile(); + File file = DataFixture.DefaultFile(FileType.Data); var liveReleaseVersion = publication.ReleaseVersions.Single(rv => rv.Published is not null); @@ -675,7 +673,7 @@ public async Task RequestedDataSetHasNoDraftVersion_Returns200_NoDraftVersion() ReleaseFile releaseFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture.DefaultFile()) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(releaseVersion); await TestApp.AddTestData(context => @@ -738,7 +736,7 @@ public async Task RequestedDataSetHasNoLiveVersion_Returns200_NoLiveVersion() ReleaseFile releaseFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture.DefaultFile()) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(releaseVersion); await TestApp.AddTestData(context => @@ -851,7 +849,7 @@ public async Task DataSetDoesNotExist_Returns404() ReleaseFile releaseFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture.DefaultFile()) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(releaseVersion); await TestApp.AddTestData(context => @@ -876,8 +874,7 @@ public async Task PublicationDoesNotExist_Returns404() .Generate(1) ); - File file = DataFixture - .DefaultFile(); + File file = DataFixture.DefaultFile(FileType.Data); var releaseVersion = publication.ReleaseVersions.Single(); @@ -912,7 +909,8 @@ private async Task GetDataSet(Guid dataSetId, HttpClient? c } } - public class CreateDataSetTests(TestApplicationFactory testApp) : DataSetsControllerTests(testApp) + public class CreateDataSetTests( + TestApplicationFactory testApp) : DataSetsControllerTests(testApp) { [Fact] public async Task Success() @@ -929,7 +927,7 @@ public async Task Success() ReleaseFile releaseFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture.DefaultFile()) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(draftReleaseVersion); await TestApp.AddTestData(context => @@ -971,15 +969,15 @@ await TestApp.AddTestData(context => return new CreateDataSetResponseViewModel { - DataSetId = dataSet!.Id, - DataSetVersionId = dataSetVersion!.Id, + DataSetId = dataSet.Id, + DataSetVersionId = dataSetVersion.Id, InstanceId = Guid.NewGuid() }; }); var client = BuildApp(processorClient.Object).CreateClient(); - var response = await CreateDataSetVersion(releaseFile.Id, client); + var response = await CreateDataSet(releaseFile.Id, client); MockUtils.VerifyAllMocks(processorClient); @@ -1010,7 +1008,7 @@ public async Task NotBauUser_Returns403() { var client = BuildApp(user: AuthenticatedUser()).CreateClient(); - var response = await CreateDataSetVersion(Guid.NewGuid(), client); + var response = await CreateDataSet(Guid.NewGuid(), client); response.AssertForbidden(); } @@ -1042,7 +1040,7 @@ public async Task ProcessorReturns400_Returns400_WithProcessorErrors() .ReturnsAsync(ValidationUtils.ValidationResult(processorErrors)); var client = BuildApp(processorClient.Object).CreateClient(); - var response = await CreateDataSetVersion(releaseFileId, client); + var response = await CreateDataSet(releaseFileId, client); MockUtils.VerifyAllMocks(processorClient); @@ -1061,16 +1059,13 @@ private WebApplicationFactory BuildApp( .SetUser(user ?? BauUser()); } - private async Task CreateDataSetVersion( + private async Task CreateDataSet( Guid releaseFileId, HttpClient? client = null) { client ??= BuildApp().CreateClient(); - var request = new DataSetVersionCreateRequest - { - ReleaseFileId = releaseFileId - }; + var request = new DataSetCreateRequest { ReleaseFileId = releaseFileId }; return await client.PostAsJsonAsync(BaseUrl, request); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/ReleasesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/ReleasesControllerTests.cs index 35d93a7790f..bed9922015e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/ReleasesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/ReleasesControllerTests.cs @@ -11,15 +11,18 @@ using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Validators; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Moq; +using System.Threading; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationErrorMessages; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; using static Moq.MockBehavior; +using ErrorViewModel = GovUk.Education.ExploreEducationStatistics.Common.ViewModels.ErrorViewModel; namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Controllers.Api { @@ -280,10 +283,10 @@ public async Task GetDeleteDataFilePlan() var fileId = Guid.NewGuid(); - var deleteDataFilePlan = new DeleteDataFilePlan(); + var deleteDataFilePlan = new DeleteDataFilePlanViewModel(); releaseService - .Setup(s => s.GetDeleteDataFilePlan(_releaseVersionId, fileId)) + .Setup(s => s.GetDeleteDataFilePlan(_releaseVersionId, fileId, It.IsAny())) .ReturnsAsync(deleteDataFilePlan); var controller = BuildController(releaseService: releaseService.Object); @@ -296,24 +299,89 @@ public async Task GetDeleteDataFilePlan() } [Fact] - public async Task GetDeleteReleasePlan() + public async Task GetDeleteReleaseVersionPlan() { var releaseService = new Mock(Strict); - var deleteReleasePlan = new DeleteReleasePlan(); + var deleteReleasePlan = new DeleteReleasePlanViewModel(); releaseService - .Setup(s => s.GetDeleteReleasePlan(_releaseVersionId)) + .Setup(s => s.GetDeleteReleaseVersionPlan(_releaseVersionId, It.IsAny())) .ReturnsAsync(deleteReleasePlan); var controller = BuildController(releaseService: releaseService.Object); - var result = await controller.GetDeleteReleasePlan(_releaseVersionId); + var result = await controller.GetDeleteReleaseVersionPlan(_releaseVersionId, It.IsAny()); VerifyAllMocks(releaseService); result.AssertOkResult(deleteReleasePlan); } + [Fact] + public async Task DeleteReleaseVersion_Returns_NoContent() + { + var releaseService = new Mock(Strict); + + var fileId = Guid.NewGuid(); + + releaseService + .Setup(service => service.DeleteReleaseVersion(_releaseVersionId)) + .ReturnsAsync(Unit.Instance); + + var controller = BuildController(releaseService: releaseService.Object); + + var result = await controller.DeleteReleaseVersion(_releaseVersionId); + VerifyAllMocks(releaseService); + + Assert.IsAssignableFrom(result); + } + + [Fact] + public async Task DeleteReleaseVersion_Returns_NotFound() + { + var releaseService = new Mock(Strict); + + var fileId = Guid.NewGuid(); + + releaseService + .Setup(service => service.DeleteReleaseVersion(_releaseVersionId)) + .ReturnsAsync(new NotFoundResult()); + + var controller = BuildController(releaseService: releaseService.Object); + + var result = await controller.DeleteReleaseVersion(_releaseVersionId); + VerifyAllMocks(releaseService); + + result.AssertNotFoundResult(); + } + + [Fact] + public async Task DeleteReleaseVersion_Returns_ValidationProblem() + { + var releaseService = new Mock(Strict); + + var fileId = Guid.NewGuid(); + + releaseService + .Setup(service => service.DeleteReleaseVersion(_releaseVersionId)) + .ReturnsAsync(ValidationUtils.ValidationResult(new ErrorViewModel + { + Code = "error code", + Path = "error path" + })); + + var controller = BuildController(releaseService: releaseService.Object); + + var result = await controller.DeleteReleaseVersion(_releaseVersionId); + VerifyAllMocks(releaseService); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: "error path", + expectedCode: "error code"); + } + [Fact] public async Task CreateReleaseStatus() { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Fixture/TestApplicationFactory.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Fixture/TestApplicationFactory.cs index af92fb96cca..98796d0987a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Fixture/TestApplicationFactory.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Fixture/TestApplicationFactory.cs @@ -1,6 +1,7 @@ #nullable enable using System.Linq; using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using Microsoft.EntityFrameworkCore; @@ -28,17 +29,8 @@ public override async ValueTask DisposeAsync() public async Task ClearTestData() where TDbContext : DbContext { - await using var context = GetDbContext(); - - var tables = context.Model.GetEntityTypes() - .Select(type => type.GetTableName()) - .Distinct() - .ToList(); - - foreach (var table in tables) - { - await context.Database.ExecuteSqlRawAsync(@$"TRUNCATE TABLE ""{table}"" RESTART IDENTITY CASCADE;"); - } + var context = GetDbContext(); + await context.ClearTestData(); } protected override IHostBuilder CreateHostBuilder() @@ -48,7 +40,10 @@ protected override IHostBuilder CreateHostBuilder() .ConfigureServices(services => { services.AddDbContext( - options => options.UseNpgsql(_postgreSqlContainer.GetConnectionString())); + options => options + .UseNpgsql( + _postgreSqlContainer.GetConnectionString(), + psqlOptions => psqlOptions.EnableRetryOnFailure())); using var serviceScope = services.BuildServiceProvider() .GetRequiredService() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataSetVersionServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataSetVersionServiceTests.cs index 48097aac45f..5ba009ae0b6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataSetVersionServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataSetVersionServiceTests.cs @@ -44,9 +44,7 @@ public async Task DataSetVersionForDifferentReleaseVersion() ReleaseFile dataFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture - .DefaultFile() - .WithType(FileType.Data)) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(releaseVersion); DataSetVersion dataSetVersion = DataFixture @@ -78,9 +76,7 @@ private async Task AssertDataSetVersionStatusReturnedOk(DataSetVersionStatus sta ReleaseFile dataFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture - .DefaultFile() - .WithType(FileType.Data)) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(releaseVersion); DataSetVersion dataSetVersion = DataFixture diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/FileUploadsValidatorServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/FileUploadsValidatorServiceTests.cs index a6d59ddafd7..1765c5ea222 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/FileUploadsValidatorServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/FileUploadsValidatorServiceTests.cs @@ -230,7 +230,7 @@ public async Task ValidateDataFilesForUpload_DuplicateDataFile() var releaseFile = _fixture.DefaultReleaseFile() .WithReleaseVersion(releaseVersion) - .WithFile(_fixture.DefaultFile() + .WithFile(_fixture.DefaultFile(FileType.Data) .WithFilename("test.csv")); var contextId = Guid.NewGuid().ToString(); @@ -260,7 +260,7 @@ public async Task ValidateDataFilesForUpload_ReplacingDataFileWithFileOfSameName ReleaseVersion releaseVersion = _fixture.DefaultReleaseVersion(); // The file being replaced here has the same name as the one being uploaded, but that's ok. - var fileBeingReplaced = _fixture.DefaultFile() + var fileBeingReplaced = _fixture.DefaultFile(FileType.Data) .WithFilename("test.csv"); var releaseFile = _fixture.DefaultReleaseFile() @@ -305,7 +305,7 @@ public async Task ValidateDataFilesForUpload_ReplacingDataFileWithFileOfDifferen // Create two release files, one of which is the file being replaced, and the other has the same filename // as the file being uploaded. - var (fileBeingReplaced, otherFile) = _fixture.DefaultFile() + var (fileBeingReplaced, otherFile) = _fixture.DefaultFile(FileType.Data) .ForIndex(0, s => s.SetFilename("test.csv")) .ForIndex(1, s => s.SetFilename("another.csv")) .Generate(2) @@ -521,7 +521,7 @@ public async Task ValidateDataArchiveEntriesForUpload_DuplicateDataFile() var releaseFile = _fixture.DefaultReleaseFile() .WithReleaseVersion(releaseVersion) - .WithFile(_fixture.DefaultFile() + .WithFile(_fixture.DefaultFile(FileType.Data) .WithFilename("test.csv")); var contextId = Guid.NewGuid().ToString(); @@ -550,7 +550,7 @@ public async Task ValidateDataArchiveEntriesForUpload_ReplacingDataFileWithFileO ReleaseVersion releaseVersion = _fixture.DefaultReleaseVersion(); // The file being replaced here has the same name as the one being uploaded, but that's ok. - var fileBeingReplaced = _fixture.DefaultFile() + var fileBeingReplaced = _fixture.DefaultFile(FileType.Data) .WithFilename("test.csv"); ReleaseFile releaseFile = _fixture.DefaultReleaseFile() @@ -589,7 +589,7 @@ public async Task // Create two release files, one of which is the file being replaced, and the other has the same filename // as the file being uploaded. - var (fileBeingReplaced, otherFile) = _fixture.DefaultFile() + var (fileBeingReplaced, otherFile) = _fixture.DefaultFile(FileType.Data) .ForIndex(0, s => s.SetFilename("test.csv")) .ForIndex(1, s => s.SetFilename("another.csv")) .Generate(2) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Public.Data/ProcessorClientTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Public.Data/ProcessorClientTests.cs index 953820815d5..7f132584b31 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Public.Data/ProcessorClientTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Public.Data/ProcessorClientTests.cs @@ -87,19 +87,6 @@ public async Task HttpClientBadRequest_ReturnsBadRequest() left.AssertValidationProblem(Errors.Error1); } - [Fact] - public async Task HttpClientNotFound_ReturnsNotFound() - { - _mockHttp.Expect(HttpMethod.Post, Uri.AbsoluteUri) - .Respond(HttpStatusCode.NotFound); - - var response = await _processorClient.CreateDataSet(releaseFileId: Guid.NewGuid()); - - _mockHttp.VerifyNoOutstandingExpectation(); - - var left = response.AssertLeft(); - left.AssertNotFoundResult(); - } [Theory] [InlineData(HttpStatusCode.RequestTimeout)] @@ -109,6 +96,7 @@ public async Task HttpClientNotFound_ReturnsNotFound() [InlineData(HttpStatusCode.Conflict)] [InlineData(HttpStatusCode.Forbidden)] [InlineData(HttpStatusCode.Gone)] + [InlineData(HttpStatusCode.NotFound)] [InlineData(HttpStatusCode.NotAcceptable)] public async Task HttpClientFailureStatusCode_ThrowsException( HttpStatusCode responseStatusCode) @@ -123,11 +111,6 @@ await Assert.ThrowsAsync(async () => _mockHttp.VerifyNoOutstandingExpectation(); } - - private enum Errors - { - Error1, - } } public class DeleteDataSetVersionTests : ProcessorClientTests @@ -174,7 +157,7 @@ public async Task HttpClientBadRequest_ReturnsBadRequest() var left = response.AssertLeft(); left.AssertValidationProblem(Errors.Error1); } - + [Fact] public async Task HttpClientNotFound_ReturnsNotFound() { @@ -215,10 +198,82 @@ await Assert.ThrowsAsync(async () => _mockHttp.VerifyNoOutstandingExpectation(); } + } + + public class BulkDeleteDataSetVersionsTests : ProcessorClientTests + { + private static readonly Uri Uri = new(BaseUri, "api/BulkDeleteDataSetVersions"); + + [Fact] + public async Task HttpClientSuccess() + { + var releaseVersionId = Guid.NewGuid(); + + _mockHttp.Expect(HttpMethod.Delete, $"{Uri.AbsoluteUri}/{releaseVersionId}") + .Respond(HttpStatusCode.NoContent); + + var response = await _processorClient.BulkDeleteDataSetVersions(releaseVersionId); + + _mockHttp.VerifyNoOutstandingExpectation(); + + response.AssertRight(); + } + + [Fact] + public async Task HttpClientBadRequest_ReturnsBadRequest() + { + var releaseVersionId = Guid.NewGuid(); + + _mockHttp.Expect(HttpMethod.Delete, $"{Uri.AbsoluteUri}/{releaseVersionId}") + .Respond( + HttpStatusCode.BadRequest, + JsonContent.Create(new ValidationProblemViewModel + { + Errors = new ErrorViewModel[] + { + new() { + Code = Errors.Error1.ToString() + } + } + })); + + var response = await _processorClient.BulkDeleteDataSetVersions(releaseVersionId); + + _mockHttp.VerifyNoOutstandingExpectation(); + + var left = response.AssertLeft(); + left.AssertValidationProblem(Errors.Error1); + } - private enum Errors + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.InternalServerError)] + [InlineData(HttpStatusCode.BadGateway)] + [InlineData(HttpStatusCode.Unauthorized)] + [InlineData(HttpStatusCode.Conflict)] + [InlineData(HttpStatusCode.Forbidden)] + [InlineData(HttpStatusCode.Gone)] + [InlineData(HttpStatusCode.NotAcceptable)] + [InlineData(HttpStatusCode.NotFound)] + public async Task HttpClientFailureStatusCode_ThrowsException( + HttpStatusCode responseStatusCode) { - Error1, + var releaseVersionId = Guid.NewGuid(); + + _mockHttp.Expect(HttpMethod.Delete, $"{Uri.AbsoluteUri}/{releaseVersionId}") + .Respond(responseStatusCode); + + await Assert.ThrowsAsync(async () => + { + await _processorClient.BulkDeleteDataSetVersions(releaseVersionId); + }); + + _mockHttp.VerifyNoOutstandingExpectation(); } } + + private enum Errors + { + Error1, + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServiceTests.cs index c0f24abf39f..70f176bdbab 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServiceTests.cs @@ -1,4 +1,8 @@ #nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -14,10 +18,6 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model.Tests.Fixtures; using Microsoft.EntityFrameworkCore; using Moq; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.DbUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions.AssertExtensions; @@ -36,15 +36,9 @@ public class ReleaseAmendmentServiceTests [Fact] public async Task CreateReleaseAmendment() { - var originalCreatedBy = new User - { - Id = Guid.NewGuid() - }; + var originalCreatedBy = new User { Id = Guid.NewGuid() }; - var amendmentCreator = new User - { - Id = _userId - }; + var amendmentCreator = new User { Id = _userId }; var dataBlockParents = _fixture .DefaultDataBlockParent() @@ -67,7 +61,12 @@ public async Task CreateReleaseAmendment() created: DateTime.UtcNow.AddDays(-2), createdById: originalCreatedBy.Id) .WithPublishScheduled(DateTime.Now.AddDays(1)) - .WithNextReleaseDate(new PartialDate { Day = "1", Month = "1", Year = "2040" }) + .WithNextReleaseDate(new PartialDate + { + Day = "1", + Month = "1", + Year = "2040" + }) .WithPublished(DateTime.UtcNow.AddDays(-1)) .WithApprovalStatus(ReleaseApprovalStatus.Approved) .WithPreviousVersionId(Guid.NewGuid()) @@ -113,7 +112,10 @@ public async Task CreateReleaseAmendment() CreatedById = Guid.NewGuid(), })) .WithKeyStatistics(ListOf( - new KeyStatisticText { Title = "key stat text", }, + new KeyStatisticText + { + Title = "key stat text", + }, new KeyStatisticDataBlock { DataBlock = dataBlock3Parent.LatestPublishedVersion!.ContentBlock, @@ -122,21 +124,21 @@ public async Task CreateReleaseAmendment() .DefaultContentSection() .ForIndex(0, s => s .SetContentBlocks(ListOf(_fixture - .DefaultHtmlBlock() - .WithBody("
") - .WithComments(new List - { - new() + .DefaultHtmlBlock() + .WithBody("
") + .WithComments(new List { - Id = Guid.NewGuid(), - Content = "Comment 1 Text" - }, - new() - { - Id = Guid.NewGuid(), - Content = "Comment 2 Text" - } - }), + new() + { + Id = Guid.NewGuid(), + Content = "Comment 1 Text" + }, + new() + { + Id = Guid.NewGuid(), + Content = "Comment 2 Text" + } + }), dataBlock1Parent.LatestPublishedVersion!.ContentBlock, new EmbedBlockLink { @@ -186,7 +188,10 @@ public async Task CreateReleaseAmendment() .WithDataBlockVersions(dataBlockParents .Select(dataBlockParent => dataBlockParent.LatestPublishedVersion!)) .WithKeyStatistics(ListOf( - new KeyStatisticText { Title = "key stat text", }, + new KeyStatisticText + { + Title = "key stat text", + }, new KeyStatisticDataBlock { DataBlock = dataBlock3Parent.LatestPublishedVersion!.ContentBlock, @@ -314,6 +319,8 @@ public async Task CreateReleaseAmendment() ) ], Published = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc), + PublicApiDataSetId = Guid.NewGuid(), + PublicApiDataSetVersion = "1.0.0", }, new() { @@ -384,37 +391,39 @@ public async Task CreateReleaseAmendment() var amendment = RetrieveAmendment(contentDbContext, amendmentId.Value); // Check the values that we expect to have been copied over successfully from the original Release. - amendment.AssertDeepEqualTo(originalReleaseVersion, Except( - r => r.Id, - r => r.Amendment, - r => r.Publication, - r => r.Release, - r => r.PreviousVersion!, - r => r.PreviousVersionId!, - r => r.Version, - r => r.Published!, - r => r.PublishScheduled!, - r => r.Live, - r => r.ApprovalStatus, - r => r.Created, - r => r.CreatedBy, - r => r.CreatedById, - r => r.NotifiedOn!, - r => r.NotifySubscribers, - r => r.UpdatePublishedDate, - r => r.LatestInternalReleaseNote!, - r => r.RelatedInformation, - r => r.Updates, - r => r.Content, - r => r.ReleaseStatuses, - r => r.DataBlockVersions, - r => r.KeyStatistics, - r => r.GenericContent, - r => r.HeadlinesSection, - r => r.KeyStatisticsSecondarySection, - r => r.RelatedDashboardsSection, - r => r.SummarySection, - r => r.FeaturedTables)); + amendment.AssertDeepEqualTo( + originalReleaseVersion, + notEqualProperties: Except( + r => r.Id, + r => r.Amendment, + r => r.Publication, + r => r.Release, + r => r.PreviousVersion!, + r => r.PreviousVersionId!, + r => r.Version, + r => r.Published!, + r => r.PublishScheduled!, + r => r.Live, + r => r.ApprovalStatus, + r => r.Created, + r => r.CreatedBy, + r => r.CreatedById, + r => r.NotifiedOn!, + r => r.NotifySubscribers, + r => r.UpdatePublishedDate, + r => r.LatestInternalReleaseNote!, + r => r.RelatedInformation, + r => r.Updates, + r => r.Content, + r => r.ReleaseStatuses, + r => r.DataBlockVersions, + r => r.KeyStatistics, + r => r.GenericContent, + r => r.HeadlinesSection, + r => r.KeyStatisticsSecondarySection, + r => r.RelatedDashboardsSection, + r => r.SummarySection, + r => r.FeaturedTables)); // Check fields that should be set to new values for an amendment, rather than copied from the original // Release. @@ -551,7 +560,8 @@ public async Task CreateReleaseAmendment() .OfType() .SingleAsync(block => block.ReleaseVersionId == amendment.Id); - var originalEmbedBlockLink = Assert.IsType(originalReleaseVersion.Content[0].Content[2]); + var originalEmbedBlockLink = + Assert.IsType(originalReleaseVersion.Content[0].Content[2]); Assert.NotEqual(originalEmbedBlockLink.Id, amendmentEmbedBlockLink.Id); Assert.NotEqual(originalEmbedBlockLink.EmbedBlockId, amendmentEmbedBlockLink.EmbedBlockId); Assert.Equal(originalEmbedBlockLink.EmbedBlock.Title, amendmentEmbedBlockLink.EmbedBlock.Title); @@ -601,22 +611,24 @@ public async Task CreateReleaseAmendment() amendment.FeaturedTables.ForEach((amendedTable, index) => { var originalTable = originalReleaseVersion.FeaturedTables[index]; - amendedTable.AssertDeepEqualTo(originalTable, Except( - ft => ft.Id, - ft => ft.DataBlock, - ft => ft.DataBlockId, - // Note that we're ignoring DataBlockParent here only, not DataBlockParentId. - // This is because the LatestPublishedVersion and LatestDraftVersion hanging from - // the representation of DataBlockParent on the amendment is more up-to-date than - // that of the original Featured Table's setup state since going through the amendment - // process. We expect both versions of the FeaturedTable to have the same - // DataBlockParentId though. - ft => ft.DataBlockParent, - ft => ft.ReleaseVersion, - ft => ft.ReleaseVersionId, - ft => ft.Created, - ft => ft.CreatedById!, - ft => ft.Updated!)); + amendedTable.AssertDeepEqualTo( + originalTable, + notEqualProperties: Except( + ft => ft.Id, + ft => ft.DataBlock, + ft => ft.DataBlockId, + // Note that we're ignoring DataBlockParent here only, not DataBlockParentId. + // This is because the LatestPublishedVersion and LatestDraftVersion hanging from + // the representation of DataBlockParent on the amendment is more up-to-date than + // that of the original Featured Table's setup state since going through the amendment + // process. We expect both versions of the FeaturedTable to have the same + // DataBlockParentId though. + ft => ft.DataBlockParent, + ft => ft.ReleaseVersion, + ft => ft.ReleaseVersionId, + ft => ft.Created, + ft => ft.CreatedById!, + ft => ft.Updated!)); Assert.NotEqual(Guid.Empty, amendedTable.Id); Assert.NotEqual(Guid.Empty, amendedTable.DataBlockParentId); @@ -641,7 +653,8 @@ public async Task CreateReleaseAmendment() { // Check the Statistics Release has been amended OK. It should have the same Id as the new Content // Release amendment. - var statsReleaseVersionAmendment = statisticsDbContext.ReleaseVersion.SingleOrDefault(rv => rv.Id == amendmentId); + var statsReleaseVersionAmendment = + statisticsDbContext.ReleaseVersion.SingleOrDefault(rv => rv.Id == amendmentId); Assert.NotNull(statsReleaseVersionAmendment); Assert.Equal(originalReleaseVersion.PublicationId, statsReleaseVersionAmendment.PublicationId); @@ -654,13 +667,15 @@ public async Task CreateReleaseAmendment() var releaseSubjectAmendment = Assert.Single(releaseSubjectLinks); - releaseSubjectAmendment.AssertDeepEqualTo(releaseSubject, Except( - rs => rs.Created!, - rs => rs.Updated!, - // SubjectId will be the same despite a different instance of Subject itself. - rs => rs.Subject, - rs => rs.ReleaseVersion, - rs => rs.ReleaseVersionId)); + releaseSubjectAmendment.AssertDeepEqualTo( + releaseSubject, + notEqualProperties: Except( + rs => rs.Created!, + rs => rs.Updated!, + // SubjectId will be the same despite a different instance of Subject itself. + rs => rs.Subject, + rs => rs.ReleaseVersion, + rs => rs.ReleaseVersionId)); releaseSubjectAmendment.Created.AssertUtcNow(withinMillis: 1500); Assert.Null(releaseSubjectAmendment.Updated); @@ -781,10 +796,7 @@ Content 1 await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { contentDbContext.ReleaseVersions.Add(originalReleaseVersion); - contentDbContext.Users.Add(new User - { - Id = _userId - }); + contentDbContext.Users.Add(new User { Id = _userId }); await contentDbContext.SaveChangesAsync(); } @@ -849,10 +861,7 @@ public async Task NullHtmlBlockBody() await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { contentDbContext.ReleaseVersions.Add(originalReleaseVersion); - contentDbContext.Users.Add(new User - { - Id = _userId - }); + contentDbContext.Users.Add(new User { Id = _userId }); await contentDbContext.SaveChangesAsync(); } @@ -895,10 +904,7 @@ public async Task CreatesRelatedDashboardsSectionIfNotOnOriginal() await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { contentDbContext.ReleaseVersions.Add(originalReleaseVersion); - contentDbContext.Users.Add(new User - { - Id = _userId - }); + contentDbContext.Users.Add(new User { Id = _userId }); await contentDbContext.SaveChangesAsync(); } @@ -980,10 +986,7 @@ public async Task CopyFootnotes() await using (var contentDbContext = InMemoryApplicationDbContext(contextId)) { contentDbContext.ReleaseVersions.AddRange(originalReleaseVersion); - contentDbContext.Users.AddRange(new User - { - Id = _userId - }); + contentDbContext.Users.AddRange(new User { Id = _userId }); await contentDbContext.SaveChangesAsync(); @@ -1076,7 +1079,8 @@ private void AssertFootnoteDetailsCopiedCorrectly(Footnote originalFootnote, Foo .SelectNullSafe(f => f.IndicatorId)); } - private DataBlock GetMatchingDataBlock(List amendmentDataBlockVersions, DataBlockParent dataBlockToFind) + private DataBlock GetMatchingDataBlock(List amendmentDataBlockVersions, + DataBlockParent dataBlockToFind) { return amendmentDataBlockVersions .Where(dataBlockVersion => dataBlockVersion.Name == dataBlockToFind.LatestDraftVersion!.Name) @@ -1086,13 +1090,16 @@ private DataBlock GetMatchingDataBlock(List amendmentDataBlock private void AssertAmendedLinkCorrect(Link amendedLink, Link originalLink) { - amendedLink.AssertDeepEqualTo(originalLink, Except(l => l.Id)); + amendedLink.AssertDeepEqualTo( + originalLink, + notEqualProperties: Except(l => l.Id)); } private void AssertAmendedUpdateCorrect(Update amendedUpdate, Update originalUpdate, ReleaseVersion amendment) { - amendedUpdate.AssertDeepEqualTo(originalUpdate, - Except( + amendedUpdate.AssertDeepEqualTo( + originalUpdate, + notEqualProperties: Except( u => u.Id, u => u.ReleaseVersion, u => u.ReleaseVersionId, @@ -1157,7 +1164,9 @@ private static void AssertAmendedReleaseRoleCorrect( Assert.Equal(originalReleaseRole.DeletedById, amendedReleaseRole.DeletedById); } - private static void AssertAmendedReleaseFileCorrect(ReleaseFile originalFile, ReleaseFile amendmentDataFile, + private static void AssertAmendedReleaseFileCorrect( + ReleaseFile originalFile, + ReleaseFile amendmentDataFile, ReleaseVersion amendment) { // Assert it's a new link table entry between the Release amendment and the data file reference @@ -1170,6 +1179,9 @@ private static void AssertAmendedReleaseFileCorrect(ReleaseFile originalFile, Re originalFile.FilterSequence.AssertDeepEqualTo(amendmentDataFile.FilterSequence); originalFile.IndicatorSequence.AssertDeepEqualTo(amendmentDataFile.IndicatorSequence); + Assert.Equal(originalFile.PublicApiDataSetId, amendmentDataFile.PublicApiDataSetId); + Assert.Equal(originalFile.PublicApiDataSetVersion, amendmentDataFile.PublicApiDataSetVersion); + // And assert that the file referenced is the SAME file reference as linked from the original Release's // link table entry Assert.Equal(originalFile.File.Id, amendmentDataFile.File.Id); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseChecklistServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseChecklistServiceTests.cs index 1bcd4c1aefb..85b95f45d51 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseChecklistServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseChecklistServiceTests.cs @@ -57,52 +57,38 @@ public async Task GetChecklist_AllErrors() PreviousVersion = originalReleaseVersion, Version = 1, Created = DateTime.UtcNow.AddMonths(-1), - GenericContent = new List - { - new() - { - Type = ContentSectionType.Generic, - Content = new List() - }, - new() + GenericContent = + new List { - Type = ContentSectionType.Generic, - Content = new List + new() { - new HtmlBlock - { - Body = "

Test

" - }, - new DataBlock(), - new HtmlBlock - { - Body = "" - } - } - } - }, - RelatedDashboardsSection = new ContentSection - { - Type = ContentSectionType.RelatedDashboards, - Content = new List - { - new HtmlBlock + Type = ContentSectionType.Generic, + Content = new List() + }, + new() { - Body = "" + Type = ContentSectionType.Generic, + Content = + new List + { + new HtmlBlock { Body = "

Test

" }, + new DataBlock(), + new HtmlBlock { Body = "" } + } } - } - }, - SummarySection = new ContentSection - { - Type = ContentSectionType.ReleaseSummary, - Content = new List + }, + RelatedDashboardsSection = + new ContentSection { - new HtmlBlock - { - Body = "" - } - } - }, + Type = ContentSectionType.RelatedDashboards, + Content = new List { new HtmlBlock { Body = "" } } + }, + SummarySection = + new ContentSection + { + Type = ContentSectionType.ReleaseSummary, + Content = new List { new HtmlBlock { Body = "" } } + }, Updates = new List { new() @@ -140,10 +126,7 @@ public async Task GetChecklist_AllErrors() releaseDataFileRepository .Setup(r => r.ListReplacementDataFiles(releaseVersion.Id)) .ReturnsAsync( - new List - { - new() - } + new List { new() } ); dataImportService @@ -155,7 +138,7 @@ public async Task GetChecklist_AllErrors() .ReturnsAsync(ValidationActionResult(PublicDataGuidanceRequired)); dataSetVersionService - .Setup(s => s.GetStatusesForReleaseVersion(releaseVersion.Id)) + .Setup(s => s.GetStatusesForReleaseVersion(releaseVersion.Id, default)) .ReturnsAsync([ new DataSetVersionStatusSummary( Id: Guid.NewGuid(), @@ -221,10 +204,7 @@ public async Task GetChecklist_AllErrors() [Fact] public async Task GetChecklist_AllWarningsWithNoDataFiles() { - var releaseVersion = new ReleaseVersion - { - Publication = new Publication() - }; + var releaseVersion = new ReleaseVersion { Publication = new Publication() }; var contextId = Guid.NewGuid().ToString(); @@ -238,7 +218,7 @@ public async Task GetChecklist_AllWarningsWithNoDataFiles() var methodologyVersionRepository = new Mock(MockBehavior.Strict); var releaseDataFileRepository = new Mock(MockBehavior.Strict); var dataSetVersionService = new Mock(MockBehavior.Strict); - + await using (var context = InMemoryContentDbContext(contextId)) { methodologyVersionRepository @@ -256,9 +236,9 @@ public async Task GetChecklist_AllWarningsWithNoDataFiles() dataGuidanceService .Setup(s => s.ValidateForReleaseChecklist(releaseVersion.Id, default)) .ReturnsAsync(ValidationActionResult(PublicDataGuidanceRequired)); - + dataSetVersionService - .Setup(s => s.GetStatusesForReleaseVersion(releaseVersion.Id)) + .Setup(s => s.GetStatusesForReleaseVersion(releaseVersion.Id, default)) .ReturnsAsync([]); var service = BuildReleaseChecklistService( @@ -291,10 +271,7 @@ public async Task GetChecklist_AllWarningsWithNoDataFiles() [Fact] public async Task GetChecklist_AllWarningsWithDataFiles() { - var releaseVersion = new ReleaseVersion - { - Publication = new Publication() - }; + var releaseVersion = new ReleaseVersion { Publication = new Publication() }; var contextId = Guid.NewGuid().ToString(); @@ -313,14 +290,9 @@ public async Task GetChecklist_AllWarningsWithDataFiles() await using (var context = InMemoryContentDbContext(contextId)) { - var subject = new Subject - { - Id = Guid.NewGuid() - }; - var otherSubject = new Subject - { - Id = Guid.NewGuid(), - }; + var subject = new Subject { Id = Guid.NewGuid() }; + + var otherSubject = new Subject { Id = Guid.NewGuid() }; methodologyVersionRepository .Setup(mock => mock.GetLatestVersionByPublication(releaseVersion.PublicationId)) @@ -374,9 +346,9 @@ public async Task GetChecklist_AllWarningsWithDataFiles() new(), } ); - + dataSetVersionService - .Setup(s => s.GetStatusesForReleaseVersion(releaseVersion.Id)) + .Setup(s => s.GetStatusesForReleaseVersion(releaseVersion.Id, default)) .ReturnsAsync([]); var service = BuildReleaseChecklistService( @@ -418,15 +390,9 @@ public async Task GetChecklist_AllWarningsWithDataFiles() [Fact] public async Task GetChecklist_AllWarningsWithUnapprovedMethodology() { - var releaseVersion = new ReleaseVersion - { - Publication = new Publication() - }; + var releaseVersion = new ReleaseVersion { Publication = new Publication() }; - var methodologyVersion = new MethodologyVersion - { - Status = Draft - }; + var methodologyVersion = new MethodologyVersion { Status = Draft }; var contextId = Guid.NewGuid().ToString(); @@ -458,9 +424,9 @@ public async Task GetChecklist_AllWarningsWithUnapprovedMethodology() dataGuidanceService .Setup(s => s.ValidateForReleaseChecklist(releaseVersion.Id, default)) .ReturnsAsync(ValidationActionResult(PublicDataGuidanceRequired)); - + dataSetVersionService - .Setup(s => s.GetStatusesForReleaseVersion(releaseVersion.Id)) + .Setup(s => s.GetStatusesForReleaseVersion(releaseVersion.Id, default)) .ReturnsAsync([]); var service = BuildReleaseChecklistService( @@ -501,10 +467,7 @@ public async Task GetChecklist_FullyValid() { var publication = new Publication(); - var methodologyVersion = new MethodologyVersion - { - Status = Approved - }; + var methodologyVersion = new MethodologyVersion { Status = Approved }; var originalReleaseVersion = new ReleaseVersion { @@ -523,64 +486,38 @@ public async Task GetChecklist_FullyValid() Created = DateTime.UtcNow.AddMonths(-1), DataGuidance = "Test guidance", PreReleaseAccessList = "Test access list", - GenericContent = new List - { - new() + GenericContent = + new List { - Type = ContentSectionType.Generic, - Content = new List + new() { - new HtmlBlock - { - Body = "

test

" - } - } - }, - new() - { - Type = ContentSectionType.Generic, - Content = new List + Type = ContentSectionType.Generic, + Content = new List { new HtmlBlock { Body = "

test

" } } + }, + new() { - new DataBlock - { - Id = dataBlockId - } + Type = ContentSectionType.Generic, + Content = new List { new DataBlock { Id = dataBlockId } } } - } - }, - HeadlinesSection = new ContentSection - { - Type = ContentSectionType.Headlines, - Content = new List + }, + HeadlinesSection = + new ContentSection { - new HtmlBlock - { - Body = "Not empty" - } - } - }, - RelatedDashboardsSection = new ContentSection - { - Type = ContentSectionType.RelatedDashboards, - Content = new List + Type = ContentSectionType.Headlines, + Content = new List { new HtmlBlock { Body = "Not empty" } } + }, + RelatedDashboardsSection = + new ContentSection { - new HtmlBlock - { - Body = "Not empty" - } - } - }, - SummarySection = new ContentSection - { - Type = ContentSectionType.ReleaseSummary, - Content = new List + Type = ContentSectionType.RelatedDashboards, + Content = new List { new HtmlBlock { Body = "Not empty" } } + }, + SummarySection = + new ContentSection { - new HtmlBlock - { - Body = "Not empty" - } - } - }, + Type = ContentSectionType.ReleaseSummary, + Content = new List { new HtmlBlock { Body = "Not empty" } } + }, NextReleaseDate = new PartialDate { Month = "12", @@ -621,10 +558,7 @@ public async Task GetChecklist_FullyValid() await using (var context = InMemoryContentDbContext(contextId)) { - var subject = new Subject - { - Id = Guid.NewGuid() - }; + var subject = new Subject { Id = Guid.NewGuid() }; methodologyVersionRepository .Setup(mock => mock.GetLatestVersionByPublication(releaseVersion.PublicationId)) @@ -660,23 +594,15 @@ public async Task GetChecklist_FullyValid() dataBlockService .Setup(s => s.ListDataBlocks(releaseVersion.Id)) .ReturnsAsync( - new List - { - new() - { - Id = dataBlockId - } - } + new List { new() { Id = dataBlockId } } ); dataSetVersionService - .Setup(s => s.GetStatusesForReleaseVersion(releaseVersion.Id)) + .Setup(s => s.GetStatusesForReleaseVersion(releaseVersion.Id, default)) .ReturnsAsync(new[] { - DataSetVersionStatus.Draft, - DataSetVersionStatus.Withdrawn, - DataSetVersionStatus.Published, - DataSetVersionStatus.Deprecated + DataSetVersionStatus.Draft, DataSetVersionStatus.Withdrawn, + DataSetVersionStatus.Published, DataSetVersionStatus.Deprecated } .Select(status => new DataSetVersionStatusSummary( Id: Guid.NewGuid(), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServicePermissionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServicePermissionTests.cs index 7a7230023fa..29860902761 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServicePermissionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServicePermissionTests.cs @@ -6,6 +6,7 @@ using GovUk.Education.ExploreEducationStatistics.Admin.Security; using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; @@ -21,7 +22,6 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using Moq; -using Xunit; using static GovUk.Education.ExploreEducationStatistics.Admin.Security.SecurityPolicies; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.MapperUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.PermissionTestUtils; @@ -94,7 +94,7 @@ await PolicyCheckBuilder() } [Fact] - public async Task GetDeleteReleasePlan() + public async Task GetDeleteReleaseVersionPlan() { await PolicyCheckBuilder() .SetupResourceCheckToFail(_releaseVersion, CanDeleteSpecificRelease) @@ -102,7 +102,7 @@ await PolicyCheckBuilder() userService => { var service = BuildReleaseService(userService.Object); - return service.GetDeleteReleasePlan(_releaseVersion.Id); + return service.GetDeleteReleaseVersionPlan(_releaseVersion.Id); } ); } @@ -116,7 +116,7 @@ await PolicyCheckBuilder() userService => { var service = BuildReleaseService(userService.Object); - return service.DeleteRelease(_releaseVersion.Id); + return service.DeleteReleaseVersion(_releaseVersion.Id); } ); } @@ -314,6 +314,8 @@ private ReleaseService BuildReleaseService( Mock.Of(), Mock.Of(), Mock.Of(), + Mock.Of(), + Mock.Of(), new SequentialGuidGenerator(), Mock.Of() ); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServiceTests.cs index 7f4256076e4..0ee8e18d8ef 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServiceTests.cs @@ -3,10 +3,13 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Requests; using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data; +using GovUk.Education.ExploreEducationStatistics.Admin.Validators; using GovUk.Education.ExploreEducationStatistics.Common.Cache; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services; @@ -14,6 +17,7 @@ using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository; @@ -24,10 +28,14 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model; using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Data.Services.Cache; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; +using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Moq; -using Xunit; +using System.Net.Http; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.DbUtils; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.MapperUtils; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationErrorMessages; @@ -38,11 +46,14 @@ using IReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IReleaseVersionRepository; using ReleaseVersion = GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion; using Unit = GovUk.Education.ExploreEducationStatistics.Common.Model.Unit; +using GovUk.Education.ExploreEducationStatistics.Admin.Validators.ErrorDetails; +using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services { public class ReleaseServiceTests { + private readonly DataFixture _dataFixture = new(); private static readonly User User = new() { Id = Guid.NewGuid() @@ -287,31 +298,194 @@ await context.AddAsync( } [Fact] - public async Task RemoveDataFiles() + public async Task GetDeleteDataFilePlan() { - var releaseVersion = new ReleaseVersion + ReleaseVersion releaseVersion = _dataFixture + .DefaultReleaseVersion(); + + Subject subject = _dataFixture + .DefaultSubject(); + + File file = _dataFixture + .DefaultFile() + .WithSubjectId(subject.Id) + .WithType(FileType.Data); + + ReleaseFile releaseFile = _dataFixture + .DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(file); + + var contextId = Guid.NewGuid().ToString(); + + await using (var contentDbContext = InMemoryApplicationDbContext(contextId)) + await using (var statisticsDbContext = InMemoryStatisticsDbContext(contextId)) { - ApprovalStatus = ReleaseApprovalStatus.Draft - }; + contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.ReleaseFiles.Add(releaseFile); + await contentDbContext.SaveChangesAsync(); - var subject = new Subject + statisticsDbContext.Subject.Add(subject); + await statisticsDbContext.SaveChangesAsync(); + } + + var dataBlockService = new Mock(Strict); + var footnoteRepository = new Mock(Strict); + + var deleteDataBlockPlan = new DeleteDataBlockPlanViewModel(); + dataBlockService.Setup(service => service.GetDeletePlan(releaseVersion.Id, It.Is(s => s.Id == subject.Id))) + .ReturnsAsync(deleteDataBlockPlan); + + var footnote = new Footnote { Id = Guid.NewGuid() }; + footnoteRepository.Setup(service => service.GetFootnotes(releaseVersion.Id, subject.Id)) + .ReturnsAsync([footnote]); + + await using (var contentDbContext = InMemoryApplicationDbContext(contextId)) + await using (var statisticsDbContext = InMemoryStatisticsDbContext(contextId)) { - Id = Guid.NewGuid() - }; + var releaseService = BuildReleaseService( + contentDbContext: contentDbContext, + statisticsDbContext: statisticsDbContext, + dataBlockService: dataBlockService.Object, + footnoteRepository: footnoteRepository.Object); + + var result = await releaseService.GetDeleteDataFilePlan( + releaseVersionId: releaseVersion.Id, + fileId: file.Id); + + VerifyAllMocks(dataBlockService, + footnoteRepository); + + var deleteDataFilePlan = result.AssertRight(); + + Assert.Equal(releaseVersion.Id, deleteDataFilePlan.ReleaseId); + Assert.Equal(subject.Id, deleteDataFilePlan.SubjectId); + Assert.Equal(deleteDataBlockPlan, deleteDataFilePlan.DeleteDataBlockPlan); + Assert.Equal([footnote.Id], deleteDataFilePlan.FootnoteIds); + Assert.Null(deleteDataFilePlan.DeleteApiDataSetVersionPlan); + Assert.True(deleteDataFilePlan.Valid); + } + } + + [Fact] + public async Task GetDeleteDataFilePlan_FileIsLinkedToPublicApiDataSet_InvalidPlan() + { + DataSet dataSet = _dataFixture + .DefaultDataSet(); + + DataSetVersion dataSetVersion = _dataFixture + .DefaultDataSetVersion() + .WithDataSet(dataSet); + + ReleaseVersion releaseVersion = _dataFixture + .DefaultReleaseVersion(); + + Subject subject = _dataFixture + .DefaultSubject(); - var file = new File + File file = _dataFixture + .DefaultFile() + .WithSubjectId(subject.Id) + .WithType(FileType.Data); + + ReleaseFile releaseFile = _dataFixture + .DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(file) + .WithPublicApiDataSetId(dataSet.Id) + .WithPublicApiDataSetVersion(dataSetVersion.FullSemanticVersion()); + + var contextId = Guid.NewGuid().ToString(); + + await using (var contentDbContext = InMemoryApplicationDbContext(contextId)) + await using (var statisticsDbContext = InMemoryStatisticsDbContext(contextId)) { - Filename = "data.csv", - Type = FileType.Data, - SubjectId = subject.Id - }; + contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.ReleaseFiles.Add(releaseFile); + await contentDbContext.SaveChangesAsync(); + + statisticsDbContext.Subject.Add(subject); + await statisticsDbContext.SaveChangesAsync(); + } + + var dataSetVersionService = new Mock(Strict); + var dataBlockService = new Mock(Strict); + var footnoteRepository = new Mock(Strict); + + dataSetVersionService.Setup(service => service.GetDataSetVersion( + releaseFile.PublicApiDataSetId!.Value, + releaseFile.PublicApiDataSetVersion!, + It.IsAny())) + .ReturnsAsync(dataSetVersion); + + var deleteDataBlockPlan = new DeleteDataBlockPlanViewModel(); + dataBlockService.Setup(service => service.GetDeletePlan(releaseVersion.Id, It.Is(s => s.Id == subject.Id))) + .ReturnsAsync(deleteDataBlockPlan); + + var footnote = new Footnote { Id = Guid.NewGuid() }; + footnoteRepository.Setup(service => service.GetFootnotes(releaseVersion.Id, subject.Id)) + .ReturnsAsync([footnote]); + + await using (var contentDbContext = InMemoryApplicationDbContext(contextId)) + await using (var statisticsDbContext = InMemoryStatisticsDbContext(contextId)) + { + var releaseService = BuildReleaseService( + contentDbContext: contentDbContext, + statisticsDbContext: statisticsDbContext, + dataSetVersionService: dataSetVersionService.Object, + dataBlockService: dataBlockService.Object, + footnoteRepository: footnoteRepository.Object); + + var result = await releaseService.GetDeleteDataFilePlan( + releaseVersionId: releaseVersion.Id, + fileId: file.Id); + + VerifyAllMocks(dataSetVersionService, + dataBlockService, + footnoteRepository); + + var deleteDataFilePlan = result.AssertRight(); + + Assert.Equal(releaseVersion.Id, deleteDataFilePlan.ReleaseId); + Assert.Equal(subject.Id, deleteDataFilePlan.SubjectId); + Assert.Equal(deleteDataBlockPlan, deleteDataFilePlan.DeleteDataBlockPlan); + Assert.Equal([footnote.Id], deleteDataFilePlan.FootnoteIds); + Assert.NotNull(deleteDataFilePlan.DeleteApiDataSetVersionPlan); + Assert.Equal(dataSet.Id, deleteDataFilePlan.DeleteApiDataSetVersionPlan.DataSetId); + Assert.Equal(dataSet.Title, deleteDataFilePlan.DeleteApiDataSetVersionPlan.DataSetTitle); + Assert.Equal(dataSetVersion.Id, deleteDataFilePlan.DeleteApiDataSetVersionPlan.Id); + Assert.Equal(dataSetVersion.Version, deleteDataFilePlan.DeleteApiDataSetVersionPlan.Version); + Assert.Equal(dataSetVersion.Status, deleteDataFilePlan.DeleteApiDataSetVersionPlan.Status); + Assert.False(deleteDataFilePlan.DeleteApiDataSetVersionPlan.Valid); + Assert.False(deleteDataFilePlan.Valid); + } + } + + [Fact] + public async Task RemoveDataFiles() + { + ReleaseVersion releaseVersion = _dataFixture + .DefaultReleaseVersion(); + + Subject subject = _dataFixture + .DefaultSubject(); + + File file = _dataFixture + .DefaultFile() + .WithSubjectId(subject.Id) + .WithType(FileType.Data); + + ReleaseFile releaseFile = _dataFixture + .DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(file); var contextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contextId)) await using (var statisticsDbContext = InMemoryStatisticsDbContext(contextId)) { contentDbContext.ReleaseVersions.Add(releaseVersion); - contentDbContext.Files.Add(file); + contentDbContext.ReleaseFiles.Add(releaseFile); await contentDbContext.SaveChangesAsync(); statisticsDbContext.Subject.Add(subject); @@ -332,9 +506,9 @@ public async Task RemoveDataFiles() dataBlockService.Setup(service => service.GetDeletePlan(releaseVersion.Id, It.Is(s => s.Id == subject.Id))) - .ReturnsAsync(new DeleteDataBlockPlan()); + .ReturnsAsync(new DeleteDataBlockPlanViewModel()); - dataBlockService.Setup(service => service.DeleteDataBlocks(It.IsAny())) + dataBlockService.Setup(service => service.DeleteDataBlocks(It.IsAny())) .ReturnsAsync(Unit.Instance); dataImportService.Setup(service => service.GetImport(file.Id)) @@ -382,29 +556,28 @@ public async Task RemoveDataFiles() [Fact] public async Task RemoveDataFiles_FileImporting() { - var releaseVersion = new ReleaseVersion - { - ApprovalStatus = ReleaseApprovalStatus.Draft - }; + ReleaseVersion releaseVersion = _dataFixture + .DefaultReleaseVersion(); - var subject = new Subject - { - Id = Guid.NewGuid() - }; + Subject subject = _dataFixture + .DefaultSubject(); - var file = new File - { - Filename = "data.csv", - Type = FileType.Data, - SubjectId = subject.Id - }; + File file = _dataFixture + .DefaultFile() + .WithSubjectId(subject.Id) + .WithType(FileType.Data); + + ReleaseFile releaseFile = _dataFixture + .DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(file); var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { contentDbContext.ReleaseVersions.Add(releaseVersion); - contentDbContext.Files.Add(file); + contentDbContext.ReleaseFiles.Add(releaseFile); await contentDbContext.SaveChangesAsync(); } @@ -433,44 +606,44 @@ public async Task RemoveDataFiles_FileImporting() [Fact] public async Task RemoveDataFiles_ReplacementExists() { - var releaseVersion = new ReleaseVersion - { - ApprovalStatus = ReleaseApprovalStatus.Draft - }; + ReleaseVersion releaseVersion = _dataFixture + .DefaultReleaseVersion(); - var subject = new Subject - { - Id = Guid.NewGuid() - }; + Subject subject = _dataFixture + .DefaultSubject(); - var replacementSubject = new Subject - { - Id = Guid.NewGuid() - }; + Subject replacementSubject = _dataFixture + .DefaultSubject(); - var file = new File - { - Filename = "data.csv", - Type = FileType.Data, - SubjectId = subject.Id - }; + File file = _dataFixture + .DefaultFile() + .WithSubjectId(subject.Id) + .WithType(FileType.Data); - var replacementFile = new File - { - Filename = "replacement.csv", - Type = FileType.Data, - SubjectId = replacementSubject.Id, - Replacing = file - }; + File replacementFile = _dataFixture + .DefaultFile() + .WithSubjectId(replacementSubject.Id) + .WithType(FileType.Data) + .WithReplacing(file); file.ReplacedBy = replacementFile; + ReleaseFile releaseFile = _dataFixture + .DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(file); + + ReleaseFile replacementReleaseFile = _dataFixture + .DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(replacementFile); + var contextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contextId)) await using (var statisticsDbContext = InMemoryStatisticsDbContext(contextId)) { contentDbContext.ReleaseVersions.Add(releaseVersion); - contentDbContext.Files.AddRange(file, replacementFile); + contentDbContext.ReleaseFiles.AddRange(releaseFile, replacementReleaseFile); await contentDbContext.SaveChangesAsync(); statisticsDbContext.Subject.AddRange(subject, replacementSubject); @@ -494,9 +667,9 @@ public async Task RemoveDataFiles_ReplacementExists() dataBlockService.Setup(service => service.GetDeletePlan(releaseVersion.Id, It.Is(s => new[] { subject.Id, replacementSubject.Id }.Contains(s.Id)))) - .ReturnsAsync(new DeleteDataBlockPlan()); + .ReturnsAsync(new DeleteDataBlockPlanViewModel()); - dataBlockService.Setup(service => service.DeleteDataBlocks(It.IsAny())) + dataBlockService.Setup(service => service.DeleteDataBlocks(It.IsAny())) .ReturnsAsync(Unit.Instance); dataImportService.Setup(service => service.GetImport(It.IsIn(file.Id, replacementFile.Id))) @@ -520,7 +693,8 @@ public async Task RemoveDataFiles_ReplacementExists() await using (var context = InMemoryApplicationDbContext(contextId)) await using (var statisticsDbContext = InMemoryStatisticsDbContext(contextId)) { - var releaseService = BuildReleaseService(context, + var releaseService = BuildReleaseService( + context, statisticsDbContext, cacheService: cacheService.Object, dataBlockService: dataBlockService.Object, @@ -529,10 +703,12 @@ public async Task RemoveDataFiles_ReplacementExists() releaseDataFileService: releaseDataFileService.Object, releaseSubjectRepository: releaseSubjectRepository.Object); - var result = await releaseService.RemoveDataFiles(releaseVersionId: releaseVersion.Id, + var result = await releaseService.RemoveDataFiles( + releaseVersionId: releaseVersion.Id, fileId: file.Id); - VerifyAllMocks(cacheService, + VerifyAllMocks( + cacheService, dataBlockService, dataImportService, footnoteRepository, @@ -540,7 +716,7 @@ public async Task RemoveDataFiles_ReplacementExists() releaseSubjectRepository); dataBlockService.Verify( - mock => mock.DeleteDataBlocks(It.IsAny()), + mock => mock.DeleteDataBlocks(It.IsAny()), Times.Exactly(2)); releaseDataFileService.Verify( @@ -564,43 +740,43 @@ public async Task RemoveDataFiles_ReplacementExists() [Fact] public async Task RemoveDataFiles_ReplacementFileImporting() { - var releaseVersion = new ReleaseVersion - { - ApprovalStatus = ReleaseApprovalStatus.Draft - }; + ReleaseVersion releaseVersion = _dataFixture + .DefaultReleaseVersion(); - var subject = new Subject - { - Id = Guid.NewGuid(), - }; + Subject subject = _dataFixture + .DefaultSubject(); - var replacementSubject = new Subject - { - Id = Guid.NewGuid() - }; + Subject replacementSubject = _dataFixture + .DefaultSubject(); - var file = new File - { - Filename = "data.csv", - Type = FileType.Data, - SubjectId = subject.Id - }; + File file = _dataFixture + .DefaultFile() + .WithSubjectId(subject.Id) + .WithType(FileType.Data); - var replacementFile = new File - { - Filename = "replacement.csv", - Type = FileType.Data, - SubjectId = replacementSubject.Id, - Replacing = file - }; + File replacementFile = _dataFixture + .DefaultFile() + .WithSubjectId(replacementSubject.Id) + .WithType(FileType.Data) + .WithReplacing(file); file.ReplacedBy = replacementFile; + ReleaseFile releaseFile = _dataFixture + .DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(file); + + ReleaseFile replacementReleaseFile = _dataFixture + .DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(replacementFile); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { contentDbContext.ReleaseVersions.Add(releaseVersion); - contentDbContext.Files.AddRange(file, replacementFile); + contentDbContext.ReleaseFiles.AddRange(releaseFile, replacementReleaseFile); await contentDbContext.SaveChangesAsync(); } @@ -631,6 +807,62 @@ public async Task RemoveDataFiles_ReplacementFileImporting() } } + [Fact] + public async Task RemoveDataFiles_FileIsLinkedToPublicApiDataSet_ValidationProblem() + { + ReleaseVersion releaseVersion = _dataFixture + .DefaultReleaseVersion(); + + File file = _dataFixture + .DefaultFile() + .WithType(FileType.Data); + + ReleaseFile releaseFile = _dataFixture + .DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(file) + .WithPublicApiDataSetId(Guid.NewGuid()) + .WithPublicApiDataSetVersion(1, 0); + + var contentDbContextId = Guid.NewGuid().ToString(); + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.ReleaseFiles.Add(releaseFile); + await contentDbContext.SaveChangesAsync(); + } + + var dataImportService = new Mock(Strict); + + dataImportService.Setup(service => service.GetImport(file.Id)) + .ReturnsAsync(new DataImport + { + Status = DataImportStatus.COMPLETE + }); + + await using (var context = InMemoryApplicationDbContext(contentDbContextId)) + { + var releaseService = BuildReleaseService(context, + dataImportService: dataImportService.Object); + + var result = await releaseService.RemoveDataFiles(releaseVersionId: releaseVersion.Id, + fileId: file.Id); + + VerifyAllMocks(dataImportService); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + var errorDetail = validationProblem.AssertHasError( + expectedPath: null, + expectedCode: ValidationMessages.CannotDeleteApiDataSetReleaseFile.Code); + + var apiDataSetErrorDetail = Assert.IsType(errorDetail.Detail); + + Assert.Equal(releaseFile.PublicApiDataSetId, apiDataSetErrorDetail.DataSetId); + } + } + [Fact] public async Task UpdateReleaseVersion() { @@ -1075,7 +1307,7 @@ public async Task GetLatestPublishedRelease_NoPublishedRelease() } [Fact] - public async Task GetDeleteReleasePlan() + public async Task GetDeleteReleaseVersionPlan() { var releaseBeingDeleted = new ReleaseVersion { @@ -1137,7 +1369,8 @@ public async Task GetDeleteReleasePlan() { var releaseService = BuildReleaseService(context); - var result = await releaseService.GetDeleteReleasePlan(releaseBeingDeleted.Id); + var result = await releaseService.GetDeleteReleaseVersionPlan(releaseBeingDeleted.Id); + var plan = result.AssertRight(); // Assert that only the 2 Methodologies that were scheduled with the Release being deleted are flagged @@ -1152,7 +1385,7 @@ public async Task GetDeleteReleasePlan() } [Fact] - public async Task DeleteRelease() + public async Task DeleteReleaseVersion() { var publication = new Publication(); @@ -1232,6 +1465,7 @@ public async Task DeleteRelease() var releaseFileService = new Mock(Strict); var releaseSubjectRepository = new Mock(Strict); var cacheService = new Mock(Strict); + var processorClient = new Mock(Strict); releaseDataFilesService.Setup(mock => mock.DeleteAll(releaseVersion.Id, false)).ReturnsAsync(Unit.Instance); @@ -1247,15 +1481,21 @@ public async Task DeleteRelease() ItIs.DeepEqualTo(new PrivateReleaseContentFolderCacheKey(releaseVersion.Id)))) .Returns(Task.CompletedTask); + processorClient.Setup(mock => mock.BulkDeleteDataSetVersions( + releaseVersion.Id, + It.IsAny())) + .ReturnsAsync(Unit.Instance); + await using (var context = InMemoryApplicationDbContext(contextId)) { var releaseService = BuildReleaseService(context, releaseDataFileService: releaseDataFilesService.Object, releaseFileService: releaseFileService.Object, releaseSubjectRepository: releaseSubjectRepository.Object, - cacheService: cacheService.Object); + cacheService: cacheService.Object, + processorClient: processorClient.Object); - var result = await releaseService.DeleteRelease(releaseVersion.Id); + var result = await releaseService.DeleteReleaseVersion(releaseVersion.Id); releaseDataFilesService.Verify(mock => mock.DeleteAll(releaseVersion.Id, false), Times.Once); @@ -1265,7 +1505,8 @@ public async Task DeleteRelease() VerifyAllMocks(cacheService, releaseDataFilesService, - releaseFileService + releaseFileService, + processorClient ); result.AssertRight(); @@ -1365,6 +1606,91 @@ public async Task DeleteRelease() } } + [Fact] + public async Task DeleteReleaseVersion_ProcessorReturns400_Returns400() + { + var releaseVersion = new ReleaseVersion + { + Id = Guid.NewGuid() + }; + + var contextId = Guid.NewGuid().ToString(); + + await using (var context = InMemoryApplicationDbContext(contextId)) + { + context.ReleaseVersions.Add(releaseVersion); + await context.SaveChangesAsync(); + } + + var processorClient = new Mock(Strict); + + processorClient.Setup(mock => mock.BulkDeleteDataSetVersions( + releaseVersion.Id, + It.IsAny())) + .ReturnsAsync(new BadRequestObjectResult(new ValidationProblemViewModel + { + Errors = new ErrorViewModel[] + { + new() { + Path ="error path", + Code = "error code" + } + } + })); + + await using (var context = InMemoryApplicationDbContext(contextId)) + { + var releaseService = BuildReleaseService( + context, + processorClient: processorClient.Object); + + var result = await releaseService.DeleteReleaseVersion(releaseVersion.Id); + + VerifyAllMocks(processorClient); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: "error path", + expectedCode: "error code"); + } + } + + [Fact] + public async Task DeleteReleaseVersion_ProcessorThrows_Throws() + { + var releaseVersion = new ReleaseVersion + { + Id = Guid.NewGuid() + }; + + var contextId = Guid.NewGuid().ToString(); + + await using (var context = InMemoryApplicationDbContext(contextId)) + { + context.ReleaseVersions.Add(releaseVersion); + await context.SaveChangesAsync(); + } + + var processorClient = new Mock(Strict); + + processorClient.Setup(mock => mock.BulkDeleteDataSetVersions( + releaseVersion.Id, + It.IsAny())) + .ThrowsAsync(new HttpRequestException()); + + await using (var context = InMemoryApplicationDbContext(contextId)) + { + var releaseService = BuildReleaseService( + context, + processorClient: processorClient.Object); + + await Assert.ThrowsAsync(async () => await releaseService.DeleteReleaseVersion(releaseVersion.Id)); + + VerifyAllMocks(processorClient); + } + } + [Fact] public async Task UpdateReleasePublished() { @@ -1889,6 +2215,8 @@ private static ReleaseService BuildReleaseService( IFootnoteRepository? footnoteRepository = null, IDataBlockService? dataBlockService = null, IReleaseSubjectRepository? releaseSubjectRepository = null, + IDataSetVersionService? dataSetVersionService = null, + IProcessorClient? processorClient = null, IBlobCacheService? cacheService = null) { var userService = AlwaysTrueUserService(); @@ -1912,6 +2240,8 @@ private static ReleaseService BuildReleaseService( footnoteRepository ?? Mock.Of(Strict), dataBlockService ?? Mock.Of(Strict), releaseSubjectRepository ?? Mock.Of(Strict), + dataSetVersionService ?? Mock.Of(Strict), + processorClient ?? Mock.Of(Strict), new SequentialGuidGenerator(), cacheService ?? Mock.Of(Strict) ); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReplacementServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReplacementServiceTests.cs index 195613e8803..fada409b36f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReplacementServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReplacementServiceTests.cs @@ -1,11 +1,8 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Cache; using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Cache; +using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Model.Chart; @@ -23,9 +20,15 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Data.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using Microsoft.EntityFrameworkCore; using Moq; -using Xunit; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.DbUtils; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationErrorMessages; using static GovUk.Education.ExploreEducationStatistics.Common.Model.TimeIdentifier; @@ -1695,6 +1698,105 @@ public async Task GetReplacementPlan_ReplacementHasDifferentLocation_LocationMat } } + [Fact] + public async Task GetReplacementPlan_FileIsLinkedToPublicApiDataSet_ReplacementInvalid() + { + DataSet dataSet = _fixture + .DefaultDataSet(); + + DataSetVersion dataSetVersion = _fixture + .DefaultDataSetVersion() + .WithDataSet(dataSet); + + Content.Model.ReleaseVersion releaseVersion = _fixture + .DefaultReleaseVersion(); + + var statsReleaseVersion = _fixture.DefaultStatsReleaseVersion() + .WithId(releaseVersion.Id) + .Generate(); + + var (originalReleaseSubject, replacementReleaseSubject) = _fixture.DefaultReleaseSubject() + .WithReleaseVersion(statsReleaseVersion) + .WithSubjects(_fixture.DefaultSubject().Generate(2)) + .Generate(2) + .ToTuple2(); + + File originalFile = _fixture + .DefaultFile() + .WithSubjectId(originalReleaseSubject.SubjectId) + .WithType(FileType.Data); + + File replacementFile = _fixture + .DefaultFile() + .WithSubjectId(replacementReleaseSubject.SubjectId) + .WithType(FileType.Data); + + var (originalReleaseFile, replacementReleaseFile) = _fixture.DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .ForIndex(0, rv => + rv.SetFile(originalFile) + .SetPublicApiDataSetId(dataSet.Id) + .SetPublicApiDataSetVersion(dataSetVersion.FullSemanticVersion())) + .ForIndex(1, rv => rv.SetFile(replacementFile)) + .Generate(2) + .ToTuple2(); + + var dataSetVersionService = new Mock(Strict); + dataSetVersionService.Setup(mock => mock.GetDataSetVersion( + originalReleaseFile.PublicApiDataSetId!.Value, + originalReleaseFile.PublicApiDataSetVersion!, + It.IsAny())) + .ReturnsAsync(dataSetVersion); + + var locationRepository = new Mock(Strict); + locationRepository.Setup(service => service.GetDistinctForSubject(replacementReleaseSubject.SubjectId)) + .ReturnsAsync(new List()); + + var timePeriodService = new Mock(Strict); + timePeriodService.Setup(service => service.GetTimePeriods(replacementReleaseSubject.SubjectId)) + .ReturnsAsync(new List<(int Year, TimeIdentifier TimeIdentifier)>()); + + var contentDbContextId = Guid.NewGuid().ToString(); + var statisticsDbContextId = Guid.NewGuid().ToString(); + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.ReleaseFiles.AddRange(originalReleaseFile, replacementReleaseFile); + await contentDbContext.SaveChangesAsync(); + } + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + await using (var statisticsDbContext = InMemoryStatisticsDbContext(statisticsDbContextId)) + { + var replacementService = BuildReplacementService( + contentDbContext, + statisticsDbContext, + dataSetVersionService: dataSetVersionService.Object, + timePeriodService: timePeriodService.Object, + locationRepository: locationRepository.Object); + + var result = await replacementService.GetReplacementPlan( + releaseVersionId: releaseVersion.Id, + originalFileId: originalFile.Id, + replacementFileId: replacementFile.Id); + + VerifyAllMocks(dataSetVersionService); + + var replacementPlan = result.AssertRight(); + + Assert.NotNull(replacementPlan.DeleteApiDataSetVersionPlan); + Assert.Equal(dataSet.Id, replacementPlan.DeleteApiDataSetVersionPlan.DataSetId); + Assert.Equal(dataSet.Title, replacementPlan.DeleteApiDataSetVersionPlan.DataSetTitle); + Assert.Equal(dataSetVersion.Id, replacementPlan.DeleteApiDataSetVersionPlan.Id); + Assert.Equal(dataSetVersion.Version, replacementPlan.DeleteApiDataSetVersionPlan.Version); + Assert.Equal(dataSetVersion.Status, replacementPlan.DeleteApiDataSetVersionPlan.Status); + Assert.False(replacementPlan.DeleteApiDataSetVersionPlan.Valid); + + Assert.False(replacementPlan.Valid); + } + } + [Fact] public async Task GetReplacementPlan_AllReplacementDataPresent_ReplacementValid() { @@ -2267,6 +2369,8 @@ public async Task GetReplacementPlan_AllReplacementDataPresent_ReplacementValid( Assert.True(footnoteForSubjectPlan.Valid); + Assert.Null(replacementPlan.DeleteApiDataSetVersionPlan); + Assert.True(replacementPlan.Valid); } } @@ -2593,6 +2697,98 @@ public async Task Replace_ReplacementFileIsNotAssociatedWithOriginalFile() } } + [Fact] + public async Task Replace_FileIsLinkedToPublicApiDataSet_ValidationProblem() + { + DataSet dataSet = _fixture + .DefaultDataSet(); + + DataSetVersion dataSetVersion = _fixture + .DefaultDataSetVersion() + .WithDataSet(dataSet); + + Content.Model.ReleaseVersion releaseVersion = _fixture + .DefaultReleaseVersion(); + + var statsReleaseVersion = _fixture.DefaultStatsReleaseVersion() + .WithId(releaseVersion.Id) + .Generate(); + + var (originalReleaseSubject, replacementReleaseSubject) = _fixture.DefaultReleaseSubject() + .WithReleaseVersion(statsReleaseVersion) + .WithSubjects(_fixture.DefaultSubject().Generate(2)) + .Generate(2) + .ToTuple2(); + + File originalFile = _fixture + .DefaultFile() + .WithSubjectId(originalReleaseSubject.SubjectId) + .WithType(FileType.Data); + + File replacementFile = _fixture + .DefaultFile() + .WithSubjectId(replacementReleaseSubject.SubjectId) + .WithType(FileType.Data); + + var (originalReleaseFile, replacementReleaseFile) = _fixture.DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .ForIndex(0, rv => + rv.SetFile(originalFile) + .SetPublicApiDataSetId(dataSet.Id) + .SetPublicApiDataSetVersion(dataSetVersion.FullSemanticVersion())) + .ForIndex(1, rv => rv.SetFile(replacementFile)) + .Generate(2) + .ToTuple2(); + + var dataSetVersionService = new Mock(Strict); + dataSetVersionService.Setup(mock => mock.GetDataSetVersion( + originalReleaseFile.PublicApiDataSetId!.Value, + originalReleaseFile.PublicApiDataSetVersion!, + It.IsAny())) + .ReturnsAsync(dataSetVersion); + + var locationRepository = new Mock(Strict); + locationRepository.Setup(service => service.GetDistinctForSubject(replacementReleaseSubject.SubjectId)) + .ReturnsAsync(new List()); + + var timePeriodService = new Mock(Strict); + timePeriodService.Setup(service => service.GetTimePeriods(replacementReleaseSubject.SubjectId)) + .ReturnsAsync(new List<(int Year, TimeIdentifier TimeIdentifier)>()); + + var contentDbContextId = Guid.NewGuid().ToString(); + var statisticsDbContextId = Guid.NewGuid().ToString(); + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.ReleaseFiles.AddRange(originalReleaseFile, replacementReleaseFile); + await contentDbContext.SaveChangesAsync(); + } + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + await using (var statisticsDbContext = InMemoryStatisticsDbContext(statisticsDbContextId)) + { + var replacementService = BuildReplacementService( + contentDbContext, + statisticsDbContext, + locationRepository: locationRepository.Object, + timePeriodService: timePeriodService.Object, + dataSetVersionService: dataSetVersionService.Object); + + var result = await replacementService.Replace( + releaseVersionId: releaseVersion.Id, + originalFileId: originalFile.Id, + replacementFileId: replacementFile.Id); + + VerifyAllMocks( + locationRepository, + timePeriodService, + dataSetVersionService); + + result.AssertBadRequest(ReplacementMustBeValid); + } + } + [Fact] public async Task Replace() { @@ -4291,6 +4487,7 @@ private static ReplacementService BuildReplacementService( StatisticsDbContext statisticsDbContext, ILocationRepository? locationRepository = null, IReleaseService? releaseService = null, + IDataSetVersionService dataSetVersionService = null, ITimePeriodService? timePeriodService = null, ICacheKeyService? cacheKeyService = null, IBlobCacheService? blobCacheService = null) @@ -4304,6 +4501,7 @@ private static ReplacementService BuildReplacementService( locationRepository ?? Mock.Of(Strict), new FootnoteRepository(statisticsDbContext), releaseService ?? Mock.Of(Strict), + dataSetVersionService ?? Mock.Of(), timePeriodService ?? Mock.Of(Strict), AlwaysTrueUserService().Object, cacheKeyService ?? Mock.Of(Strict), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataBlocksController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataBlocksController.cs index b69ee370683..e225b04c0ae 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataBlocksController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataBlocksController.cs @@ -43,7 +43,7 @@ public async Task DeleteDataBlock(Guid releaseVersionId, } [HttpGet("releases/{releaseVersionId:guid}/data-blocks/{dataBlockVersionId:guid}/delete-plan")] - public async Task> GetDeletePlan(Guid releaseVersionId, + public async Task> GetDeletePlan(Guid releaseVersionId, Guid dataBlockVersionId) { return await _dataBlockService diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataReplacementController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataReplacementController.cs index dad03a0c9d6..4f19f4bfde7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataReplacementController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataReplacementController.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; @@ -24,12 +25,14 @@ public DataReplacementController(IReplacementService replacementService) [HttpGet("releases/{releaseVersionId:guid}/data/{fileId:guid}/replacement-plan/{replacementFileId:guid}")] public async Task> GetReplacementPlan(Guid releaseVersionId, Guid fileId, - Guid replacementFileId) + Guid replacementFileId, + CancellationToken cancellationToken = default) { return await _replacementService.GetReplacementPlan( releaseVersionId: releaseVersionId, originalFileId: fileId, - replacementFileId: replacementFileId + replacementFileId: replacementFileId, + cancellationToken: cancellationToken ) .OnSuccess(plan => plan.ToSummary()) .HandleFailuresOrOk(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Public.Data/DataSetVersionsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Public.Data/DataSetVersionsController.cs index bd822680ad0..fed054441f0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Public.Data/DataSetVersionsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Public.Data/DataSetVersionsController.cs @@ -2,7 +2,9 @@ using System; using System.Threading; using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Admin.Requests.Public.Data; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data; +using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.Public.Data; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; @@ -14,6 +16,20 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Controllers.Api.Publi [Route("api/public-data/data-set-versions")] public class DataSetVersionsController(IDataSetVersionService dataSetVersionService) : ControllerBase { + [HttpPost] + [Produces("application/json")] + public async Task> CreateNextVersion( + [FromBody] NextDataSetVersionCreateRequest nextDataSetVersionCreateRequest, + CancellationToken cancellationToken) + { + return await dataSetVersionService + .CreateNextVersion( + releaseFileId: nextDataSetVersionCreateRequest.ReleaseFileId, + dataSetId: nextDataSetVersionCreateRequest.DataSetId, + cancellationToken: cancellationToken) + .HandleFailuresOrOk(); + } + [HttpDelete("{dataSetVersionId:guid}")] [Produces("application/json")] public async Task DeleteVersion( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Public.Data/DataSetsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Public.Data/DataSetsController.cs index c56f98798a1..1ad95e72d22 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Public.Data/DataSetsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Public.Data/DataSetsController.cs @@ -48,7 +48,7 @@ public async Task> GetDataSet( [HttpPost] [Produces("application/json")] public async Task> CreateDataSet( - [FromBody] DataSetVersionCreateRequest request, + [FromBody] DataSetCreateRequest request, CancellationToken cancellationToken) { return await dataSetService diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs index 128ad59daa3..475b7339ec1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Models; using GovUk.Education.ExploreEducationStatistics.Admin.Requests; @@ -60,11 +61,11 @@ public async Task> CreateRelease(ReleaseCreateReq } [HttpDelete("release/{releaseVersionId:guid}")] - public async Task> DeleteRelease(Guid releaseVersionId) + public async Task DeleteReleaseVersion(Guid releaseVersionId) { return await _releaseService - .DeleteRelease(releaseVersionId) - .HandleFailuresOrNoContent(); + .DeleteReleaseVersion(releaseVersionId) + .HandleFailuresOrNoContent(convertNotFoundToNoContent: false); } [HttpPost("release/{releaseVersionId:guid}/amendment")] @@ -236,19 +237,26 @@ public Task> GetDataUploadStatus(Guid re } [HttpGet("release/{releaseVersionId:guid}/delete-plan")] - public async Task> GetDeleteReleasePlan(Guid releaseVersionId) + public async Task> GetDeleteReleaseVersionPlan( + Guid releaseVersionId, + CancellationToken cancellationToken) { return await _releaseService - .GetDeleteReleasePlan(releaseVersionId) + .GetDeleteReleaseVersionPlan(releaseVersionId, cancellationToken) .HandleFailuresOrOk(); } [HttpGet("release/{releaseVersionId:guid}/data/{fileId:guid}/delete-plan")] - public async Task> GetDeleteDataFilePlan(Guid releaseVersionId, Guid fileId) + public async Task> GetDeleteDataFilePlan( + Guid releaseVersionId, + Guid fileId, + CancellationToken cancellationToken = default) { return await _releaseService - .GetDeleteDataFilePlan(releaseVersionId: releaseVersionId, - fileId: fileId) + .GetDeleteDataFilePlan( + releaseVersionId: releaseVersionId, + fileId: fileId, + cancellationToken: cancellationToken) .HandleFailuresOrOk(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240617005412_EES4993_AddPublicApiDataSetIdVersionToReleaseFile.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240617005412_EES4993_AddPublicApiDataSetIdVersionToReleaseFile.Designer.cs new file mode 100644 index 00000000000..be2713c500e --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240617005412_EES4993_AddPublicApiDataSetIdVersionToReleaseFile.Designer.cs @@ -0,0 +1,2198 @@ +// +using System; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + [DbContext(typeof(ContentDbContext))] + [Migration("20240617005412_EES4993_AddPublicApiDataSetIdVersionToReleaseFile")] + partial class EES4993_AddPublicApiDataSetIdVersionToReleaseFile + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.4") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContactTelNo") + .HasColumnType("nvarchar(max)"); + + b.Property("TeamEmail") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TeamName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Contacts"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.FreeTextRank", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Rank") + .HasColumnType("int"); + + b.ToTable((string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("LegacyCreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("Resolved") + .HasColumnType("datetime2"); + + b.Property("ResolvedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ResolvedById"); + + b.ToTable("Comment"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentSectionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Locked") + .HasColumnType("datetime2"); + + b.Property("LockedById") + .IsConcurrencyToken() + .HasColumnType("uniqueidentifier"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentSectionId"); + + b.HasIndex("LockedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentBlock", (string)null); + + b.HasDiscriminator("Type").HasValue("ContentBlock"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Caption") + .HasColumnType("nvarchar(max)"); + + b.Property("Heading") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentSections"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestDraftVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("LatestDraftVersionId") + .IsUnique() + .HasFilter("[LatestDraftVersionId] IS NOT NULL"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("DataBlocks", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier") + .HasColumnName("DataBlockId"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("DataBlockVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("ExpectedImportedRows") + .HasColumnType("int"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("GeographicLevels") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ImportedRows") + .HasColumnType("int"); + + b.Property("LastProcessedRowIndex") + .HasColumnType("int"); + + b.Property("MetaFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("StagePercentageComplete") + .HasColumnType("int"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("TotalRows") + .HasColumnType("int"); + + b.Property("ZipFileId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId") + .IsUnique(); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("FileId"), new[] { "Status" }); + + b.HasIndex("MetaFileId") + .IsUnique(); + + b.HasIndex("ZipFileId") + .IsUnique() + .HasFilter("[ZipFileId] IS NOT NULL"); + + b.ToTable("DataImports"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataImportId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("DataImportId"); + + b.ToTable("DataImportErrors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("EmbedBlocks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DataBlockId") + .IsUnique(); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("FeaturedTables"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentLength") + .HasColumnType("bigint"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileMeta") + .HasColumnType("nvarchar(max)"); + + b.Property("DataSetFileVersion") + .HasColumnType("int"); + + b.Property("Filename") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReplacedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReplacingId") + .HasColumnType("uniqueidentifier"); + + b.Property("RootPath") + .HasColumnType("uniqueidentifier"); + + b.Property("SourceId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReplacedById") + .IsUnique() + .HasFilter("[ReplacedById] IS NOT NULL"); + + b.HasIndex("ReplacingId") + .IsUnique() + .HasFilter("[ReplacingId] IS NOT NULL"); + + b.HasIndex("SourceId"); + + b.HasIndex("Type"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.ToTable("GlossaryEntries"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("GuidanceText") + .HasColumnType("nvarchar(max)"); + + b.Property("GuidanceTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Trend") + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("KeyStatistics"); + + b.UseTptMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("OwningPublicationSlug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OwningPublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("Methodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DisplayDate") + .HasColumnType("datetime2"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("MethodologyNotes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("MethodologyVersionId", "Slug"); + + b.ToTable("MethodologyRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("AlternativeSlug") + .HasColumnType("nvarchar(max)"); + + b.Property("AlternativeTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("PublishingStrategy") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ScheduledWithReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyId"); + + b.HasIndex("PreviousVersionId"); + + b.HasIndex("ScheduledWithReleaseVersionId"); + + b.ToTable("MethodologyVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Annexes") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("MethodologyVersionId"); + + b.ToTable("MethodologyVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Permalink", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataSetTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MigratedFromLegacy") + .HasColumnType("bit"); + + b.Property("PublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("SubjectId"); + + b.ToTable("Permalinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseSeries") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .IsRequired() + .HasMaxLength(160) + .HasColumnType("nvarchar(160)"); + + b.Property("SupersededById") + .HasColumnType("uniqueidentifier"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TopicId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContactId"); + + b.HasIndex("LatestPublishedReleaseVersionId") + .IsUnique() + .HasFilter("[LatestPublishedReleaseVersionId] IS NOT NULL"); + + b.HasIndex("SupersededById"); + + b.HasIndex("TopicId"); + + b.ToTable("Publications"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("Owner") + .HasColumnType("bit"); + + b.HasKey("PublicationId", "MethodologyId"); + + b.HasIndex("MethodologyId"); + + b.ToTable("PublicationMethodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("PublicationId", "Slug"); + + b.ToTable("PublicationRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.ToTable("Releases"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("FilterSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("IndicatorSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("PublicApiDataSetId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicApiDataSetVersion") + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("ReleaseVersionId", "FileId") + .IsUnique(); + + b.HasIndex("ReleaseVersionId", "PublicApiDataSetId", "PublicApiDataSetVersion") + .IsUnique() + .HasFilter("[PublicApiDataSetId] IS NOT NULL AND [PublicApiDataSetVersion] IS NOT NULL"); + + b.ToTable("ReleaseFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("ReleaseStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataGuidance") + .HasColumnType("nvarchar(max)"); + + b.Property("NextReleaseDate") + .HasColumnType("nvarchar(max)"); + + b.Property("NotifiedOn") + .HasColumnType("datetime2"); + + b.Property("NotifySubscribers") + .HasColumnType("bit"); + + b.Property("PreReleaseAccessList") + .HasColumnType("nvarchar(max)"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublishScheduled") + .HasColumnType("datetime2"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("RelatedInformation") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseName") + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(6) + .HasColumnType("nvarchar(6)"); + + b.Property("Type") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("UpdatePublishedDate") + .HasColumnType("bit"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("ReleaseId"); + + b.HasIndex("Type"); + + b.HasIndex("PreviousVersionId", "Version"); + + b.ToTable("ReleaseVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Themes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Topic", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("ThemeId") + .HasColumnType("uniqueidentifier"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("ThemeId"); + + b.ToTable("Topics"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("On") + .HasColumnType("datetime2"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("Update"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstName") + .HasColumnType("nvarchar(max)"); + + b.Property("LastName") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.ToTable("UserPublicationInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("UserId"); + + b.ToTable("UserPublicationRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EmailSent") + .HasColumnType("bit"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("UserReleaseInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UserId"); + + b.ToTable("UserReleaseRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Charts") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Charts"); + + b.Property("Heading") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Heading"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Query") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Query"); + + b.Property("Source") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Table") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Table"); + + b.HasDiscriminator().HasValue("DataBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("EmbedBlockId") + .HasColumnType("uniqueidentifier") + .HasColumnName("EmbedBlockId"); + + b.HasIndex("EmbedBlockId") + .IsUnique() + .HasFilter("[EmbedBlockId] IS NOT NULL"); + + b.HasDiscriminator().HasValue("EmbedBlockLink"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.HtmlBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("HtmlBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MarkDownBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("MarkDownBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.HasIndex("DataBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.ToTable("KeyStatisticsDataBlock", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("Statistic") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.ToTable("KeyStatisticsText", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", "ContentBlock") + .WithMany("Comments") + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "ResolvedBy") + .WithMany() + .HasForeignKey("ResolvedById"); + + b.Navigation("ContentBlock"); + + b.Navigation("CreatedBy"); + + b.Navigation("ResolvedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", "ContentSection") + .WithMany("Content") + .HasForeignKey("ContentSectionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "LockedBy") + .WithMany() + .HasForeignKey("LockedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentSection"); + + b.Navigation("LockedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Content") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestDraftVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestDraftVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestPublishedVersionId"); + + b.Navigation("LatestDraftVersion"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "ContentBlock") + .WithMany() + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("DataBlockVersions") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "FileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "MetaFile") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "MetaFileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ZipFile") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "ZipFileId") + .OnDelete(DeleteBehavior.Restrict); + + b.Navigation("File"); + + b.Navigation("MetaFile"); + + b.Navigation("ZipFile"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "DataImport") + .WithMany("Errors") + .HasForeignKey("DataImportId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataImport"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", "DataBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("FeaturedTables") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedBy") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Replacing") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacingId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Source") + .WithMany() + .HasForeignKey("SourceId"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReplacedBy"); + + b.Navigation("Replacing"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("CreatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("KeyStatistics") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "LatestPublishedVersionId"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany("Notes") + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Versions") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ScheduledWithReleaseVersion") + .WithMany() + .HasForeignKey("ScheduledWithReleaseVersionId"); + + b.Navigation("CreatedBy"); + + b.Navigation("Methodology"); + + b.Navigation("PreviousVersion"); + + b.Navigation("ScheduledWithReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", null) + .WithOne("MethodologyContent") + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", "MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", "Contact") + .WithMany() + .HasForeignKey("ContactId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "LatestPublishedReleaseVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "LatestPublishedReleaseVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "SupersededBy") + .WithMany() + .HasForeignKey("SupersededById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Topic", "Topic") + .WithMany("Publications") + .HasForeignKey("TopicId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ExternalMethodology", "ExternalMethodology", b1 => + { + b1.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b1.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.HasKey("PublicationId"); + + b1.ToTable("ExternalMethodology", (string)null); + + b1.WithOwner() + .HasForeignKey("PublicationId"); + }); + + b.Navigation("Contact"); + + b.Navigation("ExternalMethodology"); + + b.Navigation("LatestPublishedReleaseVersion"); + + b.Navigation("SupersededBy"); + + b.Navigation("Topic"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Publications") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Methodologies") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Methodology"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("ReleaseStatuses") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("ReleaseVersions") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") + .WithMany("Versions") + .HasForeignKey("ReleaseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("PreviousVersion"); + + b.Navigation("Publication"); + + b.Navigation("Release"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Topic", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", "Theme") + .WithMany("Topics") + .HasForeignKey("ThemeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Theme"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Updates") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("Publication"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", "EmbedBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", "EmbedBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("EmbedBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithMany() + .HasForeignKey("DataBlockId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Navigation("Comments"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Navigation("Content"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Navigation("Errors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Navigation("Publications"); + + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Navigation("MethodologyContent") + .IsRequired(); + + b.Navigation("Notes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Navigation("Methodologies"); + + b.Navigation("ReleaseVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Navigation("Content"); + + b.Navigation("DataBlockVersions"); + + b.Navigation("FeaturedTables"); + + b.Navigation("KeyStatistics"); + + b.Navigation("ReleaseStatuses"); + + b.Navigation("Updates"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Navigation("Topics"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Topic", b => + { + b.Navigation("Publications"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240617005412_EES4993_AddPublicApiDataSetIdVersionToReleaseFile.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240617005412_EES4993_AddPublicApiDataSetIdVersionToReleaseFile.cs new file mode 100644 index 00000000000..3852a5219a9 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20240617005412_EES4993_AddPublicApiDataSetIdVersionToReleaseFile.cs @@ -0,0 +1,125 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + /// + public partial class EES4993_AddPublicApiDataSetIdVersionToReleaseFile : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "PublicApiDataSetId", + table: "ReleaseFiles", + type: "uniqueidentifier", + nullable: true); + + migrationBuilder.AddColumn( + name: "PublicApiDataSetVersion", + table: "ReleaseFiles", + type: "nvarchar(20)", + maxLength: 20, + nullable: true); + + migrationBuilder.CreateIndex( + name: "IX_ReleaseFiles_ReleaseVersionId_FileId", + table: "ReleaseFiles", + columns: new[] { "ReleaseVersionId", "FileId" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_ReleaseFiles_ReleaseVersionId_PublicApiDataSetId_PublicApiDataSetVersion", + table: "ReleaseFiles", + columns: new[] { "ReleaseVersionId", "PublicApiDataSetId", "PublicApiDataSetVersion" }, + unique: true, + filter: "[PublicApiDataSetId] IS NOT NULL AND [PublicApiDataSetVersion] IS NOT NULL"); + + // This can incorrectly set these columns if an amendment is created and any + // of its ReleaseFiles are promoted to a draft API data set. This needs to + // be rectified by an additional endpoint migration. + migrationBuilder.Sql( + """ + UPDATE ReleaseFiles + SET ReleaseFiles.PublicApiDataSetId = Files.PublicApiDataSetId, + ReleaseFiles.PublicApiDataSetVersion = Files.PublicApiDataSetVersion + FROM dbo.ReleaseFiles + INNER JOIN dbo.Files ON Files.Id = ReleaseFiles.FileId + """); + + migrationBuilder.DropIndex( + name: "IX_ReleaseFiles_ReleaseVersionId", + table: "ReleaseFiles"); + + migrationBuilder.DropIndex( + name: "IX_Files_PublicApiDataSetId_PublicApiDataSetVersion", + table: "Files"); + + migrationBuilder.DropColumn( + name: "PublicApiDataSetId", + table: "Files"); + + migrationBuilder.DropColumn( + name: "PublicApiDataSetVersion", + table: "Files"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "PublicApiDataSetId", + table: "Files", + type: "uniqueidentifier", + nullable: true); + + migrationBuilder.AddColumn( + name: "PublicApiDataSetVersion", + table: "Files", + type: "nvarchar(20)", + maxLength: 20, + nullable: true); + + migrationBuilder.CreateIndex( + name: "IX_Files_PublicApiDataSetId_PublicApiDataSetVersion", + table: "Files", + columns: new[] { "PublicApiDataSetId", "PublicApiDataSetVersion" }, + unique: true, + filter: "[PublicApiDataSetId] IS NOT NULL AND [PublicApiDataSetVersion] IS NOT NULL"); + + migrationBuilder.Sql( + """ + UPDATE Files + SET Files.PublicApiDataSetId = ReleaseFiles.PublicApiDataSetId, + Files.PublicApiDataSetVersion = ReleaseFiles.PublicApiDataSetVersion + FROM dbo.ReleaseFiles + INNER JOIN dbo.Files ON Files.Id = ReleaseFiles.FileId + WHERE ReleaseFiles.PublicApiDataSetId IS NOT NULL + AND ReleaseFiles.PublicApiDataSetVersion IS NOT NULL + """); + + migrationBuilder.DropIndex( + name: "IX_ReleaseFiles_ReleaseVersionId_FileId", + table: "ReleaseFiles"); + + migrationBuilder.DropIndex( + name: "IX_ReleaseFiles_ReleaseVersionId_PublicApiDataSetId_PublicApiDataSetVersion", + table: "ReleaseFiles"); + + migrationBuilder.DropColumn( + name: "PublicApiDataSetId", + table: "ReleaseFiles"); + + migrationBuilder.DropColumn( + name: "PublicApiDataSetVersion", + table: "ReleaseFiles"); + + migrationBuilder.CreateIndex( + name: "IX_ReleaseFiles_ReleaseVersionId", + table: "ReleaseFiles", + column: "ReleaseVersionId"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs index 68a27808664..ee8bd036d31 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs @@ -17,7 +17,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "8.0.3") + .HasAnnotation("ProductVersion", "8.0.4") .HasAnnotation("Relational:MaxIdentifierLength", 128); SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); @@ -441,13 +441,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsRequired() .HasColumnType("nvarchar(max)"); - b.Property("PublicApiDataSetId") - .HasColumnType("uniqueidentifier"); - - b.Property("PublicApiDataSetVersion") - .HasMaxLength(20) - .HasColumnType("nvarchar(20)"); - b.Property("ReplacedById") .HasColumnType("uniqueidentifier"); @@ -484,10 +477,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("Type"); - b.HasIndex("PublicApiDataSetId", "PublicApiDataSetVersion") - .IsUnique() - .HasFilter("[PublicApiDataSetId] IS NOT NULL AND [PublicApiDataSetVersion] IS NOT NULL"); - b.ToTable("Files"); }); @@ -936,6 +925,13 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Order") .HasColumnType("int"); + b.Property("PublicApiDataSetId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicApiDataSetVersion") + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + b.Property("Published") .HasColumnType("datetime2"); @@ -949,7 +945,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("FileId"); - b.HasIndex("ReleaseVersionId"); + b.HasIndex("ReleaseVersionId", "FileId") + .IsUnique(); + + b.HasIndex("ReleaseVersionId", "PublicApiDataSetId", "PublicApiDataSetVersion") + .IsUnique() + .HasFilter("[PublicApiDataSetId] IS NOT NULL AND [PublicApiDataSetVersion] IS NOT NULL"); b.ToTable("ReleaseFiles"); }); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/Public.Data/DataSetCreateRequest.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/Public.Data/DataSetCreateRequest.cs new file mode 100644 index 00000000000..fa76e06dba8 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/Public.Data/DataSetCreateRequest.cs @@ -0,0 +1,19 @@ +#nullable enable +using System; +using FluentValidation; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests.Public.Data; + +public record DataSetCreateRequest +{ + public required Guid ReleaseFileId { get; init; } + + public class Validator : AbstractValidator + { + public Validator() + { + RuleFor(request => request.ReleaseFileId) + .NotEmpty(); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/Public.Data/DataSetVersionCreateRequest.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/Public.Data/DataSetVersionCreateRequest.cs deleted file mode 100644 index 121d9cd6bb9..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/Public.Data/DataSetVersionCreateRequest.cs +++ /dev/null @@ -1,9 +0,0 @@ -#nullable enable -using System; - -namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests.Public.Data; - -public record DataSetVersionCreateRequest -{ - public required Guid ReleaseFileId { get; init; } -} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/Public.Data/NextDataSetVersionCreateRequest.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/Public.Data/NextDataSetVersionCreateRequest.cs new file mode 100644 index 00000000000..7c390beaa9e --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/Public.Data/NextDataSetVersionCreateRequest.cs @@ -0,0 +1,24 @@ +#nullable enable +using System; +using FluentValidation; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests.Public.Data; + +public record NextDataSetVersionCreateRequest +{ + public required Guid DataSetId { get; init; } + + public required Guid ReleaseFileId { get; init; } + + public class Validator : AbstractValidator + { + public Validator() + { + RuleFor(request => request.DataSetId) + .NotEmpty(); + + RuleFor(request => request.ReleaseFileId) + .NotEmpty(); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataBlockService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataBlockService.cs index 867990fddc2..19e799a36b0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataBlockService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataBlockService.cs @@ -100,7 +100,7 @@ public async Task> Delete(Guid releaseVersionId, Guid .OnSuccessVoid(DeleteDataBlocks); } - public Task> DeleteDataBlocks(DeleteDataBlockPlan deletePlan) + public Task> DeleteDataBlocks(DeleteDataBlockPlanViewModel deletePlan) { return InvalidateDataBlockCaches(deletePlan) .OnSuccessVoid(async () => @@ -239,7 +239,7 @@ await _releaseFileService.Delete(releaseVersionId: dataBlockVersion.ReleaseVersi .OnSuccess(() => Get(dataBlockVersionId)); } - public async Task> GetDeletePlan(Guid releaseVersionId, + public async Task> GetDeletePlan(Guid releaseVersionId, Guid dataBlockVersionId) { return await _persistenceHelper @@ -253,7 +253,7 @@ public async Task> GetDeletePlan(Guid ) .OnSuccessDo(dataBlockVersion => _userService.CheckCanUpdateReleaseVersion(dataBlockVersion.ReleaseVersion)) .OnSuccess(async dataBlockVersion => - new DeleteDataBlockPlan + new DeleteDataBlockPlanViewModel { ReleaseId = releaseVersionId, DependentDataBlocks = new List @@ -276,7 +276,7 @@ public Task> GetDataBlockVersionForReleas .OrNotFound(); } - public async Task GetDeletePlan(Guid releaseVersionId, Subject? subject) + public async Task GetDeletePlan(Guid releaseVersionId, Subject? subject) { var dataBlockVersions = subject == null ? new List() @@ -287,7 +287,7 @@ public async Task GetDeletePlan(Guid releaseVersionId, Subj dependentBlocks.Add(await CreateDependentDataBlock(block)); } - return new DeleteDataBlockPlan() + return new DeleteDataBlockPlanViewModel() { ReleaseId = releaseVersionId, DependentDataBlocks = dependentBlocks @@ -374,7 +374,7 @@ private async Task> RemoveInfographicChartFromDataBlo return true; } - private async Task RemoveChartFileReleaseLinks(DeleteDataBlockPlan deletePlan) + private async Task RemoveChartFileReleaseLinks(DeleteDataBlockPlanViewModel deletePlan) { var chartFileIds = deletePlan.DependentDataBlocks.SelectMany( block => block.InfographicFilesInfo.Select(f => f.Id)); @@ -382,7 +382,7 @@ private async Task RemoveChartFileReleaseLinks(DeleteDataBlockPlan deletePlan) await _releaseFileService.Delete(deletePlan.ReleaseId, chartFileIds); } - private async Task> DeleteDependentDataBlocks(DeleteDataBlockPlan deletePlan) + private async Task> DeleteDependentDataBlocks(DeleteDataBlockPlanViewModel deletePlan) { var blockIdsToDelete = deletePlan .DependentDataBlocks @@ -442,7 +442,7 @@ private async Task> GetDataBlockVersion(G ); } - private Task> InvalidateDataBlockCaches(DeleteDataBlockPlan deletePlan) + private Task> InvalidateDataBlockCaches(DeleteDataBlockPlanViewModel deletePlan) { return deletePlan .DependentDataBlocks @@ -503,26 +503,4 @@ public async Task> ListDataBlocks(Guid releaseVersionId) .ToListAsync(); } } - - public class DependentDataBlock - { - [JsonIgnore] public Guid Id { get; set; } - public string Name { get; set; } = string.Empty; - public string? ContentSectionHeading { get; set; } - public List InfographicFilesInfo { get; set; } = new(); - public bool IsKeyStatistic { get; set; } - public FeaturedTableBasicViewModel? FeaturedTable { get; set; } - } - - public class InfographicFileInfo - { - public Guid Id { get; set; } - public string Filename { get; set; } = ""; - } - - public class DeleteDataBlockPlan - { - [JsonIgnore] public Guid ReleaseId { get; set; } - public List DependentDataBlocks { get; set; } = new(); - } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IDataBlockService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IDataBlockService.cs index 63c6292559f..c3fb9a5585b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IDataBlockService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IDataBlockService.cs @@ -26,12 +26,12 @@ Task> Delete(Guid releaseVersionId, Task> Update(Guid dataBlockVersionId, DataBlockUpdateViewModel dataBlockUpdate); - Task> DeleteDataBlocks(DeleteDataBlockPlan deletePlan); + Task> DeleteDataBlocks(DeleteDataBlockPlanViewModel deletePlan); - Task> GetDeletePlan(Guid releaseVersionId, + Task> GetDeletePlan(Guid releaseVersionId, Guid dataBlockVersionId); - Task GetDeletePlan(Guid releaseVersionId, + Task GetDeletePlan(Guid releaseVersionId, Subject? subject); Task> RemoveChartFile(Guid releaseVersionId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReleaseService.cs index fc791d8265e..de36b4e97a0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReleaseService.cs @@ -1,6 +1,7 @@ #nullable enable using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Requests; using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; @@ -15,9 +16,11 @@ public interface IReleaseService { Task> CreateRelease(ReleaseCreateRequest releaseCreate); - Task> GetDeleteReleasePlan(Guid releaseVersionId); + Task> GetDeleteReleaseVersionPlan( + Guid releaseVersionId, + CancellationToken cancellationToken = default); - Task> DeleteRelease(Guid releaseVersionId); + Task> DeleteReleaseVersion(Guid releaseVersionId); Task> GetRelease(Guid releaseVersionId); @@ -38,7 +41,10 @@ Task>> ListReleasesWithStatus Task>> ListScheduledReleases(); - Task> GetDeleteDataFilePlan(Guid releaseVersionId, Guid fileId); + Task> GetDeleteDataFilePlan( + Guid releaseVersionId, + Guid fileId, + CancellationToken cancellationToken = default); Task> RemoveDataFiles(Guid releaseVersionId, Guid fileId); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReplacementService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReplacementService.cs index b8ad547cb2f..c24a21df32c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReplacementService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReplacementService.cs @@ -1,5 +1,6 @@ #nullable enable using System; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -12,7 +13,8 @@ public interface IReplacementService Task> GetReplacementPlan( Guid releaseVersionId, Guid originalFileId, - Guid replacementFileId); + Guid replacementFileId, + CancellationToken cancellationToken = default); Task> Replace( Guid releaseVersionId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/Public.Data/IDataSetVersionService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/Public.Data/IDataSetVersionService.cs index a71b5aa43de..7d5aa389571 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/Public.Data/IDataSetVersionService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/Public.Data/IDataSetVersionService.cs @@ -4,16 +4,31 @@ using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Public.Data; +using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.Public.Data; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using Microsoft.AspNetCore.Mvc; +using Semver; namespace GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data; public interface IDataSetVersionService { - Task> GetStatusesForReleaseVersion(Guid releaseVersionId); + Task> GetStatusesForReleaseVersion( + Guid releaseVersionId, + CancellationToken cancellationToken = default); + + Task> GetDataSetVersion( + Guid dataSetId, + SemVersion version, + CancellationToken cancellationToken = default); Task> DeleteVersion( Guid dataSetVersionId, CancellationToken cancellationToken = default); + + Task> CreateNextVersion( + Guid releaseFileId, + Guid dataSetId, + CancellationToken cancellationToken = default); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/Public.Data/IProcessorClient.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/Public.Data/IProcessorClient.cs index 9888e0186da..5fbb7871263 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/Public.Data/IProcessorClient.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/Public.Data/IProcessorClient.cs @@ -14,6 +14,15 @@ Task> CreateDataSet( Guid releaseFileId, CancellationToken cancellationToken = default); + Task> CreateNextDataSetVersion( + Guid dataSetId, + Guid releaseFileId, + CancellationToken cancellationToken = default); + + Task> BulkDeleteDataSetVersions( + Guid releaseVersionId, + CancellationToken cancellationToken = default); + Task> DeleteDataSetVersion( Guid dataSetVersionId, CancellationToken cancellationToken = default); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetCandidateService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetCandidateService.cs index c261f75b1b1..59e4135f614 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetCandidateService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetCandidateService.cs @@ -48,9 +48,17 @@ private async Task .AsNoTracking() .Where(rf => rf.ReleaseVersionId == releaseVersionId) .Where(rf => rf.File.Type == FileType.Data) - .Where(rf => rf.File.PublicApiDataSetId == null) + .Where(rf => rf.PublicApiDataSetId == null) .Where(rf => rf.File.ReplacedById == null) .Where(rf => rf.File.ReplacingId == null) + .Join( + contentDbContext.DataImports, + rf => rf.FileId, + di => di.FileId, + (rf, di) => new { ReleaseFile = rf, DataImport = di } + ) + .Where(tuple => tuple.DataImport.Status == DataImportStatus.COMPLETE) + .Select(tuple => tuple.ReleaseFile) .Select(rf => new DataSetCandidateViewModel { ReleaseFileId = rf.Id, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetVersionService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetVersionService.cs index 01fc15f850c..058a1440c54 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetVersionService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetVersionService.cs @@ -7,6 +7,8 @@ using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Security; +using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.Public.Data; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; @@ -14,6 +16,7 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Semver; namespace GovUk.Education.ExploreEducationStatistics.Admin.Services.Public.Data; @@ -24,13 +27,15 @@ public class DataSetVersionService( IUserService userService) : IDataSetVersionService { - public async Task> GetStatusesForReleaseVersion(Guid releaseVersionId) + public async Task> GetStatusesForReleaseVersion( + Guid releaseVersionId, + CancellationToken cancellationToken = default) { var releaseFileIds = await contentDbContext .ReleaseFiles .Where(rf => rf.ReleaseVersionId == releaseVersionId && rf.File.Type == FileType.Data) .Select(rf => rf.Id) - .ToListAsync(); + .ToListAsync(cancellationToken); return await publicDataDbContext .DataSetVersions @@ -41,13 +46,59 @@ public async Task> GetStatusesForReleaseVersio dataSetVersion.DataSet.Title, dataSetVersion.Status) ) - .ToListAsync(); + .ToListAsync(cancellationToken); } - public async Task> DeleteVersion(Guid dataSetVersionId, CancellationToken cancellationToken = default) + public async Task> CreateNextVersion( + Guid releaseFileId, + Guid dataSetId, + CancellationToken cancellationToken = default) { return await userService.CheckIsBauUser() - .OnSuccessVoid(async () => await processorClient.DeleteDataSetVersion(dataSetVersionId, cancellationToken)); + .OnSuccess(async _ => await processorClient.CreateNextDataSetVersion( + dataSetId: dataSetId, + releaseFileId: releaseFileId, + cancellationToken: cancellationToken)) + .OnSuccess(async processorResponse => await publicDataDbContext + .DataSetVersions + .SingleAsync( + dataSetVersion => dataSetVersion.Id == processorResponse.DataSetVersionId, + cancellationToken)) + .OnSuccess(MapDraftSummaryVersion); + } + + public async Task> GetDataSetVersion( + Guid dataSetId, + SemVersion version, + CancellationToken cancellationToken = default) + { + return await publicDataDbContext.DataSetVersions + .AsNoTracking() + .Include(dsv => dsv.DataSet) + .Where(dsv => dsv.DataSetId == dataSetId) + .Where(dsv => dsv.VersionMajor == version!.Major) + .Where(dsv => dsv.VersionMinor == version!.Minor) + .SingleOrNotFoundAsync(cancellationToken); + } + + public async Task> DeleteVersion(Guid dataSetVersionId, + CancellationToken cancellationToken = default) + { + return await userService.CheckIsBauUser() + .OnSuccessVoid(async () => await processorClient.DeleteDataSetVersion( + dataSetVersionId: dataSetVersionId, + cancellationToken: cancellationToken)); + } + + private static DataSetVersionSummaryViewModel MapDraftSummaryVersion(DataSetVersion dataSetVersion) + { + return new DataSetVersionSummaryViewModel + { + Id = dataSetVersion.Id, + Version = dataSetVersion.Version, + Status = dataSetVersion.Status, + Type = dataSetVersion.VersionType, + }; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/ProcessorClient.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/ProcessorClient.cs index e44f6f7062d..cd979107e35 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/ProcessorClient.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/ProcessorClient.cs @@ -32,52 +32,96 @@ public async Task> CreateDa Guid releaseFileId, CancellationToken cancellationToken = default) { - await AddBearerToken(cancellationToken); + var request = new DataSetCreateRequest { ReleaseFileId = releaseFileId }; + + return await SendPost( + "api/CreateDataSet", + request, + cancellationToken: cancellationToken); + } - var request = new DataSetCreateRequest + public async Task> CreateNextDataSetVersion( + Guid dataSetId, + Guid releaseFileId, + CancellationToken cancellationToken = default) + { + var request = new NextDataSetVersionCreateRequest { ReleaseFileId = releaseFileId, + DataSetId = dataSetId }; - var response = await httpClient - .PostAsJsonAsync("api/CreateDataSet", request, cancellationToken: cancellationToken); + return await SendPost( + "api/CreateNextDataSetVersion", + request, + cancellationToken: cancellationToken); + } + + public async Task> BulkDeleteDataSetVersions( + Guid releaseVersionId, + CancellationToken cancellationToken = default) + { + return await SendDelete( + $"api/BulkDeleteDataSetVersions/{releaseVersionId}", + cancellationToken: cancellationToken); + } - if (!response.IsSuccessStatusCode) - { - switch (response.StatusCode) - { - case HttpStatusCode.BadRequest: - return new BadRequestObjectResult( - await response.Content - .ReadFromJsonAsync(cancellationToken: cancellationToken) - ); - case HttpStatusCode.NotFound: - return new NotFoundResult(); - default: - var message = await response.Content.ReadAsStringAsync(cancellationToken); - logger.LogError($""" - Failed to create data set with status code: {response.StatusCode}. Message: - {message} - """); - response.EnsureSuccessStatusCode(); - break; - } - } + public async Task> DeleteDataSetVersion( + Guid dataSetVersionId, + CancellationToken cancellationToken = default) + { + return await SendDelete( + $"api/DeleteDataSetVersion/{dataSetVersionId}", + response => + response.StatusCode == HttpStatusCode.NotFound ? new NotFoundResult() : null, + cancellationToken: cancellationToken); + } - var content = await response.Content.ReadFromJsonAsync( - cancellationToken: cancellationToken - ); + private async Task> SendPost( + string url, + TRequest request, + Func? customResponseHandler = null, + CancellationToken cancellationToken = default) + where TResponse : class + { + return await SendRequest(() => + httpClient.PostAsJsonAsync( + url, + request, + cancellationToken: cancellationToken), + customResponseHandler, + cancellationToken); + } - return content - ?? throw new Exception("Could not deserialize the response from the Public Data Processor."); + private async Task> SendDelete( + string url, + Func? customResponseHandler = null, + CancellationToken cancellationToken = default) + { + return await SendRequest( + () => httpClient.DeleteAsync( + url, + cancellationToken: cancellationToken), + customResponseHandler, + cancellationToken); } - public async Task> DeleteDataSetVersion( - Guid dataSetVersionId, + private async Task> SendRequest( + Func> requestFunction, + Func? customResponseHandler = null, CancellationToken cancellationToken = default) + where TResponse : class { - var response = await httpClient - .DeleteAsync($"api/DeleteDataSetVersion/{dataSetVersionId}", cancellationToken: cancellationToken); + await AddBearerToken(cancellationToken); + + var response = await requestFunction.Invoke(); + + var customHandlerResponse = customResponseHandler?.Invoke(response); + + if (customHandlerResponse is not null) + { + return customHandlerResponse; + } if (!response.IsSuccessStatusCode) { @@ -88,20 +132,34 @@ public async Task> DeleteDataSetVersion( await response.Content .ReadFromJsonAsync(cancellationToken: cancellationToken) ); - case HttpStatusCode.NotFound: - return new NotFoundResult(); default: - var message = await response.Content.ReadAsStringAsync(cancellationToken); - logger.LogError($""" - Failed to delete data set version with status code: {response.StatusCode}. Message: - {message} - """); + var body = await response.Content.ReadAsStringAsync(cancellationToken); + var request = response.RequestMessage!; + + logger.LogError(""" + Request {Method} {AbsolutePath} failed with status code {StatusCode}. + + Body: {Body} + """, + request.Method, + request.RequestUri!.AbsolutePath, + response.StatusCode, + body); + response.EnsureSuccessStatusCode(); break; } } - return Unit.Instance; + if (typeof(TResponse) == typeof(Unit)) + { + return (Unit.Instance as TResponse)!; + } + + var content = await response.Content.ReadFromJsonAsync(cancellationToken); + + return content + ?? throw new Exception("Could not deserialize the response from the Public Data Processor."); } /// diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseAmendmentService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseAmendmentService.cs index 26a27178e76..8a34619c752 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseAmendmentService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseAmendmentService.cs @@ -624,6 +624,8 @@ private async Task> CopyFileLinks(ReleaseVe FilterSequence = originalFile.FilterSequence, IndicatorSequence = originalFile.IndicatorSequence, Published = originalFile.Published, + PublicApiDataSetId = originalFile.PublicApiDataSetId, + PublicApiDataSetVersion = originalFile.PublicApiDataSetVersion, }) .ToList(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseApprovalService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseApprovalService.cs index 4de0202deb7..1eaca09d01d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseApprovalService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseApprovalService.cs @@ -253,7 +253,7 @@ private async Task> RemoveUnusedImages(Guid releaseVe HtmlImageUtil.GetReleaseImages(contentBlock.Body)) .Distinct(); - var imageFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, FileType.Image); + var imageFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, types: FileType.Image); var unusedImages = imageFiles .Where(file => !contentImageIds.Contains(file.File.Id)) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs index c4a5fe0f282..d8aa7d39030 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs @@ -148,7 +148,7 @@ await _privateBlobStorageService.DeleteBlob( public async Task> DeleteAll(Guid releaseVersionId, bool forceDelete = false) { - var releaseFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, FileType.Data); + var releaseFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, types: FileType.Data); return await Delete(releaseVersionId, releaseFiles.Select(releaseFile => releaseFile.File.Id), @@ -175,7 +175,7 @@ public async Task>> ListAll(Guid release .OnSuccess(_userService.CheckCanViewReleaseVersion) .OnSuccess(async () => { - var files = await _releaseFileRepository.GetByFileType(releaseVersionId, FileType.Data); + var files = await _releaseFileRepository.GetByFileType(releaseVersionId, types: FileType.Data); // Exclude files that are replacements in progress var filesExcludingReplacements = files diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseFileService.cs index 2e995396094..bcccd819c60 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseFileService.cs @@ -140,7 +140,7 @@ await _privateBlobStorageService.DeleteBlob( public async Task> DeleteAll(Guid releaseVersionId, bool forceDelete = false) { - var releaseFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, DeletableFileTypes); + var releaseFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, types: DeletableFileTypes); return await Delete(releaseVersionId, releaseFiles.Select(releaseFile => releaseFile.File.Id), @@ -155,7 +155,7 @@ public async Task>> ListAll(Guid rele .OnSuccess(_userService.CheckCanViewReleaseVersion) .OnSuccess(async _ => { - var releaseFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, types); + var releaseFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, types: types); return releaseFiles .Select(releaseFile => releaseFile.ToFileInfo()) @@ -320,7 +320,7 @@ public async Task>> GetAncillaryFiles .OnSuccess(_userService.CheckCanViewReleaseVersion) .OnSuccess(async _ => { - var releaseFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, Ancillary); + var releaseFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, types: Ancillary); var filesWithMetadata = await releaseFiles .SelectAsync(async releaseFile => await ToAncillaryFileInfo(releaseFile)); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs index c840122d58e..60bfa822b40 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs @@ -24,14 +24,17 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.Services.Cache; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using Newtonsoft.Json; +using System.Threading; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationErrorMessages; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationUtils; using static GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyApprovalStatus; using static GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyPublishingStrategy; using IReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IReleaseVersionRepository; +using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data; +using GovUk.Education.ExploreEducationStatistics.Admin.Validators.ErrorDetails; namespace GovUk.Education.ExploreEducationStatistics.Admin.Services { @@ -51,6 +54,8 @@ public class ReleaseService : IReleaseService private readonly IFootnoteRepository _footnoteRepository; private readonly IDataBlockService _dataBlockService; private readonly IReleaseSubjectRepository _releaseSubjectRepository; + private readonly IDataSetVersionService _dataSetVersionService; + private readonly IProcessorClient _processorClient; private readonly IGuidGenerator _guidGenerator; private readonly IBlobCacheService _cacheService; @@ -71,6 +76,8 @@ public ReleaseService( IFootnoteRepository footnoteRepository, IDataBlockService dataBlockService, IReleaseSubjectRepository releaseSubjectRepository, + IDataSetVersionService dataSetVersionService, + IProcessorClient processorClient, IGuidGenerator guidGenerator, IBlobCacheService cacheService) { @@ -88,6 +95,8 @@ public ReleaseService( _footnoteRepository = footnoteRepository; _dataBlockService = dataBlockService; _releaseSubjectRepository = releaseSubjectRepository; + _dataSetVersionService = dataSetVersionService; + _processorClient = processorClient; _guidGenerator = guidGenerator; _cacheService = cacheService; } @@ -174,7 +183,9 @@ await CreateGenericContentFromTemplate(releaseCreate.TemplateReleaseId.Value, }); } - public Task> GetDeleteReleasePlan(Guid releaseVersionId) + public Task> GetDeleteReleaseVersionPlan( + Guid releaseVersionId, + CancellationToken cancellationToken = default) { return _persistenceHelper .CheckEntityExists(releaseVersionId) @@ -186,18 +197,19 @@ public Task> GetDeleteReleasePlan(Guid r .Select(m => new IdTitleViewModel(m.Id, m.Title)) .ToList(); - return new DeleteReleasePlan + return new DeleteReleasePlanViewModel { ScheduledMethodologies = methodologiesScheduledWithRelease }; }); } - public Task> DeleteRelease(Guid releaseVersionId) + public Task> DeleteReleaseVersion(Guid releaseVersionId) { return _persistenceHelper .CheckEntityExists(releaseVersionId) .OnSuccess(_userService.CheckCanDeleteReleaseVersion) // only allows unapproved amendments to be removed + .OnSuccessDo(async () => await _processorClient.BulkDeleteDataSetVersions(releaseVersionId)) .OnSuccessDo(async release => await _cacheService.DeleteCacheFolderAsync( new PrivateReleaseContentFolderCacheKey(release.Id))) .OnSuccessDo(async () => await _releaseDataFileService.DeleteAll(releaseVersionId)) @@ -436,29 +448,49 @@ public async Task>> ListSched }); } - public async Task> GetDeleteDataFilePlan(Guid releaseVersionId, - Guid fileId) + public async Task> GetDeleteDataFilePlan( + Guid releaseVersionId, + Guid fileId, + CancellationToken cancellationToken = default) { return await _context.ReleaseVersions .FirstOrNotFoundAsync(rv => rv.Id == releaseVersionId) .OnSuccess(_userService.CheckCanUpdateReleaseVersion) - .OnSuccess(() => CheckFileExists(fileId)) - .OnSuccessCombineWith(file => _statisticsDbContext.Subject - .FirstOrNotFoundAsync(s => s.Id == file.SubjectId)) + .OnSuccess(() => CheckReleaseDataFileExists(releaseVersionId: releaseVersionId, fileId: fileId)) + .OnSuccessCombineWith(releaseFile => _statisticsDbContext.Subject + .FirstOrNotFoundAsync(s => s.Id == releaseFile.File.SubjectId)) .OnSuccess(async tuple => { - var (file, subject) = tuple; + var (releaseFile, subject) = tuple; + return await GetLinkedDataSetVersion(releaseFile, cancellationToken) + .OnSuccess(apiDataSetVersion => (releaseFile, subject, apiDataSetVersion)); + }) + .OnSuccess(async tuple => + { var footnotes = await _footnoteRepository.GetFootnotes(releaseVersionId: releaseVersionId, - subjectId: file.SubjectId); + subjectId: tuple.releaseFile.File.SubjectId); + + var linkedApiDataSetVersionDeletionPlan = tuple.apiDataSetVersion is null + ? null + : new DeleteApiDataSetVersionPlanViewModel + { + DataSetId = tuple.apiDataSetVersion.DataSetId, + DataSetTitle = tuple.apiDataSetVersion.DataSet.Title, + Id = tuple.apiDataSetVersion.Id, + Version = tuple.apiDataSetVersion.Version, + Status = tuple.apiDataSetVersion.Status, + Valid = false + }; - return new DeleteDataFilePlan + return new DeleteDataFilePlanViewModel { ReleaseId = releaseVersionId, - SubjectId = subject.Id, - DeleteDataBlockPlan = await _dataBlockService.GetDeletePlan(releaseVersionId, subject), - FootnoteIds = footnotes.Select(footnote => footnote.Id).ToList() + SubjectId = tuple.subject.Id, + DeleteDataBlockPlan = await _dataBlockService.GetDeletePlan(releaseVersionId, tuple.subject), + FootnoteIds = footnotes.Select(footnote => footnote.Id).ToList(), + DeleteApiDataSetVersionPlan = linkedApiDataSetVersionDeletionPlan }; }); } @@ -468,14 +500,16 @@ public async Task> RemoveDataFiles(Guid releaseVersio return await _persistenceHelper .CheckEntityExists(releaseVersionId) .OnSuccess(_userService.CheckCanUpdateReleaseVersion) - .OnSuccess(() => CheckFileExists(fileId)) - .OnSuccessDo(file => CheckCanDeleteDataFiles(releaseVersionId, file)) - .OnSuccessDo(async file => + .OnSuccess(() => CheckReleaseDataFileExists(releaseVersionId: releaseVersionId, fileId: fileId)) + .OnSuccessDo(releaseFile => CheckCanDeleteDataFiles(releaseVersionId, releaseFile)) + .OnSuccessDo(async releaseFile => { // Delete any replacement that might exist - if (file.ReplacedById.HasValue) + if (releaseFile.File.ReplacedById.HasValue) { - return await RemoveDataFiles(releaseVersionId, file.ReplacedById.Value); + return await RemoveDataFiles( + releaseVersionId: releaseVersionId, + fileId: releaseFile.File.ReplacedById.Value); } return Unit.Instance; @@ -490,7 +524,7 @@ await _cacheService.DeleteItemAsync(new PrivateSubjectMetaCacheKey( releaseVersionId: releaseVersionId, subjectId: deletePlan.SubjectId)); }) - .OnSuccess(() => _releaseDataFileService.Delete(releaseVersionId, fileId)); + .OnSuccessVoid(() => _releaseDataFileService.Delete(releaseVersionId, fileId)); } public async Task> GetDataFileImportStatus( @@ -522,13 +556,16 @@ private async Task> CheckReleaseVersionExis .SingleOrNotFoundAsync(rv => rv.Id == releaseVersionId); } - private async Task> CheckFileExists(Guid id) + private async Task> CheckReleaseDataFileExists(Guid releaseVersionId, Guid fileId) { - return await _persistenceHelper.CheckEntityExists(id) - .OnSuccess(file => file.Type != FileType.Data - ? new Either( - ValidationActionResult(FileTypeMustBeData)) - : file); + return await _context.ReleaseFiles + .Include(rf => rf.File) + .Where(rf => rf.ReleaseVersionId == releaseVersionId) + .Where(rf => rf.FileId == fileId) + .SingleOrNotFoundAsync() + .OnSuccess(rf => rf.File.Type != FileType.Data + ? new Either(ValidationActionResult(FileTypeMustBeData)) + : rf); } private async Task> ValidateReleaseSlugUniqueToPublication(string slug, @@ -598,9 +635,27 @@ private async Task CanUpdateDataFiles(Guid releaseVersionId) return releaseVersion.ApprovalStatus != ReleaseApprovalStatus.Approved; } - private async Task> CheckCanDeleteDataFiles(Guid releaseVersionId, File file) + private async Task> GetLinkedDataSetVersion( + ReleaseFile releaseFile, + CancellationToken cancellationToken = default) + { + if (releaseFile.PublicApiDataSetId is null) + { + return (DataSetVersion)null!; + } + + return await _dataSetVersionService.GetDataSetVersion( + releaseFile.PublicApiDataSetId.Value, + releaseFile.PublicApiDataSetVersion!, + cancellationToken) + .OnSuccess(dsv => (DataSetVersion?)dsv) + .OnFailureDo(_ => throw new ApplicationException( + $"API data set version could not be found. Data set ID: '{releaseFile.PublicApiDataSetId}', version: '{releaseFile.PublicApiDataSetVersion}'")); + } + + private async Task> CheckCanDeleteDataFiles(Guid releaseVersionId, ReleaseFile releaseFile) { - var import = await _dataImportService.GetImport(file.Id); + var import = await _dataImportService.GetImport(releaseFile.FileId); var importStatus = import?.Status ?? DataImportStatus.NOT_FOUND; if (!importStatus.IsFinished()) @@ -613,6 +668,16 @@ private async Task> CheckCanDeleteDataFiles(Guid rele return ValidationActionResult(CannotRemoveDataFilesOnceReleaseApproved); } + if (releaseFile.PublicApiDataSetId is not null) + { + return Common.Validators.ValidationUtils.ValidationResult(new ErrorViewModel + { + Code = ValidationMessages.CannotDeleteApiDataSetReleaseFile.Code, + Message = ValidationMessages.CannotDeleteApiDataSetReleaseFile.Message, + Detail = new ApiDataSetErrorDetail(releaseFile.PublicApiDataSetId.Value) + }); + } + return Unit.Instance; } @@ -634,22 +699,4 @@ private IList GetMethodologiesScheduledWithRelease(Guid rele .ToList(); } } - - public class DeleteDataFilePlan - { - [JsonIgnore] - public Guid ReleaseId { get; set; } - - [JsonIgnore] - public Guid SubjectId { get; set; } - - public DeleteDataBlockPlan DeleteDataBlockPlan { get; set; } = null!; - - public List FootnoteIds { get; set; } = null!; - } - - public class DeleteReleasePlan - { - public List ScheduledMethodologies { get; set; } = new(); - } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReplacementService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReplacementService.cs index 05ea25cfe57..7ab80d9511c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReplacementService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReplacementService.cs @@ -2,9 +2,11 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Cache; +using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; @@ -20,6 +22,7 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.Services; using GovUk.Education.ExploreEducationStatistics.Data.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationErrorMessages; @@ -39,6 +42,7 @@ public class ReplacementService : IReplacementService private readonly ILocationRepository _locationRepository; private readonly IFootnoteRepository _footnoteRepository; private readonly IReleaseService _releaseService; + private readonly IDataSetVersionService _dataSetVersionService; private readonly ITimePeriodService _timePeriodService; private readonly IUserService _userService; private readonly ICacheKeyService _cacheKeyService; @@ -54,6 +58,7 @@ public ReplacementService(ContentDbContext contentDbContext, ILocationRepository locationRepository, IFootnoteRepository footnoteRepository, IReleaseService releaseService, + IDataSetVersionService dataSetVersionService, ITimePeriodService timePeriodService, IUserService userService, ICacheKeyService cacheKeyService, @@ -67,6 +72,7 @@ public ReplacementService(ContentDbContext contentDbContext, _locationRepository = locationRepository; _footnoteRepository = footnoteRepository; _releaseService = releaseService; + _dataSetVersionService = dataSetVersionService; _timePeriodService = timePeriodService; _userService = userService; _cacheKeyService = cacheKeyService; @@ -76,7 +82,8 @@ public ReplacementService(ContentDbContext contentDbContext, public async Task> GetReplacementPlan( Guid releaseVersionId, Guid originalFileId, - Guid replacementFileId) + Guid replacementFileId, + CancellationToken cancellationToken = default) { return await _contentDbContext.ReleaseVersions .FirstOrNotFoundAsync(rv => rv.Id == releaseVersionId) @@ -84,10 +91,17 @@ public async Task> GetReplace .OnSuccess(() => CheckReleaseFilesExist(releaseVersionId: releaseVersionId, originalFileId: originalFileId, replacementFileId: replacementFileId)) - .OnSuccess(async releaseFiles => + .OnSuccess(async tuple => { - var originalFile = releaseFiles.originalReleaseFile.File; - var replacementFile = releaseFiles.replacementReleaseFile.File; + var (originalReleaseFile, replacementReleaseFile) = tuple; + + return await GetLinkedDataSetVersion(originalReleaseFile, cancellationToken) + .OnSuccess(apiDataSetVersion => (originalReleaseFile, replacementReleaseFile, apiDataSetVersion)); + }) + .OnSuccess(async tuple => + { + var originalFile = tuple.originalReleaseFile.File; + var replacementFile = tuple.replacementReleaseFile.File; var originalSubjectId = originalFile.SubjectId!.Value; var replacementSubjectId = replacementFile.SubjectId!.Value; @@ -101,11 +115,26 @@ public async Task> GetReplace subjectId: originalSubjectId, replacementSubjectMeta); - return new DataReplacementPlanViewModel( - dataBlocks, - footnotes, - originalSubjectId, - replacementSubjectId); + var apiDataSetVersionDeletionPlan = tuple.apiDataSetVersion is null + ? null + : new DeleteApiDataSetVersionPlanViewModel + { + DataSetId = tuple.apiDataSetVersion.DataSetId, + DataSetTitle = tuple.apiDataSetVersion.DataSet.Title, + Id = tuple.apiDataSetVersion.Id, + Version = tuple.apiDataSetVersion.Version, + Status = tuple.apiDataSetVersion.Status, + Valid = false, + }; + + return new DataReplacementPlanViewModel + { + DataBlocks = dataBlocks, + Footnotes = footnotes, + DeleteApiDataSetVersionPlan = apiDataSetVersionDeletionPlan, + OriginalSubjectId = originalSubjectId, + ReplacementSubjectId = replacementSubjectId, + }; }); } @@ -120,7 +149,7 @@ public async Task> Replace( .OnSuccessCombineWith(_ => CheckReleaseFilesExist(releaseVersionId: releaseVersionId, originalFileId: originalFileId, replacementFileId: replacementFileId)) - .OnSuccess(async planAndReleaseFiles => + .OnSuccess(planAndReleaseFiles => { var (plan, (originalReleaseFile, replacementReleaseFile)) = planAndReleaseFiles; @@ -141,6 +170,12 @@ public async Task> Replace( "Replacement file has no link with the original file"); } + return (plan, originalReleaseFile, replacementReleaseFile); + }) + .OnSuccess(async planAndReleaseFiles => + { + var (plan, originalReleaseFile, replacementReleaseFile) = planAndReleaseFiles; + var originalSubjectId = plan.OriginalSubjectId; var replacementSubjectId = plan.ReplacementSubjectId; @@ -173,6 +208,24 @@ await plan.Footnotes }); } + private async Task> GetLinkedDataSetVersion( + ReleaseFile releaseFile, + CancellationToken cancellationToken = default) + { + if (releaseFile.PublicApiDataSetId is null) + { + return (DataSetVersion)null!; + } + + return await _dataSetVersionService.GetDataSetVersion( + releaseFile.PublicApiDataSetId.Value, + releaseFile.PublicApiDataSetVersion!, + cancellationToken) + .OnSuccess(dsv => (DataSetVersion?)dsv) + .OnFailureDo(_ => throw new ApplicationException( + $"API data set version could not be found. Data set ID: '{releaseFile.PublicApiDataSetId}', version: '{releaseFile.PublicApiDataSetVersion}'")); + } + private async Task> CheckReleaseFilesExist( Guid releaseVersionId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs index 5814a9dabd6..a756a571679 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs @@ -22,11 +22,13 @@ using GovUk.Education.ExploreEducationStatistics.Admin.Services.Methodologies; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Public.Data; using GovUk.Education.ExploreEducationStatistics.Admin.Settings; +using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.Public.Data; using GovUk.Education.ExploreEducationStatistics.Common.Cache; using GovUk.Education.ExploreEducationStatistics.Common.Cancellation; using GovUk.Education.ExploreEducationStatistics.Common.Config; using GovUk.Education.ExploreEducationStatistics.Common.Database; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Services; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; @@ -56,6 +58,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Identity.EntityFrameworkCore; +using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Authorization; using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.SignalR; @@ -81,21 +84,29 @@ using DataSetService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Public.Data.DataSetService; using GlossaryService = GovUk.Education.ExploreEducationStatistics.Admin.Services.GlossaryService; using IContentGlossaryService = GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.IGlossaryService; -using IContentMethodologyService = GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.IMethodologyService; -using IContentPublicationService = GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.IPublicationService; +using IContentMethodologyService = + GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.IMethodologyService; +using IContentPublicationService = + GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.IPublicationService; using IContentReleaseService = GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.IReleaseService; using IDataGuidanceService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IDataGuidanceService; -using IDataSetService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data.IDataSetService; +using IDataSetService = + GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Public.Data.IDataSetService; using IGlossaryService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IGlossaryService; -using IMethodologyImageService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Methodologies.IMethodologyImageService; -using IMethodologyService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Methodologies.IMethodologyService; -using IPublicationRepository = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IPublicationRepository; +using IMethodologyImageService = + GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Methodologies.IMethodologyImageService; +using IMethodologyService = + GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Methodologies.IMethodologyService; +using IPublicationRepository = + GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IPublicationRepository; using IPublicationService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IPublicationService; using IReleaseFileService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IReleaseFileService; using IReleaseService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IReleaseService; -using IReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IReleaseVersionRepository; +using IReleaseVersionRepository = + GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IReleaseVersionRepository; using IThemeService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IThemeService; -using MethodologyImageService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Methodologies.MethodologyImageService; +using MethodologyImageService = + GovUk.Education.ExploreEducationStatistics.Admin.Services.Methodologies.MethodologyImageService; using MethodologyService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Methodologies.MethodologyService; using PublicationRepository = GovUk.Education.ExploreEducationStatistics.Admin.Services.PublicationRepository; using PublicationService = GovUk.Education.ExploreEducationStatistics.Admin.Services.PublicationService; @@ -107,10 +118,14 @@ using HeaderNames = Microsoft.Net.Http.Headers.HeaderNames; using Microsoft.AspNetCore.Mvc; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using Semver; namespace GovUk.Education.ExploreEducationStatistics.Admin { - public class Startup(IConfiguration configuration, IHostEnvironment hostEnvironment) + public class Startup( + IConfiguration configuration, + IHostEnvironment hostEnvironment) { // This method gets called by the runtime. Use this method to add services to the container. public virtual void ConfigureServices(IServiceCollection services) @@ -264,8 +279,8 @@ public virtual void ConfigureServices(IServiceCollection services) if (!hostEnvironment.IsIntegrationTest()) { services - // This tells Identity Framework to look for Bearer tokens in incoming requests' Authorization - // headers as a way of identifying users. + // This tells Identity Framework to look for Bearer tokens in incoming requests' Authorization + // headers as a way of identifying users. .AddAuthentication(options => { // This line tells Identity Framework to use the JWT mechanism for verifying users based upon @@ -331,7 +346,8 @@ public virtual void ConfigureServices(IServiceCollection services) * Configuration options */ - services.Configure(configuration.GetRequiredSection(PublicDataProcessorOptions.Section)); + services.Configure( + configuration.GetRequiredSection(PublicDataProcessorOptions.Section)); services.Configure(configuration); services.Configure(configuration.GetRequiredSection(LocationsOptions.Locations)); services.Configure( @@ -460,7 +476,7 @@ public virtual void ConfigureServices(IServiceCollection services) provider.GetService(), provider.GetRequiredService(), provider.GetRequiredService())); - + services.AddTransient(); } @@ -562,7 +578,11 @@ public virtual void ConfigureServices(IServiceCollection services) services.AddSwaggerGen(c => { c.SwaggerDoc("v1", - new OpenApiInfo {Title = "Explore education statistics - Admin API", Version = "v1"}); + new OpenApiInfo + { + Title = "Explore education statistics - Admin API", + Version = "v1" + }); c.CustomSchemaIds((type) => type.FullName); c.AddSecurityDefinition("Bearer", new OpenApiSecurityScheme { @@ -577,9 +597,13 @@ public virtual void ConfigureServices(IServiceCollection services) { new OpenApiSecurityScheme { - Reference = new OpenApiReference {Type = ReferenceType.SecurityScheme, Id = "Bearer"} + Reference = new OpenApiReference + { + Type = ReferenceType.SecurityScheme, + Id = "Bearer" + } }, - new[] {string.Empty} + new[] { string.Empty } } }); }); @@ -649,7 +673,8 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) .FontSources(s => s.Self()) .FormActions(s => { - var loginAuthorityUrl = configuration.GetRequiredSection("OpenIdConnectIdentityFramework").GetValue("Authority"); + var loginAuthorityUrl = configuration.GetRequiredSection("OpenIdConnectIdentityFramework") + .GetValue("Authority"); var loginAuthorityUri = new Uri(loginAuthorityUrl); s .CustomSources(loginAuthorityUri.GetLeftPart(UriPartial.Authority)) @@ -745,16 +770,26 @@ private static void ApplyCustomMigrations(params ICustomMigration[] migrations) internal class NoOpDataSetVersionService : IDataSetVersionService { - public Task> GetStatusesForReleaseVersion(Guid releaseVersionId) + public Task> GetStatusesForReleaseVersion( + Guid releaseVersionId, + CancellationToken cancellationToken = default) { return Task.FromResult(new List()); } - - public Task FileHasVersion(Guid releaseFileId, CancellationToken cancellationToken = default) + + public Task> GetDataSetVersion( + Guid dataSetId, + SemVersion version, + CancellationToken cancellationToken = default) { - return Task.FromResult(false); + return Task.FromResult(new Either(new NotFoundResult())); } + public Task> CreateNextVersion( + Guid releaseFileId, + Guid dataSetId, + CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task> DeleteVersion(Guid dataSetVersionId, CancellationToken cancellationToken = default) { return Task.FromResult(new Either(Unit.Instance)); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ErrorDetails/ApiDataSetErrorDetail.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ErrorDetails/ApiDataSetErrorDetail.cs new file mode 100644 index 00000000000..b639f406a08 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ErrorDetails/ApiDataSetErrorDetail.cs @@ -0,0 +1,5 @@ +using System; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Validators.ErrorDetails; + +public record ApiDataSetErrorDetail(Guid DataSetId); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs index a7e285cba54..1cda44bcb2a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs @@ -1,7 +1,12 @@ #nullable enable +using GovUk.Education.ExploreEducationStatistics.Common.Model; namespace GovUk.Education.ExploreEducationStatistics.Admin.Validators; public static class ValidationMessages { + public static readonly LocalizableMessage CannotDeleteApiDataSetReleaseFile = new( + Code: nameof(CannotDeleteApiDataSetReleaseFile), + Message: "The file cannot be deleted as it is linked to an API data set." + ); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DataReplacementPlanViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DataReplacementPlanViewModel.cs index a5ea2d6abb5..ad662caae3d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DataReplacementPlanViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DataReplacementPlanViewModel.cs @@ -1,36 +1,26 @@ -#nullable enable -using System; -using System.Collections.Generic; -using System.Linq; +#nullable enable using GovUk.Education.ExploreEducationStatistics.Common.Converters; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; namespace GovUk.Education.ExploreEducationStatistics.Admin.ViewModels { public class DataReplacementPlanViewModel { - public IEnumerable DataBlocks { get; } - public IEnumerable Footnotes { get; } - public Guid OriginalSubjectId { get; } - public Guid ReplacementSubjectId { get; } - - public DataReplacementPlanViewModel( - IEnumerable dataBlocks, - IEnumerable footnotes, - Guid originalSubjectId, - Guid replacementSubjectId) - { - DataBlocks = dataBlocks; - Footnotes = footnotes; - OriginalSubjectId = originalSubjectId; - ReplacementSubjectId = replacementSubjectId; - } + public IEnumerable DataBlocks { get; init; } = []; + public IEnumerable Footnotes { get; init; } = []; + public DeleteApiDataSetVersionPlanViewModel? DeleteApiDataSetVersionPlan { get; init; } + public Guid OriginalSubjectId { get; init; } + public Guid ReplacementSubjectId { get; init; } public bool Valid => DataBlocks.All(info => info.Valid) - && Footnotes.All(info => info.Valid); + && Footnotes.All(info => info.Valid) + && (DeleteApiDataSetVersionPlan?.Valid ?? true); /** * Trimmed down version of the data replacement plan that @@ -38,12 +28,14 @@ public DataReplacementPlanViewModel( */ public DataReplacementPlanViewModel ToSummary() { - return new DataReplacementPlanViewModel( - DataBlocks.Select(block => block.ToSummary()), - Footnotes.Select(footnote => footnote.ToSummary()), - OriginalSubjectId, - ReplacementSubjectId - ); + return new DataReplacementPlanViewModel + { + DataBlocks = DataBlocks.Select(block => block.ToSummary()), + Footnotes = Footnotes.Select(footnote => footnote.ToSummary()), + DeleteApiDataSetVersionPlan = DeleteApiDataSetVersionPlan, + OriginalSubjectId = OriginalSubjectId, + ReplacementSubjectId = ReplacementSubjectId + }; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteApiDataSetVersionPlanViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteApiDataSetVersionPlanViewModel.cs new file mode 100644 index 00000000000..fc897a65875 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteApiDataSetVersionPlanViewModel.cs @@ -0,0 +1,20 @@ +#nullable enable +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using System; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; + +public record DeleteApiDataSetVersionPlanViewModel +{ + public Guid DataSetId { get; init; } + + public string DataSetTitle { get; init; } = null!; + + public Guid Id { get; init; } + + public string Version { get; init; } = null!; + + public DataSetVersionStatus Status { get; init; } + + public bool Valid { get; init; } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteDataBlockPlanViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteDataBlockPlanViewModel.cs new file mode 100644 index 00000000000..c348e3ffe7e --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteDataBlockPlanViewModel.cs @@ -0,0 +1,28 @@ +#nullable enable +using System.Collections.Generic; +using System; +using Newtonsoft.Json; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; + +public class DeleteDataBlockPlanViewModel +{ + [JsonIgnore] public Guid ReleaseId { get; set; } + public List DependentDataBlocks { get; set; } = new(); +} + +public class DependentDataBlock +{ + [JsonIgnore] public Guid Id { get; set; } + public string Name { get; set; } = string.Empty; + public string? ContentSectionHeading { get; set; } + public List InfographicFilesInfo { get; set; } = new(); + public bool IsKeyStatistic { get; set; } + public FeaturedTableBasicViewModel? FeaturedTable { get; set; } +} + +public class InfographicFileInfo +{ + public Guid Id { get; set; } + public string Filename { get; set; } = ""; +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteDataFilePlanViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteDataFilePlanViewModel.cs new file mode 100644 index 00000000000..45310feb61a --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteDataFilePlanViewModel.cs @@ -0,0 +1,23 @@ +#nullable enable +using System.Collections.Generic; +using System; +using Newtonsoft.Json; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; + +public record DeleteDataFilePlanViewModel +{ + [JsonIgnore] + public Guid ReleaseId { get; init; } + + [JsonIgnore] + public Guid SubjectId { get; init; } + + public DeleteDataBlockPlanViewModel DeleteDataBlockPlan { get; init; } = null!; + + public List FootnoteIds { get; init; } = null!; + + public DeleteApiDataSetVersionPlanViewModel? DeleteApiDataSetVersionPlan { get; init; } + + public bool Valid => DeleteApiDataSetVersionPlan?.Valid ?? true; +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteReleasePlanViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteReleasePlanViewModel.cs new file mode 100644 index 00000000000..262af92833c --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/DeleteReleasePlanViewModel.cs @@ -0,0 +1,10 @@ +#nullable enable +using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; +using System.Collections.Generic; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; + +public record DeleteReleasePlanViewModel +{ + public IReadOnlyList ScheduledMethodologies { get; init; } = []; +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/AssertExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/AssertExtensions.cs index 13842195212..ee80c79ba95 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/AssertExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/AssertExtensions.cs @@ -8,10 +8,10 @@ namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions { - public static class AssertExtensions { - public const int TimeWithinMillis = 750; + public const int TimeWithinMillis = 1500; + /** * Calling this method causes a Test to fail with the given message. The equivalent of `Assert.Fail()` in * other testing frameworks. @@ -24,11 +24,14 @@ public static XunitException AssertFail(string message) public static bool AssertDeepEqualTo( this T actual, T expected, + bool ignoreCollectionOrders = false, Expression>[]? notEqualProperties = null) { var compareLogic = new CompareLogic(); notEqualProperties?.ForEach(compareLogic.Config.IgnoreProperty); - var comparison = compareLogic.Compare(actual, expected); + compareLogic.Config.MaxDifferences = 100; + compareLogic.Config.IgnoreCollectionOrder = ignoreCollectionOrders; + var comparison = compareLogic.Compare(expected, actual); Assert.True(comparison.AreEqual, comparison.DifferencesString); notEqualProperties?.ForEach(notEqualField => { @@ -45,7 +48,6 @@ public static bool AssertDeepEqualTo( throw new XunitException($"Expected values for expression {notEqualField} to not be equal, " + $"but they were both of value \"{expectedValue}\"."); } - }); return true; } @@ -67,7 +69,8 @@ public static bool IsDeepEqualTo( { var compareLogic = new CompareLogic(); ignoreProperties?.ForEach(compareLogic.Config.IgnoreProperty); - var comparison = compareLogic.Compare(actual, expected); + compareLogic.Config.MaxDifferences = 100; + var comparison = compareLogic.Compare(expected, actual); return comparison.AreEqual; } @@ -87,7 +90,7 @@ public static void AssertUtcNow(this DateTime? dateTime, int withinMillis = Time Assert.NotNull(dateTime); dateTime.Value.AssertUtcNow(withinMillis: withinMillis); } - + /// /// Assert that the given DateTimeOffset is effectively "now", within a given tolerance of milliseconds. /// diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/AssertExtensionsTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/AssertExtensionsTests.cs index 5010f8c2918..26b379d58a0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/AssertExtensionsTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/AssertExtensionsTests.cs @@ -70,7 +70,7 @@ public void ExceptField() // Assert that all fields in Person are expected to be equal apart from the "Name" field, which is // expected to be unequal. - person1.AssertDeepEqualTo(person2, Except(person => person.Name)); + person1.AssertDeepEqualTo(person2, notEqualProperties: Except(person => person.Name)); } [Fact] @@ -82,7 +82,7 @@ public void ExceptField_ButFieldIsEqual() // Assert that all fields in Person are expected to be equal apart from the "Name" field, which is // expected to be unequal. It is, however, equal. var exception = Assert.Throws(() => - person1.AssertDeepEqualTo(person2, Except(person => person.Name))); + person1.AssertDeepEqualTo(person2, notEqualProperties: Except(person => person.Name))); Assert.Equal( "Expected values for expression person => person.Name to not be equal," + diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/DbContextTestExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/DbContextTestExtensions.cs new file mode 100644 index 00000000000..3cc63341f9b --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/DbContextTestExtensions.cs @@ -0,0 +1,41 @@ +#nullable enable +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; + +public static class DbContextTestExtensions +{ + public static async Task ClearTestData(this TDbContext context) where TDbContext : DbContext + { + if (context.Database.IsNpgsql()) + { +#pragma warning disable EF1002 + var tables = context.Model.GetEntityTypes() + .Select(type => type.GetTableName()) + .OfType() + .Distinct() + .ToList(); + + foreach (var table in tables) + { + await context.Database.ExecuteSqlRawAsync($"""TRUNCATE TABLE "{table}" RESTART IDENTITY CASCADE;"""); + } + + var sequences = context.Model.GetSequences(); + + foreach (var sequence in sequences) + { + await context.Database.ExecuteSqlRawAsync($"""ALTER SEQUENCE "{sequence.Name}" RESTART WITH 1;"""); + } +#pragma warning restore EF1002 + } + else + { + throw new NotImplementedException( + $"Clearing test data is not supported for type {context.Database.ProviderName}"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/DbContextTransactionExtensionsTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/DbContextTransactionExtensionsTests.cs new file mode 100644 index 00000000000..9f2a3c8e0fe --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/DbContextTransactionExtensionsTests.cs @@ -0,0 +1,401 @@ +#nullable enable +using System; +using System.Linq; +using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Testcontainers.PostgreSql; +using Xunit; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions.DbContextTransactionExtensionsTests; + +namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; + +/// +/// This test suite covers convenience extension methods for removing the verbosity of +/// creating shared transactions between different DbContexts, and the behaviours around +/// how they interact and some of the issues to look out for when using. +/// +/// From the tests covered below, we establish that: +/// +/// * multiple DbContexts are able to coordinate under the same transaction boundary and +/// all roll back if a failure occurs. +/// * Transactions can be nested within each other and the parent has control of completing +/// or failing the transaction. If the child transaction fails, the parent needs to +/// acknowledge that failure in order to fail itself (e.g. rethrow an exception, return a +/// failing Either etc). +/// * only a single DbContext that supports RetryOnFailure need be the one to create an +/// ExecutionContext, and thereafter all DbContexts supporting RetryOnFailure will +/// operate successfully. +/// * If a DbContext that doesn't support RetryOnFailure is used as the one that creates +/// the ExecutionStrategy and subsequently a DbContext that *does* support RetryOnFailure +/// is used, it will throw an InvalidOperationException, showing therefore that it is +/// best to use a RetryOnFailure-supporting DbContext if possible to originally create the +/// transaction. +/// +public abstract class DbContextTransactionExtensionsTests + : IClassFixture, + IAsyncLifetime +{ + private readonly WebApplicationFactory _testApp; + + public DbContextTransactionExtensionsTests(DbContextTransactionExtensionsTestFixture fixture) + { + _testApp = new TestApplicationFactory() + .ConfigureServices(services => services + .AddDbContext(options => + options.UseNpgsql(fixture.PostgreSqlContainers[0].GetConnectionString(), + psqlOptions => psqlOptions.EnableRetryOnFailure())) + .AddDbContext(options => + options.UseNpgsql(fixture.PostgreSqlContainers[1].GetConnectionString(), + psqlOptions => psqlOptions.EnableRetryOnFailure())) + .AddDbContext(options => + options.UseNpgsql(fixture.PostgreSqlContainers[2].GetConnectionString()))); + } + + public Task InitializeAsync() + { + return Task.CompletedTask; + } + + public async Task DisposeAsync() + { + await ClearTestData(); + await ClearTestData(); + await ClearTestData(); + } + + public class NoTransactionTests( + DbContextTransactionExtensionsTestFixture fixture) + : DbContextTransactionExtensionsTests(fixture) + { + [Fact] + public async Task SucceedWithoutTransaction() + { + await WriteToAllDbContexts(); + AssertEntitiesInAllDbContexts(); + } + } + + public class RequireTransactionTests( + DbContextTransactionExtensionsTestFixture fixture) + : DbContextTransactionExtensionsTests(fixture) + { + public class FlatTransactionTests( + DbContextTransactionExtensionsTestFixture fixture) + : RequireTransactionTests(fixture) + { + [Fact] + public async Task Succeed() + { + await using var dbContext1 = GetDbContext1(); + + await dbContext1.RequireTransaction(() => + WriteToAllDbContexts()); + + AssertEntitiesInAllDbContexts(); + } + + [Fact] + public async Task SucceedWithEither() + { + await using var dbContext1 = GetDbContext1(); + + await dbContext1.RequireTransaction(async () => + { + await WriteToAllDbContexts(); + return new Either("success!"); + }); + + AssertEntitiesInAllDbContexts(); + } + + [Fact] + public async Task Fail() + { + await using var dbContext1 = GetDbContext1(); + + await Assert.ThrowsAsync(() => + dbContext1.RequireTransaction(async () => + { + await WriteToAllDbContexts(); + throw new SimulateFailureException(); + })); + + AssertNoEntitiesInAnyDbContexts(); + } + + [Fact] + public async Task FailWithEither() + { + await using var dbContext1 = GetDbContext1(); + + await dbContext1.RequireTransaction(async () => + { + await WriteToAllDbContexts(); + return new Either(1); + }); + + AssertNoEntitiesInAnyDbContexts(); + } + } + + public class NestedTransactionTests( + DbContextTransactionExtensionsTestFixture fixture) + : RequireTransactionTests(fixture) + { + [Fact] + public async Task SucceedWithinNestedTransaction() + { + await using var dbContext1 = GetDbContext1(); + + await dbContext1.RequireTransaction(async () => + { + await WriteToAllDbContexts(1); + await dbContext1.RequireTransaction(() => WriteToAllDbContexts(2)); + }); + + AssertEntitiesInAllDbContexts(1); + AssertEntitiesInAllDbContexts(2); + } + + [Fact] + public async Task SucceedWithinNestedTransaction_MultipleContextsRequestTransaction() + { + await using var dbContext1 = GetDbContext1(); + await using var dbContext2 = GetDbContext2(); + await using var dbContext3WithoutRetry = GetDbContext3WithoutRetry(); + + await dbContext1.RequireTransaction(async () => + { + await WriteToAllDbContexts(); + await dbContext2.RequireTransaction(async () => + { + await WriteToAllDbContexts(2); + await dbContext3WithoutRetry.RequireTransaction( + () => WriteToAllDbContexts(3)); + }); + }); + + AssertEntitiesInAllDbContexts(1); + AssertEntitiesInAllDbContexts(2); + AssertEntitiesInAllDbContexts(3); + } + + [Fact] + public async Task TransactionInitiatedByNonRetryDbContext_ThrowsException() + { + await using var dbContext2 = GetDbContext2(); + await using var dbContext3WithoutRetry = GetDbContext3WithoutRetry(); + + await Assert.ThrowsAsync(() => + dbContext3WithoutRetry.RequireTransaction(async () => + { + await WriteToAllDbContexts(); + // ReSharper disable once AccessToDisposedClosure + await dbContext2.RequireTransaction(() => WriteToAllDbContexts(2)); + })); + + AssertNoEntitiesInAnyDbContexts(); + } + + [Fact] + public async Task FailWithinNestedTransaction() + { + await using var dbContext1 = GetDbContext1(); + + await Assert.ThrowsAsync(() => + dbContext1.RequireTransaction(async () => + { + await WriteToAllDbContexts(); + // ReSharper disable once AccessToDisposedClosure + await dbContext1.RequireTransaction(async () => + { + await WriteToAllDbContexts(2); + throw new SimulateFailureException(); + }); + })); + + AssertNoEntitiesInAnyDbContexts(); + } + + [Fact] + public async Task FailWithinNestedTransaction_WithEither() + { + await using var dbContext1 = GetDbContext1(); + await using var dbContext2 = GetDbContext1(); + + await dbContext1.RequireTransaction(async () => + { + await WriteToAllDbContexts(); + return await dbContext2.RequireTransaction(async () => + { + await WriteToAllDbContexts(2); + return new Either(1); + }); + }); + + AssertNoEntitiesInAnyDbContexts(); + } + + [Fact] + public async Task FailAtTopLevelWithNestedTransaction() + { + await using var dbContext1 = GetDbContext1(); + + await Assert.ThrowsAsync(() => dbContext1 + .RequireTransaction(async () => + { + await WriteToAllDbContexts(); + // ReSharper disable once AccessToDisposedClosure + await dbContext1.RequireTransaction(() => WriteToAllDbContexts(2)); + throw new SimulateFailureException(); + })); + + AssertNoEntitiesInAnyDbContexts(); + } + + [Fact] + public async Task FailAtTopLevelWithNestedTransaction_WithEither() + { + await using var dbContext1 = GetDbContext1(); + + await dbContext1.RequireTransaction(async () => + { + await WriteToAllDbContexts(); + await dbContext1.RequireTransaction(() => WriteToAllDbContexts(2)); + return new Either(1); + }); + + AssertNoEntitiesInAnyDbContexts(); + } + } + } + + private void AssertEntitiesInAllDbContexts(int expectedId = 1) + { + using var dbContext1 = GetDbContext1(); + using var dbContext2 = GetDbContext2(); + using var dbContext3 = GetDbContext3WithoutRetry(); + + Assert.NotNull(dbContext1 + .Entities + .SingleOrDefault(entity => entity.Id == expectedId)); + + Assert.NotNull(dbContext2 + .Entities + .SingleOrDefault(entity => entity.Id == expectedId)); + + Assert.NotNull(dbContext3 + .Entities + .SingleOrDefault(entity => entity.Id == expectedId)); + } + + private void AssertNoEntitiesInAnyDbContexts() + { + using var dbContext1 = GetDbContext1(); + using var dbContext2 = GetDbContext2(); + using var dbContext3 = GetDbContext3WithoutRetry(); + + Assert.Empty(dbContext1.Entities); + Assert.Empty(dbContext2.Entities); + Assert.Empty(dbContext3.Entities); + } + + internal class TestEntity + { + public int Id { get; set; } + } + + internal class TestDbContext1( + DbContextOptions options) : DbContext(options) + { + public DbSet Entities { get; init; } = null!; + } + + internal class TestDbContext2( + DbContextOptions options) : DbContext(options) + { + public DbSet Entities { get; init; } = null!; + } + + internal class TestDbContext3WithoutRetry( + DbContextOptions options) : DbContext(options) + { + public DbSet Entities { get; init; } = null!; + } + + private class SimulateFailureException : Exception; + + // ReSharper disable once ClassNeverInstantiated.Global + public class DbContextTransactionExtensionsTestFixture : + IAsyncLifetime + { + public readonly PostgreSqlContainer[] PostgreSqlContainers = + Enumerable.Range(0, 3).Select(_ => new PostgreSqlBuilder() + .WithImage("postgres:16.1-alpine") + .WithCommand("-c", "max_prepared_transactions=100") + .Build()) + .ToArray(); + + /// + /// Add prepared transaction support to the PostgreSQL containers so that a shared transaction + /// can be created for the various DbContexts in this test. + /// + /// Create an "Entities" table in each database for Entity Framework to read and write to. + /// + public async Task InitializeAsync() => + await Task.WhenAll(PostgreSqlContainers.Select(async container => + { + await container.StartAsync(); + await container.ExecScriptAsync("""CREATE TABLE IF NOT EXISTS "Entities" ("Id" int PRIMARY KEY)"""); + return Task.CompletedTask; + })); + + public async Task DisposeAsync() => + await Task.WhenAll(PostgreSqlContainers.Select(async container => + await container.DisposeAsync())); + } + + private async Task WriteToAllDbContexts(int id = 1) + { + var dbContext1 = _testApp.Services.GetRequiredService(); + var dbContext2 = _testApp.Services.GetRequiredService(); + var dbContext3WithoutRetry = _testApp.Services.GetRequiredService(); + await dbContext1.Entities.AddAsync(new TestEntity {Id = id}); + await dbContext1.SaveChangesAsync(); + await dbContext2.Entities.AddAsync(new TestEntity {Id = id}); + await dbContext2.SaveChangesAsync(); + await dbContext3WithoutRetry.Entities.AddAsync(new TestEntity {Id = id}); + await dbContext3WithoutRetry.SaveChangesAsync(); + } + + private async Task ClearTestData() where TDbContext : DbContext + { + var context = _testApp.GetDbContext(); + await context.ClearTestData(); + } + + private TDbContext GetDbContext() where TDbContext : DbContext + { + return _testApp.GetDbContext(); + } + + private TestDbContext1 GetDbContext1() + { + return _testApp.GetDbContext(); + } + + private TestDbContext2 GetDbContext2() + { + return _testApp.GetDbContext(); + } + + private TestDbContext3WithoutRetry GetDbContext3WithoutRetry() + { + return _testApp.GetDbContext(); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EitherTestExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EitherTestExtensions.cs index e231b350a76..1f550e88748 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EitherTestExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EitherTestExtensions.cs @@ -1,6 +1,7 @@ -using System; +using System; using System.Net; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using Microsoft.AspNetCore.Mvc; using Xunit; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions.AssertExtensions; @@ -80,6 +81,12 @@ public static ActionResult AssertBadRequest(this Either(this Either either) + { + var badRequest = either.AssertActionResultOfType(); + return Assert.IsAssignableFrom(badRequest.Value); + } + private static TActionResult AssertActionResultOfType(this Either result) where TActionResult : ActionResult { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/TypeExtensionsTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/TypeExtensionsTests.cs index a74b43a9e60..0c2b5ff3815 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/TypeExtensionsTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/TypeExtensionsTests.cs @@ -145,6 +145,76 @@ public void ActionResult_Void() } } + public class IsSimpleTests + { + public static readonly TheoryData SimpleTypes = new() + { + typeof(int), + typeof(int?), + typeof(double), + typeof(double?), + typeof(bool), + typeof(bool?), + typeof(char), + typeof(char?), + typeof(string), + typeof(decimal), + typeof(decimal?), + typeof(TestEnum), + typeof(TestEnum?), + typeof(DateTime), + typeof(DateTime?), + typeof(DateTimeOffset), + typeof(DateTimeOffset?), + typeof(TimeSpan), + typeof(Uri), + typeof(Guid), + typeof(Guid?), + }; + + public static readonly TheoryData ComplexTypes = new() + { + typeof(List), + typeof(ISet), + typeof(TypeExtensionsTests), + typeof(Type), + typeof(TestStruct), + typeof(TestStruct?), + + }; + + [Theory] + [MemberData(nameof(SimpleTypes))] + public void SimpleType_ReturnsTrue(Type simpleType) + { + Assert.True(simpleType.IsSimple()); + } + + [Theory] + [MemberData(nameof(ComplexTypes))] + public void ComplexType_ReturnsFalse(Type complexType) + { + Assert.False(complexType.IsSimple()); + } + } + + public class IsComplexTests + { + [Theory] + [MemberData(nameof(IsSimpleTests.SimpleTypes), MemberType = typeof(IsSimpleTests))] + public void SimpleType_ReturnsFalse(Type simpleType) + { + Assert.False(simpleType.IsComplex()); + } + + [Theory] + [MemberData(nameof(IsSimpleTests.ComplexTypes), MemberType = typeof(IsSimpleTests))] + public void ComplexType_ReturnsTrue(Type complexType) + { + Assert.True(complexType.IsComplex()); + } + } + public class IsNullableTypeTests { [Theory] @@ -246,8 +316,8 @@ private class NullableReferenceTypeClass public NullableReferenceTypeClass? NullableReferenceType { get; set; } }; - private enum TestEnum - { - } + private enum TestEnum; + + private struct TestStruct; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/ValidationProblemViewModelTestExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/ValidationProblemViewModelTestExtensions.cs index 70e8ba3897d..6f17c6df641 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/ValidationProblemViewModelTestExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/ValidationProblemViewModelTestExtensions.cs @@ -341,36 +341,50 @@ public static ErrorViewModel AssertHasAllowedValueError( public static ErrorViewModel AssertHasNotEmptyBodyError( this ValidationProblemViewModel validationProblem) { - var error = validationProblem.AssertHasError( + return validationProblem.AssertHasError( expectedPath: null, expectedCode: ValidationMessages.NotEmptyBody.Code ); - - return error; } public static ErrorViewModel AssertHasInvalidInputError( this ValidationProblemViewModel validationProblem, string expectedPath) { - var error = validationProblem.AssertHasError( + return validationProblem.AssertHasError( expectedPath: expectedPath, - expectedCode: ValidationMessages.InvalidInput.Code + expectedCode: ValidationMessages.InvalidValue.Code ); + } - return error; + public static ErrorViewModel AssertHasRequiredValueError( + this ValidationProblemViewModel validationProblem, + string expectedPath) + { + return validationProblem.AssertHasError( + expectedPath: expectedPath, + expectedCode: ValidationMessages.RequiredValue.Code + ); } - public static ErrorViewModel AssertHasRequiredFieldError( + public static ErrorViewModel AssertHasInvalidValueError( this ValidationProblemViewModel validationProblem, string expectedPath) { - var error = validationProblem.AssertHasError( + return validationProblem.AssertHasError( expectedPath: expectedPath, - expectedCode: ValidationMessages.RequiredField.Code + expectedCode: ValidationMessages.InvalidValue.Code ); + } - return error; + public static ErrorViewModel AssertHasUnknownFieldError( + this ValidationProblemViewModel validationProblem, + string expectedPath) + { + return validationProblem.AssertHasError( + expectedPath: expectedPath, + expectedCode: ValidationMessages.UnknownField.Code + ); } public static ErrorViewModel AssertHasError( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/WebApplicationFactoryExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/WebApplicationFactoryExtensions.cs index 81e717a2a63..151bcdb5bb0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/WebApplicationFactoryExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/WebApplicationFactoryExtensions.cs @@ -21,7 +21,16 @@ this WebApplicationFactory app where TDbContext : DbContext where TEntrypoint : class { - return app.WithWebHostBuilder(builder => builder.ResetDbContext()); + return app.WithWebHostBuilder(builder => builder + .ResetDbContext()); + } + + public static TDbContext GetDbContext(this WebApplicationFactory app) + where TDbContext : DbContext + where TEntrypoint : class + { + var scope = app.Services.CreateScope(); + return scope.ServiceProvider.GetRequiredService(); } /// diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/DataFixtureTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/DataFixtureTests.cs index 1edc0e2dae0..3fc20929e77 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/DataFixtureTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/DataFixtureTests.cs @@ -45,8 +45,8 @@ public void Generator_Single_RandomIsDeterministic() public void Generator_Single_GenerateRandomIsDeterministic() { var fixture = new DataFixture(); - var generator = fixture.Generator(); + var items = generator .ForInstance(s => s .Set(p => p.FirstName, "Test person")) @@ -99,7 +99,7 @@ public void Generator_Single_Index_IncrementsFromZeroOnMultipleGenerates() var items = generator .ForInstance(s => s - .Set(p => p.FirstName, (faker, _, context) => $"Jane {context.Index}")) + .Set(p => p.FirstName, (_, _, context) => $"Jane {context.Index}")) .GenerateList(3); Assert.Equal("Jane 0", items[0].FirstName); @@ -113,6 +113,69 @@ public void Generator_Single_Index_IncrementsFromZeroOnMultipleGenerates() Assert.Equal("Jane 2", items[2].FirstName); } + [Fact] + public void Generator_Single_DefaultDataFixture_Index_IncrementsFromZeroOnMultipleGenerates() + { + var generator = new Generator(); + + var items = generator + .ForInstance(s => s + .Set(p => p.FirstName, (_, _, context) => $"Jane {context.Index}")) + .GenerateList(3); + + Assert.Equal("Jane 0", items[0].FirstName); + Assert.Equal("Jane 1", items[1].FirstName); + Assert.Equal("Jane 2", items[2].FirstName); + + items = generator.GenerateList(3); + + Assert.Equal("Jane 0", items[0].FirstName); + Assert.Equal("Jane 1", items[1].FirstName); + Assert.Equal("Jane 2", items[2].FirstName); + } + + [Fact] + public void Generator_Single_DefaultDataFixture_FixtureIndex_IncrementSharedAcrossMultipleGenerates() + { + var generator = new Generator(); + + var items = generator + .ForInstance(s => s + .Set(p => p.FirstName, (_, _, context) => $"Jane {context.FixtureIndex}")) + .GenerateList(3); + + Assert.Equal("Jane 0", items[0].FirstName); + Assert.Equal("Jane 1", items[1].FirstName); + Assert.Equal("Jane 2", items[2].FirstName); + + items = generator.GenerateList(3); + + Assert.Equal("Jane 3", items[0].FirstName); + Assert.Equal("Jane 4", items[1].FirstName); + Assert.Equal("Jane 5", items[2].FirstName); + } + + [Fact] + public void Generator_Single_BuiltInDataFixture_FixtureTypeIndex_IncrementSharedAcrossMultipleGenerates() + { + var generator = new Generator(); + + var items = generator + .ForInstance(s => s + .Set(p => p.FirstName, (_, _, context) => $"Jane {context.FixtureTypeIndex}")) + .GenerateList(3); + + Assert.Equal("Jane 0", items[0].FirstName); + Assert.Equal("Jane 1", items[1].FirstName); + Assert.Equal("Jane 2", items[2].FirstName); + + items = generator.GenerateList(3); + + Assert.Equal("Jane 3", items[0].FirstName); + Assert.Equal("Jane 4", items[1].FirstName); + Assert.Equal("Jane 5", items[2].FirstName); + } + [Fact] public void Generator_Single_IndexFaker_DoesNotIncrementFromZeroOnMultipleGenerates() { @@ -324,6 +387,35 @@ public void Generator_Multiple_FixtureTypeIndex_IncrementsForGeneratorsOfSameTyp Assert.Equal("Doe 5", items[2].LastName); } + [Fact] + public void Generate_Multiple_DefaultDataFixtures_FixtureTypeIndex_IncrementIndependently() + { + // Instantiate generators without a shared `DataFixture` instance, + // meaning each generator gets its own default instance. + var persons = new Generator() + .ForInstance(s => s + .Set(p => p.FirstName, (_, _, context) => $"Jane {context.FixtureTypeIndex}") + .Set(p => p.LastName, (_, _, context) => $"Doe {context.FixtureTypeIndex}")) + .GenerateList(3); + + Assert.Equal("Jane 0", persons[0].FirstName); + Assert.Equal("Jane 1", persons[1].FirstName); + Assert.Equal("Jane 2", persons[2].FirstName); + + Assert.Equal("Doe 0", persons[0].LastName); + Assert.Equal("Doe 1", persons[1].LastName); + Assert.Equal("Doe 2", persons[2].LastName); + + var companies = new Generator() + .ForInstance(s => s + .Set(c => c.Name, (_, _, context) => $"Acme {context.FixtureTypeIndex}")) + .GenerateList(3); + + Assert.Equal("Acme 0", companies[0].Name); + Assert.Equal("Acme 1", companies[1].Name); + Assert.Equal("Acme 2", companies[2].Name); + } + [Fact] public void Generate_Multiple_FixtureIndex_IncrementsForGeneratorsOfAnyType() { @@ -350,13 +442,44 @@ public void Generate_Multiple_FixtureIndex_IncrementsForGeneratorsOfAnyType() .Set(c => c.Name, (_, _, context) => $"Acme {context.FixtureIndex}")) .GenerateList(3); - // FixtureTypeIndex increments when generators + // FixtureIndex increments when generators // of any type generate new instances. Assert.Equal("Acme 3", companies[0].Name); Assert.Equal("Acme 4", companies[1].Name); Assert.Equal("Acme 5", companies[2].Name); } + [Fact] + public void Generate_Multiple_DefaultDataFixtures_FixtureIndex_IncrementIndependently() + { + // Instantiate generators without a shared `DataFixture` instance, + // meaning each generator gets its own default instance. + var persons = new Generator() + .ForInstance(s => s + .Set(p => p.FirstName, (_, _, context) => $"Jane {context.FixtureIndex}") + .Set(p => p.LastName, (_, _, context) => $"Doe {context.FixtureIndex}")) + .GenerateList(3); + + Assert.Equal("Jane 0", persons[0].FirstName); + Assert.Equal("Jane 1", persons[1].FirstName); + Assert.Equal("Jane 2", persons[2].FirstName); + + Assert.Equal("Doe 0", persons[0].LastName); + Assert.Equal("Doe 1", persons[1].LastName); + Assert.Equal("Doe 2", persons[2].LastName); + + var companies = new Generator() + .ForInstance(s => s + .Set(c => c.Name, (_, _, context) => $"Acme {context.FixtureIndex}")) + .GenerateList(3); + + // FixtureIndex increments when generators + // of any type generate new instances. + Assert.Equal("Acme 0", companies[0].Name); + Assert.Equal("Acme 1", companies[1].Name); + Assert.Equal("Acme 2", companies[2].Name); + } + [Fact] public void Generator_Multiple_SetDefaults_IncrementsForGeneratorsOfSameType() { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/Generator.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/Generator.cs index 9f0e49c26ca..b26d33c7677 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/Generator.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/Generator.cs @@ -15,7 +15,7 @@ public class Generator where T : class { private readonly Faker _faker; private readonly Func? _seeder; - private readonly DataFixture? _fixture; + private readonly DataFixture _fixture; private Func _factory = Activator.CreateInstance; @@ -32,7 +32,7 @@ public Generator( { _faker = faker ?? new Faker(); _seeder = seeder; - _fixture = fixture; + _fixture = fixture ?? new DataFixture(); } /// @@ -219,7 +219,7 @@ private int GetMaximumIndex() : 0; } - private T GenerateSingle(int index, int? fixtureTypeIndex, int? fixtureIndex) + private T GenerateSingle(int index, int fixtureTypeIndex, int fixtureIndex) { if (_seeder is not null) { @@ -236,10 +236,10 @@ private T GenerateSingle(int index, int? fixtureTypeIndex, int? fixtureIndex) _faker, instance, new SetterContext( - index: index, - fixture: _fixture, - fixtureTypeIndex: fixtureTypeIndex, - fixtureIndex: fixtureIndex + Index: index, + Fixture: _fixture, + FixtureTypeIndex: fixtureTypeIndex, + FixtureIndex: fixtureIndex ) ); } @@ -292,10 +292,10 @@ private T GenerateWithRange(int index, int length) _faker, instance, new SetterContext( - index: index, - fixture: _fixture, - fixtureTypeIndex: fixtureTypeIndex, - fixtureIndex: fixtureIndex + Index: index, + Fixture: _fixture, + FixtureTypeIndex: fixtureTypeIndex, + FixtureIndex: fixtureIndex ) ); } @@ -309,9 +309,9 @@ private T GenerateWithRange(int index, int length) private void InvokeFinalizers(T instance) => _finalizers.ForEach(finalizer => finalizer(instance, _faker)); - private int? NextFixtureTypeIndex() => _fixture?.NextTypeIndex(); + private int NextFixtureTypeIndex() => _fixture.NextTypeIndex(); - private int? NextFixtureIndex() => _fixture?.NextIndex(); + private int NextFixtureIndex() => _fixture.NextIndex(); private interface ISetter { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/InstanceSetters.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/InstanceSetters.cs index a435866bad5..4c27b344b2b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/InstanceSetters.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/InstanceSetters.cs @@ -136,39 +136,27 @@ public int GetDisplayIndex(SetterContext context, Faker faker) /// /// Provides the setter with contextual data whilst generating instances. /// -public record SetterContext +public record SetterContext(int Index, DataFixture Fixture, int FixtureTypeIndex, int FixtureIndex) { /// /// The instance's index in the generated sequence. /// - public readonly int Index; + public readonly int Index = Index; /// /// The associated fixture for this generator (if there is one). - /// If no associated fixture, this will be null. /// - public readonly DataFixture? Fixture; - - private readonly int? _fixtureTypeIndex; - private readonly int? _fixtureIndex; - - public SetterContext(int index, DataFixture? fixture, int? fixtureTypeIndex, int? fixtureIndex) - { - Index = index; - Fixture = fixture; - _fixtureTypeIndex = fixtureTypeIndex; - _fixtureIndex = fixtureIndex; - } + public readonly DataFixture Fixture = Fixture; /// /// The instance's index across all generators for this type in the - /// associated fixture. If no associated fixture, this will be -1. + /// associated fixture. /// - public int FixtureTypeIndex => _fixtureTypeIndex ?? -1; + public readonly int FixtureTypeIndex = FixtureTypeIndex; /// /// The instance's index across all instances generated by the - /// associated fixture. If no associated fixture, this will be -1. + /// associated fixture. /// - public int FixtureIndex => _fixtureIndex ?? -1; + public readonly int FixtureIndex = FixtureIndex; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/TestApplicationFactory.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/TestApplicationFactory.cs index 71c5ba13c03..dbfa14e0437 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/TestApplicationFactory.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Fixtures/TestApplicationFactory.cs @@ -2,6 +2,7 @@ using System; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.TestHost; @@ -32,14 +33,13 @@ public async Task AddTestData(Action supplier) where TDb public async Task EnsureDatabaseDeleted() where TDbContext : DbContext { - await using var context = GetDbContext(); + await using var context = this.GetDbContext(); await context.Database.EnsureDeletedAsync(); } public TDbContext GetDbContext() where TDbContext : DbContext { - var scope = Services.CreateScope(); - return scope.ServiceProvider.GetRequiredService(); + return this.GetDbContext(); } protected override IHostBuilder CreateHostBuilder() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/FunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/FunctionsIntegrationTest.cs index cf2668240ad..87b4e15b424 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/FunctionsIntegrationTest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/FunctionsIntegrationTest.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; @@ -18,6 +19,7 @@ public abstract class FunctionsIntegrationTest where TFunctionsIntegrationTestFixture : FunctionsIntegrationTestFixture { protected readonly DataFixture DataFixture = new(); + private readonly IHost _host = fixture .ConfigureTestHostBuilder() .Build(); @@ -42,21 +44,10 @@ protected TDbContext GetDbContext() where TDbContext : DbContext return scope.ServiceProvider.GetRequiredService(); } - protected void ClearTestData() where TDbContext : DbContext + protected async Task ClearTestData() where TDbContext : DbContext { - using var context = GetDbContext(); - - var tables = context.Model.GetEntityTypes() - .Select(type => type.GetTableName()) - .Distinct() - .ToList(); - - foreach (var table in tables) - { -#pragma warning disable EF1002 - context.Database.ExecuteSqlRaw($@"TRUNCATE TABLE ""{table}"" RESTART IDENTITY CASCADE;"); -#pragma warning restore EF1002 - } + var context = GetDbContext(); + await context.ClearTestData(); } protected void ResetDbContext() where TDbContext : DbContext diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/GovUk.Education.ExploreEducationStatistics.Common.Tests.csproj b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/GovUk.Education.ExploreEducationStatistics.Common.Tests.csproj index 0dfbb9e71d4..3093d690753 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/GovUk.Education.ExploreEducationStatistics.Common.Tests.csproj +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/GovUk.Education.ExploreEducationStatistics.Common.Tests.csproj @@ -1,43 +1,46 @@  - - net8.0 - false - true - + + net8.0 + false + true + - - - - + + + + - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + - - - + + + - - - - Always - - + + + + Always + + diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/DbContextTransactionExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/DbContextTransactionExtensions.cs new file mode 100644 index 00000000000..e871f89672b --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/DbContextTransactionExtensions.cs @@ -0,0 +1,114 @@ +#nullable enable +using System; +using System.Threading.Tasks; +using System.Transactions; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Common.Extensions; + +/// +/// These extension methods provide a convenient way to ensure a transaction is +/// created for a given unit of work. +/// +public static class DbContextTransactionExtensions +{ + /// + /// This method ensures that a transaction is available for the given unit of work and + /// is shared between any DbContexts in use during the unit of work. + /// + /// If an existing transaction already exists, the unit of work will use the + /// existing transaction rather than this method creating a new one. + /// + /// If an uncaught exception is thrown during the unit of work, this will result in + /// a rollback, unless caught and suppressed by any parent unit of work that has its + /// own transaction open. + /// + /// Note that if you have a unit of work that requires interaction with multiple + /// DbContexts and any use the EnableRetryOnFailure() strategy, you should use any + /// one of the DbContexts that use this strategy to call "RequireTransaction" on, + /// and thereafter any other DbContexts that also use the EnableRetryOnFailure() + /// strategy will be functional. Not doing this will result in + /// InvalidOperationExceptions when those DbContexts are used. + /// + public static async Task RequireTransaction( + this TDbContext context, + Func> transactionalUnit) + where TDbContext : DbContext + { + var strategy = context.Database.CreateExecutionStrategy(); + + return await strategy.ExecuteAsync( + async () => + { + using var transactionScope = CreateTransactionScope(); + var result = await transactionalUnit.Invoke(); + transactionScope.Complete(); + return result; + }); + } + + /// + /// See the description for . + /// The only difference with this method is that it accepts a Func that does not have + /// a return value. + /// + public static async Task RequireTransaction( + this TDbContext context, + Func transactionalUnit) + where TDbContext : DbContext + { + await RequireTransaction(context, async () => + { + await transactionalUnit.Invoke(); + return Unit.Instance; + }); + } + + /// + /// This method ensures that a transaction is available for the given unit of work and + /// is shared between any DbContexts in use during the unit of work. + /// + /// If an existing transaction already exists, the unit of work will use the + /// existing transaction rather than this method creating a new one. + /// + /// If the unit of work returns a Left Either, this will result in a rollback + /// unless ignored by any parent unit of work that has its own transaction + /// open. It is good practice therefore for any parent unit of work that has also + /// opened its own transaction to propagate the Left Either back to the top of the + /// call chain. + /// + /// Note that if you have a unit of work that requires interaction with multiple + /// DbContexts and any use the EnableRetryOnFailure() strategy, you should use any + /// one of the DbContexts that use this strategy to call "RequireTransaction" on, + /// and thereafter any other DbContexts that also use the EnableRetryOnFailure() + /// strategy will be functional. Not doing this will result in + /// InvalidOperationExceptions when those DbContexts are used. + /// + public static async Task> RequireTransaction( + this TDbContext context, + Func>> transactionalUnit) + where TDbContext : DbContext + { + var strategy = context.Database.CreateExecutionStrategy(); + + return await strategy.ExecuteAsync( + async () => + { + using var transactionScope = CreateTransactionScope(); + return await transactionalUnit + .Invoke() + .OnSuccessDo(transactionScope.Complete); + }); + } + + private static TransactionScope CreateTransactionScope( + TransactionScopeOption transactionScopeOption = TransactionScopeOption.Required, + IsolationLevel isolationLevel = IsolationLevel.ReadCommitted) + { + return new( + scopeOption: transactionScopeOption, + transactionOptions: new TransactionOptions {IsolationLevel = isolationLevel}, + asyncFlowOption: TransactionScopeAsyncFlowOption.Enabled); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/ServiceCollectionExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/ServiceCollectionExtensions.cs index fa0eacd4456..0f9e34bf515 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/ServiceCollectionExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/ServiceCollectionExtensions.cs @@ -76,7 +76,9 @@ private static IServiceCollection AddDevelopmentPsqlDbContext( services.AddDbContext(options => { options - .UseNpgsql(dataSource) + .UseNpgsql( + dataSource, + psqlOptions => psqlOptions.EnableRetryOnFailure()) .EnableSensitiveDataLogging(); optionsConfiguration?.Invoke(options); @@ -111,7 +113,10 @@ private static IServiceCollection RegisterManagedIdentityPsqlDbContext(options => { - options.UseNpgsql(dataSource); + options + .UseNpgsql( + dataSource, + psqlOptions => psqlOptions.EnableRetryOnFailure()); optionsConfiguration?.Invoke(options); }); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/TypeExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/TypeExtensions.cs index 63a5b98e664..f43f8e929a9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/TypeExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/TypeExtensions.cs @@ -86,6 +86,32 @@ public static IEnumerable GetLoadableTypes(this Assembly assembly) } } + /// + /// Returns true if the type is considered 'simple' i.e. it can be + /// represented as a single value e.g. a string, number, date, etc. + /// + public static bool IsSimple(this Type type) + { + var underlyingType = type.GetUnderlyingType(); + + return underlyingType.IsPrimitive + || underlyingType.IsEnum + || underlyingType == typeof(string) + || underlyingType == typeof(decimal) + || underlyingType == typeof(DateTime) + || underlyingType == typeof(DateTimeOffset) + || underlyingType == typeof(TimeSpan) + || underlyingType == typeof(Uri) + || underlyingType == typeof(Guid); + } + + /// + /// Returns true if the type is considered 'complex' i.e. it cannot be + /// represented as a single value e.g. an object, list, etc. + /// This is the opposite of . + /// + public static bool IsComplex(this Type type) => !type.IsSimple(); + public static bool IsNullableType(this Type type) => Nullable.GetUnderlyingType(type) is not null; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/ModelBinding/EmptyStringMetadataDetailsProvider.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/ModelBinding/EmptyStringMetadataDetailsProvider.cs new file mode 100644 index 00000000000..8d035f69d71 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/ModelBinding/EmptyStringMetadataDetailsProvider.cs @@ -0,0 +1,19 @@ +using Microsoft.AspNetCore.Mvc.ModelBinding.Metadata; + +namespace GovUk.Education.ExploreEducationStatistics.Common.ModelBinding; + +/// +/// Modifies the model metadata to stop empty strings being converted to null. +/// This is mostly for query strings with empty parameters (e.g. `?foo=`) where +/// the presence of the query parameter should bind an empty string to the model. +/// +public class EmptyStringMetadataDetailsProvider : IDisplayMetadataProvider +{ + public void CreateDisplayMetadata(DisplayMetadataProviderContext context) + { + if (context.Key.MetadataKind == ModelMetadataKind.Property) + { + context.DisplayMetadata.ConvertEmptyStringToNull = false; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/DatabaseHelper.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/DatabaseHelper.cs index 2b174005546..959403265d7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/DatabaseHelper.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/DatabaseHelper.cs @@ -39,7 +39,7 @@ public Task DoInTransaction( where TDbContext : DbContext { var strategy = context.Database.CreateExecutionStrategy(); - + return strategy.Execute(async () => { // We use a delegate context here to allow us to retry on failure successfully when defining our own @@ -49,7 +49,7 @@ public Task DoInTransaction( await using var transaction = await ctxDelegate.Database.BeginTransactionAsync(); var result = await transactionalUnit.Invoke(ctxDelegate); - + await transaction.CommitAsync(); await ctxDelegate.Database.CloseConnectionAsync(); @@ -58,7 +58,7 @@ public Task DoInTransaction( } public Task DoInTransaction( - TDbContext context, + TDbContext context, Action transactionalUnit) where TDbContext : DbContext { @@ -70,13 +70,13 @@ public Task DoInTransaction( } public Task DoInTransaction( - TDbContext context, + TDbContext context, Func transactionalUnit) where TDbContext : DbContext { return DoInTransaction(context, ctx => Task.FromResult(transactionalUnit.Invoke(ctx))); } - + public Task ExecuteWithExclusiveLock( TDbContext dbContext, string lockName, @@ -85,8 +85,10 @@ public Task ExecuteWithExclusiveLock( { return DoInTransaction(dbContext, async ctx => { +#pragma warning disable EF1002 await ctx.Database.ExecuteSqlRawAsync($"exec sp_getapplock '{lockName}', 'exclusive'"); +#pragma warning restore EF1002 await action.Invoke(ctx); }); } -} \ No newline at end of file +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Validators/ValidationMessages.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Validators/ValidationMessages.cs index 73d83847843..2dc8efffa04 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Validators/ValidationMessages.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Validators/ValidationMessages.cs @@ -11,9 +11,9 @@ public static class ValidationMessages Message: "Must be one of the allowed values." ); - public static readonly LocalizableMessage InvalidInput = new( - Code: "InvalidInput", - Message: "The input is not valid. Check that it is in the expected format." + public static readonly LocalizableMessage InvalidValue = new( + Code: "InvalidValue", + Message: "Must be a valid value. Check that the type and format are correct." ); public static readonly LocalizableMessage NotEmptyBody = new( @@ -21,8 +21,13 @@ public static class ValidationMessages Message: "The request body must not be empty." ); - public static readonly LocalizableMessage RequiredField = new( - Code: "RequiredField", - Message: "The field is required." + public static readonly LocalizableMessage RequiredValue = new( + Code: "RequiredValue", + Message: "A value is required for this field." + ); + + public static readonly LocalizableMessage UnknownField = new( + Code: "UnknownField", + Message: "The field was not expected in the request and should be removed." ); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerCachingTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerCachingTests.cs index c075faa7bc2..651ba30ac9d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerCachingTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerCachingTests.cs @@ -82,6 +82,7 @@ public class ListDataSetsTests : DataSetFilesControllerCachingTests Indicators = ["Indicator 1"], }, LatestData = true, + IsSuperseded = false, Published = DateTime.UtcNow, } ], diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index 3380c7b435c..90d757366b1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -680,15 +680,14 @@ public async Task FilterByDataSetType_Api_ReturnsOnlyDataSetsWithAssociatedApiDa var releaseVersionFiles = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) - .WithFiles(_fixture.DefaultFile() - .ForIndex(0, s => s - .SetPublicApiDataSetId(Guid.NewGuid()) - .SetPublicApiDataSetVersion(major: 1, minor: 0)) - .ForIndex(1, s => s - .SetPublicApiDataSetId(Guid.NewGuid()) - .SetPublicApiDataSetVersion(major: 2, minor: 0)) - .GenerateList(5)) - .GenerateList(); + .WithFile(() => _fixture.DefaultFile(FileType.Data)) + .ForIndex(0, s => s + .SetPublicApiDataSetId(Guid.NewGuid()) + .SetPublicApiDataSetVersion(major: 1, minor: 0)) + .ForIndex(1, s => s + .SetPublicApiDataSetId(Guid.NewGuid()) + .SetPublicApiDataSetVersion(major: 2, minor: 0)) + .GenerateList(5); await TestApp.AddTestData(context => { @@ -710,7 +709,7 @@ await TestApp.AddTestData(context => var pagedResult = response.AssertOk>(); var expectedReleaseFiles = releaseVersionFiles - .Where(rf => rf.File.PublicApiDataSetId.HasValue) + .Where(rf => rf.PublicApiDataSetId.HasValue) .OrderBy(rf => rf.Name) .ToList(); @@ -733,14 +732,13 @@ public async Task FilterByDataSetType_AllOrUnset_ReturnsAllDataSets(DataSetType? var releaseVersionFiles = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) - .WithFiles(_fixture.DefaultFile() - .ForIndex(0, s => s - .SetPublicApiDataSetId(Guid.NewGuid()) - .SetPublicApiDataSetVersion(major: 1, minor: 1)) - .ForIndex(1, s => s - .SetPublicApiDataSetId(Guid.NewGuid()) - .SetPublicApiDataSetVersion(major: 2, minor: 0)) - .GenerateList(5)) + .WithFile(() => _fixture.DefaultFile(FileType.Data)) + .ForIndex(0, s => s + .SetPublicApiDataSetId(Guid.NewGuid()) + .SetPublicApiDataSetVersion(major: 1, minor: 1)) + .ForIndex(1, s => s + .SetPublicApiDataSetId(Guid.NewGuid()) + .SetPublicApiDataSetVersion(major: 2, minor: 0)) .GenerateList(); await TestApp.AddTestData(context => @@ -1558,7 +1556,7 @@ public async Task DataSetFileMetaCorrectlyReturned_Success() var releaseFile = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) - .WithFile(_fixture.DefaultFile() + .WithFile(_fixture.DefaultFile(FileType.Data) .WithType(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() .WithGeographicLevels([GeographicLevel.Country, GeographicLevel.LocalAuthority]) @@ -1723,7 +1721,6 @@ private static void AssertResultsForExpectedReleaseFiles( var releaseVersion = releaseFile.ReleaseVersion; var publication = releaseVersion.Publication; var theme = publication.Topic.Theme; - var publicApiDataSetVersion = releaseFile.File.PublicApiDataSetVersion; Assert.Multiple( () => Assert.Equal(releaseFile.FileId, viewModel.FileId), @@ -1740,13 +1737,12 @@ private static void AssertResultsForExpectedReleaseFiles( () => Assert.Equal(theme.Title, viewModel.Theme.Title), () => Assert.Equal(releaseVersion.Id == publication.LatestPublishedReleaseVersionId, viewModel.LatestData), + () => Assert.Equal(publication.SupersededBy != null + && publication.SupersededBy.LatestPublishedReleaseVersionId != null, + viewModel.IsSuperseded), () => Assert.Equal(releaseFile.ReleaseVersion.Published!.Value, viewModel.Published), - () => Assert.Equal(releaseFile.File.PublicApiDataSetId, viewModel.Api?.Id), - () => Assert.Equal( - publicApiDataSetVersion is not null - ? $"{publicApiDataSetVersion.Major}.{publicApiDataSetVersion.Minor}" - : null, - viewModel.Api?.Version) + () => Assert.Equal(releaseFile.PublicApiDataSetId, viewModel.Api?.Id), + () => Assert.Equal(releaseFile.PublicApiVersionString, viewModel.Api?.Version) ); }); } @@ -1757,7 +1753,7 @@ private List GenerateDataSetFilesForReleaseVersion( { return _fixture.DefaultReleaseFile() .WithReleaseVersion(releaseVersion) - .WithFiles(_fixture.DefaultFile() + .WithFiles(_fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta()) .GenerateList(numberOfDataSets)) .GenerateList(); @@ -1803,15 +1799,15 @@ public async Task ListSitemapItems() ReleaseFile releaseFile = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) - .WithFile(_fixture.DefaultFile() + .WithPublicApiDataSetId(Guid.NewGuid()) + .WithPublicApiDataSetVersion(major: 1, minor: 0) + .WithFile(_fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() .WithTimePeriodRange( _fixture.DefaultTimePeriodRangeMeta() .WithStart("2000", TimeIdentifier.CalendarYear) .WithEnd("2001", TimeIdentifier.CalendarYear) )) - .WithPublicApiDataSetId(Guid.NewGuid()) - .WithPublicApiDataSetVersion(major: 1, minor: 0) ); await TestApp.StartAzurite(); @@ -1850,7 +1846,6 @@ await TestApp.AddTestData(context => } } - public class GetDataSetFileTests(TestApplicationFactory testApp) : DataSetFilesControllerTests(testApp) { public override async Task InitializeAsync() @@ -1870,15 +1865,15 @@ public async Task FetchDataSetDetails_Success() ReleaseFile releaseFile = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) - .WithFile(_fixture.DefaultFile() + .WithPublicApiDataSetId(Guid.NewGuid()) + .WithPublicApiDataSetVersion(major: 1, minor: 0) + .WithFile(_fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() .WithTimePeriodRange( _fixture.DefaultTimePeriodRangeMeta() .WithStart("2000", TimeIdentifier.CalendarYear) .WithEnd("2001", TimeIdentifier.CalendarYear) )) - .WithPublicApiDataSetId(Guid.NewGuid()) - .WithPublicApiDataSetVersion(major: 1, minor: 0) ); await TestApp.AddTestData(context => @@ -1925,6 +1920,7 @@ await publicBlobStorageService.UploadFile( Assert.Equal(releaseFile.ReleaseVersion.Slug, viewModel.Release.Slug); Assert.Equal(releaseFile.ReleaseVersion.Type, viewModel.Release.Type); Assert.True(viewModel.Release.IsLatestPublishedRelease); + Assert.False(viewModel.Release.IsSuperseded); Assert.Equal(releaseFile.ReleaseVersion.Published, viewModel.Release.Published); Assert.Equal(publication.Id, viewModel.Release.Publication.Id); @@ -1933,9 +1929,9 @@ await publicBlobStorageService.UploadFile( Assert.Equal(publication.Topic.Theme.Title, viewModel.Release.Publication.ThemeTitle); Assert.NotNull(viewModel.Api); - Assert.Equal(file.PublicApiDataSetId, viewModel.Api.Id); + Assert.Equal(releaseFile.PublicApiDataSetId, viewModel.Api.Id); Assert.Equal( - $"{file.PublicApiDataSetVersion!.Major}.{file.PublicApiDataSetVersion.Minor}", + $"{releaseFile.PublicApiDataSetVersion!.Major}.{releaseFile.PublicApiDataSetVersion.Minor}", viewModel.Api.Version); var dataSetFileMeta = file.DataSetFileMeta; @@ -2000,7 +1996,7 @@ public async Task FetchCsvWithSingleDataRow_Success() ReleaseFile releaseFile = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) - .WithFile(_fixture.DefaultFile() + .WithFile(_fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta())); await TestApp.AddTestData(context => @@ -2060,7 +2056,7 @@ public async Task FetchDataSetFiltersOrdered_Success() new FilterSequenceEntry(filter2Id, []), new FilterSequenceEntry(filter3Id, []), ]) - .WithFile(_fixture.DefaultFile() + .WithFile(_fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() .WithFilters([ new FilterMeta { Id = filter3Id, Label = "Filter 3", ColumnName = "filter_3", }, @@ -2120,7 +2116,7 @@ public async Task FetchDataSetIndicatorsOrdered_Success() new IndicatorGroupSequenceEntry(Guid.NewGuid(), [indicator2Id]), new IndicatorGroupSequenceEntry(Guid.NewGuid(), [indicator3Id, indicator4Id]) ]) - .WithFile(_fixture.DefaultFile() + .WithFile(_fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() .WithIndicators([ new IndicatorMeta { Id = indicator3Id, Label = "Indicator 3", ColumnName = "indicator_3", }, @@ -2172,7 +2168,7 @@ public async Task FetchVariables_Success() ReleaseFile releaseFile = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) - .WithFile(_fixture.DefaultFile() + .WithFile(_fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() .WithFilters([ new FilterMeta { Id = Guid.NewGuid(), Label = "Filter 1", ColumnName = "A_filter_1", Hint = "hint", }, @@ -2242,7 +2238,7 @@ public async Task FetchDataSetFootnotes_Success() var releaseFile = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) - .WithFile(_fixture.DefaultFile() + .WithFile(_fixture.DefaultFile(FileType.Data) .WithSubjectId(subject.Id)) .Generate(); @@ -2310,7 +2306,7 @@ public async Task NoDataSetFile_ReturnsNotFound() ReleaseFile releaseFile = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) - .WithFile(_fixture.DefaultFile()); + .WithFile(_fixture.DefaultFile(FileType.Data)); await TestApp.AddTestData(context => { @@ -2334,7 +2330,7 @@ public async Task ReleaseNotPublished_ReturnsNotFound() ReleaseFile releaseFile = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) - .WithFile(_fixture.DefaultFile()); + .WithFile(_fixture.DefaultFile(FileType.Data)); await TestApp.AddTestData(context => { @@ -2356,7 +2352,7 @@ public async Task AmendmentNotPublished_ReturnsOk() .WithTopic(_fixture.DefaultTopic() .WithTheme(_fixture.DefaultTheme())); - File file = _fixture.DefaultFile() + File file = _fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta()); ReleaseFile releaseFile0 = _fixture.DefaultReleaseFile() @@ -2409,7 +2405,7 @@ public async Task DataSetFileRemovedOnAmendment_ReturnsNotFound() .WithTopic(_fixture.DefaultTopic() .WithTheme(_fixture.DefaultTheme())); - File file = _fixture.DefaultFile(); + File file = _fixture.DefaultFile(FileType.Data); ReleaseFile releaseFile0 = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication.ReleaseVersions[0]) // the previous published version diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/MethodologyControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/MethodologyControllerTests.cs index 75fc102c178..65e022245e1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/MethodologyControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/MethodologyControllerTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils; @@ -62,7 +63,7 @@ public async Task ListSitemapItems() { var methodologyService = new Mock(MockBehavior.Strict); - methodologyService.Setup(mock => mock.ListSitemapItems()) + methodologyService.Setup(mock => mock.ListSitemapItems(It.IsAny())) .ReturnsAsync(new List() { new() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/PublicationControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/PublicationControllerTests.cs index 566cac61bf9..41cfaddb9d5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/PublicationControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/PublicationControllerTests.cs @@ -1,7 +1,7 @@ #nullable enable using System; using System.Collections.Generic; -using System.Linq; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Content.Api.Controllers; @@ -109,7 +109,7 @@ public async Task ListSitemapItems() { var publicationService = new Mock(Strict); - publicationService.Setup(mock => mock.ListSitemapItems()) + publicationService.Setup(mock => mock.ListSitemapItems(It.IsAny())) .ReturnsAsync(new List() { new() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/ReleaseFileControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/ReleaseFileControllerTests.cs index c7d0585f331..b7fe5d47dd4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/ReleaseFileControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/ReleaseFileControllerTests.cs @@ -42,8 +42,7 @@ public async Task Success_FiltersByIds() .GenerateList(1)); var releaseFiles = DataFixture.DefaultReleaseFile() - .WithFile(() => DataFixture.DefaultFile() - .WithType(FileType.Data)) + .WithFile(() => DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(publication.ReleaseVersions[0]) .GenerateList(4); @@ -91,8 +90,7 @@ public async Task Success_FiltersUnpublishedReleaseFiles() var unpublishedReleaseVersion = publication.ReleaseVersions[1]; var releaseFiles = DataFixture.DefaultReleaseFile() - .WithFile(() => DataFixture.DefaultFile() - .WithType(FileType.Data)) + .WithFile(() => DataFixture.DefaultFile(FileType.Data)) .ForRange(..2, rf => rf .SetReleaseVersion(unpublishedReleaseVersion)) .ForRange(2..4, rf => rf @@ -144,8 +142,7 @@ public async Task Success_FiltersNotLatestPublishedReleaseFiles() var latestPublishedReleaseVersion = publication.ReleaseVersions[2]; var releaseFiles = DataFixture.DefaultReleaseFile() - .WithFile(() => DataFixture.DefaultFile() - .WithType(FileType.Data)) + .WithFile(() => DataFixture.DefaultFile(FileType.Data)) .ForRange(..2, rf => rf .SetReleaseVersion(publication.ReleaseVersions[0])) .ForRange(2..4, rf => rf diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/DataSetFilesController.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/DataSetFilesController.cs index 2d5d54ccbf6..fb9e8b0cfab 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/DataSetFilesController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/DataSetFilesController.cs @@ -60,9 +60,8 @@ public async Task> GetDataSetFile( } [HttpGet("data-set-files/sitemap-items")] - public async Task>> ListSitemapItems() - { - return await _dataSetFileService.ListSitemapItems() + public async Task>> ListSitemapItems( + CancellationToken cancellationToken = default) => + await _dataSetFileService.ListSitemapItems(cancellationToken) .HandleFailuresOrOk(); - } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/MethodologyController.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/MethodologyController.cs index a71e31ff2b8..d100a0d05f7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/MethodologyController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/MethodologyController.cs @@ -1,6 +1,7 @@ #nullable enable using System.Collections.Generic; using System.Net.Mime; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces; @@ -28,10 +29,9 @@ public async Task> GetLatestMethodolog } [HttpGet("methodologies/sitemap-items")] - public async Task>> ListSitemapItems() - { - return await _methodologyService.ListSitemapItems() + public async Task>> ListSitemapItems( + CancellationToken cancellationToken = default) => + await _methodologyService.ListSitemapItems(cancellationToken) .HandleFailuresOrOk(); - } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/PublicationController.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/PublicationController.cs index 83c61aea1d9..91770b19697 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/PublicationController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/PublicationController.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Net.Mime; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Cache; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; @@ -105,10 +106,9 @@ public async Task> GetPublicationTitle(s } [HttpGet("publications/sitemap-items")] - public async Task>> ListSitemapItems() - { - return await _publicationService.ListSitemapItems() + public async Task>> ListSitemapItems( + CancellationToken cancellationToken = default) => + await _publicationService.ListSitemapItems(cancellationToken) .HandleFailuresOrOk(); - } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs index e0a8bd00cb4..7a02d7410cf 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs @@ -12,55 +12,140 @@ public static Generator DefaultDataImport(this DataFixture fixture) public static Generator WithDefaults(this Generator generator) => generator.ForInstance(d => d.SetDefaults()); - public static InstanceSetters SetDefaults(this InstanceSetters setters) - => setters - .SetDefault(d => d.Id) - .SetFiles("data"); - public static Generator WithSubjectId( this Generator generator, Guid subjectId) - => generator.ForInstance(s => s.Set(d => d.SubjectId, subjectId)); + => generator.ForInstance(s => s.SetSubjectId(subjectId)); - public static Generator WithFiles( + public static Generator WithDefaultFiles( this Generator generator, string dataFileName) - => generator.ForInstance(s => s.SetFiles(dataFileName)); + => generator.ForInstance(s => s.SetDefaultFiles(dataFileName)); + + public static Generator WithFile( + this Generator generator, + File file) + => generator.ForInstance(s => s.SetFile(file)); + + public static Generator WithMetaFile( + this Generator generator, + File metaFile) + => generator.ForInstance(s => s.SetMetaFile(metaFile)); + + public static Generator WithZipFile( + this Generator generator, + File zipFile) + => generator.ForInstance(s => s.SetZipFile(zipFile)); public static Generator WithStatus( this Generator generator, DataImportStatus status) - => generator.ForInstance(s => s.Set(d => d.Status, status)); + => generator.ForInstance(s => s.SetStatus(status)); - public static Generator WithRowCounts( + public static Generator WithStagePercentageComplete( this Generator generator, - int? totalRows = null, - int? expectedImportedRows = null, - int? importedRows = null, - int? lastProcessedRowIndex = null) - => generator.ForInstance(s => s - .Set(d => d.TotalRows, totalRows) - .Set(d => d.ExpectedImportedRows, expectedImportedRows) - .Set(d => d.ImportedRows, importedRows ?? 0) - .Set(d => d.LastProcessedRowIndex, lastProcessedRowIndex)); - - public static InstanceSetters SetFiles( + int stagePercentageComplete) + => generator.ForInstance(s => s.SetStagePercentageComplete(stagePercentageComplete)); + + public static Generator WithTotalRows( + this Generator generator, + int totalRows) + => generator.ForInstance(s => s.SetTotalRows(totalRows)); + + public static Generator WithExpectedImportedRows( + this Generator generator, + int expectedImportedRows) + => generator.ForInstance(s => s.SetExpectedImportedRows(expectedImportedRows)); + + public static Generator WithImportedRows( + this Generator generator, + int importedRows) + => generator.ForInstance(s => s.SetImportedRows(importedRows)); + + public static Generator WithLastProcessedRowIndex( + this Generator generator, + int lastProcessedRowIndex) + => generator.ForInstance(s => s.SetLastProcessedRowIndex(lastProcessedRowIndex)); + + public static InstanceSetters SetDefaults(this InstanceSetters setters) + => setters + .SetDefault(d => d.Id) + .SetDefaultFiles("data"); + + public static InstanceSetters SetSubjectId( + this InstanceSetters setters, + Guid subjectId) + => setters.Set(d => d.SubjectId, subjectId); + + public static InstanceSetters SetDefaultFiles( this InstanceSetters setters, string dataFileName) => setters - .Set(d => d.File, (_, dataImport) => new File - { - Id = Guid.NewGuid(), - Filename = $"{dataFileName}.csv", - Type = FileType.Data, - SubjectId = dataImport.SubjectId, - }) - .Set(d => d.MetaFile, (_, dataImport) => new File - { - Id = Guid.NewGuid(), - Filename = $"{dataFileName}.meta.csv", - Type = FileType.Metadata, - SubjectId = dataImport.SubjectId, - }); + .Set( + d => d.File, + (_, d, context) => context.Fixture + .DefaultFile(FileType.Data) + .WithFilename($"{dataFileName}.csv") + .WithSubjectId(d.SubjectId) + ) + .Set(d => d.FileId, (_, d) => d.File.Id) + .Set( + d => d.MetaFile, + (_, d, context) => context.Fixture + .DefaultFile(FileType.Metadata) + .WithFilename($"{dataFileName}.meta.csv") + .WithSubjectId(d.SubjectId) + ) + .Set(d => d.MetaFileId, (_, d) => d.MetaFile.Id); + + public static InstanceSetters SetFile( + this InstanceSetters setters, + File file) + => setters + .Set(d => d.File, file) + .Set(d => d.FileId, (_, d) => d.File.Id); + + public static InstanceSetters SetMetaFile( + this InstanceSetters setters, + File file) + => setters + .Set(d => d.MetaFile, file) + .Set(d => d.MetaFileId, (_, d) => d.MetaFile.Id); + + public static InstanceSetters SetZipFile( + this InstanceSetters setters, + File file) + => setters + .Set(d => d.ZipFile, file) + .Set(d => d.ZipFileId, (_, d) => d.ZipFile?.Id); + public static InstanceSetters SetStatus( + this InstanceSetters setters, + DataImportStatus status) + => setters.Set(d => d.Status, status); + + public static InstanceSetters SetStagePercentageComplete( + this InstanceSetters setters, + int stagePercentageComplete) + => setters.Set(d => d.StagePercentageComplete, stagePercentageComplete); + + public static InstanceSetters SetTotalRows( + this InstanceSetters setters, + int totalRows) + => setters.Set(d => d.TotalRows, totalRows); + + public static InstanceSetters SetExpectedImportedRows( + this InstanceSetters setters, + int expectedImportedRows) + => setters.Set(d => d.ExpectedImportedRows, expectedImportedRows); + + public static InstanceSetters SetImportedRows( + this InstanceSetters setters, + int importedRows) + => setters.Set(d => d.ImportedRows, importedRows); + + public static InstanceSetters SetLastProcessedRowIndex( + this InstanceSetters setters, + int lastProcessedRowIndex) + => setters.Set(d => d.LastProcessedRowIndex, lastProcessedRowIndex); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs index 01dfdce7a13..f45b0e99adc 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs @@ -8,35 +8,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixture public static class FileGeneratorExtensions { - public static Generator DefaultFile(this DataFixture fixture) - => fixture.Generator().WithDefaults(); + public static Generator DefaultFile(this DataFixture fixture, FileType? fileType = null) + => fixture.Generator().WithDefaults(fileType); - public static Generator WithDefaults(this Generator generator) - => generator.ForInstance(d => d.SetDefaults()); - - public static InstanceSetters SetDefaults(this InstanceSetters setters) - => setters - .SetDefault(f => f.Id) - .SetDefault(f => f.RootPath) - .SetDefault(f => f.DataSetFileId) - .SetDefault(f => f.SubjectId) - .SetContentLength(1024 * 1024) - .SetContentType("text/csv") - .SetType(FileType.Data) - .SetDefault(f => f.Filename) - .SetDefault(f => f.DataSetFileId) - .SetDataSetFileMeta(new DataSetFileMeta - { - GeographicLevels = [GeographicLevel.Country], - TimePeriodRange = new TimePeriodRangeMeta - { - Start = new TimePeriodRangeBoundMeta { TimeIdentifier = TimeIdentifier.CalendarYear, Period = "2000", }, - End = new TimePeriodRangeBoundMeta { TimeIdentifier = TimeIdentifier.CalendarYear, Period = "2001", } - }, - Filters = [new() { Id = Guid.NewGuid(), Label = "Filter 1", ColumnName = "filter_1", }], - Indicators = [new() { Id = Guid.NewGuid(), Label = "Indicator 1", ColumnName = "indicator_1", }], - }) - .Set(f => f.Filename, (_, f) => $"{f.Filename}.csv"); + public static Generator WithDefaults(this Generator generator, FileType? fileType = null) + => generator.ForInstance(d => d.SetDefaults(fileType)); public static Generator WithContentLength( this Generator generator, @@ -73,23 +49,6 @@ public static Generator WithReplacedById( Guid replacedById) => generator.ForInstance(s => s.SetReplacedById(replacedById)); - public static Generator WithPublicApiDataSetId( - this Generator generator, - Guid publicDataSetId) - => generator.ForInstance(s => s.SetPublicApiDataSetId(publicDataSetId)); - - public static Generator WithPublicApiDataSetVersion( - this Generator generator, - int major, - int minor, - int patch = 0) - => generator.ForInstance(s => s.SetPublicApiDataSetVersion(major, minor, patch)); - - public static Generator WithPublicApiDataSetVersion( - this Generator generator, - SemVersion version) - => generator.ForInstance(s => s.SetPublicApiDataSetVersion(version)); - public static Generator WithRootPath( this Generator generator, Guid rootPath) @@ -122,9 +81,53 @@ public static Generator WithDataSetFileId( public static Generator WithDataSetFileMeta( this Generator generator, - DataSetFileMeta dataSetFileMeta) + DataSetFileMeta? dataSetFileMeta) => generator.ForInstance(s => s.SetDataSetFileMeta(dataSetFileMeta)); + public static InstanceSetters SetDefaults(this InstanceSetters setters, FileType? fileType) + => fileType switch + { + FileType.Data => setters.SetDataFileDefaults(), + FileType.Metadata => setters.SetMetaFileDefaults(), + FileType.Ancillary => setters.SetAncillaryFileDefaults(), + null => setters.SetDefaults(), + // TODO: Implement other file types + _ => throw new ArgumentOutOfRangeException(nameof(fileType), fileType, null) + }; + + public static InstanceSetters SetDefaults(this InstanceSetters setters) + => setters + .SetDefault(f => f.Id) + .SetDefault(f => f.RootPath) + .SetDefault(f => f.Filename) + .Set(f => f.ContentLength, f => f.Random.Int(1, 1024) * 1024); + + public static InstanceSetters SetDataFileDefaults(this InstanceSetters setters) + => setters + .SetDefaults() + .SetType(FileType.Data) + .Set(f => f.Filename, (_, f) => $"{f.Filename}.csv") + .SetDefault(f => f.SubjectId) + .SetContentType("text/csv") + .SetDefault(f => f.DataSetFileId) + .Set(f => f.DataSetFileMeta, (_, _, context) => context.Fixture.DefaultDataSetFileMeta()); + + public static InstanceSetters SetMetaFileDefaults(this InstanceSetters setters) + => setters + .SetDefaults() + .SetType(FileType.Metadata) + .SetDefault(f => f.SubjectId) + .SetContentType("text/csv") + .Set(f => f.Filename, (_, f) => $"{f.Filename}.meta.csv"); + + public static InstanceSetters SetAncillaryFileDefaults(this InstanceSetters setters) + => setters + .SetDefaults() + .SetType(FileType.Ancillary) + .SetContentType("application/pdf") + .SetDefault(f => f.Filename) + .Set(f => f.Filename, (_, f) => $"{f.Filename}.pdf"); + public static InstanceSetters SetContentLength( this InstanceSetters setters, long contentLength) @@ -151,25 +154,6 @@ public static InstanceSetters SetReplacedById( Guid replacedById) => setters.Set(f => f.ReplacedById, replacedById); - public static InstanceSetters SetPublicApiDataSetId( - this InstanceSetters setters, - Guid publicDataSetId) - => setters.Set(f => f.PublicApiDataSetId, publicDataSetId); - - public static InstanceSetters SetPublicApiDataSetVersion( - this InstanceSetters setters, - int major, - int minor, - int patch = 0) - => setters.Set( - f => f.PublicApiDataSetVersion, - new SemVersion(major: major, minor: minor, patch: patch)); - - public static InstanceSetters SetPublicApiDataSetVersion( - this InstanceSetters setters, - SemVersion version) - => setters.Set(f => f.PublicApiDataSetVersion, version); - public static InstanceSetters SetReplacing( this InstanceSetters setters, File replacing) @@ -214,6 +198,6 @@ public static InstanceSetters SetDataSetFileId( public static InstanceSetters SetDataSetFileMeta( this InstanceSetters setters, - DataSetFileMeta dataSetFileMeta) + DataSetFileMeta? dataSetFileMeta) => setters.Set(f => f.DataSetFileMeta, dataSetFileMeta); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs index 74b06bd623b..b3d81b05b22 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs @@ -22,6 +22,11 @@ public static InstanceSetters SetDefaults(this InstanceSetters p.Summary) .SetDefault(p => p.Title); + public static Generator WithId( + this Generator generator, + Guid id) + => generator.ForInstance(s => s.SetId(id)); + public static Generator WithLatestPublishedReleaseVersion( this Generator generator, ReleaseVersion releaseVersion) @@ -77,6 +82,11 @@ public static Generator WithTopic( Topic topic) => generator.ForInstance(s => s.SetTopic(topic)); + public static InstanceSetters SetId( + this InstanceSetters setters, + Guid id) + => setters.Set(p => p.Id, id); + public static InstanceSetters SetLatestPublishedReleaseVersion( this InstanceSetters setters, ReleaseVersion? releaseVersion) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseFileGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseFileGeneratorExtensions.cs index b5362ad4f56..2011202833f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseFileGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseFileGeneratorExtensions.cs @@ -2,6 +2,7 @@ using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using System; using System.Collections.Generic; +using Semver; namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; @@ -89,6 +90,23 @@ public static Generator WithIndicatorSequence( List sequence) => generator.ForInstance(s => s.SetIndicatorSequence(sequence)); + public static Generator WithPublicApiDataSetId( + this Generator generator, + Guid publicApiDataSetId) + => generator.ForInstance(s => s.SetPublicApiDataSetId(publicApiDataSetId)); + + public static Generator WithPublicApiDataSetVersion( + this Generator generator, + SemVersion version) + => generator.ForInstance(s => s.SetPublicApiDataSetVersion(version)); + + public static Generator WithPublicApiDataSetVersion( + this Generator generator, + int major, + int minor, + int patch = 0) + => generator.ForInstance(s => s.SetPublicApiDataSetVersion(major, minor, patch)); + public static InstanceSetters SetFile( this InstanceSetters setters, File file) @@ -141,4 +159,23 @@ public static InstanceSetters SetIndicatorSequence( this InstanceSetters setters, List sequence) => setters.Set(rf => rf.IndicatorSequence, sequence); + + public static InstanceSetters SetPublicApiDataSetId( + this InstanceSetters setters, + Guid publicApiDataSetId) + => setters.Set(rf => rf.PublicApiDataSetId, publicApiDataSetId); + + public static InstanceSetters SetPublicApiDataSetVersion( + this InstanceSetters setters, + int major, + int minor, + int patch = 0) + => setters.Set( + rf => rf.PublicApiDataSetVersion, + new SemVersion(major: major, minor: minor, patch: patch)); + + public static InstanceSetters SetPublicApiDataSetVersion( + this InstanceSetters setters, + SemVersion version) + => setters.Set(rf => rf.PublicApiDataSetVersion, version); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseVersionGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseVersionGeneratorExtensions.cs index d67ca16e7e0..622c8e73943 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseVersionGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseVersionGeneratorExtensions.cs @@ -35,6 +35,11 @@ public static Generator WithPublication( this Generator generator, Publication publication) => generator.ForInstance(s => s.SetPublication(publication)); + + public static Generator WithPublicationId( + this Generator generator, + Guid publicationId) + => generator.ForInstance(s => s.SetPublicationId(publicationId)); public static Generator WithPublications(this Generator generator, IEnumerable publications) @@ -49,6 +54,11 @@ public static Generator WithRelease( this Generator generator, Release release) => generator.ForInstance(s => s.SetRelease(release)); + + public static Generator WithReleaseId( + this Generator generator, + Guid releaseId) + => generator.ForInstance(s => s.SetReleaseId(releaseId)); public static Generator WithApprovalStatus( this Generator generator, @@ -194,6 +204,8 @@ public static InstanceSetters SetDefaults(this InstanceSetters p.Slug) .SetDefault(p => p.DataGuidance) .SetDefault(p => p.Type) + .SetDefault(p => p.PublicationId) + .SetDefault(p => p.ReleaseId) .SetApprovalStatus(ReleaseApprovalStatus.Draft) .SetTimePeriodCoverage(TimeIdentifier.AcademicYear) .Set(p => p.ReleaseName, (_, _, context) => $"{2000 + context.Index}") @@ -210,11 +222,14 @@ public static InstanceSetters SetId( public static InstanceSetters SetPublication( this InstanceSetters setters, Publication publication) - => setters.Set((_, releaseVersion, _) => - { - releaseVersion.Publication = publication; - releaseVersion.PublicationId = publication.Id; - }); + => setters + .Set(releaseVersion => releaseVersion.Publication, publication) + .SetPublicationId(publication.Id); + + public static InstanceSetters SetPublicationId( + this InstanceSetters setters, + Guid publicationId) + => setters.Set(releaseVersion => releaseVersion.PublicationId, publicationId); public static InstanceSetters SetRelease( this InstanceSetters setters, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs index 442d65d4af9..2e2120bdeda 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs @@ -409,6 +409,27 @@ private static void ConfigureReleaseFile(ModelBuilder modelBuilder) v => v.HasValue ? DateTime.SpecifyKind(v.Value, DateTimeKind.Utc) : null); + entity.Property(f => f.PublicApiDataSetVersion) + .HasMaxLength(20) + .HasConversion( + v => v.ToString(), + v => SemVersion.Parse(v, SemVersionStyles.Strict, 20) + ); + + entity.HasIndex(rf => new + { + rf.ReleaseVersionId, + rf.FileId, + }) + .IsUnique(); + + entity.HasIndex(rf => new + { + rf.ReleaseVersionId, + rf.PublicApiDataSetId, + rf.PublicApiDataSetVersion + }) + .IsUnique(); }); } @@ -440,19 +461,6 @@ private static void ConfigureFile(ModelBuilder modelBuilder) .HasConversion( // You might want to use EF8 JSON support instead of this v => JsonConvert.SerializeObject(v), v => JsonConvert.DeserializeObject(v)); - - entity.Property(f => f.PublicApiDataSetVersion) - .HasMaxLength(20) - .HasConversion( - v => v.ToString(), - v => SemVersion.Parse(v, SemVersionStyles.Strict, 20) - ); - - entity.HasIndex(f => new - { - PublicDataSetId = f.PublicApiDataSetId, PublicDataSetVersion = f.PublicApiDataSetVersion - }) - .IsUnique(); }); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs index 974a6215366..50fecf2eb20 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs @@ -25,10 +25,6 @@ public class File : ICreatedTimestamp public DataSetFileMeta? DataSetFileMeta { get; set; } - public Guid? PublicApiDataSetId { get; set; } - - public SemVersion? PublicApiDataSetVersion { get; set; } - public Guid? ReplacedById { get; set; } public File? ReplacedBy { get; set; } @@ -48,9 +44,5 @@ public class File : ICreatedTimestamp public User? CreatedBy { get; set; } public Guid? CreatedById { get; set; } - - public string? PublicApiVersionString => PublicApiDataSetVersion is not null - ? $"{PublicApiDataSetVersion.Major}.{PublicApiDataSetVersion.Minor}" - : null; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseFile.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseFile.cs index 11928e38914..a47334de81b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseFile.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseFile.cs @@ -1,6 +1,7 @@ #nullable enable using System; using System.Collections.Generic; +using Semver; namespace GovUk.Education.ExploreEducationStatistics.Content.Model; @@ -22,6 +23,14 @@ public class ReleaseFile public int Order { get; set; } + public Guid? PublicApiDataSetId { get; set; } + + public SemVersion? PublicApiDataSetVersion { get; set; } + + public string? PublicApiVersionString => PublicApiDataSetVersion is not null + ? $"{PublicApiDataSetVersion.Major}.{PublicApiDataSetVersion.Minor}" + : null; + public List? FilterSequence { get; set; } public List? IndicatorSequence { get; set; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseFileRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseFileRepository.cs index 98479080350..70283f06a81 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseFileRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseFileRepository.cs @@ -1,6 +1,7 @@ #nullable enable using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model; using Microsoft.AspNetCore.Mvc; @@ -30,6 +31,7 @@ Task> FindOrNotFound(Guid releaseVersionId, Guid fileId); Task> GetByFileType(Guid releaseVersionId, + CancellationToken cancellationToken = default, params FileType[] types); Task FileIsLinkedToOtherReleases(Guid releaseVersionId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseFileRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseFileRepository.cs index 67cfc798e1c..e779d36a731 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseFileRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseFileRepository.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; @@ -97,6 +98,7 @@ public async Task> FindOrNotFound(Guid release } public async Task> GetByFileType(Guid releaseVersionId, + CancellationToken cancellationToken = default, params FileType[] types) { return await _contentDbContext.ReleaseFiles @@ -104,7 +106,7 @@ public async Task> GetByFileType(Guid releaseVersionId, .Where(releaseFile => releaseFile.ReleaseVersionId == releaseVersionId && types.Contains(releaseFile.File.Type)) - .ToListAsync(); + .ToListAsync(cancellationToken); } public async Task FileIsLinkedToOtherReleases(Guid releaseVersionId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/MethodologyServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/MethodologyServiceTests.cs index bffa5227cbe..feec1ac0cc8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/MethodologyServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/MethodologyServiceTests.cs @@ -740,8 +740,6 @@ public async Task ListSitemapItems() await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { - contentDbContext.Attach(methodology.Versions[0]); - var service = SetupMethodologyService(contentDbContext); var result = (await service.ListSitemapItems()).AssertRight(); @@ -799,8 +797,6 @@ public async Task ListSitemapItems_AlternativeSlugTakesPriority() await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { - contentDbContext.Attach(methodology.Versions[0]); - var service = SetupMethodologyService(contentDbContext); var result = (await service.ListSitemapItems()).AssertRight(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index e43f99fe54c..534a06b1917 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -1,4 +1,12 @@ #nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; using CsvHelper; using GovUk.Education.ExploreEducationStatistics.Common; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; @@ -18,14 +26,6 @@ using GovUk.Education.ExploreEducationStatistics.Data.Services.Utils; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Linq.Expressions; -using System.Threading; -using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Common.Model.SortDirection; using static GovUk.Education.ExploreEducationStatistics.Content.Requests.DataSetsListRequestSortBy; using File = GovUk.Education.ExploreEducationStatistics.Content.Model.File; @@ -130,9 +130,11 @@ private static Expression, DataSetFileSumm }, LatestData = result.Value.ReleaseVersionId == result.Value.ReleaseVersion.Publication.LatestPublishedReleaseVersionId, + IsSuperseded = result.Value.ReleaseVersion.Publication.SupersededBy != null + && result.Value.ReleaseVersion.Publication.SupersededBy.LatestPublishedReleaseVersionId != null, Published = result.Value.ReleaseVersion.Published!.Value, LastUpdated = result.Value.Published!.Value, - Api = BuildDataSetFileApiViewModel(result.Value.File), + Api = BuildDataSetFileApiViewModel(result.Value), Meta = BuildDataSetFileMetaViewModel( result.Value.File.DataSetFileMeta, result.Value.FilterSequence, @@ -140,7 +142,8 @@ private static Expression, DataSetFileSumm }; } - public async Task>> ListSitemapItems() + public async Task>> ListSitemapItems( + CancellationToken cancellationToken = default) { var latestReleaseVersions = _contentDbContext.ReleaseVersions .LatestReleaseVersions(publishedOnly: true); @@ -157,7 +160,7 @@ public async Task>> ListS Id = rf.File.DataSetFileId!.Value.ToString(), LastModified = rf.Published }) - .ToListAsync(); + .ToListAsync(cancellationToken); } private static async Task> ChangeSummaryHtmlToText( @@ -177,6 +180,7 @@ public async Task> GetDataSetFile( { var releaseFile = await _contentDbContext.ReleaseFiles .Include(rf => rf.ReleaseVersion.Publication.Topic.Theme) + .Include(rf => rf.ReleaseVersion.Publication.SupersededBy) .Include(rf => rf.File) .Where(rf => rf.File.DataSetFileId == dataSetId @@ -214,6 +218,8 @@ public async Task> GetDataSetFile( IsLatestPublishedRelease = releaseFile.ReleaseVersion.Publication.LatestPublishedReleaseVersionId == releaseFile.ReleaseVersionId, + IsSuperseded = releaseFile.ReleaseVersion.Publication.SupersededBy != null + && releaseFile.ReleaseVersion.Publication.SupersededBy.LatestPublishedReleaseVersionId != null, Published = releaseFile.ReleaseVersion.Published!.Value, LastUpdated = releaseFile.Published!.Value, Publication = new DataSetFilePublicationViewModel @@ -238,7 +244,7 @@ public async Task> GetDataSetFile( SubjectId = releaseFile.File.SubjectId!.Value, }, Footnotes = FootnotesViewModelBuilder.BuildFootnotes(footnotes), - Api = BuildDataSetFileApiViewModel(releaseFile.File) + Api = BuildDataSetFileApiViewModel(releaseFile) }; } @@ -351,17 +357,17 @@ private static List GetOrderedIndicators(List metaIndicat return indicators.Select(i => i.Label).ToList(); } - private static DataSetFileApiViewModel? BuildDataSetFileApiViewModel(File file) + private static DataSetFileApiViewModel? BuildDataSetFileApiViewModel(ReleaseFile releaseFile) { - if (file.PublicApiDataSetId is null || file.PublicApiVersionString is null) + if (releaseFile.PublicApiDataSetId is null || releaseFile.PublicApiVersionString is null) { return null; } return new DataSetFileApiViewModel { - Id = file.PublicApiDataSetId.Value, - Version = file.PublicApiVersionString, + Id = releaseFile.PublicApiDataSetId.Value, + Version = releaseFile.PublicApiVersionString, }; } } @@ -462,7 +468,7 @@ internal static IQueryable OfDataSetType( return dataSetType switch { DataSetType.All => query, - DataSetType.Api => query.Where(rf => rf.File.PublicApiDataSetId.HasValue), + DataSetType.Api => query.Where(rf => rf.PublicApiDataSetId.HasValue), _ => throw new ArgumentOutOfRangeException(nameof(dataSetType)), }; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IDataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IDataSetFileService.cs index 7414878bc36..bebcf5129fb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IDataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IDataSetFileService.cs @@ -29,5 +29,6 @@ Task>> Task> GetDataSetFile( Guid dataSetId); - Task>> ListSitemapItems(); + Task>> ListSitemapItems( + CancellationToken cancellationToken = default); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IMethodologyService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IMethodologyService.cs index cf72204332c..a4bd8274e06 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IMethodologyService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IMethodologyService.cs @@ -1,5 +1,6 @@ #nullable enable using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Content.ViewModels; @@ -13,5 +14,6 @@ public interface IMethodologyService Task>> GetSummariesTree(); - Task>> ListSitemapItems(); + Task>> ListSitemapItems( + CancellationToken cancellationToken = default); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IPublicationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IPublicationService.cs index 12c27c1cb56..67bb9041c47 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IPublicationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IPublicationService.cs @@ -1,6 +1,7 @@ #nullable enable using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; @@ -29,5 +30,6 @@ Task? publicationIds = null); - Task>> ListSitemapItems(); + Task>> ListSitemapItems( + CancellationToken cancellationToken = default); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/MethodologyService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/MethodologyService.cs index 27a82529791..a6112053102 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/MethodologyService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/MethodologyService.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using AutoMapper; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -134,10 +135,12 @@ private async Task> BuildMethodologiesF return _mapper.Map>(latestPublishedMethodologies); } - public async Task>> ListSitemapItems() + public async Task>> ListSitemapItems( + CancellationToken cancellationToken = default) { return await _contentDbContext.Methodologies .Include(m => m.LatestPublishedVersion) + .ThenInclude(mv => mv!.Methodology) .Where(m => m.LatestPublishedVersion != null) .Select(m => m.LatestPublishedVersion) .OfType() @@ -147,7 +150,7 @@ public async Task>> L Slug = mv.Slug, LastModified = mv.Published }) - .ToListAsync(); + .ToListAsync(cancellationToken); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs index 704d5c8598c..75141da7ca5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -341,7 +342,8 @@ private async Task HasAnyDataFiles(Guid releaseVersionId) .AnyAsync(rf => rf.ReleaseVersionId == releaseVersionId && rf.File.Type == FileType.Data); } - public async Task>> ListSitemapItems() => + public async Task>> ListSitemapItems( + CancellationToken cancellationToken = default) => await _contentDbContext.Publications .Where(p => p.LatestPublishedReleaseVersionId.HasValue && @@ -353,7 +355,7 @@ public async Task>> L LastModified = p.Updated, Releases = ListUniqueReleaseVersionSitemapItems(p) }) - .ToListAsync(); + .ToListAsync(cancellationToken); private static List ListUniqueReleaseVersionSitemapItems(Publication publication) => publication.ReleaseVersions diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs index 79ca4396bac..97d0b74d267 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs @@ -139,8 +139,10 @@ private static void FilterContentBlock(IContentBlockViewModel block) private async Task> GetDownloadFiles(ReleaseVersion releaseVersion) { - var files = await _releaseFileRepository.GetByFileType(releaseVersion.Id, FileType.Ancillary, - FileType.Data); + var files = await _releaseFileRepository.GetByFileType( + releaseVersion.Id, + types: [FileType.Ancillary, FileType.Data]); + return files .Select(rf => rf.ToPublicFileInfo()) .OrderBy(file => file.Name) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileSummaryViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileSummaryViewModel.cs index 81510ab2303..957ee92e11c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileSummaryViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileSummaryViewModel.cs @@ -26,6 +26,8 @@ public record DataSetFileSummaryViewModel public required bool LatestData { get; init; } + public required bool IsSuperseded { get; init; } + public required DateTime Published { get; init; } public DateTime LastUpdated { get; init; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs index 2b33236100d..6113392e2d7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/DataSetFileViewModel.cs @@ -47,6 +47,8 @@ public record DataSetFileReleaseViewModel public required bool IsLatestPublishedRelease { get; init; } + public required bool IsSuperseded { get; init; } + public required DateTime Published { get; init; } public DateTime LastUpdated { get; init; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage1Tests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage1Tests.cs index 1f96a8e222d..e44d9740711 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage1Tests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage1Tests.cs @@ -46,10 +46,7 @@ private async Task AssertStage1ItemsValidatedCorrectly(IProcessorStage1TestScena { var metaFileUnderTest = scenario.GetFilenameUnderTest().Replace(".csv", ".meta.csv"); - var subject = new Subject - { - Id = scenario.GetSubjectId() - }; + var subject = new Subject { Id = scenario.GetSubjectId() }; var import = new DataImport { @@ -117,7 +114,7 @@ private async Task AssertStage1ItemsValidatedCorrectly(IProcessorStage1TestScena var importerMetaService = new ImporterMetaService(guidGenerator, transactionHelper); var importerService = new ImporterService( - Options.Create(new AppSettingOptions()), + Options.Create(new AppSettingsOptions()), guidGenerator, new ImporterLocationService( guidGenerator, @@ -159,10 +156,7 @@ private async Task AssertStage1ItemsValidatedCorrectly(IProcessorStage1TestScena VerifyAllMocks(privateBlobStorageService); // Verify that the message will be queued to trigger the next stage. - Assert.Equal(new[] - { - importMessage - }, outputMessages); + Assert.Equal(new[] { importMessage }, outputMessages); await using (var contentDbContext = InMemoryContentDbContext(_contentDbContextId)) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage2Tests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage2Tests.cs index 7998ca537cf..d8361789c4f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage2Tests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage2Tests.cs @@ -162,10 +162,7 @@ private async Task AssertStage2ItemsImportedCorrectly( var metaFileUnderTest = scenario.GetFilenameUnderTest().Replace(".csv", ".meta.csv"); - var subject = new Subject - { - Id = scenario.GetSubjectId() - }; + var subject = new Subject { Id = scenario.GetSubjectId() }; var import = new DataImport { @@ -232,7 +229,7 @@ private async Task AssertStage2ItemsImportedCorrectly( var importerMetaService = new ImporterMetaService(guidGenerator, transactionHelper); var importerService = new ImporterService( - Options.Create(new AppSettingOptions()), + Options.Create(new AppSettingsOptions()), guidGenerator, new ImporterLocationService( guidGenerator, @@ -267,10 +264,7 @@ private async Task AssertStage2ItemsImportedCorrectly( VerifyAllMocks(privateBlobStorageService); // Verify that the message will be queued to trigger the next stage. - Assert.Equal(new[] - { - importMessage - }, outputMessages); + Assert.Equal(new[] { importMessage }, outputMessages); await using (var statisticsDbContext = InMemoryStatisticsDbContext(_statisticsDbContextId)) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage3Tests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage3Tests.cs index 4be42ea9b6c..91dbf91c918 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage3Tests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage3Tests.cs @@ -106,12 +106,10 @@ public async Task ProcessStage3() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithFiles("small-csv") + .WithDefaultFiles("small-csv") .WithStatus(STAGE_3) - .WithRowCounts( - totalRows: 16, - expectedImportedRows: 16 - ) + .WithTotalRows(16) + .WithExpectedImportedRows(16) .Generate(); await using (var contentDbContext = InMemoryContentDbContext(_contentDbContextId)) @@ -171,10 +169,7 @@ public async Task ProcessStage3() var observationBatchImporter = new TestObservationBatchImporter(); var importerService = new ImporterService( - Options.Create(new AppSettingOptions - { - RowsPerBatch = 5000 - }), + Options.Create(new AppSettingsOptions { RowsPerBatch = 5000 }), guidGenerator, new ImporterLocationService( guidGenerator, @@ -312,13 +307,15 @@ public async Task ProcessStage3() .Select(f => (f.Label, f.Hint, f.ColumnName)).ToList() .AssertDeepEqualTo([ ("Filter one", "Hint 1", "filter_one"), - ("Filter two", null, "filter_two")]); + ("Filter two", null, "filter_two") + ]); file.DataSetFileMeta.Indicators .Select(i => (i.Label, i.ColumnName)).ToList() .AssertDeepEqualTo([ ("Indicator one", "indicator_one"), - ("Indicator two", "indicator_two")]); + ("Indicator two", "indicator_two") + ]); } } @@ -328,12 +325,10 @@ public async Task ProcessStage3_FailsImportIfRowCountsDontMatch() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithFiles("small-csv") + .WithDefaultFiles("small-csv") .WithStatus(STAGE_3) - .WithRowCounts( - totalRows: 16, - expectedImportedRows: 16 - ) + .WithTotalRows(16) + .WithExpectedImportedRows(16) .Generate(); var unexpectedImportedObservation = _fixture @@ -399,10 +394,7 @@ public async Task ProcessStage3_FailsImportIfRowCountsDontMatch() var observationBatchImporter = new TestObservationBatchImporter(); var importerService = new ImporterService( - Options.Create(new AppSettingOptions - { - RowsPerBatch = 5000 - }), + Options.Create(new AppSettingsOptions { RowsPerBatch = 5000 }), guidGenerator, new ImporterLocationService( guidGenerator, @@ -462,14 +454,12 @@ public async Task ProcessStage3_PartiallyImportedAlready() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithFiles("small-csv") + .WithDefaultFiles("small-csv") .WithStatus(STAGE_3) - .WithRowCounts( - totalRows: 16, - expectedImportedRows: 16, - importedRows: 4, - lastProcessedRowIndex: 3 - ) + .WithTotalRows(16) + .WithExpectedImportedRows(16) + .WithImportedRows(4) + .WithLastProcessedRowIndex(3) .Generate(); var alreadyImportedObservations = _fixture @@ -536,10 +526,7 @@ public async Task ProcessStage3_PartiallyImportedAlready() var observationBatchImporter = new TestObservationBatchImporter(); var importerService = new ImporterService( - Options.Create(new AppSettingOptions - { - RowsPerBatch = 5000 - }), + Options.Create(new AppSettingsOptions { RowsPerBatch = 5000 }), guidGenerator, new ImporterLocationService( guidGenerator, @@ -617,14 +604,12 @@ public async Task ProcessStage3_PartiallyImportedAlready_BatchSizeChanged() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithFiles("small-csv") + .WithDefaultFiles("small-csv") .WithStatus(STAGE_3) - .WithRowCounts( - totalRows: 16, - expectedImportedRows: 16, - importedRows: 10, - lastProcessedRowIndex: 9 - ) + .WithTotalRows(16) + .WithExpectedImportedRows(16) + .WithImportedRows(10) + .WithLastProcessedRowIndex(9) .Generate(); var alreadyImportedObservations = _fixture @@ -691,10 +676,7 @@ public async Task ProcessStage3_PartiallyImportedAlready_BatchSizeChanged() var observationBatchImporter = new TestObservationBatchImporter(); var importerService = new ImporterService( - Options.Create(new AppSettingOptions - { - RowsPerBatch = 3 - }), + Options.Create(new AppSettingsOptions { RowsPerBatch = 3 }), guidGenerator, new ImporterLocationService( guidGenerator, @@ -767,12 +749,10 @@ public async Task ProcessStage3_IgnoredRows() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithFiles("ignored-school-rows") + .WithDefaultFiles("ignored-school-rows") .WithStatus(STAGE_3) - .WithRowCounts( - totalRows: 16, - expectedImportedRows: 8 - ) + .WithTotalRows(16) + .WithExpectedImportedRows(8) .Generate(); await using (var contentDbContext = InMemoryContentDbContext(_contentDbContextId)) @@ -832,10 +812,7 @@ public async Task ProcessStage3_IgnoredRows() var observationBatchImporter = new TestObservationBatchImporter(); var importerService = new ImporterService( - Options.Create(new AppSettingOptions - { - RowsPerBatch = 5000 - }), + Options.Create(new AppSettingsOptions { RowsPerBatch = 5000 }), guidGenerator, new ImporterLocationService( guidGenerator, @@ -917,14 +894,12 @@ public async Task ProcessStage3_IgnoredRows_PartiallyImported() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithFiles("ignored-school-rows") + .WithDefaultFiles("ignored-school-rows") .WithStatus(STAGE_3) - .WithRowCounts( - totalRows: 16, - expectedImportedRows: 8, - importedRows: 4, - lastProcessedRowIndex: 6 - ) + .WithTotalRows(16) + .WithExpectedImportedRows(8) + .WithImportedRows(4) + .WithLastProcessedRowIndex(6) .Generate(); // Generate already-imported Observations with alternating CsvRow numbers @@ -999,10 +974,7 @@ public async Task ProcessStage3_IgnoredRows_PartiallyImported() var observationBatchImporter = new TestObservationBatchImporter(); var importerService = new ImporterService( - Options.Create(new AppSettingOptions - { - RowsPerBatch = 3 - }), + Options.Create(new AppSettingsOptions { RowsPerBatch = 3 }), guidGenerator, new ImporterLocationService( guidGenerator, @@ -1080,12 +1052,10 @@ public async Task ProcessStage3_Cancelling() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithFiles("small-csv") + .WithDefaultFiles("small-csv") .WithStatus(STAGE_3) - .WithRowCounts( - totalRows: 16, - expectedImportedRows: 16 - ) + .WithTotalRows(16) + .WithExpectedImportedRows(16) .Generate(); await using (var contentDbContext = InMemoryContentDbContext(_contentDbContextId)) @@ -1143,16 +1113,10 @@ public async Task ProcessStage3_Cancelling() guidGenerator, databaseHelper); - var observationBatchImporterMock = new Mock - { - CallBase = true - }; + var observationBatchImporterMock = new Mock { CallBase = true }; var importerService = new ImporterService( - Options.Create(new AppSettingOptions - { - RowsPerBatch = 3 - }), + Options.Create(new AppSettingsOptions { RowsPerBatch = 3 }), guidGenerator, new ImporterLocationService( guidGenerator, @@ -1226,12 +1190,10 @@ public async Task ProcessStage3_CancelledAlready() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithFiles("small-csv") + .WithDefaultFiles("small-csv") .WithStatus(CANCELLED) - .WithRowCounts( - totalRows: 16, - expectedImportedRows: 16 - ) + .WithTotalRows(16) + .WithExpectedImportedRows(16) .Generate(); await using (var contentDbContext = InMemoryContentDbContext(_contentDbContextId)) @@ -1276,10 +1238,7 @@ public async Task ProcessStage3_CancelledAlready() var observationBatchImporter = new TestObservationBatchImporter(); var importerService = new ImporterService( - Options.Create(new AppSettingOptions - { - RowsPerBatch = 3 - }), + Options.Create(new AppSettingsOptions { RowsPerBatch = 3 }), guidGenerator, new ImporterLocationService( guidGenerator, @@ -1365,12 +1324,10 @@ public async Task ProcessStage3_AdditionalFiltersAndIndicators() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithFiles("additional-filters-and-indicators") + .WithDefaultFiles("additional-filters-and-indicators") .WithStatus(STAGE_3) - .WithRowCounts( - totalRows: 16, - expectedImportedRows: 16 - ) + .WithTotalRows(16) + .WithExpectedImportedRows(16) .Generate(); await using (var contentDbContext = InMemoryContentDbContext(_contentDbContextId)) @@ -1431,10 +1388,7 @@ public async Task ProcessStage3_AdditionalFiltersAndIndicators() var observationBatchImporter = new TestObservationBatchImporter(); var importerService = new ImporterService( - Options.Create(new AppSettingOptions - { - RowsPerBatch = 5000 - }), + Options.Create(new AppSettingsOptions { RowsPerBatch = 5000 }), guidGenerator, new ImporterLocationService( guidGenerator, @@ -1539,12 +1493,10 @@ public async Task ProcessStage3_SpecialFilterItemAndIndicatorValues() var import = _fixture .DefaultDataImport() .WithSubjectId(subject.Id) - .WithFiles("small-csv-with-special-data") + .WithDefaultFiles("small-csv-with-special-data") .WithStatus(STAGE_3) - .WithRowCounts( - totalRows: 5, - expectedImportedRows: 5 - ) + .WithTotalRows(5) + .WithExpectedImportedRows(5) .Generate(); await using (var contentDbContext = InMemoryContentDbContext(_contentDbContextId)) @@ -1604,10 +1556,7 @@ public async Task ProcessStage3_SpecialFilterItemAndIndicatorValues() var observationBatchImporter = new TestObservationBatchImporter(); var importerService = new ImporterService( - Options.Create(new AppSettingOptions - { - RowsPerBatch = 5000 - }), + Options.Create(new AppSettingsOptions { RowsPerBatch = 5000 }), guidGenerator, new ImporterLocationService( guidGenerator, @@ -1691,7 +1640,8 @@ public async Task ProcessStage3_SpecialFilterItemAndIndicatorValues() // ReSharper disable once ClassWithVirtualMembersNeverInherited.Global public class TestObservationBatchImporter : IObservationBatchImporter { - public virtual async Task ImportObservationBatch(StatisticsDbContext context, IEnumerable observations) + public virtual async Task ImportObservationBatch( + StatisticsDbContext context, IEnumerable observations) { await context.Observation.AddRangeAsync(observations); await context.SaveChangesAsync(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs index c76f23b5a4d..f0aabc50536 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; @@ -219,7 +218,7 @@ public async Task WriteDataSetMetaFile_Success() var subject = _fixture.DefaultSubject() .Generate(); - var file = _fixture.DefaultFile() + var file = _fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(null) .WithSubjectId(subject.Id) .Generate(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Configuration/AppSettingOptions.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Configuration/AppSettingsOptions.cs similarity index 85% rename from src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Configuration/AppSettingOptions.cs rename to src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Configuration/AppSettingsOptions.cs index 60a43e82f6e..04424c681e3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Configuration/AppSettingOptions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Configuration/AppSettingsOptions.cs @@ -1,6 +1,6 @@ namespace GovUk.Education.ExploreEducationStatistics.Data.Processor.Configuration; -public class AppSettingOptions +public class AppSettingsOptions { public const string AppSettings = "AppSettings"; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Program.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Program.cs index 4870aa67a4a..5b5b112a39d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Program.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Program.cs @@ -51,7 +51,7 @@ .AddSingleton() .AddSingleton() .AddSingleton() - .Configure(hostContext.Configuration.GetSection(AppSettingOptions.AppSettings)); + .Configure(hostContext.Configuration.GetSection(AppSettingsOptions.AppSettings)); }) .Build(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/ImporterService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/ImporterService.cs index 3cce7249aaf..5f2d65bf07c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/ImporterService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/ImporterService.cs @@ -24,7 +24,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Data.Processor.Services { public class ImporterService : IImporterService { - private readonly AppSettingOptions _appSettingOptions; + private readonly AppSettingsOptions _appSettingsOptions; private readonly IGuidGenerator _guidGenerator; private readonly ImporterLocationService _importerLocationService; private readonly IImporterMetaService _importerMetaService; @@ -36,16 +36,16 @@ public class ImporterService : IImporterService private const int Stage2RowCheck = 1000; public ImporterService( - IOptions appSettingOptions, + IOptions appSettingsOptions, IGuidGenerator guidGenerator, ImporterLocationService importerLocationService, IImporterMetaService importerMetaService, - IDataImportService dataImportService, - ILogger logger, - IDatabaseHelper databaseHelper, + IDataImportService dataImportService, + ILogger logger, + IDatabaseHelper databaseHelper, IObservationBatchImporter? observationBatchImporter = null) { - _appSettingOptions = appSettingOptions.Value; + _appSettingsOptions = appSettingsOptions.Value; _guidGenerator = guidGenerator; _importerLocationService = importerLocationService; _importerMetaService = importerMetaService; @@ -57,7 +57,7 @@ public ImporterService( public Task ImportMeta( List metaFileCsvHeaders, - List> metaFileRows, + List> metaFileRows, Subject subject, StatisticsDbContext context) { @@ -70,8 +70,8 @@ public virtual bool Equals(FilterItemMeta? other) { if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; - return FilterId.Equals(other.FilterId) - && string.Equals(FilterGroupLabel, other.FilterGroupLabel, CurrentCultureIgnoreCase) + return FilterId.Equals(other.FilterId) + && string.Equals(FilterGroupLabel, other.FilterGroupLabel, CurrentCultureIgnoreCase) && string.Equals(FilterItemLabel, other.FilterItemLabel, CurrentCultureIgnoreCase); } @@ -87,7 +87,7 @@ public virtual bool Equals(FilterGroupMeta? other) { if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; - return FilterId.Equals(other.FilterId) + return FilterId.Equals(other.FilterId) && string.Equals(FilterGroupLabel, other.FilterGroupLabel, CurrentCultureIgnoreCase); } @@ -157,7 +157,7 @@ await _dataImportService.UpdateStatus(dataImport.Id, .Select(filterItemMeta => { var (filterId, filterGroupLabel, filterItemLabel) = filterItemMeta; - + var filterGroup = filterGroups.Single(fg => fg.FilterId.Equals(filterId) && string.Equals(fg.Label, filterGroupLabel, CurrentCultureIgnoreCase)); @@ -183,26 +183,27 @@ await _databaseHelper.DoInTransaction(context, async ctxDelegate => // This also lets us take advantage of the performance gains of saving new Locations in a single batch // rather than in separate SaveChanges() calls, which introduces quite a penalty. await _databaseHelper.ExecuteWithExclusiveLock( - context, - "Importer_AddNewLocations", + context, + "Importer_AddNewLocations", ctxDelegate => _importerLocationService.CreateIfNotExistsAndCache(ctxDelegate, locations.ToList())); } - public async Task ImportObservations(DataImport import, + public async Task ImportObservations( + DataImport import, Func> dataFileStreamProvider, Func> metaFileStreamProvider, Subject subject, StatisticsDbContext context) { - var importObservationsBatchSize = _appSettingOptions.RowsPerBatch; + var importObservationsBatchSize = _appSettingsOptions.RowsPerBatch; var soleGeographicLevel = import.HasSoleGeographicLevel(); var csvHeaders = await CsvUtils.GetCsvHeaders(dataFileStreamProvider); - var totalBatches = Math.Ceiling((decimal) import.TotalRows!.Value / importObservationsBatchSize); + var totalBatches = Math.Ceiling((decimal)import.TotalRows!.Value / importObservationsBatchSize); var importedRowsSoFar = import.ImportedRows; var lastProcessedRowIndex = import.LastProcessedRowIndex ?? -1; var startingRowIndex = lastProcessedRowIndex + 1; var startingBatchIndex = startingRowIndex / importObservationsBatchSize; - + var metaFileCsvHeaders = await CsvUtils.GetCsvHeaders(metaFileStreamProvider); var metaFileCsvRows = await CsvUtils.GetCsvRows(metaFileStreamProvider); @@ -214,7 +215,7 @@ public async Task ImportObservations(DataImport import, var fixedInformationReader = new FixedInformationDataFileReader(csvHeaders); var filterAndIndicatorReader = new FilterAndIndicatorValuesReader(csvHeaders, subjectMeta); - + await CsvUtils.Batch( dataFileStreamProvider, importObservationsBatchSize, @@ -226,13 +227,14 @@ await CsvUtils.Batch( { _logger.LogInformation( "Import for {FileName} has finished or is being aborted, " + - "so finishing importing Observations early", import.File.Filename); + "so finishing importing Observations early", + import.File.Filename); return false; } - + _logger.LogInformation( - "Importing Observation batch {BatchNumber} of {TotalBatches}", - batchIndex + 1, + "Importing Observation batch {BatchNumber} of {TotalBatches}", + batchIndex + 1, totalBatches); // Find the subset of this batch that hasn't yet been processed. We can use the @@ -242,26 +244,27 @@ await CsvUtils.Batch( // changed, so that there is now some overlap between the rows in a new batch and the end of // a previous batch using the old batch size. var startOfBatchRowIndex = batchIndex * importObservationsBatchSize; - var firstRowIndexOfBatchToProcess = + var firstRowIndexOfBatchToProcess = Math.Max(startOfBatchRowIndex, lastProcessedRowIndex + 1) - startOfBatchRowIndex; var unprocessedRows = batchOfRows.GetRange( - firstRowIndexOfBatchToProcess, batchOfRows.Count - firstRowIndexOfBatchToProcess); + firstRowIndexOfBatchToProcess, + batchOfRows.Count - firstRowIndexOfBatchToProcess); if (unprocessedRows.Count != batchOfRows.Count) { _logger.LogInformation( "Skipping first {SkippedRowCount} rows of batch {BatchNumber} as it is already " + - "partially imported", + "partially imported", batchOfRows.Count() - unprocessedRows.Count, batchIndex + 1); } - + var allowedRows = unprocessedRows.Select((cells, rowIndex) => { if (IsRowAllowed(soleGeographicLevel, cells, fixedInformationReader)) { var csvRow = startOfBatchRowIndex + firstRowIndexOfBatchToProcess + rowIndex + 2; - + return ObservationFromCsv( cells, subject, @@ -277,19 +280,20 @@ await CsvUtils.Batch( return null; }).WhereNotNull(); - await _databaseHelper.DoInTransaction(context, async contextDelegate => - await _observationBatchImporter.ImportObservationBatch(contextDelegate, allowedRows) + await _databaseHelper.DoInTransaction(context, + async contextDelegate => + await _observationBatchImporter.ImportObservationBatch(contextDelegate, allowedRows) ); importedRowsSoFar += allowedRows.Count(); lastProcessedRowIndex += unprocessedRows.Count; await _dataImportService.Update( - import.Id, + import.Id, importedRows: importedRowsSoFar, lastProcessedRowIndex: lastProcessedRowIndex); - - var percentageComplete = (double) ((batchIndex + 1) / totalBatches) * 100; + + var percentageComplete = (double)((batchIndex + 1) / totalBatches) * 100; await _dataImportService.UpdateStatus(import.Id, DataImportStatus.STAGE_3, percentageComplete); @@ -302,7 +306,8 @@ await _dataImportService.Update( /// Determines if a row should be imported based on geographic level. /// If a file contains a sole level then any row is allowed, otherwise rows for 'solo' importable levels are ignored. /// - private static bool IsRowAllowed(bool soleGeographicLevel, + private static bool IsRowAllowed( + bool soleGeographicLevel, IReadOnlyList rowValues, FixedInformationDataFileReader fixedInformationReader) { @@ -351,7 +356,8 @@ private List GetFilterItems( }).ToList(); } - private Guid GetLocationId(IReadOnlyList rowValues, FixedInformationDataFileReader fixedInformationReader) + private Guid GetLocationId( + IReadOnlyList rowValues, FixedInformationDataFileReader fixedInformationReader) { var location = fixedInformationReader.GetLocation(rowValues); return _importerLocationService.Get(location).Id; @@ -400,18 +406,21 @@ public async Task ImportObservationBatch(StatisticsDbContext context, IEnumerabl var parameter = new SqlParameter("@Observations", SqlDbType.Structured) { - Value = observationsTable, TypeName = "[dbo].[ObservationType]" + Value = observationsTable, + TypeName = "[dbo].[ObservationType]" }; await context.Database.ExecuteSqlRawAsync("EXEC [dbo].[InsertObservations] @Observations", parameter); parameter = new SqlParameter("@ObservationFilterItems", SqlDbType.Structured) { - Value = observationsFilterItemsTable, TypeName = "[dbo].[ObservationFilterItemType]" + Value = observationsFilterItemsTable, + TypeName = "[dbo].[ObservationFilterItemType]" }; await context.Database.ExecuteSqlRawAsync( - "EXEC [dbo].[InsertObservationFilterItems] @ObservationFilterItems", parameter); + "EXEC [dbo].[InsertObservationFilterItems] @ObservationFilterItems", + parameter); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Notifier.Tests/Functions/ReleaseNotifierTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Notifier.Tests/Functions/ReleaseNotifierTests.cs index 502bcdaa715..e5872e974b4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Notifier.Tests/Functions/ReleaseNotifierTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Notifier.Tests/Functions/ReleaseNotifierTests.cs @@ -21,7 +21,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Notifier.Tests.Functions; public class ReleaseNotifierTests { - private static readonly AppSettingOptions AppSettingOptions = new() + private static readonly AppSettingsOptions AppSettingsOptions = new() { BaseUrl = "https://notifier.func/api", PublicAppUrl = "https://public.app" @@ -610,9 +610,9 @@ private static bool AssertEmailTemplateValues(Dictionary values { Assert.Equal(pubName, values["publication_name"]); Assert.Equal(releaseName, values["release_name"]); - Assert.Equal($"{AppSettingOptions.PublicAppUrl}/find-statistics/{pubSlug}/{releaseSlug}", + Assert.Equal($"{AppSettingsOptions.PublicAppUrl}/find-statistics/{pubSlug}/{releaseSlug}", values["release_link"]); - Assert.Equal($"{AppSettingOptions.PublicAppUrl}/subscriptions/{pubSlug}/confirm-unsubscription/{unsubToken}", values["unsubscribe_link"]); + Assert.Equal($"{AppSettingsOptions.PublicAppUrl}/subscriptions/{pubSlug}/confirm-unsubscription/{unsubToken}", values["unsubscribe_link"]); if (updateNote != null) { @@ -643,7 +643,7 @@ private static ReleaseNotifier BuildFunction( { return new ReleaseNotifier( Mock.Of>(), - Options.Create(AppSettingOptions), + Options.Create(AppSettingsOptions), Options.Create(new GovUkNotifyOptions { ApiKey = "", diff --git a/src/GovUk.Education.ExploreEducationStatistics.Notifier.Tests/Functions/SubscriptionManagerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Notifier.Tests/Functions/SubscriptionManagerTests.cs index adb61cbb5af..0b4f0ca7a3d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Notifier.Tests/Functions/SubscriptionManagerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Notifier.Tests/Functions/SubscriptionManagerTests.cs @@ -38,7 +38,7 @@ public class SubscriptionManagerFunctionTests(NotifierFunctionsIntegrationTestFi public async Task SendsSubscriptionVerificationEmail() { // Arrange (mocks) - var storageTableService = new StorageTableService(Options.Create(new AppSettingOptions + var storageTableService = new StorageTableService(Options.Create(new AppSettingsOptions { TableStorageConnectionString = fixture.TableStorageConnectionString() })); @@ -57,7 +57,9 @@ public async Task SendsSubscriptionVerificationEmail() var emailService = new Mock(MockBehavior.Strict); emailService.Setup(mock => - mock.SendEmail(notificationClient, "test1@test.com", "subscription-verification-id", + mock.SendEmail(notificationClient, + "test1@test.com", + "subscription-verification-id", It.IsAny>())); var notifierFunction = BuildFunction( @@ -76,19 +78,24 @@ public async Task SendsSubscriptionVerificationEmail() // Act var result = - await notifierFunction.RequestPendingSubscriptionFunc(request, new TestFunctionContext(), + await notifierFunction.RequestPendingSubscriptionFunc(request, + new TestFunctionContext(), new CancellationToken()); // Assert Assert.IsType(result); emailService.Verify(mock => - mock.SendEmail(notificationClient, "test1@test.com", "subscription-verification-id", - It.Is>(d => - AssertEmailTemplateValues(d, "Test Publication Title 1", - "https://localhost:3000/subscriptions/test-publication-slug-1/confirm-subscription/activation-code-1", - null) - )), Times.Once); + mock.SendEmail(notificationClient, + "test1@test.com", + "subscription-verification-id", + It.Is>(d => + AssertEmailTemplateValues(d, + "Test Publication Title 1", + "https://localhost:3000/subscriptions/test-publication-slug-1/confirm-subscription/activation-code-1", + null) + )), + Times.Once); } [Fact] @@ -96,11 +103,14 @@ public async Task DoesNotSendEmailAgainIfSubIsPending() { // Arrange (data) await fixture.AddTestSubscription(NotifierPendingSubscriptionsTableName, - new SubscriptionEntity("test-id-2", "test2@test.com", "Test Publication Title 2", "test-publication-slug-2", + new SubscriptionEntity("test-id-2", + "test2@test.com", + "Test Publication Title 2", + "test-publication-slug-2", DateTime.UtcNow)); // Arrange (mocks) - var storageTableService = new StorageTableService(Options.Create(new AppSettingOptions + var storageTableService = new StorageTableService(Options.Create(new AppSettingsOptions { TableStorageConnectionString = fixture.TableStorageConnectionString() })); @@ -119,7 +129,9 @@ await fixture.AddTestSubscription(NotifierPendingSubscriptionsTableName, var emailService = new Mock(MockBehavior.Strict); emailService.Setup(mock => - mock.SendEmail(notificationClient, "test2@test.com", "subscription-verification-id", + mock.SendEmail(notificationClient, + "test2@test.com", + "subscription-verification-id", It.IsAny>())); var notifierFunction = BuildFunction( @@ -138,7 +150,8 @@ await fixture.AddTestSubscription(NotifierPendingSubscriptionsTableName, // Act var result = - await notifierFunction.RequestPendingSubscriptionFunc(request, new TestFunctionContext(), + await notifierFunction.RequestPendingSubscriptionFunc(request, + new TestFunctionContext(), new CancellationToken()); // Assert @@ -147,12 +160,16 @@ await fixture.AddTestSubscription(NotifierPendingSubscriptionsTableName, Assert.Equal(200, okResult.StatusCode); emailService.Verify(mock => - mock.SendEmail(notificationClient, "test2@test.com", "subscription-verification-id", - It.Is>(d => - AssertEmailTemplateValues(d, "Test Publication Title 2", - "https://localhost:3000/subscriptions/test-publication-slug-2/confirm-subscription/activation-code-2", - null) - )), Times.Never); + mock.SendEmail(notificationClient, + "test2@test.com", + "subscription-verification-id", + It.Is>(d => + AssertEmailTemplateValues(d, + "Test Publication Title 2", + "https://localhost:3000/subscriptions/test-publication-slug-2/confirm-subscription/activation-code-2", + null) + )), + Times.Never); } [Fact] @@ -160,11 +177,14 @@ public async Task SendsConfirmationEmailIfUserAlreadySubscribed() { // Arrange (data) await fixture.AddTestSubscription(NotifierSubscriptionsTableName, - new SubscriptionEntity("test-id-3", "test3@test.com", "Test Publication Title 3", "test-publication-slug-3", + new SubscriptionEntity("test-id-3", + "test3@test.com", + "Test Publication Title 3", + "test-publication-slug-3", DateTime.UtcNow.AddDays(-4))); // Arrange (mocks) - var storageTableService = new StorageTableService(Options.Create(new AppSettingOptions + var storageTableService = new StorageTableService(Options.Create(new AppSettingsOptions { TableStorageConnectionString = fixture.TableStorageConnectionString() })); @@ -183,7 +203,9 @@ await fixture.AddTestSubscription(NotifierSubscriptionsTableName, var emailService = new Mock(MockBehavior.Strict); emailService.Setup(mock => - mock.SendEmail(notificationClient, "test3@test.com", "subscription-confirmation-id", + mock.SendEmail(notificationClient, + "test3@test.com", + "subscription-confirmation-id", It.IsAny>())); var notifierFunction = BuildFunction( @@ -202,7 +224,8 @@ await fixture.AddTestSubscription(NotifierSubscriptionsTableName, // Act var result = - await notifierFunction.RequestPendingSubscriptionFunc(request, new TestFunctionContext(), + await notifierFunction.RequestPendingSubscriptionFunc(request, + new TestFunctionContext(), new CancellationToken()); // Assert @@ -211,18 +234,23 @@ await fixture.AddTestSubscription(NotifierSubscriptionsTableName, Assert.Equal(200, okResult.StatusCode); emailService.Verify(mock => - mock.SendEmail(notificationClient, "test3@test.com", "subscription-confirmation-id", - It.Is>(d => - AssertEmailTemplateValues(d, "Test Publication Title 3", null, - "https://localhost:3000/subscriptions/test-publication-slug-3/confirm-unsubscription/activation-code-3") - )), Times.Once); + mock.SendEmail(notificationClient, + "test3@test.com", + "subscription-confirmation-id", + It.Is>(d => + AssertEmailTemplateValues(d, + "Test Publication Title 3", + null, + "https://localhost:3000/subscriptions/test-publication-slug-3/confirm-unsubscription/activation-code-3") + )), + Times.Once); } [Fact] public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Id_Is_Blank() { // Arrange (mocks) - var storageTableService = new StorageTableService(Options.Create(new AppSettingOptions + var storageTableService = new StorageTableService(Options.Create(new AppSettingsOptions { TableStorageConnectionString = fixture.TableStorageConnectionString() })); @@ -241,7 +269,9 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Id_Is var emailService = new Mock(MockBehavior.Strict); emailService.Setup(mock => - mock.SendEmail(notificationClient, "test@test.com", "subscription-verification-id", + mock.SendEmail(notificationClient, + "test@test.com", + "subscription-verification-id", It.IsAny>())); var notifierFunction = BuildFunction( @@ -252,12 +282,16 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Id_Is var request = new NewPendingSubscriptionRequest { - Id = "", Slug = "test-publication-slug", Email = "test@test.com", Title = "Test Publication Title" + Id = "", + Slug = "test-publication-slug", + Email = "test@test.com", + Title = "Test Publication Title" }; // Act var result = - await notifierFunction.RequestPendingSubscriptionFunc(request, new TestFunctionContext(), + await notifierFunction.RequestPendingSubscriptionFunc(request, + new TestFunctionContext(), new CancellationToken()); // Assert @@ -269,7 +303,7 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Id_Is public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Title_Is_Blank() { // Arrange (mocks) - var storageTableService = new StorageTableService(Options.Create(new AppSettingOptions + var storageTableService = new StorageTableService(Options.Create(new AppSettingsOptions { TableStorageConnectionString = fixture.TableStorageConnectionString() })); @@ -288,7 +322,9 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Title var emailService = new Mock(MockBehavior.Strict); emailService.Setup(mock => - mock.SendEmail(notificationClient, "test@test.com", "subscription-verification-id", + mock.SendEmail(notificationClient, + "test@test.com", + "subscription-verification-id", It.IsAny>())); var notifierFunction = BuildFunction( @@ -299,12 +335,16 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Title var request = new NewPendingSubscriptionRequest { - Id = "123abc", Slug = "test-publication-slug", Email = "test@test.com", Title = "" + Id = "123abc", + Slug = "test-publication-slug", + Email = "test@test.com", + Title = "" }; // Act var result = - await notifierFunction.RequestPendingSubscriptionFunc(request, new TestFunctionContext(), + await notifierFunction.RequestPendingSubscriptionFunc(request, + new TestFunctionContext(), new CancellationToken()); // Assert @@ -316,7 +356,7 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Title public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Email_Is_Blank() { // Arrange (mocks) - var storageTableService = new StorageTableService(Options.Create(new AppSettingOptions + var storageTableService = new StorageTableService(Options.Create(new AppSettingsOptions { TableStorageConnectionString = fixture.TableStorageConnectionString() })); @@ -335,7 +375,9 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Email var emailService = new Mock(MockBehavior.Strict); emailService.Setup(mock => - mock.SendEmail(notificationClient, "test@test.com", "subscription-verification-id", + mock.SendEmail(notificationClient, + "test@test.com", + "subscription-verification-id", It.IsAny>())); var notifierFunction = BuildFunction( @@ -346,12 +388,16 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Email var request = new NewPendingSubscriptionRequest { - Id = "123abc", Slug = "test-publication-slug", Email = "", Title = "Test Publication Title" + Id = "123abc", + Slug = "test-publication-slug", + Email = "", + Title = "Test Publication Title" }; // Act var result = - await notifierFunction.RequestPendingSubscriptionFunc(request, new TestFunctionContext(), + await notifierFunction.RequestPendingSubscriptionFunc(request, + new TestFunctionContext(), new CancellationToken()); // Assert @@ -363,7 +409,7 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Email public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Slug_Is_Blank() { // Arrange (mocks) - var storageTableService = new StorageTableService(Options.Create(new AppSettingOptions + var storageTableService = new StorageTableService(Options.Create(new AppSettingsOptions { TableStorageConnectionString = fixture.TableStorageConnectionString() })); @@ -382,7 +428,9 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Slug_ var emailService = new Mock(MockBehavior.Strict); emailService.Setup(mock => - mock.SendEmail(notificationClient, "test@test.com", "subscription-verification-id", + mock.SendEmail(notificationClient, + "test@test.com", + "subscription-verification-id", It.IsAny>())); var notifierFunction = BuildFunction( @@ -393,12 +441,16 @@ public async Task RequestPendingSubscription_ReturnsValidationProblem_When_Slug_ var request = new NewPendingSubscriptionRequest { - Id = "123abc", Slug = "", Email = "test@test.com", Title = "Test Publication Title" + Id = "123abc", + Slug = "", + Email = "test@test.com", + Title = "Test Publication Title" }; // Act var result = - await notifierFunction.RequestPendingSubscriptionFunc(request, new TestFunctionContext(), + await notifierFunction.RequestPendingSubscriptionFunc(request, + new TestFunctionContext(), new CancellationToken()); // Assert @@ -412,12 +464,15 @@ public async Task SendsSubscriptionConfirmationEmail() { // Arrange (data) await fixture.AddTestSubscription(NotifierPendingSubscriptionsTableName, - new SubscriptionEntity("test-id-4", "test4@test.com", "Test Publication Title 4", "test-publication-slug-4", + new SubscriptionEntity("test-id-4", + "test4@test.com", + "Test Publication Title 4", + "test-publication-slug-4", DateTime.UtcNow.AddDays(-4))); // Arrange (mocks) - var storageTableService = new StorageTableService(Options.Create(new AppSettingOptions + var storageTableService = new StorageTableService(Options.Create(new AppSettingsOptions { TableStorageConnectionString = fixture.TableStorageConnectionString() })); @@ -439,7 +494,9 @@ await fixture.AddTestSubscription(NotifierPendingSubscriptionsTableName, var emailService = new Mock(MockBehavior.Strict); emailService.Setup(mock => - mock.SendEmail(notificationClient, "test4@test.com", "subscription-confirmation-id", + mock.SendEmail(notificationClient, + "test4@test.com", + "subscription-confirmation-id", It.IsAny>())); @@ -451,19 +508,24 @@ await fixture.AddTestSubscription(NotifierPendingSubscriptionsTableName, // Act var result = - await notifierFunction.VerifySubscriptionFunc(new TestFunctionContext(), "test-id-4", + await notifierFunction.VerifySubscriptionFunc(new TestFunctionContext(), + "test-id-4", "verification-code-4"); // Assert Assert.IsType(result); emailService.Verify(mock => - mock.SendEmail(notificationClient, "test4@test.com", "subscription-confirmation-id", - It.Is>(d => - AssertEmailTemplateValues(d, "Test Publication Title 4", - null, - "https://localhost:3000/subscriptions/test-publication-slug-4/confirm-unsubscription/unsubscription-code-4") - )), Times.Once); + mock.SendEmail(notificationClient, + "test4@test.com", + "subscription-confirmation-id", + It.Is>(d => + AssertEmailTemplateValues(d, + "Test Publication Title 4", + null, + "https://localhost:3000/subscriptions/test-publication-slug-4/confirm-unsubscription/unsubscription-code-4") + )), + Times.Once); } [Fact] @@ -471,12 +533,15 @@ public async Task Unsubscribes() { // Arrange (data) await fixture.AddTestSubscription(NotifierSubscriptionsTableName, - new SubscriptionEntity("test-id-5", "test5@test.com", "Test Publication Title 5", "test-publication-slug-5", + new SubscriptionEntity("test-id-5", + "test5@test.com", + "Test Publication Title 5", + "test-publication-slug-5", DateTime.UtcNow.AddDays(-4))); // Arrange (mocks) - var storageTableService = new StorageTableService(Options.Create(new AppSettingOptions + var storageTableService = new StorageTableService(Options.Create(new AppSettingsOptions { TableStorageConnectionString = fixture.TableStorageConnectionString() })); @@ -503,7 +568,8 @@ await fixture.AddTestSubscription(NotifierSubscriptionsTableName, // Act var result = - await notifierFunction.PublicationUnsubscribeFunc(new TestFunctionContext(), "test-id-5", + await notifierFunction.PublicationUnsubscribeFunc(new TestFunctionContext(), + "test-id-5", "unsubscription-code-5"); var okResult = Assert.IsAssignableFrom(result); @@ -513,7 +579,8 @@ await notifierFunction.PublicationUnsubscribeFunc(new TestFunctionContext(), "te Assert.Equal("Test Publication Title 5", subscription.Title); } - private static bool AssertEmailTemplateValues(Dictionary values, + private static bool AssertEmailTemplateValues( + Dictionary values, string publicationName, string? verificationLink, string? unsubscribeLink) @@ -540,8 +607,12 @@ private static SubscriptionManager BuildFunction( INotificationClientProvider? notificationClientProvider = null) => new( Mock.Of>(), - Options.Create(new AppSettingOptions { PublicAppUrl = "https://localhost:3000" }), - Options.Create(new GovUkNotifyOptions { ApiKey = "", EmailTemplates = EmailTemplateOptions }), + Options.Create(new AppSettingsOptions { PublicAppUrl = "https://localhost:3000" }), + Options.Create(new GovUkNotifyOptions + { + ApiKey = "", + EmailTemplates = EmailTemplateOptions + }), tokenService ?? Mock.Of(MockBehavior.Strict), emailService ?? Mock.Of(MockBehavior.Strict), storageTableService ?? Mock.Of(MockBehavior.Strict), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Configuration/AppSettingOptions.cs b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Configuration/AppSettingsOptions.cs similarity index 92% rename from src/GovUk.Education.ExploreEducationStatistics.Notifier/Configuration/AppSettingOptions.cs rename to src/GovUk.Education.ExploreEducationStatistics.Notifier/Configuration/AppSettingsOptions.cs index 25209813f6d..451c50e0cde 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Configuration/AppSettingOptions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Configuration/AppSettingsOptions.cs @@ -1,6 +1,6 @@ namespace GovUk.Education.ExploreEducationStatistics.Notifier.Configuration; -public class AppSettingOptions +public class AppSettingsOptions { public const string AppSettings = "AppSettings"; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Functions/ReleaseNotifier.cs b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Functions/ReleaseNotifier.cs index 6599f96523d..9948a4a7893 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Functions/ReleaseNotifier.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Functions/ReleaseNotifier.cs @@ -19,7 +19,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Notifier.Functions; public class ReleaseNotifier { private readonly ILogger _logger; - private readonly AppSettingOptions _appSettingOptions; + private readonly AppSettingsOptions _appSettingsOptions; private readonly GovUkNotifyOptions.EmailTemplateOptions _emailTemplateOptions; private readonly ITokenService _tokenService; private readonly IEmailService _emailService; @@ -28,7 +28,7 @@ public class ReleaseNotifier public ReleaseNotifier( ILogger logger, - IOptions appSettingOptions, + IOptions appSettingsOptions, IOptions govUkNotifyOptions, ITokenService tokenService, IEmailService emailService, @@ -36,19 +36,18 @@ public ReleaseNotifier( INotificationClientProvider notificationClientProvider) { _logger = logger; - _appSettingOptions = appSettingOptions.Value; + _appSettingsOptions = appSettingsOptions.Value; _emailTemplateOptions = govUkNotifyOptions.Value.EmailTemplates; _tokenService = tokenService ?? throw new ArgumentNullException(nameof(tokenService)); _emailService = emailService ?? throw new ArgumentNullException(nameof(emailService)); _storageTableService = storageTableService ?? throw new ArgumentNullException(nameof(storageTableService)); _notificationClientProvider = notificationClientProvider - ?? throw new ArgumentNullException(nameof(notificationClientProvider)); + ?? throw new ArgumentNullException(nameof(notificationClientProvider)); } [Function("ReleaseNotifier")] public async Task ReleaseNotifierFunc( - [QueueTrigger(ReleaseNotificationQueue)] - ReleaseNotificationMessage msg, + [QueueTrigger(ReleaseNotificationQueue)] ReleaseNotificationMessage msg, FunctionContext context) { _logger.LogInformation("{FunctionName} triggered", context.FunctionDefinition.Name); @@ -61,7 +60,8 @@ public async Task ReleaseNotifierFunc( // Send emails to subscribers of publication var releaseSubscriberQuery = new TableQuery() - .Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, + .Where(TableQuery.GenerateFilterCondition("PartitionKey", + QueryComparisons.Equal, msg.PublicationId.ToString())); var releaseSubscriberEmails = await GetSubscriberEmails(subscribersTable, releaseSubscriberQuery); @@ -78,10 +78,12 @@ public async Task ReleaseNotifierFunc( foreach (var supersededPublication in msg.SupersededPublications) { var releaseSupersededPubSubsQuery = new TableQuery() - .Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, + .Where(TableQuery.GenerateFilterCondition("PartitionKey", + QueryComparisons.Equal, supersededPublication.Id.ToString())); var supersededPublicationSubscriberEmails = await GetSubscriberEmails( - subscribersTable, releaseSupersededPubSubsQuery); + subscribersTable, + releaseSupersededPubSubsQuery); var numSupersededSubscriberEmailsSent = 0; @@ -100,7 +102,8 @@ public async Task ReleaseNotifierFunc( numSupersededSubscriberEmailsSent++; } - _logger.LogInformation("Emailed {NumSupersededPublicationEmailsSent} subscribers from a superseded publication", + _logger.LogInformation( + "Emailed {NumSupersededPublicationEmailsSent} subscribers from a superseded publication", numSupersededSubscriberEmailsSent); } @@ -140,19 +143,15 @@ private void SendSubscriberEmail( var values = new Dictionary { - { - "publication_name", msg.PublicationName - }, - { - "release_name", msg.ReleaseName - }, + { "publication_name", msg.PublicationName }, + { "release_name", msg.ReleaseName }, { "release_link", - $"{_appSettingOptions.PublicAppUrl}/find-statistics/{msg.PublicationSlug}/{msg.ReleaseSlug}" + $"{_appSettingsOptions.PublicAppUrl}/find-statistics/{msg.PublicationSlug}/{msg.ReleaseSlug}" }, { "unsubscribe_link", - $"{_appSettingOptions.PublicAppUrl}/subscriptions/{msg.PublicationSlug}/confirm-unsubscription/{unsubscribeToken}" + $"{_appSettingsOptions.PublicAppUrl}/subscriptions/{msg.PublicationSlug}/confirm-unsubscription/{unsubscribeToken}" } }; @@ -181,23 +180,17 @@ private void SendSupersededSubscriberEmail( var values = new Dictionary { - { - "publication_name", msg.PublicationName - }, - { - "release_name", msg.ReleaseName - }, + { "publication_name", msg.PublicationName }, + { "release_name", msg.ReleaseName }, { "release_link", - $"{_appSettingOptions.PublicAppUrl}/find-statistics/{msg.PublicationSlug}/{msg.ReleaseSlug}" + $"{_appSettingsOptions.PublicAppUrl}/find-statistics/{msg.PublicationSlug}/{msg.ReleaseSlug}" }, { "unsubscribe_link", - $"{_appSettingOptions.PublicAppUrl}/subscriptions/{msg.PublicationSlug}/confirm-unsubscription/{unsubscribeToken}" + $"{_appSettingsOptions.PublicAppUrl}/subscriptions/{msg.PublicationSlug}/confirm-unsubscription/{unsubscribeToken}" }, - { - "superseded_publication_title", supersededPublication.Title - } + { "superseded_publication_title", supersededPublication.Title } }; if (msg.Amendment) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Functions/SubscriptionManager.cs b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Functions/SubscriptionManager.cs index 8becbde4bc4..ea90ea2f568 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Functions/SubscriptionManager.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Functions/SubscriptionManager.cs @@ -19,8 +19,9 @@ namespace GovUk.Education.ExploreEducationStatistics.Notifier.Functions { - public class SubscriptionManager(ILogger logger, - IOptions appSettingOptions, + public class SubscriptionManager( + ILogger logger, + IOptions appSettingsOptions, IOptions govUkNotifyOptions, ITokenService tokenService, IEmailService emailService, @@ -28,13 +29,13 @@ public class SubscriptionManager(ILogger logger, INotificationClientProvider notificationClientProvider, IValidator requestValidator) { - private readonly AppSettingOptions _appSettingOptions = appSettingOptions.Value; + private readonly AppSettingsOptions _appSettingsOptions = appSettingsOptions.Value; private readonly GovUkNotifyOptions.EmailTemplateOptions _emailTemplateOptions = govUkNotifyOptions.Value.EmailTemplates; private readonly ITokenService _tokenService = tokenService ?? throw new ArgumentNullException(nameof(tokenService)); private readonly IEmailService _emailService = emailService ?? throw new ArgumentNullException(nameof(emailService)); private readonly IStorageTableService _storageTableService = storageTableService ?? throw new ArgumentNullException(nameof(storageTableService)); private readonly INotificationClientProvider _notificationClientProvider = notificationClientProvider ?? - throw new ArgumentNullException(nameof(notificationClientProvider)); + throw new ArgumentNullException(nameof(notificationClientProvider)); [Function("RequestPendingSubscription")] // ReSharper disable once UnusedMember.Global @@ -79,12 +80,10 @@ public async Task RequestPendingSubscriptionFunc( var confirmationValues = new Dictionary { - { - "publication_name", subscription.Subscriber.Title - }, + { "publication_name", subscription.Subscriber.Title }, { "unsubscribe_link", - $"{_appSettingOptions.PublicAppUrl}/subscriptions/{req.Slug}/confirm-unsubscription/{unsubscribeToken}" + $"{_appSettingsOptions.PublicAppUrl}/subscriptions/{req.Slug}/confirm-unsubscription/{unsubscribeToken}" } }; @@ -105,12 +104,10 @@ await _storageTableService.UpdateSubscriber(pendingSubscriptionTable, var values = new Dictionary { - { - "publication_name", req.Title - }, + { "publication_name", req.Title }, { "verification_link", - $"{_appSettingOptions.PublicAppUrl}/subscriptions/{req.Slug}/confirm-subscription/{activationCode}" + $"{_appSettingsOptions.PublicAppUrl}/subscriptions/{req.Slug}/confirm-subscription/{activationCode}" } }; @@ -220,12 +217,10 @@ public async Task VerifySubscriptionFunc( var values = new Dictionary { - { - "publication_name", sub.Title - }, + { "publication_name", sub.Title }, { "unsubscribe_link", - $"{_appSettingOptions.PublicAppUrl}/subscriptions/{sub.Slug}/confirm-unsubscription/{unsubscribeToken}" + $"{_appSettingsOptions.PublicAppUrl}/subscriptions/{sub.Slug}/confirm-unsubscription/{unsubscribeToken}" } }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Program.cs b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Program.cs index 4cba11adef5..1994f1dcd35 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Program.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Program.cs @@ -28,7 +28,7 @@ .AddTransient() .AddScoped, NewPendingSubscriptionRequest.Validator>() - .Configure(hostContext.Configuration.GetSection(AppSettingOptions.AppSettings)) + .Configure(hostContext.Configuration.GetSection(AppSettingsOptions.AppSettings)) .Configure(hostContext.Configuration.GetSection(GovUkNotifyOptions.GovUkNotify)); }) .Build(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Services/StorageTableService.cs b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Services/StorageTableService.cs index 565e3a8ead9..6498195c26d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Services/StorageTableService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Services/StorageTableService.cs @@ -9,9 +9,9 @@ namespace GovUk.Education.ExploreEducationStatistics.Notifier.Services { - public class StorageTableService(IOptions appSettingOptions) : IStorageTableService + public class StorageTableService(IOptions appSettingsOptions) : IStorageTableService { - private readonly AppSettingOptions _appSettingOptions = appSettingOptions.Value; + private readonly AppSettingsOptions _appSettingsOptions = appSettingsOptions.Value; public async Task UpdateSubscriber(CloudTable table, SubscriptionEntity subscription) { @@ -27,9 +27,11 @@ public async Task RemoveSubscriber(CloudTable table, SubscriptionEntity subscrip public async Task RetrieveSubscriber(CloudTable table, SubscriptionEntity subscription) { // Need to define the extra columns to retrieve - var columns = new List() + var columns = new List { - "Verified", "Slug", "Title" + "Verified", + "Slug", + "Title" }; var result = await table.ExecuteAsync( TableOperation.Retrieve(subscription.PartitionKey, subscription.RowKey, columns)); @@ -38,7 +40,7 @@ public async Task RemoveSubscriber(CloudTable table, SubscriptionEntity subscrip public async Task GetTable(string storageTableName) { - var storageAccount = CloudStorageAccount.Parse(_appSettingOptions.TableStorageConnectionString); + var storageAccount = CloudStorageAccount.Parse(_appSettingsOptions.TableStorageConnectionString); var tableClient = storageAccount.CreateCloudTableClient(); var table = tableClient.GetTableReference(storageTableName); await table.CreateIfNotExistsAsync(); @@ -49,19 +51,28 @@ public async Task GetSubscription(string id, string email) { var pendingSub = await RetrieveSubscriber(await GetTable(NotifierPendingSubscriptionsTableName), - new SubscriptionEntity(id, email)); + new SubscriptionEntity(id, email)); if (pendingSub is not null) { - return new Subscription() { Subscriber = pendingSub, Status = SubscriptionStatus.SubscriptionPending }; + return new Subscription + { + Subscriber = pendingSub, + Status = SubscriptionStatus.SubscriptionPending + }; } - - var activeSubscriber = await RetrieveSubscriber(await GetTable(NotifierSubscriptionsTableName), new SubscriptionEntity(id, email)); + + var activeSubscriber = await RetrieveSubscriber(await GetTable(NotifierSubscriptionsTableName), + new SubscriptionEntity(id, email)); if (activeSubscriber is not null) { - return new Subscription() { Subscriber = activeSubscriber, Status = SubscriptionStatus.Subscribed }; + return new Subscription + { + Subscriber = activeSubscriber, + Status = SubscriptionStatus.Subscribed + }; } - - return new Subscription() { Status = SubscriptionStatus.NotSubscribed }; + + return new Subscription { Status = SubscriptionStatus.NotSubscribed }; } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Services/TokenService.cs b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Services/TokenService.cs index 2014e114e33..ba9bc432278 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Notifier/Services/TokenService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Notifier/Services/TokenService.cs @@ -10,23 +10,20 @@ namespace GovUk.Education.ExploreEducationStatistics.Notifier.Services { - public class TokenService(IOptions appSettingOptions) : ITokenService + public class TokenService(IOptions appSettingsOptions) : ITokenService { - private readonly AppSettingOptions _appSettingOptions = appSettingOptions.Value; + private readonly AppSettingsOptions _appSettingsOptions = appSettingsOptions.Value; public string GenerateToken(string email, DateTime expiryDateTime) { - var securityKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(_appSettingOptions.TokenSecretKey)); + var securityKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(_appSettingsOptions.TokenSecretKey)); var credentials = new SigningCredentials(securityKey, SecurityAlgorithms.HmacSha256); var secToken = new JwtSecurityToken( signingCredentials: credentials, issuer: "Sample", audience: "Sample", - claims: new[] - { - new Claim(JwtRegisteredClaimNames.Email, email) - }, + claims: new[] { new Claim(JwtRegisteredClaimNames.Email, email) }, expires: expiryDateTime); var handler = new JwtSecurityTokenHandler(); @@ -36,7 +33,7 @@ public string GenerateToken(string email, DateTime expiryDateTime) public string? GetEmailFromToken(string authToken) { var tokenHandler = new JwtSecurityTokenHandler(); - var validationParameters = GetValidationParameters(_appSettingOptions.TokenSecretKey); + var validationParameters = GetValidationParameters(_appSettingsOptions.TokenSecretKey); string? email = null; try diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs new file mode 100644 index 00000000000..f5709d0b8c7 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs @@ -0,0 +1,863 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils; +using GovUk.Education.ExploreEducationStatistics.Content.Requests; +using GovUk.Education.ExploreEducationStatistics.Content.ViewModels; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Fixture; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.ViewModels; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.WebUtilities; +using Moq; +using PublicationSummaryViewModel = GovUk.Education.ExploreEducationStatistics.Content.ViewModels.PublicationSummaryViewModel; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Controllers; + +public abstract class DataSetVersionsControllerTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) +{ + private const string BaseUrl = "api/v1/data-sets"; + + public class ListDataSetVersionsTests(TestApplicationFactory testApp) : DataSetVersionsControllerTests(testApp) + { + [Theory] + [InlineData(1, 2, 1)] + [InlineData(1, 2, 2)] + [InlineData(1, 2, 9)] + [InlineData(1, 3, 2)] + [InlineData(2, 2, 9)] + public async Task MultipleAvailableVersionsForRequestedDataSet_Returns200_PaginatedCorrectly( + int page, + int pageSize, + int numberOfAvailableDataSetVersions) + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + var dataSetVersions = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) + .WithStatusPublished() + .WithDataSetId(dataSet.Id) + .GenerateList(numberOfAvailableDataSetVersions); + + await TestApp.AddTestData(context => + context.DataSetVersions.AddRange(dataSetVersions)); + + var pagedDataSetVersions = dataSetVersions + .OrderByDescending(dsv => dsv.VersionMajor) + .ThenByDescending(dsv => dsv.VersionMinor) + .Skip((page - 1) * pageSize) + .Take(pageSize) + .ToList(); + + var releaseFiles = pagedDataSetVersions + .Select( + dsv => DefaultReleaseFileViewModel() + .ForInstance(s => s.Set(rf => rf.Id, dsv.ReleaseFileId)) + .Generate() + ) + .ToList(); + + var contentApiClient = new Mock(MockBehavior.Strict); + + contentApiClient + .Setup(c => c.ListReleaseFiles( + It.Is(req => + releaseFiles.Select(rf => rf.Id).SequenceEqual(req.Ids)), + It.IsAny() + )) + .ReturnsAsync(releaseFiles); + + var response = await ListDataSetVersions( + dataSetId: dataSet.Id, + page: page, + pageSize: pageSize, + contentApiClient: contentApiClient.Object); + + var viewModel = response.AssertOk(useSystemJson: true); + + MockUtils.VerifyAllMocks(contentApiClient); + + Assert.NotNull(viewModel); + Assert.Equal(page, viewModel.Paging.Page); + Assert.Equal(pageSize, viewModel.Paging.PageSize); + Assert.Equal(numberOfAvailableDataSetVersions, viewModel.Paging.TotalResults); + Assert.Equal(pagedDataSetVersions.Count, viewModel.Results.Count); + } + + [Fact] + public async Task MultipleAvailableVersionsForRequestedDataSet_Returns200_OrderedCorrectly() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + var dataSetVersions = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) + .WithStatusPublished() + .WithDataSetId(dataSet.Id) + .ForIndex(0, dsv => dsv.SetVersionNumber(1, 0)) + .ForIndex(1, dsv => dsv.SetVersionNumber(1, 1)) + .ForIndex(2, dsv => dsv.SetVersionNumber(3, 1)) + .ForIndex(3, dsv => dsv.SetVersionNumber(3, 0)) + .ForIndex(4, dsv => dsv.SetVersionNumber(2, 0)) + .ForIndex(5, dsv => dsv.SetVersionNumber(2, 1)) + .GenerateList(); + + await TestApp.AddTestData(context => + { + context.DataSetVersions.AddRange(dataSetVersions); + }); + + var releaseFiles = dataSetVersions + .OrderByDescending(dsv => dsv.VersionMajor) + .ThenByDescending(dsv => dsv.VersionMinor) + .Select(dsv => DefaultReleaseFileViewModel() + .ForInstance(s => s.Set(rf => rf.Id, dsv.ReleaseFileId)) + .Generate()) + .ToList(); + + var releaseFileIds = releaseFiles + .Select(rf => rf.Id) + .ToHashSet(); + + var contentApiClient = new Mock(MockBehavior.Strict); + + contentApiClient + .Setup(c => c.ListReleaseFiles( + It.Is(req => + req.Ids.All(id => releaseFileIds.Contains(id))), + It.IsAny() + )) + .ReturnsAsync( releaseFiles[..3]); + + var page1Response = await ListDataSetVersions( + dataSetId: dataSet.Id, + page: 1, + pageSize: 3, + contentApiClient: contentApiClient.Object); + + var page1ViewModel = page1Response.AssertOk(useSystemJson: true); + + Assert.Equal(3, page1ViewModel.Results.Count); + Assert.Equal("3.1", page1ViewModel.Results[0].Version); + Assert.Equal("3.0", page1ViewModel.Results[1].Version); + Assert.Equal("2.1", page1ViewModel.Results[2].Version); + + contentApiClient + .Setup(c => c.ListReleaseFiles( + It.Is(req => + req.Ids.All(id => releaseFileIds.Contains(id))), + It.IsAny() + )) + .ReturnsAsync(releaseFiles[3..6]); + + var page2Response = await ListDataSetVersions( + dataSetId: dataSet.Id, + page: 2, + pageSize: 3, + contentApiClient: contentApiClient.Object); + + var page2ViewModel = page2Response.AssertOk(useSystemJson: true); + + Assert.Equal(3, page2ViewModel.Results.Count); + Assert.Equal("2.0", page2ViewModel.Results[0].Version); + Assert.Equal("1.1", page2ViewModel.Results[1].Version); + Assert.Equal("1.0", page2ViewModel.Results[2].Version); + } + + [Theory] + [InlineData(DataSetVersionStatus.Published)] + [InlineData(DataSetVersionStatus.Withdrawn)] + [InlineData(DataSetVersionStatus.Deprecated)] + public async Task DataSetVersionIsAvailable_Returns200_CorrectViewModel(DataSetVersionStatus dataSetVersionStatus) + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + var dataSetVersionGenerator = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) + .WithStatus(dataSetVersionStatus) + .WithPublished(DateTimeOffset.UtcNow) + .WithDataSetId(dataSet.Id); + + DataSetVersion dataSetVersion = dataSetVersionStatus == DataSetVersionStatus.Withdrawn + ? dataSetVersionGenerator.WithWithdrawn(DateTimeOffset.UtcNow) + : dataSetVersionGenerator; + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var releaseFiles = DefaultReleaseFileViewModel() + .ForInstance(s => s.Set(rf => rf.Id, dataSetVersion.ReleaseFileId)) + .GenerateList(1); + + var releaseFileIds = releaseFiles + .Select(rf => rf.Id) + .ToHashSet(); + + var contentApiClient = new Mock(MockBehavior.Strict); + + contentApiClient + .Setup(c => c.ListReleaseFiles( + It.Is(req => + req.Ids.All(id => releaseFileIds.Contains(id))), + It.IsAny() + )) + .ReturnsAsync(releaseFiles); + + var response = await ListDataSetVersions( + dataSetId: dataSet.Id, + page: 1, + pageSize: 1, + contentApiClient: contentApiClient.Object); + + var viewModel = response.AssertOk(useSystemJson: true); + + Assert.NotNull(viewModel); + Assert.Equal(1, viewModel.Paging.Page); + Assert.Equal(1, viewModel.Paging.PageSize); + Assert.Equal(1, viewModel.Paging.TotalResults); + + var result = Assert.Single(viewModel.Results); + + Assert.Equal(dataSetVersion.Version, result.Version); + Assert.Equal(dataSetVersion.VersionType, result.Type); + Assert.Equal(dataSetVersion.Status, result.Status); + Assert.Equal( + dataSetVersion.Published.TruncateNanoseconds(), + result.Published + ); + if (dataSetVersionStatus == DataSetVersionStatus.Withdrawn) + { + Assert.Equal( + dataSetVersion.Withdrawn.TruncateNanoseconds(), + result.Withdrawn + ); + } + Assert.Equal(dataSetVersion.Notes, result.Notes); + Assert.Equal(dataSetVersion.TotalResults, result.TotalResults); + + Assert.Equal(releaseFiles[0].DataSetFileId, result.File.Id); + + Assert.Equal(releaseFiles[0].Release.Title, result.Release.Title); + Assert.Equal(releaseFiles[0].Release.Slug, result.Release.Slug); + + Assert.Equal( + TimePeriodFormatter.FormatLabel( + dataSetVersion.MetaSummary!.TimePeriodRange.Start.Period, + dataSetVersion.MetaSummary.TimePeriodRange.Start.Code), + result.TimePeriods.Start); + Assert.Equal( + TimePeriodFormatter.FormatLabel( + dataSetVersion.MetaSummary.TimePeriodRange.End.Period, + dataSetVersion.MetaSummary.TimePeriodRange.End.Code), + result.TimePeriods.End); + Assert.Equal(dataSetVersion.MetaSummary.GeographicLevels, result.GeographicLevels); + Assert.Equal(dataSetVersion.MetaSummary.Filters, result.Filters); + Assert.Equal(dataSetVersion.MetaSummary.Indicators, result.Indicators); + } + + [Fact] + public async Task AvailableVersionForOtherDataSet_Returns200_OnlyVersionForRequestedDataSet() + { + DataSet dataSet1 = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + DataSet dataSet2 = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => + context.DataSets.AddRange(dataSet1, dataSet2)); + + DataSetVersion dataSet1Version = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) + .WithStatusPublished() + .WithVersionNumber(1, 1) + .WithDataSetId(dataSet1.Id); + + DataSetVersion dataSet2Version = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) + .WithStatusPublished() + .WithVersionNumber(2, 2) + .WithDataSetId(dataSet2.Id); + + await TestApp.AddTestData(context => + context.DataSetVersions.AddRange(dataSet1Version, dataSet2Version)); + + var releaseFiles = DefaultReleaseFileViewModel() + .ForInstance(s => s.Set(rf => rf.Id, dataSet1Version.ReleaseFileId)) + .GenerateList(1); + + var releaseFileIds = releaseFiles + .Select(rf => rf.Id) + .ToHashSet(); + + var contentApiClient = new Mock(MockBehavior.Strict); + + contentApiClient + .Setup(c => c.ListReleaseFiles( + It.Is(req => + req.Ids.All(id => releaseFileIds.Contains(id))), + It.IsAny() + )) + .ReturnsAsync(releaseFiles); + + var response = await ListDataSetVersions( + dataSetId: dataSet1.Id, + page: 1, + pageSize: 1, + contentApiClient: contentApiClient.Object); + + var viewModel = response.AssertOk(useSystemJson: true); + + Assert.NotNull(viewModel); + Assert.Equal(1, viewModel.Paging.Page); + Assert.Equal(1, viewModel.Paging.PageSize); + Assert.Equal(1, viewModel.Paging.TotalResults); + var result = Assert.Single(viewModel.Results); + Assert.Equal(dataSet1Version.Version, result.Version); + } + + [Theory] + [InlineData(DataSetVersionStatus.Processing)] + [InlineData(DataSetVersionStatus.Failed)] + [InlineData(DataSetVersionStatus.Mapping)] + [InlineData(DataSetVersionStatus.Draft)] + public async Task DataSetVersionUnavailable_Returns200_EmptyList(DataSetVersionStatus dataSetVersionStatus) + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) + .WithStatus(dataSetVersionStatus) + .WithDataSetId(dataSet.Id); + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var response = await ListDataSetVersions( + dataSetId: dataSet.Id, + page: 1, + pageSize: 1); + + var viewModel = response.AssertOk(useSystemJson: true); + + Assert.NotNull(viewModel); + Assert.Equal(1, viewModel.Paging.Page); + Assert.Equal(1, viewModel.Paging.PageSize); + Assert.Equal(0, viewModel.Paging.TotalResults); + Assert.Empty(viewModel.Results); + } + + [Fact] + public async Task NoDataSetVersions_Returns200_EmptyList() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + var response = await ListDataSetVersions( + dataSetId: dataSet.Id, + page: 1, + pageSize: 1); + + var viewModel = response.AssertOk(useSystemJson: true); + + Assert.NotNull(viewModel); + Assert.Equal(1, viewModel.Paging.Page); + Assert.Equal(1, viewModel.Paging.PageSize); + Assert.Equal(0, viewModel.Paging.TotalResults); + Assert.Empty(viewModel.Results); + } + + [Fact] + public async Task ReleaseFilesDoNotExist_Returns500() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + var dataSetVersions = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) + .WithStatus(DataSetVersionStatus.Published) + .WithDataSetId(dataSet.Id) + .GenerateList(2); + + await TestApp.AddTestData(context => + context.DataSetVersions.AddRange(dataSetVersions)); + + var contentApiClient = new Mock(MockBehavior.Strict); + + contentApiClient + .Setup(c => c.ListReleaseFiles( + It.IsAny(), + It.IsAny() + )) + .ReturnsAsync([]); + + var response = await ListDataSetVersions( + dataSetId: dataSet.Id, + page: 1, + pageSize: 1, + contentApiClient: contentApiClient.Object); + + response.AssertInternalServerError(); + } + + [Fact] + public async Task ContentApiClientThrows_Returns500() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + var dataSetVersions = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) + .WithStatus(DataSetVersionStatus.Published) + .WithDataSetId(dataSet.Id) + .GenerateList(2); + + await TestApp.AddTestData(context => + context.DataSetVersions.AddRange(dataSetVersions)); + + var contentApiClient = new Mock(MockBehavior.Strict); + + contentApiClient + .Setup(c => c.ListReleaseFiles( + It.IsAny(), + It.IsAny() + )) + .ThrowsAsync(new Exception("Something went wrong")); + + var response = await ListDataSetVersions( + dataSetId: dataSet.Id, + page: 1, + pageSize: 1, + contentApiClient: contentApiClient.Object); + + response.AssertInternalServerError(); + } + + [Fact] + public async Task PageTooBig_Returns200_EmptyList() + { + var page = 2; + var pageSize = 2; + var numberOfDataSetVersions = 2; + + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + var dataSetVersions = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) + .WithStatusPublished() + .WithDataSetId(dataSet.Id) + .GenerateList(numberOfDataSetVersions); + + await TestApp.AddTestData(context => context.DataSetVersions.AddRange(dataSetVersions)); + + var response = await ListDataSetVersions( + dataSetId: dataSet.Id, + page: page, + pageSize: pageSize); + + var viewModel = response.AssertOk(useSystemJson: true); + + Assert.NotNull(viewModel); + Assert.Equal(page, viewModel.Paging.Page); + Assert.Equal(pageSize, viewModel.Paging.PageSize); + Assert.Equal(numberOfDataSetVersions, viewModel.Paging.TotalResults); + Assert.Empty(viewModel.Results); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public async Task PageTooSmall_Returns400(int page) + { + var response = await ListDataSetVersions( + dataSetId: Guid.NewGuid(), + page: page, + pageSize: 1); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + validationProblem.AssertHasGreaterThanOrEqualError("page", comparisonValue: 1); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(21)] + public async Task PageSizeOutOfBounds_Returns400(int pageSize) + { + var response = await ListDataSetVersions( + dataSetId: Guid.NewGuid(), + page: 1, + pageSize: pageSize); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + validationProblem.AssertHasInclusiveBetweenError("pageSize", from: 1, to: 20); + } + + [Theory] + [InlineData(DataSetStatus.Draft)] + [InlineData(DataSetStatus.Withdrawn)] + public async Task UnavailableDataSet_Returns503(DataSetStatus status) + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatus(status); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + var response = await ListDataSetVersions(dataSetId: dataSet.Id); + + response.AssertForbidden(); + } + + [Fact] + public async Task InvalidDataSetId_Returns404() + { + var client = TestApp.CreateClient(); + + var query = new Dictionary + { + { "page", "1" }, + { "pageSize", "1" }, + }; + + var uri = QueryHelpers.AddQueryString($"{BaseUrl}/not_a_valid_guid/versions", query); + + var response = await client.GetAsync(uri); + + response.AssertNotFound(); + } + + private async Task ListDataSetVersions( + Guid dataSetId, + int? page = null, + int? pageSize = null, + IContentApiClient? contentApiClient = null) + { + var query = new Dictionary + { + { "page", page?.ToString() }, + { "pageSize", pageSize?.ToString() }, + }; + + var uri = QueryHelpers.AddQueryString($"{BaseUrl}/{dataSetId}/versions", query); + + var client = BuildApp(contentApiClient).CreateClient(); + + return await client.GetAsync(uri); + } + } + + public class GetDataSetVersionTests(TestApplicationFactory testApp) : DataSetVersionsControllerTests(testApp) + { + [Theory] + [InlineData(DataSetVersionStatus.Published)] + [InlineData(DataSetVersionStatus.Withdrawn)] + [InlineData(DataSetVersionStatus.Deprecated)] + public async Task VersionIsAvailable_Returns200(DataSetVersionStatus dataSetVersionStatus) + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + var dataSetVersionGenerator = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) + .WithStatus(dataSetVersionStatus) + .WithPublished(DateTimeOffset.UtcNow) + .WithDataSetId(dataSet.Id); + + DataSetVersion dataSetVersion = dataSetVersionStatus == DataSetVersionStatus.Withdrawn + ? dataSetVersionGenerator.WithWithdrawn(DateTimeOffset.UtcNow) + : dataSetVersionGenerator; + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var releaseFiles = DefaultReleaseFileViewModel() + .GenerateList(1); + + var contentApiClient = new Mock(MockBehavior.Strict); + + contentApiClient + .Setup(c => c.ListReleaseFiles( + It.Is(req => req.Ids.Contains(dataSetVersion.ReleaseFileId)), + It.IsAny() + )) + .ReturnsAsync(releaseFiles); + + var response = await GetDataSetVersion( + dataSet.Id, + dataSetVersion.Version, + contentApiClient.Object + ); + + var viewModel = response.AssertOk(useSystemJson: true); + + MockUtils.VerifyAllMocks(contentApiClient); + + Assert.NotNull(viewModel); + Assert.Equal(dataSetVersion.Version, viewModel.Version); + Assert.Equal(dataSetVersion.VersionType, viewModel.Type); + Assert.Equal(dataSetVersion.Status, viewModel.Status); + Assert.Equal( + dataSetVersion.Published.TruncateNanoseconds(), + viewModel.Published + ); + if (dataSetVersionStatus == DataSetVersionStatus.Withdrawn) + { + Assert.Equal( + dataSetVersion.Withdrawn.TruncateNanoseconds(), + viewModel.Withdrawn + ); + } + Assert.Equal(dataSetVersion.Notes, viewModel.Notes); + Assert.Equal(dataSetVersion.TotalResults, viewModel.TotalResults); + + Assert.Equal(releaseFiles[0].DataSetFileId, viewModel.File.Id); + + Assert.Equal(releaseFiles[0].Release.Title, viewModel.Release.Title); + Assert.Equal(releaseFiles[0].Release.Slug, viewModel.Release.Slug); + + Assert.Equal( + TimePeriodFormatter.FormatLabel( + dataSetVersion.MetaSummary!.TimePeriodRange.Start.Period, + dataSetVersion.MetaSummary.TimePeriodRange.Start.Code), + viewModel.TimePeriods.Start); + Assert.Equal( + TimePeriodFormatter.FormatLabel( + dataSetVersion.MetaSummary.TimePeriodRange.End.Period, + dataSetVersion.MetaSummary.TimePeriodRange.End.Code), + viewModel.TimePeriods.End); + Assert.Equal(dataSetVersion.MetaSummary.GeographicLevels, viewModel.GeographicLevels); + Assert.Equal(dataSetVersion.MetaSummary.Filters, viewModel.Filters); + Assert.Equal(dataSetVersion.MetaSummary.Indicators, viewModel.Indicators); + } + + [Theory] + [InlineData(DataSetVersionStatus.Processing)] + [InlineData(DataSetVersionStatus.Failed)] + [InlineData(DataSetVersionStatus.Mapping)] + [InlineData(DataSetVersionStatus.Draft)] + public async Task VersionNotAvailable_Returns403(DataSetVersionStatus dataSetVersionStatus) + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithStatus(dataSetVersionStatus) + .WithDataSetId(dataSet.Id); + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var response = await GetDataSetVersion(dataSet.Id, dataSetVersion.Version); + + response.AssertForbidden(); + } + + [Fact] + public async Task VersionExistsForOtherDataSet_Returns404() + { + DataSet dataSet1 = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + DataSet dataSet2 = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.AddRange(dataSet1, dataSet2)); + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithStatusPublished() + .WithDataSetId(dataSet1.Id); + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var response = await GetDataSetVersion(dataSet2.Id, dataSetVersion.Version); + + response.AssertNotFound(); + } + + [Fact] + public async Task VersionDoesNotExist_Returns404() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + var response = await GetDataSetVersion(dataSet.Id, "1.0"); + + response.AssertNotFound(); + } + + [Fact] + public async Task DataSetDoesNotExist_Returns404() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithStatusPublished() + .WithDataSetId(dataSet.Id); + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var response = await GetDataSetVersion(Guid.NewGuid(), dataSetVersion.Version); + + response.AssertNotFound(); + } + + [Fact] + public async Task ReleaseFileDoesNotExist_Returns500() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithStatusPublished() + .WithDataSetId(dataSet.Id); + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var contentApiClient = new Mock(MockBehavior.Strict); + + contentApiClient + .Setup(c => c.ListReleaseFiles( + It.Is(req => req.Ids.Contains(dataSetVersion.ReleaseFileId)), + It.IsAny() + )) + .ReturnsAsync([]); + + var response = await GetDataSetVersion(dataSet.Id, dataSetVersion.Version, contentApiClient.Object); + + response.AssertInternalServerError(); + } + + [Fact] + public async Task ContentApiClientThrows_Returns500() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithStatusPublished() + .WithDataSetId(dataSet.Id); + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var contentApiClient = new Mock(MockBehavior.Strict); + + contentApiClient + .Setup(c => c.ListReleaseFiles( + It.Is(req => req.Ids.Contains(dataSetVersion.ReleaseFileId)), + It.IsAny() + )) + .ThrowsAsync(new Exception("Something went wrong")); + + var response = await GetDataSetVersion(dataSet.Id, dataSetVersion.Version, contentApiClient.Object); + + response.AssertInternalServerError(); + } + + private async Task GetDataSetVersion( + Guid dataSetId, + string dataSetVersion, + IContentApiClient? contentApiClient = null) + { + var client = BuildApp(contentApiClient).CreateClient(); + + var uri = new Uri($"{BaseUrl}/{dataSetId}/versions/{dataSetVersion}", UriKind.Relative); + + return await client.GetAsync(uri); + } + } + + private WebApplicationFactory BuildApp(IContentApiClient? contentApiClient = null) + { + return TestApp.ConfigureServices(services => + { + services.ReplaceService(contentApiClient ?? Mock.Of()); + }); + } + + private Generator DefaultReleaseFileViewModel() => + DataFixture.Generator() + .ForInstance(s => s + .SetDefault(r => r.Id) + .Set(r => r.File, () => DataFixture.DefaultFileInfo()) + .SetDefault(r => r.DataSetFileId) + .Set(r => r.Release, () => DefaultReleaseSummaryViewModel()) + ); + + private Generator DefaultReleaseSummaryViewModel() => + DataFixture.Generator() + .ForInstance(s => s + .SetDefault(r => r.Id) + .SetDefault(r => r.Title) + .SetDefault(r => r.Slug) + .Set(r => r.Publication, () => DefaultPublicationSummaryViewModel())); + + private Generator DefaultPublicationSummaryViewModel() => + DataFixture.Generator() + .ForInstance(s => s + .SetDefault(r => r.Id) + .SetDefault(r => r.Title) + .SetDefault(r => r.Slug)); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs index 24f2e93f810..5e277d9a441 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs @@ -106,13 +106,13 @@ public async Task Empty_Returns400() var client = BuildApp().CreateClient(); - var response = await client.GetAsync($"{BaseUrl}/{dataSetVersion.DataSetId}/query?indicators[]="); + var response = await client.GetAsync($"{BaseUrl}/{dataSetVersion.DataSetId}/query?indicators[0]="); var validationProblem = response.AssertValidationProblem(); Assert.Single(validationProblem.Errors); - validationProblem.AssertHasNotEmptyError("indicators"); + validationProblem.AssertHasNotEmptyError("indicators[0]"); } [Fact] @@ -165,7 +165,7 @@ public async Task MissingParam_Returns400() Assert.Single(validationProblem.Errors); - validationProblem.AssertHasNotEmptyError("indicators"); + validationProblem.AssertHasRequiredValueError("indicators"); } [Fact] @@ -191,8 +191,8 @@ public async Task NotFound_Returns400() public class FiltersValidationTests(TestApplicationFactory testApp) : DataSetsControllerGetQueryTests(testApp) { [Theory] - [InlineData("filters.in")] - [InlineData("filters.notIn")] + [InlineData("filters.in[0]")] + [InlineData("filters.notIn[0]")] public async Task Empty_Returns400(string path) { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -203,7 +203,7 @@ public async Task Empty_Returns400(string path) queryParameters: new Dictionary { { - $"{path}[]", "" + path, "" } } ); @@ -274,7 +274,7 @@ public async Task AllComparatorsInvalid_Returns400() "filters.notEq", new string('a', 12) }, { - "filters.in[]", "" + "filters.in[0]", "" }, { "filters.notIn", invalidFilters @@ -288,7 +288,7 @@ public async Task AllComparatorsInvalid_Returns400() validationProblem.AssertHasMaximumLengthError("filters.eq", maxLength: 10); validationProblem.AssertHasMaximumLengthError("filters.notEq", maxLength: 10); - validationProblem.AssertHasNotEmptyError("filters.in"); + validationProblem.AssertHasNotEmptyError("filters.in[0]"); validationProblem.AssertHasMaximumLengthError("filters.notIn[0]", maxLength: 10); validationProblem.AssertHasNotEmptyError("filters.notIn[1]"); } @@ -328,8 +328,8 @@ public async Task NotFound_Returns200_HasWarning(string path) public class GeographicLevelsValidationTests(TestApplicationFactory testApp) : DataSetsControllerGetQueryTests(testApp) { [Theory] - [InlineData("geographicLevels.in")] - [InlineData("geographicLevels.notIn")] + [InlineData("geographicLevels.in[0]")] + [InlineData("geographicLevels.notIn[0]")] public async Task Empty_Returns400(string path) { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -340,7 +340,7 @@ public async Task Empty_Returns400(string path) queryParameters: new Dictionary { { - $"{path}[]", "" + path, "" } } ); @@ -349,7 +349,11 @@ public async Task Empty_Returns400(string path) Assert.Single(validationProblem.Errors); - validationProblem.AssertHasNotEmptyError(path); + validationProblem.AssertHasAllowedValueError( + expectedPath: path, + value: null, + allowed: GeographicLevelUtils.OrderedCodes + ); } [Theory] @@ -459,7 +463,7 @@ public async Task AllComparatorsInvalid_Returns400() "geographicLevels.in", invalidLevels }, { - "geographicLevels.notIn[]", "" + "geographicLevels.notIn[0]", "" }, } ); @@ -490,7 +494,11 @@ public async Task AllComparatorsInvalid_Returns400() value: invalidLevels[1], allowed: allowed ); - validationProblem.AssertHasNotEmptyError("geographicLevels.notIn"); + validationProblem.AssertHasAllowedValueError( + expectedPath: "geographicLevels.notIn[0]", + value: null, + allowed: allowed + ); } } @@ -498,8 +506,8 @@ public class LocationsValidationTests(TestApplicationFactory testApp) : DataSetsControllerGetQueryTests(testApp) { [Theory] - [InlineData("locations.in")] - [InlineData("locations.notIn")] + [InlineData("locations.in[0]")] + [InlineData("locations.notIn[0]")] public async Task Empty_Returns400(string path) { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -510,7 +518,7 @@ public async Task Empty_Returns400(string path) queryParameters: new Dictionary { { - $"{path}[]", "" + path, "" } } ); @@ -654,7 +662,7 @@ public async Task AllComparatorsInvalid_Returns400() "locations.in", invalidLocations }, { - "locations.notIn[]", "" + "locations.notIn[0]", "" }, } ); @@ -680,7 +688,7 @@ public async Task AllComparatorsInvalid_Returns400() property: "id", maxLength: 10 ); - validationProblem.AssertHasNotEmptyError("locations.notIn"); + validationProblem.AssertHasNotEmptyError("locations.notIn[0]"); } [Theory] @@ -736,7 +744,7 @@ public async Task Empty_Returns400() queryParameters: new Dictionary { { - "timePeriods.in[]", "" + "timePeriods.in[0]", "" } } ); @@ -745,7 +753,7 @@ public async Task Empty_Returns400() Assert.Single(validationProblem.Errors); - validationProblem.AssertHasNotEmptyError("timePeriods.in"); + validationProblem.AssertHasNotEmptyError("timePeriods.in[0]"); } [Theory] @@ -826,7 +834,7 @@ public async Task AllComparatorsInvalid_Returns400() "timePeriods.in", invalidTimePeriods }, { - "timePeriods.notIn[]", "" + "timePeriods.notIn[0]", "" } } ); @@ -840,7 +848,7 @@ public async Task AllComparatorsInvalid_Returns400() validationProblem.AssertHasNotEmptyError(expectedPath: "timePeriods.in[0]"); validationProblem.AssertHasTimePeriodFormatError(expectedPath: "timePeriods.in[1]", value: "invalid"); validationProblem.AssertHasTimePeriodAllowedCodeError(expectedPath: "timePeriods.in[2]", code: ""); - validationProblem.AssertHasNotEmptyError(expectedPath: "timePeriods.notIn"); + validationProblem.AssertHasNotEmptyError(expectedPath: "timePeriods.notIn[0]"); } [Theory] @@ -892,7 +900,7 @@ public async Task Empty_Returns400() queryParameters: new Dictionary { { - "sorts[]", "" + "sorts[0]", "" } } ); @@ -901,7 +909,7 @@ public async Task Empty_Returns400() Assert.Single(validationProblem.Errors); - validationProblem.AssertHasNotEmptyError("sorts"); + validationProblem.AssertHasNotEmptyError("sorts[0]"); } [Fact] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerTests.cs index 7ba67cd352c..872ad79dc76 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerTests.cs @@ -1,13 +1,8 @@ using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; -using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; -using GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils; using GovUk.Education.ExploreEducationStatistics.Common.Utils; -using GovUk.Education.ExploreEducationStatistics.Content.Requests; -using GovUk.Education.ExploreEducationStatistics.Content.ViewModels; using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Fixture; using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.ViewModels; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; @@ -16,8 +11,6 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.WebUtilities; -using Moq; -using PublicationSummaryViewModel = GovUk.Education.ExploreEducationStatistics.Content.ViewModels.PublicationSummaryViewModel; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Controllers; @@ -115,813 +108,6 @@ private async Task GetDataSet(Guid dataSetId) } } - public class ListDataSetVersionsTests(TestApplicationFactory testApp) : DataSetsControllerTests(testApp) - { - [Theory] - [InlineData(1, 2, 1)] - [InlineData(1, 2, 2)] - [InlineData(1, 2, 9)] - [InlineData(1, 3, 2)] - [InlineData(2, 2, 9)] - public async Task MultipleAvailableVersionsForRequestedDataSet_Returns200_PaginatedCorrectly( - int page, - int pageSize, - int numberOfAvailableDataSetVersions) - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - var dataSetVersions = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) - .WithStatusPublished() - .WithDataSetId(dataSet.Id) - .GenerateList(numberOfAvailableDataSetVersions); - - await TestApp.AddTestData(context => - context.DataSetVersions.AddRange(dataSetVersions)); - - var pagedDataSetVersions = dataSetVersions - .OrderByDescending(dsv => dsv.VersionMajor) - .ThenByDescending(dsv => dsv.VersionMinor) - .Skip((page - 1) * pageSize) - .Take(pageSize) - .ToList(); - - var releaseFiles = pagedDataSetVersions - .Select( - dsv => DefaultReleaseFileViewModel() - .ForInstance(s => s.Set(rf => rf.Id, dsv.ReleaseFileId)) - .Generate() - ) - .ToList(); - - var contentApiClient = new Mock(MockBehavior.Strict); - - contentApiClient - .Setup(c => c.ListReleaseFiles( - It.Is(req => - releaseFiles.Select(rf => rf.Id).SequenceEqual(req.Ids)), - It.IsAny() - )) - .ReturnsAsync(releaseFiles); - - var response = await ListDataSetVersions( - dataSetId: dataSet.Id, - page: page, - pageSize: pageSize, - contentApiClient: contentApiClient.Object); - - var viewModel = response.AssertOk(useSystemJson: true); - - MockUtils.VerifyAllMocks(contentApiClient); - - Assert.NotNull(viewModel); - Assert.Equal(page, viewModel.Paging.Page); - Assert.Equal(pageSize, viewModel.Paging.PageSize); - Assert.Equal(numberOfAvailableDataSetVersions, viewModel.Paging.TotalResults); - Assert.Equal(pagedDataSetVersions.Count, viewModel.Results.Count); - } - - [Fact] - public async Task MultipleAvailableVersionsForRequestedDataSet_Returns200_OrderedCorrectly() - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - var dataSetVersions = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) - .WithStatusPublished() - .WithDataSetId(dataSet.Id) - .ForIndex(0, dsv => dsv.SetVersionNumber(1, 0)) - .ForIndex(1, dsv => dsv.SetVersionNumber(1, 1)) - .ForIndex(2, dsv => dsv.SetVersionNumber(3, 1)) - .ForIndex(3, dsv => dsv.SetVersionNumber(3, 0)) - .ForIndex(4, dsv => dsv.SetVersionNumber(2, 0)) - .ForIndex(5, dsv => dsv.SetVersionNumber(2, 1)) - .GenerateList(); - - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(dataSetVersions); - }); - - var releaseFiles = dataSetVersions - .OrderByDescending(dsv => dsv.VersionMajor) - .ThenByDescending(dsv => dsv.VersionMinor) - .Select(dsv => DefaultReleaseFileViewModel() - .ForInstance(s => s.Set(rf => rf.Id, dsv.ReleaseFileId)) - .Generate()) - .ToList(); - - var releaseFileIds = releaseFiles - .Select(rf => rf.Id) - .ToHashSet(); - - var contentApiClient = new Mock(MockBehavior.Strict); - - contentApiClient - .Setup(c => c.ListReleaseFiles( - It.Is(req => - req.Ids.All(id => releaseFileIds.Contains(id))), - It.IsAny() - )) - .ReturnsAsync( releaseFiles[..3]); - - var page1Response = await ListDataSetVersions( - dataSetId: dataSet.Id, - page: 1, - pageSize: 3, - contentApiClient: contentApiClient.Object); - - var page1ViewModel = page1Response.AssertOk(useSystemJson: true); - - Assert.Equal(3, page1ViewModel.Results.Count); - Assert.Equal("3.1", page1ViewModel.Results[0].Version); - Assert.Equal("3.0", page1ViewModel.Results[1].Version); - Assert.Equal("2.1", page1ViewModel.Results[2].Version); - - contentApiClient - .Setup(c => c.ListReleaseFiles( - It.Is(req => - req.Ids.All(id => releaseFileIds.Contains(id))), - It.IsAny() - )) - .ReturnsAsync(releaseFiles[3..6]); - - var page2Response = await ListDataSetVersions( - dataSetId: dataSet.Id, - page: 2, - pageSize: 3, - contentApiClient: contentApiClient.Object); - - var page2ViewModel = page2Response.AssertOk(useSystemJson: true); - - Assert.Equal(3, page2ViewModel.Results.Count); - Assert.Equal("2.0", page2ViewModel.Results[0].Version); - Assert.Equal("1.1", page2ViewModel.Results[1].Version); - Assert.Equal("1.0", page2ViewModel.Results[2].Version); - } - - [Theory] - [InlineData(DataSetVersionStatus.Published)] - [InlineData(DataSetVersionStatus.Withdrawn)] - [InlineData(DataSetVersionStatus.Deprecated)] - public async Task DataSetVersionIsAvailable_Returns200_CorrectViewModel(DataSetVersionStatus dataSetVersionStatus) - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - var dataSetVersionGenerator = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) - .WithStatus(dataSetVersionStatus) - .WithPublished(DateTimeOffset.UtcNow) - .WithDataSetId(dataSet.Id); - - DataSetVersion dataSetVersion = dataSetVersionStatus == DataSetVersionStatus.Withdrawn - ? dataSetVersionGenerator.WithWithdrawn(DateTimeOffset.UtcNow) - : dataSetVersionGenerator; - - await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); - - var releaseFiles = DefaultReleaseFileViewModel() - .ForInstance(s => s.Set(rf => rf.Id, dataSetVersion.ReleaseFileId)) - .GenerateList(1); - - var releaseFileIds = releaseFiles - .Select(rf => rf.Id) - .ToHashSet(); - - var contentApiClient = new Mock(MockBehavior.Strict); - - contentApiClient - .Setup(c => c.ListReleaseFiles( - It.Is(req => - req.Ids.All(id => releaseFileIds.Contains(id))), - It.IsAny() - )) - .ReturnsAsync(releaseFiles); - - var response = await ListDataSetVersions( - dataSetId: dataSet.Id, - page: 1, - pageSize: 1, - contentApiClient: contentApiClient.Object); - - var viewModel = response.AssertOk(useSystemJson: true); - - Assert.NotNull(viewModel); - Assert.Equal(1, viewModel.Paging.Page); - Assert.Equal(1, viewModel.Paging.PageSize); - Assert.Equal(1, viewModel.Paging.TotalResults); - - var result = Assert.Single(viewModel.Results); - - Assert.Equal(dataSetVersion.Version, result.Version); - Assert.Equal(dataSetVersion.VersionType, result.Type); - Assert.Equal(dataSetVersion.Status, result.Status); - Assert.Equal( - dataSetVersion.Published.TruncateNanoseconds(), - result.Published - ); - if (dataSetVersionStatus == DataSetVersionStatus.Withdrawn) - { - Assert.Equal( - dataSetVersion.Withdrawn.TruncateNanoseconds(), - result.Withdrawn - ); - } - Assert.Equal(dataSetVersion.Notes, result.Notes); - Assert.Equal(dataSetVersion.TotalResults, result.TotalResults); - - Assert.Equal(releaseFiles[0].DataSetFileId, result.File.Id); - - Assert.Equal(releaseFiles[0].Release.Title, result.Release.Title); - Assert.Equal(releaseFiles[0].Release.Slug, result.Release.Slug); - - Assert.Equal( - TimePeriodFormatter.FormatLabel( - dataSetVersion.MetaSummary!.TimePeriodRange.Start.Period, - dataSetVersion.MetaSummary.TimePeriodRange.Start.Code), - result.TimePeriods.Start); - Assert.Equal( - TimePeriodFormatter.FormatLabel( - dataSetVersion.MetaSummary.TimePeriodRange.End.Period, - dataSetVersion.MetaSummary.TimePeriodRange.End.Code), - result.TimePeriods.End); - Assert.Equal(dataSetVersion.MetaSummary.GeographicLevels, result.GeographicLevels); - Assert.Equal(dataSetVersion.MetaSummary.Filters, result.Filters); - Assert.Equal(dataSetVersion.MetaSummary.Indicators, result.Indicators); - } - - [Fact] - public async Task AvailableVersionForOtherDataSet_Returns200_OnlyVersionForRequestedDataSet() - { - DataSet dataSet1 = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - DataSet dataSet2 = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => - context.DataSets.AddRange(dataSet1, dataSet2)); - - DataSetVersion dataSet1Version = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) - .WithStatusPublished() - .WithVersionNumber(1, 1) - .WithDataSetId(dataSet1.Id); - - DataSetVersion dataSet2Version = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) - .WithStatusPublished() - .WithVersionNumber(2, 2) - .WithDataSetId(dataSet2.Id); - - await TestApp.AddTestData(context => - context.DataSetVersions.AddRange(dataSet1Version, dataSet2Version)); - - var releaseFiles = DefaultReleaseFileViewModel() - .ForInstance(s => s.Set(rf => rf.Id, dataSet1Version.ReleaseFileId)) - .GenerateList(1); - - var releaseFileIds = releaseFiles - .Select(rf => rf.Id) - .ToHashSet(); - - var contentApiClient = new Mock(MockBehavior.Strict); - - contentApiClient - .Setup(c => c.ListReleaseFiles( - It.Is(req => - req.Ids.All(id => releaseFileIds.Contains(id))), - It.IsAny() - )) - .ReturnsAsync(releaseFiles); - - var response = await ListDataSetVersions( - dataSetId: dataSet1.Id, - page: 1, - pageSize: 1, - contentApiClient: contentApiClient.Object); - - var viewModel = response.AssertOk(useSystemJson: true); - - Assert.NotNull(viewModel); - Assert.Equal(1, viewModel.Paging.Page); - Assert.Equal(1, viewModel.Paging.PageSize); - Assert.Equal(1, viewModel.Paging.TotalResults); - var result = Assert.Single(viewModel.Results); - Assert.Equal(dataSet1Version.Version, result.Version); - } - - [Theory] - [InlineData(DataSetVersionStatus.Processing)] - [InlineData(DataSetVersionStatus.Failed)] - [InlineData(DataSetVersionStatus.Mapping)] - [InlineData(DataSetVersionStatus.Draft)] - public async Task DataSetVersionUnavailable_Returns200_EmptyList(DataSetVersionStatus dataSetVersionStatus) - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion dataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) - .WithStatus(dataSetVersionStatus) - .WithDataSetId(dataSet.Id); - - await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); - - var response = await ListDataSetVersions( - dataSetId: dataSet.Id, - page: 1, - pageSize: 1); - - var viewModel = response.AssertOk(useSystemJson: true); - - Assert.NotNull(viewModel); - Assert.Equal(1, viewModel.Paging.Page); - Assert.Equal(1, viewModel.Paging.PageSize); - Assert.Equal(0, viewModel.Paging.TotalResults); - Assert.Empty(viewModel.Results); - } - - [Fact] - public async Task NoDataSetVersions_Returns200_EmptyList() - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - var response = await ListDataSetVersions( - dataSetId: dataSet.Id, - page: 1, - pageSize: 1); - - var viewModel = response.AssertOk(useSystemJson: true); - - Assert.NotNull(viewModel); - Assert.Equal(1, viewModel.Paging.Page); - Assert.Equal(1, viewModel.Paging.PageSize); - Assert.Equal(0, viewModel.Paging.TotalResults); - Assert.Empty(viewModel.Results); - } - - [Fact] - public async Task ReleaseFilesDoNotExist_Returns500() - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - var dataSetVersions = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) - .WithStatus(DataSetVersionStatus.Published) - .WithDataSetId(dataSet.Id) - .GenerateList(2); - - await TestApp.AddTestData(context => - context.DataSetVersions.AddRange(dataSetVersions)); - - var contentApiClient = new Mock(MockBehavior.Strict); - - contentApiClient - .Setup(c => c.ListReleaseFiles( - It.IsAny(), - It.IsAny() - )) - .ReturnsAsync([]); - - var response = await ListDataSetVersions( - dataSetId: dataSet.Id, - page: 1, - pageSize: 1, - contentApiClient: contentApiClient.Object); - - response.AssertInternalServerError(); - } - - [Fact] - public async Task ContentApiClientThrows_Returns500() - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - var dataSetVersions = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) - .WithStatus(DataSetVersionStatus.Published) - .WithDataSetId(dataSet.Id) - .GenerateList(2); - - await TestApp.AddTestData(context => - context.DataSetVersions.AddRange(dataSetVersions)); - - var contentApiClient = new Mock(MockBehavior.Strict); - - contentApiClient - .Setup(c => c.ListReleaseFiles( - It.IsAny(), - It.IsAny() - )) - .ThrowsAsync(new Exception("Something went wrong")); - - var response = await ListDataSetVersions( - dataSetId: dataSet.Id, - page: 1, - pageSize: 1, - contentApiClient: contentApiClient.Object); - - response.AssertInternalServerError(); - } - - [Fact] - public async Task PageTooBig_Returns200_EmptyList() - { - var page = 2; - var pageSize = 2; - var numberOfDataSetVersions = 2; - - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - var dataSetVersions = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) - .WithStatusPublished() - .WithDataSetId(dataSet.Id) - .GenerateList(numberOfDataSetVersions); - - await TestApp.AddTestData(context => context.DataSetVersions.AddRange(dataSetVersions)); - - var response = await ListDataSetVersions( - dataSetId: dataSet.Id, - page: page, - pageSize: pageSize); - - var viewModel = response.AssertOk(useSystemJson: true); - - Assert.NotNull(viewModel); - Assert.Equal(page, viewModel.Paging.Page); - Assert.Equal(pageSize, viewModel.Paging.PageSize); - Assert.Equal(numberOfDataSetVersions, viewModel.Paging.TotalResults); - Assert.Empty(viewModel.Results); - } - - [Theory] - [InlineData(0)] - [InlineData(-1)] - public async Task PageTooSmall_Returns400(int page) - { - var response = await ListDataSetVersions( - dataSetId: Guid.NewGuid(), - page: page, - pageSize: 1); - - var validationProblem = response.AssertValidationProblem(); - - Assert.Single(validationProblem.Errors); - - validationProblem.AssertHasGreaterThanOrEqualError("page", comparisonValue: 1); - } - - [Theory] - [InlineData(0)] - [InlineData(-1)] - [InlineData(21)] - public async Task PageSizeOutOfBounds_Returns400(int pageSize) - { - var response = await ListDataSetVersions( - dataSetId: Guid.NewGuid(), - page: 1, - pageSize: pageSize); - - var validationProblem = response.AssertValidationProblem(); - - Assert.Single(validationProblem.Errors); - - validationProblem.AssertHasInclusiveBetweenError("pageSize", from: 1, to: 20); - } - - [Theory] - [InlineData(DataSetStatus.Draft)] - [InlineData(DataSetStatus.Withdrawn)] - public async Task UnavailableDataSet_Returns503(DataSetStatus status) - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatus(status); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - var response = await ListDataSetVersions(dataSetId: dataSet.Id); - - response.AssertForbidden(); - } - - [Fact] - public async Task InvalidDataSetId_Returns404() - { - var client = TestApp.CreateClient(); - - var query = new Dictionary - { - { "page", "1" }, - { "pageSize", "1" }, - }; - - var uri = QueryHelpers.AddQueryString($"{BaseUrl}/not_a_valid_guid/versions", query); - - var response = await client.GetAsync(uri); - - response.AssertNotFound(); - } - - private async Task ListDataSetVersions( - Guid dataSetId, - int? page = null, - int? pageSize = null, - IContentApiClient? contentApiClient = null) - { - var query = new Dictionary - { - { "page", page?.ToString() }, - { "pageSize", pageSize?.ToString() }, - }; - - var uri = QueryHelpers.AddQueryString($"{BaseUrl}/{dataSetId}/versions", query); - - var client = BuildApp(contentApiClient).CreateClient(); - - return await client.GetAsync(uri); - } - } - - public class GetDataSetVersionTests(TestApplicationFactory testApp) : DataSetsControllerTests(testApp) - { - [Theory] - [InlineData(DataSetVersionStatus.Published)] - [InlineData(DataSetVersionStatus.Withdrawn)] - [InlineData(DataSetVersionStatus.Deprecated)] - public async Task VersionIsAvailable_Returns200(DataSetVersionStatus dataSetVersionStatus) - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - var dataSetVersionGenerator = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 3) - .WithStatus(dataSetVersionStatus) - .WithPublished(DateTimeOffset.UtcNow) - .WithDataSetId(dataSet.Id); - - DataSetVersion dataSetVersion = dataSetVersionStatus == DataSetVersionStatus.Withdrawn - ? dataSetVersionGenerator.WithWithdrawn(DateTimeOffset.UtcNow) - : dataSetVersionGenerator; - - await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); - - var releaseFiles = DefaultReleaseFileViewModel() - .GenerateList(1); - - var contentApiClient = new Mock(MockBehavior.Strict); - - contentApiClient - .Setup(c => c.ListReleaseFiles( - It.Is(req => req.Ids.Contains(dataSetVersion.ReleaseFileId)), - It.IsAny() - )) - .ReturnsAsync(releaseFiles); - - var response = await GetDataSetVersion( - dataSet.Id, - dataSetVersion.Version, - contentApiClient.Object - ); - - var viewModel = response.AssertOk(useSystemJson: true); - - MockUtils.VerifyAllMocks(contentApiClient); - - Assert.NotNull(viewModel); - Assert.Equal(dataSetVersion.Version, viewModel.Version); - Assert.Equal(dataSetVersion.VersionType, viewModel.Type); - Assert.Equal(dataSetVersion.Status, viewModel.Status); - Assert.Equal( - dataSetVersion.Published.TruncateNanoseconds(), - viewModel.Published - ); - if (dataSetVersionStatus == DataSetVersionStatus.Withdrawn) - { - Assert.Equal( - dataSetVersion.Withdrawn.TruncateNanoseconds(), - viewModel.Withdrawn - ); - } - Assert.Equal(dataSetVersion.Notes, viewModel.Notes); - Assert.Equal(dataSetVersion.TotalResults, viewModel.TotalResults); - - Assert.Equal(releaseFiles[0].DataSetFileId, viewModel.File.Id); - - Assert.Equal(releaseFiles[0].Release.Title, viewModel.Release.Title); - Assert.Equal(releaseFiles[0].Release.Slug, viewModel.Release.Slug); - - Assert.Equal( - TimePeriodFormatter.FormatLabel( - dataSetVersion.MetaSummary!.TimePeriodRange.Start.Period, - dataSetVersion.MetaSummary.TimePeriodRange.Start.Code), - viewModel.TimePeriods.Start); - Assert.Equal( - TimePeriodFormatter.FormatLabel( - dataSetVersion.MetaSummary.TimePeriodRange.End.Period, - dataSetVersion.MetaSummary.TimePeriodRange.End.Code), - viewModel.TimePeriods.End); - Assert.Equal(dataSetVersion.MetaSummary.GeographicLevels, viewModel.GeographicLevels); - Assert.Equal(dataSetVersion.MetaSummary.Filters, viewModel.Filters); - Assert.Equal(dataSetVersion.MetaSummary.Indicators, viewModel.Indicators); - } - - [Theory] - [InlineData(DataSetVersionStatus.Processing)] - [InlineData(DataSetVersionStatus.Failed)] - [InlineData(DataSetVersionStatus.Mapping)] - [InlineData(DataSetVersionStatus.Draft)] - public async Task VersionNotAvailable_Returns403(DataSetVersionStatus dataSetVersionStatus) - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion dataSetVersion = DataFixture - .DefaultDataSetVersion() - .WithStatus(dataSetVersionStatus) - .WithDataSetId(dataSet.Id); - - await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); - - var response = await GetDataSetVersion(dataSet.Id, dataSetVersion.Version); - - response.AssertForbidden(); - } - - [Fact] - public async Task VersionExistsForOtherDataSet_Returns404() - { - DataSet dataSet1 = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - DataSet dataSet2 = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.AddRange(dataSet1, dataSet2)); - - DataSetVersion dataSetVersion = DataFixture - .DefaultDataSetVersion() - .WithStatusPublished() - .WithDataSetId(dataSet1.Id); - - await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); - - var response = await GetDataSetVersion(dataSet2.Id, dataSetVersion.Version); - - response.AssertNotFound(); - } - - [Fact] - public async Task VersionDoesNotExist_Returns404() - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - var response = await GetDataSetVersion(dataSet.Id, "1.0"); - - response.AssertNotFound(); - } - - [Fact] - public async Task DataSetDoesNotExist_Returns404() - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion dataSetVersion = DataFixture - .DefaultDataSetVersion() - .WithStatusPublished() - .WithDataSetId(dataSet.Id); - - await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); - - var response = await GetDataSetVersion(Guid.NewGuid(), dataSetVersion.Version); - - response.AssertNotFound(); - } - - [Fact] - public async Task ReleaseFileDoesNotExist_Returns500() - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion dataSetVersion = DataFixture - .DefaultDataSetVersion() - .WithStatusPublished() - .WithDataSetId(dataSet.Id); - - await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); - - var contentApiClient = new Mock(MockBehavior.Strict); - - contentApiClient - .Setup(c => c.ListReleaseFiles( - It.Is(req => req.Ids.Contains(dataSetVersion.ReleaseFileId)), - It.IsAny() - )) - .ReturnsAsync([]); - - var response = await GetDataSetVersion(dataSet.Id, dataSetVersion.Version, contentApiClient.Object); - - response.AssertInternalServerError(); - } - - [Fact] - public async Task ContentApiClientThrows_Returns500() - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion dataSetVersion = DataFixture - .DefaultDataSetVersion() - .WithStatusPublished() - .WithDataSetId(dataSet.Id); - - await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); - - var contentApiClient = new Mock(MockBehavior.Strict); - - contentApiClient - .Setup(c => c.ListReleaseFiles( - It.Is(req => req.Ids.Contains(dataSetVersion.ReleaseFileId)), - It.IsAny() - )) - .ThrowsAsync(new Exception("Something went wrong")); - - var response = await GetDataSetVersion(dataSet.Id, dataSetVersion.Version, contentApiClient.Object); - - response.AssertInternalServerError(); - } - - private async Task GetDataSetVersion( - Guid dataSetId, - string dataSetVersion, - IContentApiClient? contentApiClient = null) - { - var client = BuildApp(contentApiClient).CreateClient(); - - var uri = new Uri($"{BaseUrl}/{dataSetId}/versions/{dataSetVersion}", UriKind.Relative); - - return await client.GetAsync(uri); - } - } - public class GetDataSetMetaTests(TestApplicationFactory testApp) : DataSetsControllerTests(testApp) { public class NoQueryParametersTests(TestApplicationFactory testApp) : GetDataSetMetaTests(testApp) @@ -1823,35 +1009,8 @@ private async Task GetDataSetMeta( } } - private WebApplicationFactory BuildApp(IContentApiClient? contentApiClient = null) + private WebApplicationFactory BuildApp() { - return TestApp.ConfigureServices(services => - { - services.ReplaceService(contentApiClient ?? Mock.Of()); - }); + return TestApp; } - - private Generator DefaultReleaseFileViewModel() => - DataFixture.Generator() - .ForInstance(s => s - .SetDefault(r => r.Id) - .Set(r => r.File, () => DataFixture.DefaultFileInfo()) - .SetDefault(r => r.DataSetFileId) - .Set(r => r.Release, () => DefaultReleaseSummaryViewModel()) - ); - - private Generator DefaultReleaseSummaryViewModel() => - DataFixture.Generator() - .ForInstance(s => s - .SetDefault(r => r.Id) - .SetDefault(r => r.Title) - .SetDefault(r => r.Slug) - .Set(r => r.Publication, () => DefaultPublicationSummaryViewModel())); - - private Generator DefaultPublicationSummaryViewModel() => - DataFixture.Generator() - .ForInstance(s => s - .SetDefault(r => r.Id) - .SetDefault(r => r.Title) - .SetDefault(r => r.Slug)); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Filters/InvalidRequestInputFilterTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Filters/InvalidRequestInputFilterTests.cs new file mode 100644 index 00000000000..0488f7a0e28 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Filters/InvalidRequestInputFilterTests.cs @@ -0,0 +1,465 @@ +using System.Net.Http.Json; +using System.Text; +using Asp.Versioning; +using GovUk.Education.ExploreEducationStatistics.Common.ModelBinding; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Validators; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Fixture; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Filters; + +public class InvalidRequestInputFilterTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) +{ + [Fact] + public async Task TestPersonBody_ValidBody_Returns200() + { + var client = BuildApp().CreateClient(); + + var response = await client.PostAsJsonAsync( + requestUri: nameof(TestController.TestPersonBody), + value: new TestPerson + { + Name = "Test name" + }); + + var person = response.AssertOk(); + + Assert.Equal("Test name", person.Name); + } + + [Theory] + [InlineData("{}", "")] + [InlineData("{ \"name\": 123 }", "name")] + [InlineData("{ \"name\": true }", "name")] + public async Task TestPersonBody_InvalidInput_Returns400(string body, string path) + { + var client = BuildApp().CreateClient(); + + var response = await client.PostAsync( + requestUri: nameof(TestController.TestPersonBody), + content: new StringContent(body, mediaType: "application/json", encoding: Encoding.UTF8)); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasInvalidInputError(path); + + Assert.Equal(ValidationMessages.InvalidValue.Message, error.Message); + } + + [Theory] + [InlineData("{ \"firstName\": \"test\" }", "firstName")] + [InlineData("{ \"firstName\": 123 }", "firstName")] + [InlineData("{ \"name\": \"Test\", \"lastName\": \"Last\" }", "lastName")] + public async Task TestPersonBody_UnknownFields_Returns400(string body, string path) + { + var client = BuildApp().CreateClient(); + + var response = await client.PostAsync( + requestUri: nameof(TestController.TestPersonBody), + content: new StringContent(body, mediaType: "application/json", encoding: Encoding.UTF8)); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasUnknownFieldError(path); + + Assert.Equal(ValidationMessages.UnknownField.Message, error.Message); + } + + [Fact] + public async Task TestPersonBody_EmptyBody_Returns400() + { + var client = BuildApp().CreateClient(); + + var response = await client.PostAsync( + requestUri: nameof(TestController.TestPersonBody), + content: new StringContent("", mediaType: "application/json", encoding: Encoding.UTF8)); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasNotEmptyBodyError(); + + Assert.Equal(ValidationMessages.NotEmptyBody.Message, error.Message); + } + + [Fact] + public async Task TestPersonBody_RequiredValue_Returns400() + { + var client = BuildApp().CreateClient(); + + var response = await client.PostAsync( + requestUri: nameof(TestController.TestPersonBody), + content: new StringContent( + "{ \"name\": null }", + mediaType: "application/json", + encoding: Encoding.UTF8)); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasRequiredValueError("name"); + + Assert.Equal(ValidationMessages.RequiredValue.Message, error.Message); + } + + [Fact] + public async Task TestGroupBody_ValidBody_Returns200() + { + var client = BuildApp().CreateClient(); + + var response = await client.PostAsJsonAsync( + requestUri: nameof(TestController.TestGroupBody), + value: new TestGroup + { + Owner = new TestPerson + { + Name = "Test name" + } + }); + + var group = response.AssertOk(); + + Assert.Equal("Test name", group.Owner.Name); + } + + [Theory] + [InlineData("{}", "")] + [InlineData("{ \"owner\": 123 }", "owner")] + [InlineData("{ \"owner\": {} }", "owner")] + [InlineData("{ \"owner\": { \"name\": 123 } }", "owner.name")] + [InlineData("{ \"owner\": { \"name\": true } }", "owner.name")] + public async Task TestGroupBody_InvalidInput_Returns400(string body, string path) + { + var client = BuildApp().CreateClient(); + + var response = await client.PostAsync( + requestUri: nameof(TestController.TestGroupBody), + content: new StringContent(body, mediaType: "application/json", encoding: Encoding.UTF8)); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasInvalidInputError(path); + + Assert.Equal(ValidationMessages.InvalidValue.Message, error.Message); + } + + [Theory] + [InlineData("{ \"owner\": { \"firstName\": \"test\" } }", "owner.firstName")] + [InlineData("{ \"owner\": { \"firstName\": 123 } }", "owner.firstName")] + [InlineData("{ \"owner\": { \"name\": \"Test\", \"lastName\": \"Last\" } }", "owner.lastName")] + public async Task TestGroupBody_UnknownFields_Returns400(string body, string path) + { + var client = BuildApp().CreateClient(); + + var response = await client.PostAsync( + requestUri: nameof(TestController.TestGroupBody), + content: new StringContent(body, mediaType: "application/json", encoding: Encoding.UTF8)); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasUnknownFieldError(path); + + Assert.Equal(ValidationMessages.UnknownField.Message, error.Message); + } + + [Fact] + public async Task TestGroupBody_EmptyBody_Returns400() + { + var client = BuildApp().CreateClient(); + + var response = await client.PostAsync( + requestUri: nameof(TestController.TestGroupBody), + content: new StringContent("", mediaType: "application/json", encoding: Encoding.UTF8)); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasNotEmptyBodyError(); + + Assert.Equal(ValidationMessages.NotEmptyBody.Message, error.Message); + } + + [Fact] + public async Task TestGroupBody_RequiredValue_Returns400() + { + var client = BuildApp().CreateClient(); + + var response = await client.PostAsync( + requestUri: nameof(TestController.TestGroupBody), + content: new StringContent( + content: "{ \"owner\": { \"name\": null } }", + mediaType: "application/json", + encoding: Encoding.UTF8)); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasRequiredValueError("owner.name"); + + Assert.Equal(ValidationMessages.RequiredValue.Message, error.Message); + } + + [Theory] + [InlineData("?name=Test+name", "Test name")] + [InlineData("?name=Test+name&age=30", "Test name", 30)] + [InlineData("?name=123", "123")] + [InlineData("?name=true", "true")] + [InlineData("?name=", "")] + [InlineData("?name=Test&aliases[0]=Alias1&aliases[1]=Alias2", "Test", null, "Alias1", "Alias2")] + [InlineData("?name=Test&aliases=Alias1,Alias2", "Test", null, "Alias1", "Alias2")] + public async Task TestPersonQuery_ValidQuery_Returns200( + string query, + string expectedName, + int? expectedAge = null, + params string[] expectedAliases) + { + var client = BuildApp().CreateClient(); + + var response = await client.GetAsync(requestUri: $"{nameof(TestController.TestPersonQuery)}{query}"); + + var person = response.AssertOk(); + + Assert.Equal(expectedName, person.Name); + Assert.Equal(expectedAge, person.Age); + Assert.Equal(expectedAliases, person.Aliases); + } + + [Fact] + public async Task TestPersonQuery_RequiredValue_Returns400() + { + var client = BuildApp().CreateClient(); + + var response = await client.GetAsync(requestUri: nameof(TestController.TestPersonQuery)); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasRequiredValueError("name"); + + Assert.Equal(ValidationMessages.RequiredValue.Message, error.Message); + } + + [Theory] + [InlineData("?name=Test&firstName=First", "firstName")] + [InlineData("?name=Test&firstName=First&lastName=Last", "firstName", "lastName")] + public async Task TestPersonQuery_UnknownFields_Returns400(string query, params string[] unknownFields) + { + var client = BuildApp().CreateClient(); + + var response = await client.GetAsync(requestUri: $"{nameof(TestController.TestPersonQuery)}{query}"); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Equal(unknownFields.Length, validationProblem.Errors.Count); + + Assert.All(unknownFields, field => + { + var error = validationProblem.AssertHasUnknownFieldError(field); + + Assert.Equal(ValidationMessages.UnknownField.Message, error.Message); + }); + } + + [Theory] + [InlineData("?name=Test&age=ten")] + public async Task TestPersonQuery_InvalidValue_Returns400(string query) + { + var client = BuildApp().CreateClient(); + + var response = await client.GetAsync(requestUri: $"{nameof(TestController.TestPersonQuery)}{query}"); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasInvalidValueError("age"); + + Assert.Equal(ValidationMessages.InvalidValue.Message, error.Message); + } + + [Theory] + [InlineData("?owner.name=Test+name", "Test name")] + [InlineData("?owner.name=Test+name&owner.age=30", "Test name", 30)] + [InlineData("?owner.name=123", "123")] + [InlineData("?owner.name=true", "true")] + [InlineData("?owner.name=", "")] + [InlineData("?owner.name=Test&owner.aliases[0]=Alias1&owner.aliases[1]=Alias2", "Test", null, "Alias1", "Alias2")] + [InlineData("?owner.name=Test&owner.aliases=Alias1,Alias2", "Test", null, "Alias1", "Alias2")] + public async Task TestGroupQuery_ValidQuery_Returns200( + string query, + string expectedName, + int? expectedAge = null, + params string[] expectedAliases) + { + var client = BuildApp().CreateClient(); + + var response = await client.GetAsync(requestUri: $"{nameof(TestController.TestGroupQuery)}{query}"); + + var group = response.AssertOk(); + + Assert.Equal(expectedName, group.Owner.Name); + Assert.Equal(expectedAge, group.Owner.Age); + Assert.Equal(expectedAliases, group.Owner.Aliases); + } + + [Fact] + public async Task TestGroupQuery_RequiredValue_Returns400() + { + var client = BuildApp().CreateClient(); + + var response = await client.GetAsync(requestUri: nameof(TestController.TestGroupQuery)); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasRequiredValueError("owner"); + + Assert.Equal(ValidationMessages.RequiredValue.Message, error.Message); + } + + [Theory] + [InlineData("?owner.name=Test&owner=Owner", "owner")] + [InlineData("?owner.name=Test&owner.firstName=First", "owner.firstName")] + [InlineData("?owner.name=Test&owner.firstName=First&owner.lastName=Last", "owner.firstName", "owner.lastName")] + [InlineData("?owner.name=Test&name=Test", "name")] + public async Task TestGroupQuery_UnknownFields_Returns400(string query, params string[] unknownFields) + { + var client = BuildApp().CreateClient(); + + var response = await client.GetAsync(requestUri: $"{nameof(TestController.TestGroupQuery)}{query}"); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Equal(unknownFields.Length, validationProblem.Errors.Count); + + Assert.All(unknownFields, field => + { + var error = validationProblem.AssertHasUnknownFieldError(field); + + Assert.Equal(ValidationMessages.UnknownField.Message, error.Message); + }); + } + + [Theory] + [InlineData("?owner.name=Test&owner.age=ten")] + public async Task TestGroupQuery_InvalidValue_Returns400(string query) + { + var client = BuildApp().CreateClient(); + + var response = await client.GetAsync(requestUri: $"{nameof(TestController.TestGroupQuery)}{query}"); + + var validationProblem = response.AssertValidationProblem(); + + Assert.Single(validationProblem.Errors); + + var error = validationProblem.AssertHasInvalidValueError("owner.age"); + + Assert.Equal(ValidationMessages.InvalidValue.Message, error.Message); + } + + [Theory] + [InlineData("?name=Test", "Test")] + [InlineData("?name=Test&age=30", "Test", 30)] + [InlineData("?name=Test&aliases[0]=Alias1&aliases[1]=Alias2", "Test", null, "Alias1", "Alias2")] + public async Task TestQueryArgs_ValidQuery_Returns200( + string query, + string expectedName, + int? expectedAge = null, + params string[] expectedAliases) + { + var client = BuildApp().CreateClient(); + + var response = await client.GetAsync(requestUri: $"{nameof(TestController.TestQueryArgs)}{query}"); + + var group = response.AssertOk(); + + Assert.Equal(expectedName, group.Name); + Assert.Equal(expectedAge, group.Age); + Assert.Equal(expectedAliases, group.Aliases); + } + + [ApiController] + private class TestController : ControllerBase + { + [HttpPost(nameof(TestPersonBody))] + public ActionResult TestPersonBody([FromBody] TestPerson testPerson, CancellationToken _) + { + return Ok(testPerson); + } + + [HttpPost(nameof(TestGroupBody))] + public ActionResult TestGroupBody([FromBody] TestGroup testGroup, CancellationToken _) + { + return Ok(testGroup); + } + + [HttpGet(nameof(TestPersonQuery))] + public ActionResult TestPersonQuery([FromQuery] TestPerson testPerson, CancellationToken _) + { + return Ok(testPerson); + } + + [HttpGet(nameof(TestGroupQuery))] + public ActionResult TestGroupQuery([FromQuery] TestGroup testGroup, CancellationToken _) + { + return Ok(testGroup); + } + + [HttpGet(nameof(TestQueryArgs))] + public ActionResult TestQueryArgs( + string name, + [FromQuery(Name = "age")] int? ageNumber, + [FromQuery] List? aliases, + CancellationToken _) + { + return Ok(new TestPerson + { + Name = name, + Age = ageNumber, + Aliases = aliases ?? [] + }); + } + } + + private class TestPerson + { + public required string Name { get; init; } + + public int? Age { get; init; } + + [FromQuery] + [QuerySeparator] + public List Aliases { get; init; } = []; + } + + private class TestGroup + { + public required TestPerson Owner { get; init; } + } + private WebApplicationFactory BuildApp() + { + return TestApp + .WithWebHostBuilder(builder => builder.WithAdditionalControllers(typeof(TestController))) + .ConfigureServices(s => + { + s.Configure(options => options.AssumeDefaultVersionWhenUnspecified = true); + }); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Filters/InvalidRequestInputResultFilterTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Filters/InvalidRequestInputResultFilterTests.cs deleted file mode 100644 index 2f7392de60d..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Filters/InvalidRequestInputResultFilterTests.cs +++ /dev/null @@ -1,191 +0,0 @@ -using System.Net.Http.Json; -using System.Text; -using Asp.Versioning; -using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; -using GovUk.Education.ExploreEducationStatistics.Common.Validators; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Fixture; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Mvc.Testing; -using Microsoft.Extensions.DependencyInjection; - -namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Filters; - -public class InvalidRequestInputResultFilterTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) -{ - [Fact] - public async Task TestPersonBody_Returns200() - { - var client = BuildApp().CreateClient(); - - var response = await client.PostAsJsonAsync( - requestUri: nameof(TestController.TestPersonBody), - value: new TestPerson - { - Name = "Test name" - }); - - response.AssertOk(); - } - - [Theory] - [InlineData("{}", "")] - [InlineData("{ \"name\": 123 }", "name")] - [InlineData("{ \"name\": true }", "name")] - [InlineData("{ \"firstName\": \"test\" }", "")] - [InlineData("{ \"firstName\": 123 }", "")] - public async Task TestPersonBody_InvalidJson_Returns400(string body, string path) - { - var client = BuildApp().CreateClient(); - - var response = await client.PostAsync( - requestUri: nameof(TestController.TestPersonBody), - content: new StringContent(body, mediaType: "application/json", encoding: Encoding.UTF8)); - - var validationProblem = response.AssertValidationProblem(); - - Assert.Single(validationProblem.Errors); - - var error = validationProblem.AssertHasInvalidInputError(path); - - Assert.Equal(ValidationMessages.InvalidInput.Message, error.Message); - } - - [Fact] - public async Task TestPersonBody_EmptyBody_Returns400() - { - var client = BuildApp().CreateClient(); - - var response = await client.PostAsync( - requestUri: nameof(TestController.TestPersonBody), - content: new StringContent("", mediaType: "application/json", encoding: Encoding.UTF8)); - - var validationProblem = response.AssertValidationProblem(); - - Assert.Single(validationProblem.Errors); - - var error = validationProblem.AssertHasNotEmptyBodyError(); - - Assert.Equal(ValidationMessages.NotEmptyBody.Message, error.Message); - } - - [Fact] - public async Task TestPersonBody_RequiredField_Returns400() - { - var client = BuildApp().CreateClient(); - - var response = await client.PostAsync( - requestUri: nameof(TestController.TestPersonBody), - content: new StringContent( - "{ \"name\": null }", - mediaType: "application/json", - encoding: Encoding.UTF8)); - - var validationProblem = response.AssertValidationProblem(); - - Assert.Single(validationProblem.Errors); - - var error = validationProblem.AssertHasRequiredFieldError("name"); - - Assert.Equal(ValidationMessages.RequiredField.Message, error.Message); - } - - [Theory] - [InlineData("{}", "")] - [InlineData("{ \"owner\": 123 }", "owner")] - [InlineData("{ \"owner\": {} }", "owner")] - [InlineData("{ \"owner\": { \"name\": 123 } }", "owner.name")] - [InlineData("{ \"owner\": { \"name\": true } }", "owner.name")] - [InlineData("{ \"owner\": { \"firstName\": \"test\" } }", "owner")] - [InlineData("{ \"owner\": { \"firstName\": 123 } }", "owner")] - public async Task TestGroupBody_InvalidJson_Returns400(string body, string path) - { - var client = BuildApp().CreateClient(); - - var response = await client.PostAsync( - requestUri: nameof(TestController.TestGroupBody), - content: new StringContent(body, mediaType: "application/json", encoding: Encoding.UTF8)); - - var validationProblem = response.AssertValidationProblem(); - - Assert.Single(validationProblem.Errors); - - var error = validationProblem.AssertHasInvalidInputError(path); - - Assert.Equal(ValidationMessages.InvalidInput.Message, error.Message); - } - - [Fact] - public async Task TestGroupBody_EmptyBody_Returns400() - { - var client = BuildApp().CreateClient(); - - var response = await client.PostAsync( - requestUri: nameof(TestController.TestGroupBody), - content: new StringContent("", mediaType: "application/json", encoding: Encoding.UTF8)); - - var validationProblem = response.AssertValidationProblem(); - - Assert.Single(validationProblem.Errors); - - var error = validationProblem.AssertHasNotEmptyBodyError(); - - Assert.Equal(ValidationMessages.NotEmptyBody.Message, error.Message); - } - - [Fact] - public async Task TestGroupBody_RequiredField_Returns400() - { - var client = BuildApp().CreateClient(); - - var response = await client.PostAsync( - requestUri: nameof(TestController.TestGroupBody), - content: new StringContent( - content: "{ \"owner\": { \"name\": null } }", - mediaType: "application/json", - encoding: Encoding.UTF8)); - - var validationProblem = response.AssertValidationProblem(); - - Assert.Single(validationProblem.Errors); - - var error = validationProblem.AssertHasRequiredFieldError("owner.name"); - - Assert.Equal(ValidationMessages.RequiredField.Message, error.Message); - } - - [ApiController] - private class TestController : ControllerBase - { - [HttpPost(nameof(TestPersonBody))] - public ActionResult TestPersonBody([FromBody] TestPerson testPerson) - { - return Ok(testPerson); - } - - [HttpPost(nameof(TestGroupBody))] - public ActionResult TestGroupBody([FromBody] TestGroup testGroup) - { - return Ok(testGroup); - } - } - - private class TestPerson - { - public required string Name { get; init; } - } - - private class TestGroup - { - public required TestPerson Owner { get; init; } - } - - private WebApplicationFactory BuildApp() - { - return TestApp - .WithWebHostBuilder(builder => builder.WithAdditionalControllers(typeof(TestController))) - .ConfigureServices(s => - { - s.Configure(options => options.AssumeDefaultVersionWhenUnspecified = true); - }); - } -} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Fixture/TestApplicationFactory.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Fixture/TestApplicationFactory.cs index aa724eaeca6..e67f90bcde9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Fixture/TestApplicationFactory.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Fixture/TestApplicationFactory.cs @@ -1,3 +1,4 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using Microsoft.EntityFrameworkCore; @@ -25,18 +26,8 @@ public override async ValueTask DisposeAsync() public async Task ClearTestData() where TDbContext : DbContext { - await using var context = GetDbContext(); - - var tables = context.Model.GetEntityTypes() - .Select(type => type.GetTableName()) - .Distinct() - .Cast() - .ToList(); - - foreach (var table in tables) - { - await context.Database.ExecuteSqlRawAsync(@$"TRUNCATE TABLE ""{table}"" RESTART IDENTITY CASCADE;"); - } + var context = this.GetDbContext(); + await context.ClearTestData(); } public async Task ClearAllTestData() @@ -51,7 +42,10 @@ protected override IHostBuilder CreateHostBuilder() .ConfigureServices(services => { services.AddDbContext( - options => options.UseNpgsql(_postgreSqlContainer.GetConnectionString())); + options => options + .UseNpgsql( + _postgreSqlContainer.GetConnectionString(), + psqlOptions => psqlOptions.EnableRetryOnFailure())); }); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs new file mode 100644 index 00000000000..00407661bb5 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs @@ -0,0 +1,67 @@ +using Asp.Versioning; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Requests; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.ViewModels; +using Microsoft.AspNetCore.Mvc; +using Swashbuckle.AspNetCore.Annotations; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Controllers; + +[ApiVersion(1.0)] +[ApiController] +[Route("api/v{version:apiVersion}/data-sets/{dataSetId:guid}/versions")] +public class DataSetVersionsController( + IDataSetService dataSetService) + : ControllerBase +{ + /// + /// List a data set’s versions + /// + /// + /// List a data set’s versions. Only provides summary information of each version. + /// + [HttpGet] + [Produces("application/json")] + [SwaggerResponse(200, "The paginated list of data set versions", type: typeof(DataSetVersionPaginatedListViewModel))] + [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] + [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] + public async Task> ListDataSetVersions( + [FromQuery] DataSetVersionListRequest request, + [SwaggerParameter("The ID of the data set.")] Guid dataSetId, + CancellationToken cancellationToken) + { + return await dataSetService + .ListVersions( + dataSetId: dataSetId, + page: request.Page, + pageSize: request.PageSize, + cancellationToken: cancellationToken) + .HandleFailuresOrOk(); + } + + /// + /// Get a data set version + /// + /// + /// Get a data set version's summary details. + /// + [HttpGet("{dataSetVersion}")] + [Produces("application/json")] + [SwaggerResponse(200, "The requested data set version", type: typeof(DataSetVersionViewModel))] + [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] + [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] + public async Task> GetDataSetVersion( + [SwaggerParameter("The ID of the data set.")] Guid dataSetId, + [SwaggerParameter("The data set version e.g. 1.0, 1.1, 2.0, etc.")] string dataSetVersion, + CancellationToken cancellationToken) + { + return await dataSetService + .GetVersion( + dataSetId: dataSetId, + dataSetVersion: dataSetVersion, + cancellationToken: cancellationToken) + .HandleFailuresOrOk(); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs index 14edabeb7d6..77ad0df428f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs @@ -39,55 +39,6 @@ public async Task> GetDataSet( .HandleFailuresOrOk(); } - /// - /// Get a data set version - /// - /// - /// Get a data set version's summary details. - /// - [HttpGet("{dataSetId:guid}/versions/{dataSetVersion}")] - [Produces("application/json")] - [SwaggerResponse(200, "The requested data set version", type: typeof(DataSetVersionViewModel))] - [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] - [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] - public async Task> GetDataSetVersion( - [SwaggerParameter("The ID of the data set.")] Guid dataSetId, - [SwaggerParameter("The data set version e.g. 1.0, 1.1, 2.0, etc.")] string dataSetVersion, - CancellationToken cancellationToken) - { - return await dataSetService - .GetVersion( - dataSetId: dataSetId, - dataSetVersion: dataSetVersion, - cancellationToken: cancellationToken) - .HandleFailuresOrOk(); - } - - /// - /// List a data set’s versions - /// - /// - /// List a data set’s versions. Only provides summary information of each version. - /// - [HttpGet("{dataSetId:guid}/versions")] - [Produces("application/json")] - [SwaggerResponse(200, "The paginated list of data set versions", type: typeof(DataSetVersionPaginatedListViewModel))] - [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] - [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] - public async Task> ListDataSetVersions( - [FromQuery] DataSetVersionListRequest request, - [SwaggerParameter("The ID of the data set.")] Guid dataSetId, - CancellationToken cancellationToken) - { - return await dataSetService - .ListVersions( - dataSetId: dataSetId, - page: request.Page, - pageSize: request.PageSize, - cancellationToken: cancellationToken) - .HandleFailuresOrOk(); - } - /// /// Get a data set’s metadata /// diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Extensions/MvcOptionsExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Extensions/MvcOptionsExtensions.cs index 55d3b64084c..958be34d7bd 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Extensions/MvcOptionsExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Extensions/MvcOptionsExtensions.cs @@ -11,6 +11,18 @@ public static void AddInvalidRequestInputResultFilter(this MvcOptions options) options.ModelBindingMessageProvider .SetMissingRequestBodyRequiredValueAccessor(() => ValidationMessages.NotEmptyBody.Message); - options.Filters.Add(); + options.ModelBindingMessageProvider + .SetValueIsInvalidAccessor(_ => ValidationMessages.InvalidValue.Message); + + options.ModelBindingMessageProvider + .SetAttemptedValueIsInvalidAccessor((_, _) => ValidationMessages.InvalidValue.Message); + + options.ModelBindingMessageProvider + .SetUnknownValueIsInvalidAccessor(_ => ValidationMessages.InvalidValue.Message); + + options.ModelBindingMessageProvider + .SetMissingBindRequiredValueAccessor(_ => ValidationMessages.InvalidValue.Message); + + options.Filters.Add(); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Filters/InvalidRequestInputFilter.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Filters/InvalidRequestInputFilter.cs new file mode 100644 index 00000000000..b4af3b7ff91 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Filters/InvalidRequestInputFilter.cs @@ -0,0 +1,267 @@ +using System.Diagnostics.CodeAnalysis; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using GovUk.Education.ExploreEducationStatistics.Common.Validators; +using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.AspNetCore.Mvc.Infrastructure; +using Microsoft.AspNetCore.Mvc.ModelBinding; +using JsonException = System.Text.Json.JsonException; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Filters; + +/// +/// Filter to convert errors caused by invalid request input into a standard +/// validation error response. Importantly, this filter fixes the error paths to +/// point to invalid part of the request and adds error codes +/// +public class InvalidRequestInputFilter : IAsyncActionFilter +{ + public Task OnActionExecutionAsync(ActionExecutingContext context, ActionExecutionDelegate next) + { + // A filter or endpoint has already created our standard + // validation error response. We can bail early. + if (context.Result is BadRequestObjectResult { Value: ValidationProblemViewModel }) + { + return next(); + } + + var errors = new List(); + + errors.AddRange(GetUnknownQueryParameterErrors(context)); + + if (context.ModelState.IsValid && errors.Count == 0) + { + return next(); + } + + var invalidModelState = context.ModelState + .Where(kv => kv.Value?.ValidationState is ModelValidationState.Invalid) + .Cast>() + .ToDictionary(); + + if (TryGetJsonError(invalidModelState, out var jsonError)) + { + errors.Add(jsonError); + } + + if (TryGetEmptyBodyError(invalidModelState, out var emptyBodyError)) + { + errors.Add(emptyBodyError); + } + + // There can be error entries for the controller method's parameters + // which need to be filtered out first (which otherwise leak internals). + var invalidErrorKeys = context.ActionDescriptor + .Parameters + .Where(param => param.ParameterType.IsComplex()) + .Select(param => param.Name) + .ToHashSet(); + + invalidModelState = invalidModelState + .Where(entry => !invalidErrorKeys.Contains(entry.Key)) + .ToDictionary( + kv => FormatErrorKey(kv.Key), + kv => kv.Value + ); + + if (TryGetRequiredValueError(invalidModelState, out var requiredFieldError)) + { + errors.Add(requiredFieldError); + } + + if (TryGetInvalidValueError(invalidModelState, out var invalidValueError)) + { + errors.Add(invalidValueError); + } + + if (errors.Count == 0) + { + return next(); + } + + var problemDetailsFactory = context.HttpContext.RequestServices.GetRequiredService(); + + var problemDetails = problemDetailsFactory.CreateValidationProblemDetails( + context.HttpContext, + new ModelStateDictionary() + ); + + context.Result = new BadRequestObjectResult(ValidationProblemViewModel.Create(problemDetails, errors)); + + return Task.CompletedTask; + } + + private static List GetUnknownQueryParameterErrors( + ActionExecutingContext context) + { + return context.HttpContext.Request + .Query + .Where(param => !context.ModelState.ContainsKey(param.Key)) + .Select(param => new ErrorViewModel + { + Code = ValidationMessages.UnknownField.Code, + Message = ValidationMessages.UnknownField.Message, + Path = param.Key + }) + .ToList(); + } + + private static bool TryGetJsonError( + IDictionary errorEntries, + [NotNullWhen(true)] out ErrorViewModel? error) + { + var modelError = errorEntries + .SelectMany(entry => entry.Value.Errors) + .FirstOrDefault(error => error.Exception is JsonException); + + if (modelError is null) + { + error = null; + return false; + } + + error = GetJsonError(modelError); + + return true; + } + + private static ErrorViewModel GetJsonError(ModelError error) + { + List paths = []; + + var currentException = error.Exception; + var completed = false; + + while (!completed) + { + if (currentException is JsonException { Path: not null } jsonException) + { + paths.Add(jsonException.Path); + } + + if (currentException?.InnerException is not null) + { + currentException = currentException.InnerException; + } + else + { + completed = true; + } + } + + // Not ideal, but there isn't any better way of figuring out if we have this type + // of JsonException as they don't provide any error codes or other identifier. + var message = currentException is not null + && currentException.Message.StartsWith("The JSON property") + && currentException.Message.Contains( + "could not be mapped to any .NET member contained in type" + ) + ? ValidationMessages.UnknownField + : ValidationMessages.InvalidValue; + + return new ErrorViewModel + { + Code = message.Code, + Message = message.Message, + Path = JsonPathUtils.Concat(paths) + }; + } + + private static bool TryGetEmptyBodyError( + IDictionary modelState, + [NotNullWhen(true)] out ErrorViewModel? error) + { + if (!modelState.Any(entry => entry.Value.Errors + .Any(e => e.ErrorMessage == ValidationMessages.NotEmptyBody.Message))) + { + error = null; + return false; + } + + error = new ErrorViewModel + { + Code = ValidationMessages.NotEmptyBody.Code, + Message = ValidationMessages.NotEmptyBody.Message, + }; + + return true; + } + + private static bool TryGetRequiredValueError( + IDictionary modelState, + [NotNullWhen(true)] out ErrorViewModel? error) + { + var requiredFieldErrorKey = modelState + // This isn't the ideal way of determining if we have a required field error. + // Despite trying a few things (including using localization files), there didn't + // seem to be an easy way to change the default error message for `RequiredAttribute`. + // In the interest of time, this approach will have to do. + // TODO: Work out better way to modify default data annotation error messages + .FirstOrDefault(entry => entry.Value.Errors + .Any(e => e.ErrorMessage.StartsWith("The") && e.ErrorMessage.EndsWith("field is required."))) + .Key; + + if (requiredFieldErrorKey.IsNullOrEmpty()) + { + error = null; + return false; + } + + error = new ErrorViewModel + { + Code = ValidationMessages.RequiredValue.Code, + Message = ValidationMessages.RequiredValue.Message, + Path = requiredFieldErrorKey + }; + + return true; + } + + private static bool TryGetInvalidValueError( + IDictionary modelState, + [NotNullWhen(true)] out ErrorViewModel? error) + { + var invalidValueErrorKey = modelState + .FirstOrDefault(entry => entry.Value.Errors + .Any(e => e.ErrorMessage == ValidationMessages.InvalidValue.Message)) + .Key; + + if (invalidValueErrorKey.IsNullOrEmpty()) + { + error = null; + return false; + } + + error = new ErrorViewModel + { + Code = ValidationMessages.InvalidValue.Code, + Message = ValidationMessages.InvalidValue.Message, + Path = invalidValueErrorKey + }; + + return true; + } + + private static string FormatErrorKey(string errorKey) + { + if (errorKey.IsNullOrEmpty()) + { + return errorKey; + } + + // Assume that the error key is camelCased already + if (errorKey[0] != '$' && !char.IsUpper(errorKey[0])) + { + return errorKey; + } + + // The error key is PascalCased, so we have to camelCase it + var parts = errorKey.Split('.'); + + return parts + .Select(part => part.ToLowerFirst()) + .JoinToString('.'); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Filters/InvalidRequestInputResultFilter.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Filters/InvalidRequestInputResultFilter.cs deleted file mode 100644 index de1b8bdad60..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Filters/InvalidRequestInputResultFilter.cs +++ /dev/null @@ -1,178 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using GovUk.Education.ExploreEducationStatistics.Common.Utils; -using GovUk.Education.ExploreEducationStatistics.Common.Validators; -using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Mvc.Abstractions; -using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.AspNetCore.Mvc.Infrastructure; -using Microsoft.AspNetCore.Mvc.ModelBinding; -using JsonException = System.Text.Json.JsonException; - -namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Filters; - -/// -/// Filter to convert errors caused by invalid request input into a standard -/// validation error response. Importantly, this filter fixes the error paths to -/// point to invalid part of the request and adds error codes -/// -public class InvalidRequestInputResultFilter : IResultFilter -{ - public void OnResultExecuting(ResultExecutingContext context) - { - if (context.ModelState.IsValid) - { - return; - } - - // A filter or endpoint has already created our standard - // validation error response. We can bail early. - if (context.Result is BadRequestObjectResult { Value: ValidationProblemViewModel }) - { - return; - } - - var errorEntries = context.ModelState - .Where(kv => kv.Value?.ValidationState is ModelValidationState.Invalid) - .Cast>() - .ToList(); - - var errors = new List(); - - if (TryGetJsonError(errorEntries, out var jsonError)) - { - errors.Add(jsonError); - } - - if (TryGetEmptyBodyError(errorEntries, out var emptyBodyError)) - { - errors.Add(emptyBodyError); - } - - if (TryGetRequiredFieldError(errorEntries, context.ActionDescriptor, out var requiredFieldError)) - { - errors.Add(requiredFieldError); - } - - if (errors.Count == 0) - { - return; - } - - var problemDetailsFactory = context.HttpContext.RequestServices.GetRequiredService(); - - var problemDetails = problemDetailsFactory.CreateValidationProblemDetails( - context.HttpContext, - new ModelStateDictionary() - ); - - context.Result = new BadRequestObjectResult(ValidationProblemViewModel.Create(problemDetails, errors)); - } - - public void OnResultExecuted(ResultExecutedContext context) - { - } - - private static bool TryGetJsonError( - IEnumerable> errorEntries, - [NotNullWhen(true)] out ErrorViewModel? error) - { - var modelError = errorEntries - .SelectMany(entry => entry.Value.Errors) - .FirstOrDefault(error => error.Exception is JsonException); - - if (modelError is null) - { - error = null; - return false; - } - - error = new ErrorViewModel - { - Code = ValidationMessages.InvalidInput.Code, - Message = ValidationMessages.InvalidInput.Message, - Path = GetJsonPath(modelError) - }; - - return true; - } - - private static string GetJsonPath(ModelError error) - { - List paths = []; - - var currentException = error.Exception; - - while (currentException is not null) - { - if (currentException is JsonException { Path: not null } jsonException) - { - paths.Add(jsonException.Path); - } - - currentException = currentException.InnerException; - } - - return JsonPathUtils.Concat(paths); - } - - private static bool TryGetEmptyBodyError( - IEnumerable> errorEntries, - [NotNullWhen(true)] out ErrorViewModel? error) - { - if (!errorEntries.Any(entry => entry.Value.Errors - .Any(e => e.ErrorMessage == ValidationMessages.NotEmptyBody.Message))) - { - error = null; - return false; - } - - error = new ErrorViewModel - { - Code = ValidationMessages.NotEmptyBody.Code, - Message = ValidationMessages.NotEmptyBody.Message, - }; - - return true; - } - - private static bool TryGetRequiredFieldError( - IEnumerable> errorEntries, - ActionDescriptor actionDescriptor, - [NotNullWhen(true)] out ErrorViewModel? error) - { - var actionParams = actionDescriptor.Parameters - .Where(param => param.BindingInfo?.BindingSource == BindingSource.Body - || param.BindingInfo?.BindingSource == BindingSource.Form) - .Select(param => param.Name) - .ToHashSet(); - - var requiredFieldError = errorEntries - // There are entries for the controller method's parameters which - // need to be filtered out first. We don't want to show errors - // for these parameters as they leak internals to API users. - .Where(entry => !actionParams.Contains(entry.Key)) - // This isn't the ideal way of determining if we have a required field error. - // Despite trying a few things (including using localization files), there didn't - // seem to be an easy way to change the default error message for `RequiredAttribute`. - // In the interest of time, this approach will have to do. - // TODO: Work out better way to modify default data annotation error messages - .FirstOrDefault(entry => entry.Value.Errors - .Any(e => e.ErrorMessage.StartsWith("The") && e.ErrorMessage.EndsWith("field is required."))); - - if (requiredFieldError.Equals(default(KeyValuePair))) - { - error = null; - return false; - } - - error = new ErrorViewModel - { - Code = ValidationMessages.RequiredField.Code, - Message = ValidationMessages.RequiredField.Message, - Path = requiredFieldError.Key - }; - - return true; - } -} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs index 075316bb6c2..d5fa00ba9a0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs @@ -5,6 +5,7 @@ using GovUk.Education.ExploreEducationStatistics.Common.Cancellation; using GovUk.Education.ExploreEducationStatistics.Common.Config; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.ModelBinding; using GovUk.Education.ExploreEducationStatistics.Common.Rules; using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Options; @@ -72,6 +73,8 @@ public void ConfigureServices(IServiceCollection services) options.AddCommaSeparatedQueryModelBinderProvider(); options.AddTrimStringBinderProvider(); + // Stop empty query string parameters being converted to null + options.ModelMetadataDetailsProviders.Add(new EmptyStringMetadataDetailsProvider()); // Adds correct camelCased paths for model errors options.ModelMetadataDetailsProviders.Add(new SystemTextJsonValidationMetadataProvider()); }) @@ -83,6 +86,7 @@ public void ConfigureServices(IServiceCollection services) .AddJsonOptions(options => { options.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; + options.JsonSerializerOptions.UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow; // This must be false to allow `JsonExceptionResultFilter` to work correctly, // otherwise, JsonExceptions can't be identified. Also, this prevents diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMetaGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMetaGeneratorExtensions.cs index 33c9c8f46ec..689f7130e5f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMetaGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMetaGeneratorExtensions.cs @@ -15,9 +15,14 @@ public static Generator DefaultFilterMeta(this DataFixture fixture, public static Generator WithDefaults(this Generator generator) => generator.ForInstance(s => s.SetDefaults()); - public static Generator WithDataSetVersion(this Generator generator, DataSetVersion dataSetVersion) + public static Generator WithDataSetVersion(this Generator generator, + DataSetVersion dataSetVersion) => generator.ForInstance(s => s.SetDataSetVersion(dataSetVersion)); + public static Generator WithDataSetVersionId(this Generator generator, + Guid dataSetVersionId) + => generator.ForInstance(s => s.SetDataSetVersionId(dataSetVersionId)); + public static Generator WithPublicId(this Generator generator, string identifier) => generator.ForInstance(s => s.SetPublicId(identifier)); @@ -53,7 +58,12 @@ public static InstanceSetters SetDataSetVersion( DataSetVersion dataSetVersion) => setters .Set(m => m.DataSetVersion, dataSetVersion) - .Set(m => m.DataSetVersionId, dataSetVersion.Id); + .SetDataSetVersionId(dataSetVersion.Id); + + public static InstanceSetters SetDataSetVersionId( + this InstanceSetters setters, + Guid dataSetVersionId) + => setters.Set(m => m.DataSetVersionId, dataSetVersionId); public static InstanceSetters SetPublicId(this InstanceSetters setters, string publicId) => setters.Set(m => m.PublicId, publicId); @@ -67,7 +77,7 @@ public static InstanceSetters SetHint(this InstanceSetters SetOptions( this InstanceSetters setters, IEnumerable options) - => setters.Set(m => m.Options, () => options); + => setters.SetOptions(() => options); public static InstanceSetters SetOptions( this InstanceSetters setters, @@ -83,6 +93,7 @@ public static InstanceSetters SetOptions( .Select(o => context.Fixture .DefaultFilterOptionMetaLink() .WithOption(o) + .WithMeta(m) .Generate()) .ToList(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/IndicatorMetaGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/IndicatorMetaGeneratorExtensions.cs index de8f72a6365..2dc8e83c0c7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/IndicatorMetaGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/IndicatorMetaGeneratorExtensions.cs @@ -16,6 +16,10 @@ public static Generator WithDataSetVersion( DataSetVersion dataSetVersion) => generator.ForInstance(s => s.SetDataSetVersion(dataSetVersion)); + public static Generator WithDataSetVersionId(this Generator generator, + Guid dataSetVersionId) + => generator.ForInstance(s => s.SetDataSetVersionId(dataSetVersionId)); + public static Generator WithLabel(this Generator generator, string label) => generator.ForInstance(s => s.SetLabel(label)); @@ -47,7 +51,12 @@ public static InstanceSetters SetDataSetVersion( DataSetVersion dataSetVersion) => setters .Set(m => m.DataSetVersion, dataSetVersion) - .Set(m => m.DataSetVersionId, dataSetVersion.Id); + .SetDataSetVersionId(dataSetVersion.Id); + + public static InstanceSetters SetDataSetVersionId( + this InstanceSetters setters, + Guid dataSetVersionId) + => setters.Set(m => m.DataSetVersionId, dataSetVersionId); public static InstanceSetters SetLabel( this InstanceSetters setters, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/LocationMetaGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/LocationMetaGeneratorExtensions.cs index b52d2026b0a..fc4c3ab74c9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/LocationMetaGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/LocationMetaGeneratorExtensions.cs @@ -30,6 +30,11 @@ public static Generator WithDataSetVersion( DataSetVersion dataSetVersion) => generator.ForInstance(s => s.SetDataSetVersion(dataSetVersion)); + public static Generator WithDataSetVersionId( + this Generator generator, + Guid dataSetVersionId) + => generator.ForInstance(s => s.SetDataSetVersionId(dataSetVersionId)); + public static Generator WithLevel( this Generator generator, GeographicLevel level) @@ -62,7 +67,12 @@ public static InstanceSetters SetDataSetVersion( DataSetVersion dataSetVersion) => setters .Set(m => m.DataSetVersion, dataSetVersion) - .Set(m => m.DataSetVersionId, dataSetVersion.Id); + .SetDataSetVersionId(dataSetVersion.Id); + + public static InstanceSetters SetDataSetVersionId( + this InstanceSetters setters, + Guid dataSetVersionId) + => setters.Set(m => m.DataSetVersionId, dataSetVersionId); public static InstanceSetters SetLevel( this InstanceSetters setters, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/TimePeriodMetaGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/TimePeriodMetaGeneratorExtensions.cs index f38766cc1d5..bb1312918e9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/TimePeriodMetaGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/TimePeriodMetaGeneratorExtensions.cs @@ -23,6 +23,10 @@ public static Generator WithDataSetVersion( DataSetVersion dataSetVersion) => generator.ForInstance(s => s.SetDataSetVersion(dataSetVersion)); + public static Generator WithDataSetVersionId(this Generator generator, + Guid dataSetVersionId) + => generator.ForInstance(s => s.SetDataSetVersionId(dataSetVersionId)); + public static Generator WithCode(this Generator generator, TimeIdentifier code) => generator.ForInstance(s => s.SetCode(code)); @@ -43,7 +47,12 @@ public static InstanceSetters SetDataSetVersion( DataSetVersion dataSetVersion) => setters .Set(m => m.DataSetVersion, dataSetVersion) - .Set(m => m.DataSetVersionId, dataSetVersion.Id); + .SetDataSetVersionId(dataSetVersion.Id); + + public static InstanceSetters SetDataSetVersionId( + this InstanceSetters setters, + Guid dataSetVersionId) + => setters.Set(m => m.DataSetVersionId, dataSetVersionId); public static InstanceSetters SetCode( this InstanceSetters setters, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs index 4884f9356b5..e532e25d41c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs @@ -9,6 +9,8 @@ public enum DataSetVersionImportStage Pending, CopyingCsvFiles, ImportingMetadata, + CreatingMappings, + AutoMapping, ImportingData, WritingDataFiles, Completing diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs new file mode 100644 index 00000000000..93426a4a392 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs @@ -0,0 +1,226 @@ +using System.Text.Json; +using GovUk.Education.ExploreEducationStatistics.Common.Database; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model; + +public class DataSetVersionMapping : ICreatedUpdatedTimestamps +{ + public Guid Id { get; init; } + + public required Guid SourceDataSetVersionId { get; set; } + + public DataSetVersion SourceDataSetVersion { get; set; } = null!; + + public required Guid TargetDataSetVersionId { get; set; } + + public DataSetVersion TargetDataSetVersion { get; set; } = null!; + + public LocationMappingPlan LocationMappingPlan { get; set; } = null!; + + public FilterMappingPlan FilterMappingPlan { get; set; } = null!; + + public bool LocationMappingsComplete { get; set; } + + public bool FilterMappingsComplete { get; set; } + + public DateTimeOffset Created { get; set; } + + public DateTimeOffset? Updated { get; set; } + + internal class Config : IEntityTypeConfiguration + { + public void Configure(EntityTypeBuilder builder) + { + builder.Property(mapping => mapping.Id) + .HasValueGenerator(); + + builder.HasIndex(mapping => new { mapping.SourceDataSetVersionId }) + .HasDatabaseName("IX_DataSetVersionMappings_SourceDataSetVersionId") + .IsUnique(); + + builder.HasIndex(mapping => new { mapping.TargetDataSetVersionId }) + .HasDatabaseName("IX_DataSetVersionMappings_TargetDataSetVersionId") + .IsUnique(); + + builder.Property(p => p.LocationMappingPlan) + .HasColumnType("jsonb") + .HasConversion( + v => + JsonSerializer.Serialize(v, (JsonSerializerOptions)null!), + v => + JsonSerializer.Deserialize(v, (JsonSerializerOptions)null!)!); + + builder.Property(p => p.FilterMappingPlan) + .HasColumnType("jsonb") + .HasConversion( + v => + JsonSerializer.Serialize(v, (JsonSerializerOptions)null!), + v => + JsonSerializer.Deserialize(v, (JsonSerializerOptions)null!)!); + } + } +} + +/// +/// This enum indicates the type of mapping that an element from the source DataSetVersion has had applied +/// so far in the mapping process. +/// +public enum MappingType +{ + /// + /// No mapping has yet been carried out, either automatically or by the user. + /// + None, + + /// + /// The user has manually chosen a mapping candidate for this source element. + /// + ManualMapped, + + /// + /// The user has manually indicated that no mapping candidate exists for this source element. + /// + ManualNone, + + /// + /// The service has automatically selected a likely mapping candidate for this source element. + /// + AutoMapped, + + /// + /// The service has automatically indicated that no likely mapping candidate exists for this + /// source element. It will still take the user to confirm these and switch them to be + /// "ManualNone" in the process before the service indicates that the mappings are complete. + /// + AutoNone +} + +/// +/// This base class represents an element from the DataSetVersions that can be mapped. +/// +public abstract record MappableElement(string Label); + +public abstract record MappableElementWithOptions(string Label) + : MappableElement(Label) + where TMappableOption : MappableElement +{ + public Dictionary Options { get; set; } = []; +} + +/// +/// This base class represents a mapping for a single mappable element e.g. a single Location. +/// This holds the source element itself from the source DataSetVersion e.g. a particular Location, +/// the type of mapping that has been performed (e.g. the user choosing a candidate Location from +/// the target DataSetVersion) and the candidate key (if a candidate has been chosen). +/// +public abstract record Mapping + where TMappableElement : MappableElement +{ + public MappingType Type { get; set; } = MappingType.None; + + public TMappableElement Source { get; set; } = null!; + + public string? CandidateKey { get; set; } +} + +/// +/// This base class represents a mapping for a parent element which then itself also contains +/// child elements (or "options") that can themselves be mapped. +/// +public abstract record ParentMapping + : Mapping + where TMappableElement : MappableElement + where TOption : MappableElement + where TOptionMapping : Mapping +{ + public Dictionary OptionMappings { get; set; } = []; +} + +/// +/// This represents a location option that is potentially mappable to another location option +/// from the same geographic level. +/// +public record MappableLocationOption(string Label) : MappableElement(Label) +{ + public string? Code { get; set; } + + public string? OldCode { get; set; } + + public string? Urn { get; set; } + + public string? LaEstab { get; set; } + + public string? Ukprn { get; set; } +}; + +/// +/// This represents the mapping, or failure to map, of a source location option to a target +/// location option from the same geographic level. +/// +public record LocationOptionMapping : Mapping; + +/// +/// This represents a single geographic level's worth of location mappings from the source +/// data set version and potential candidates to map to from the target data set version. +/// +public record LocationLevelMappings +{ + public Dictionary Mappings { get; set; } = []; + + public Dictionary Candidates { get; set; } = []; +} + +/// +/// This represents the overall mapping plan for all the geographic levels +/// and locations from the source data set version to the target version. +/// +public class LocationMappingPlan +{ + public Dictionary Levels { get; set; } = []; +} + +/// +/// This represents a filter option that is potentially mappable to another filter option. +/// +public record MappableFilterOption(string Label) : MappableElement(Label); + +/// +/// This represents a filter that is potentially mappable to another filter. +/// +public record MappableFilter(string Label) : MappableElement(Label); + +/// +/// This represents a candidate filter and all of its candidate filter options from +/// the target data set version that could be mapped to from filters and filter options +/// from the source version. +/// +public record FilterMappingCandidate(string Label) + : MappableElementWithOptions(Label); + +/// +/// This represents a potential mapping of a filter option from the source data set version +/// to a filter option in the target version. In order to be mappable, both filter options' +/// parent filters must firstly be mapped to each other. +/// +public record FilterOptionMapping : Mapping; + +/// +/// This represents a potential mapping of a filter from the source data set version +/// to a filter in the target version. +/// +public record FilterMapping : ParentMapping; + +/// +/// This represents the overall mapping plan for filters and filter options from the source +/// data set version to filters and filter options in the target data set version. +/// +public record FilterMappingPlan +{ + public Dictionary Mappings { get; set; } = []; + + public Dictionary Candidates { get; set; } = []; +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs index cf3581c4853..f62f4ac3578 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs @@ -1,8 +1,10 @@ using System.Text.Json.Serialization; +using Newtonsoft.Json.Converters; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model; [JsonConverter(typeof(JsonStringEnumConverter))] +[Newtonsoft.Json.JsonConverter(typeof(StringEnumConverter))] public enum DataSetVersionStatus { Processing, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Database/PublicDataDbContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Database/PublicDataDbContext.cs index a14da4f1546..ae46795c31a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Database/PublicDataDbContext.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Database/PublicDataDbContext.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.CodeAnalysis; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using Microsoft.EntityFrameworkCore; @@ -5,7 +6,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; public class PublicDataDbContext : DbContext { - public PublicDataDbContext(DbContextOptions options, bool updateTimestamps = true) : base(options) + public const string FilterOptionMetaLinkSequence = "FilterOptionMetaLink_seq"; + public const string LocationOptionMetasIdSequence = "LocationOptionMetas_Id_seq"; + + public PublicDataDbContext( + DbContextOptions options, bool updateTimestamps = true) : base(options) { Configure(updateTimestamps: updateTimestamps); } @@ -22,11 +27,28 @@ private void Configure(bool updateTimestamps = true) protected override void OnModelCreating(ModelBuilder modelBuilder) { modelBuilder.ApplyConfigurationsFromAssembly(typeof(PublicDataDbContext).Assembly); + + modelBuilder.HasSequence(FilterOptionMetaLinkSequence); } + [SuppressMessage("Security", "EF1002:Risk of vulnerability to SQL injection.")] + public Task NextSequenceValue(string sequenceName, CancellationToken cancellationToken = default) => + Database.SqlQueryRaw($""" + SELECT nextval('public."{sequenceName}"') AS "Value" + """) + .FirstAsync(cancellationToken); + + [SuppressMessage("Security", "EF1002:Risk of vulnerability to SQL injection.")] + public Task SetSequenceValue(string sequenceName, long value, CancellationToken cancellationToken = default) => + Database.SqlQueryRaw($""" + SELECT setval('public."{sequenceName}"', {value}) AS "Value" + """) + .FirstAsync(cancellationToken); + public DbSet DataSets { get; init; } = null!; public DbSet DataSetVersions { get; init; } = null!; public DbSet DataSetVersionImports { get; init; } = null!; + public DbSet DataSetVersionMappings { get; init; } = null!; public DbSet GeographicLevelMetas { get; init; } = null!; public DbSet LocationMetas { get; init; } = null!; public DbSet LocationOptionMetas { get; init; } = null!; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnection.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnection.cs index f1224f99e86..0692beb5390 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnection.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DuckDb/DuckDbConnection.cs @@ -10,6 +10,11 @@ public static DuckDbConnection CreateFileConnection(string filename) return new DuckDbConnection($"DataSource={filename}"); } + public static DuckDbConnection CreateFileConnectionReadOnly(string filename) + { + return new DuckDbConnection($"DataSource={filename};access_mode=read_only"); + } + public override DuckDbCommand CreateCommand() { // Bit rubbish to do this but we don't have access to the diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240618080650_EES4945_AddDataSetVersionMappingsTable.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240618080650_EES4945_AddDataSetVersionMappingsTable.Designer.cs new file mode 100644 index 00000000000..2c4744a79aa --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240618080650_EES4945_AddDataSetVersionMappingsTable.Designer.cs @@ -0,0 +1,1777 @@ +// +using System; +using System.Collections.Generic; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Migrations +{ + [DbContext(typeof(PublicDataDbContext))] + [Migration("20240618080650_EES4945_AddDataSetVersionMappingsTable")] + partial class EES4945_AddDataSetVersionMappingsTable + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.4") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetFilterOptions", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("ChangeSetFilterOptions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetFilters", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("ChangeSetFilters"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetIndicators", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("ChangeSetIndicators"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetLocations", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("ChangeSetLocations"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetTimePeriods", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("ChangeSetTimePeriods"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("LatestDraftVersionId") + .HasColumnType("uuid"); + + b.Property("LatestLiveVersionId") + .HasColumnType("uuid"); + + b.Property("PublicationId") + .HasColumnType("uuid"); + + b.Property("Published") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text"); + + b.Property("Summary") + .IsRequired() + .HasColumnType("text"); + + b.Property("SupersedingDataSetId") + .HasColumnType("uuid"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.Property("Withdrawn") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("LatestDraftVersionId") + .IsUnique(); + + b.HasIndex("LatestLiveVersionId") + .IsUnique(); + + b.HasIndex("SupersedingDataSetId"); + + b.ToTable("DataSets"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetId") + .HasColumnType("uuid"); + + b.Property("Notes") + .IsRequired() + .HasColumnType("text"); + + b.Property("Published") + .HasColumnType("timestamp with time zone"); + + b.Property("ReleaseFileId") + .HasColumnType("uuid"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text"); + + b.Property("TotalResults") + .HasColumnType("bigint"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.Property("VersionMajor") + .HasColumnType("integer"); + + b.Property("VersionMinor") + .HasColumnType("integer"); + + b.Property("VersionPatch") + .HasColumnType("integer"); + + b.Property("Withdrawn") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseFileId"); + + b.HasIndex("DataSetId", "VersionMajor", "VersionMinor", "VersionPatch") + .IsUnique() + .HasDatabaseName("IX_DataSetVersions_DataSetId_VersionNumber"); + + b.ToTable("DataSetVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionImport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Completed") + .HasColumnType("timestamp with time zone"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("InstanceId") + .HasColumnType("uuid"); + + b.Property("Stage") + .IsRequired() + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DataSetVersionImports"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("SourceDataSetVersionId") + .HasColumnType("uuid"); + + b.Property("TargetDataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("SourceDataSetVersionId") + .IsUnique() + .HasDatabaseName("IX_DataSetVersionMappings_SourceDataSetVersionId"); + + b.HasIndex("TargetDataSetVersionId") + .IsUnique() + .HasDatabaseName("IX_DataSetVersionMappings_TargetDataSetVersionId"); + + b.ToTable("DataSetVersionMappings"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Hint") + .IsRequired() + .HasColumnType("text"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b.Property("PublicId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "PublicId") + .IsUnique(); + + b.ToTable("FilterMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("IsAggregate") + .HasColumnType("boolean"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("FilterOptionMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMetaLink", b => + { + b.Property("MetaId") + .HasColumnType("integer"); + + b.Property("OptionId") + .HasColumnType("integer"); + + b.Property("PublicId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("MetaId", "OptionId"); + + b.HasIndex("OptionId"); + + b.HasIndex("PublicId"); + + b.HasIndex("MetaId", "PublicId") + .IsUnique(); + + b.ToTable("FilterOptionMetaLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property>("Levels") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId") + .IsUnique(); + + b.ToTable("GeographicLevelMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("DecimalPlaces") + .HasColumnType("smallint"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b.Property("PublicId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Unit") + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "PublicId") + .IsUnique(); + + b.ToTable("IndicatorMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Level") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("character varying(5)"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "Level") + .IsUnique(); + + b.ToTable("LocationMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("text"); + + b.Property("LaEstab") + .HasColumnType("text"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b.Property("OldCode") + .HasColumnType("text"); + + b.Property("PublicId") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Ukprn") + .HasColumnType("text"); + + b.Property("Urn") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Code"); + + b.HasIndex("LaEstab"); + + b.HasIndex("OldCode"); + + b.HasIndex("PublicId") + .IsUnique(); + + b.HasIndex("Type"); + + b.HasIndex("Ukprn"); + + b.HasIndex("Urn"); + + b.HasIndex(new[] { "Type", "Label", "Code", "OldCode", "Urn", "LaEstab", "Ukprn" }, "IX_LocationOptionMetas_All") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.AreNullsDistinct(b.HasIndex(new[] { "Type", "Label", "Code", "OldCode", "Urn", "LaEstab", "Ukprn" }, "IX_LocationOptionMetas_All"), false); + + b.ToTable("LocationOptionMetas"); + + b.HasDiscriminator("Type").HasValue("LocationOptionMeta"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMetaLink", b => + { + b.Property("MetaId") + .HasColumnType("integer"); + + b.Property("OptionId") + .HasColumnType("integer"); + + b.HasKey("MetaId", "OptionId"); + + b.HasIndex("OptionId"); + + b.ToTable("LocationOptionMetaLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Code") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Period") + .IsRequired() + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "Code", "Period") + .IsUnique(); + + b.ToTable("TimePeriodMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationCodedOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("CODE"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationLocalAuthorityOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("LA"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationProviderOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("PROV"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationRscRegionOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("RSC"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationSchoolOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("SCH"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetFilterOptions", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("FilterOptionChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Change", "Changes", b1 => + { + b1.Property("ChangeSetFilterOptionsId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Identifier") + .HasColumnType("uuid") + .HasAnnotation("Relational:JsonPropertyName", "Id"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("ChangeSetFilterOptionsId", "Id"); + + b1.ToTable("ChangeSetFilterOptions"); + + b1.ToJson("Changes"); + + b1.WithOwner() + .HasForeignKey("ChangeSetFilterOptionsId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionChangeState", "CurrentState", b2 => + { + b2.Property("ChangeSetFilterOptionsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("FilterId") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("IsAggregate") + .HasColumnType("boolean"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("ChangeSetFilterOptionsId", "ChangeId"); + + b2.ToTable("ChangeSetFilterOptions"); + + b2.WithOwner() + .HasForeignKey("ChangeSetFilterOptionsId", "ChangeId"); + }); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionChangeState", "PreviousState", b2 => + { + b2.Property("ChangeSetFilterOptionsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("FilterId") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("IsAggregate") + .HasColumnType("boolean"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("ChangeSetFilterOptionsId", "ChangeId"); + + b2.ToTable("ChangeSetFilterOptions"); + + b2.WithOwner() + .HasForeignKey("ChangeSetFilterOptionsId", "ChangeId"); + }); + + b1.Navigation("CurrentState"); + + b1.Navigation("PreviousState"); + }); + + b.Navigation("Changes"); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetFilters", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("FilterChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Change", "Changes", b1 => + { + b1.Property("ChangeSetFiltersId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Identifier") + .HasColumnType("uuid") + .HasAnnotation("Relational:JsonPropertyName", "Id"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("ChangeSetFiltersId", "Id"); + + b1.ToTable("ChangeSetFilters"); + + b1.ToJson("Changes"); + + b1.WithOwner() + .HasForeignKey("ChangeSetFiltersId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterChangeState", "CurrentState", b2 => + { + b2.Property("ChangeSetFiltersId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Hint") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("ChangeSetFiltersId", "ChangeId"); + + b2.ToTable("ChangeSetFilters"); + + b2.WithOwner() + .HasForeignKey("ChangeSetFiltersId", "ChangeId"); + }); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterChangeState", "PreviousState", b2 => + { + b2.Property("ChangeSetFiltersId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Hint") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("ChangeSetFiltersId", "ChangeId"); + + b2.ToTable("ChangeSetFilters"); + + b2.WithOwner() + .HasForeignKey("ChangeSetFiltersId", "ChangeId"); + }); + + b1.Navigation("CurrentState"); + + b1.Navigation("PreviousState"); + }); + + b.Navigation("Changes"); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetIndicators", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("IndicatorChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Change", "Changes", b1 => + { + b1.Property("ChangeSetIndicatorsId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Identifier") + .HasColumnType("uuid") + .HasAnnotation("Relational:JsonPropertyName", "Id"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("ChangeSetIndicatorsId", "Id"); + + b1.ToTable("ChangeSetIndicators"); + + b1.ToJson("Changes"); + + b1.WithOwner() + .HasForeignKey("ChangeSetIndicatorsId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorChangeState", "CurrentState", b2 => + { + b2.Property("ChangeSetIndicatorsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("DecimalPlaces") + .HasColumnType("smallint"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Unit") + .HasColumnType("text"); + + b2.HasKey("ChangeSetIndicatorsId", "ChangeId"); + + b2.ToTable("ChangeSetIndicators"); + + b2.WithOwner() + .HasForeignKey("ChangeSetIndicatorsId", "ChangeId"); + }); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorChangeState", "PreviousState", b2 => + { + b2.Property("ChangeSetIndicatorsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("DecimalPlaces") + .HasColumnType("smallint"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Unit") + .HasColumnType("text"); + + b2.HasKey("ChangeSetIndicatorsId", "ChangeId"); + + b2.ToTable("ChangeSetIndicators"); + + b2.WithOwner() + .HasForeignKey("ChangeSetIndicatorsId", "ChangeId"); + }); + + b1.Navigation("CurrentState"); + + b1.Navigation("PreviousState"); + }); + + b.Navigation("Changes"); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetLocations", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("LocationChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Change", "Changes", b1 => + { + b1.Property("ChangeSetLocationsId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Identifier") + .HasColumnType("uuid") + .HasAnnotation("Relational:JsonPropertyName", "Id"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("ChangeSetLocationsId", "Id"); + + b1.ToTable("ChangeSetLocations"); + + b1.ToJson("Changes"); + + b1.WithOwner() + .HasForeignKey("ChangeSetLocationsId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationChangeState", "CurrentState", b2 => + { + b2.Property("ChangeSetLocationsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Level") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("ChangeSetLocationsId", "ChangeId"); + + b2.ToTable("ChangeSetLocations"); + + b2.WithOwner() + .HasForeignKey("ChangeSetLocationsId", "ChangeId"); + }); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationChangeState", "PreviousState", b2 => + { + b2.Property("ChangeSetLocationsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Level") + .HasColumnType("integer"); + + b2.HasKey("ChangeSetLocationsId", "ChangeId"); + + b2.ToTable("ChangeSetLocations"); + + b2.WithOwner() + .HasForeignKey("ChangeSetLocationsId", "ChangeId"); + }); + + b1.Navigation("CurrentState"); + + b1.Navigation("PreviousState"); + }); + + b.Navigation("Changes"); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetTimePeriods", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("TimePeriodChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Change", "Changes", b1 => + { + b1.Property("ChangeSetTimePeriodsId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Identifier") + .HasColumnType("uuid") + .HasAnnotation("Relational:JsonPropertyName", "Id"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("ChangeSetTimePeriodsId", "Id"); + + b1.ToTable("ChangeSetTimePeriods"); + + b1.ToJson("Changes"); + + b1.WithOwner() + .HasForeignKey("ChangeSetTimePeriodsId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodChangeState", "CurrentState", b2 => + { + b2.Property("ChangeSetTimePeriodsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Year") + .HasColumnType("integer"); + + b2.HasKey("ChangeSetTimePeriodsId", "ChangeId"); + + b2.ToTable("ChangeSetTimePeriods"); + + b2.WithOwner() + .HasForeignKey("ChangeSetTimePeriodsId", "ChangeId"); + }); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodChangeState", "PreviousState", b2 => + { + b2.Property("ChangeSetTimePeriodsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Year") + .HasColumnType("integer"); + + b2.HasKey("ChangeSetTimePeriodsId", "ChangeId"); + + b2.ToTable("ChangeSetTimePeriods"); + + b2.WithOwner() + .HasForeignKey("ChangeSetTimePeriodsId", "ChangeId"); + }); + + b1.Navigation("CurrentState"); + + b1.Navigation("PreviousState"); + }); + + b.Navigation("Changes"); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "LatestDraftVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "LatestDraftVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "LatestLiveVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "LatestLiveVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "SupersedingDataSet") + .WithMany() + .HasForeignKey("SupersedingDataSetId"); + + b.Navigation("LatestDraftVersion"); + + b.Navigation("LatestLiveVersion"); + + b.Navigation("SupersedingDataSet"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "DataSet") + .WithMany("Versions") + .HasForeignKey("DataSetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionMetaSummary", "MetaSummary", b1 => + { + b1.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b1.Property>("Filters") + .IsRequired() + .HasColumnType("text[]"); + + b1.Property>("GeographicLevels") + .IsRequired() + .HasColumnType("text[]"); + + b1.Property>("Indicators") + .IsRequired() + .HasColumnType("text[]"); + + b1.HasKey("DataSetVersionId"); + + b1.ToTable("DataSetVersions"); + + b1.ToJson("MetaSummary"); + + b1.WithOwner() + .HasForeignKey("DataSetVersionId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodRange", "TimePeriodRange", b2 => + { + b2.Property("DataSetVersionMetaSummaryDataSetVersionId") + .HasColumnType("uuid"); + + b2.HasKey("DataSetVersionMetaSummaryDataSetVersionId"); + + b2.ToTable("DataSetVersions"); + + b2.WithOwner() + .HasForeignKey("DataSetVersionMetaSummaryDataSetVersionId"); + + b2.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodRangeBound", "End", b3 => + { + b3.Property("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId") + .HasColumnType("uuid"); + + b3.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b3.Property("Period") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + + b3.ToTable("DataSetVersions"); + + b3.WithOwner() + .HasForeignKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + }); + + b2.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodRangeBound", "Start", b3 => + { + b3.Property("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId") + .HasColumnType("uuid"); + + b3.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b3.Property("Period") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + + b3.ToTable("DataSetVersions"); + + b3.WithOwner() + .HasForeignKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + }); + + b2.Navigation("End") + .IsRequired(); + + b2.Navigation("Start") + .IsRequired(); + }); + + b1.Navigation("TimePeriodRange") + .IsRequired(); + }); + + b.Navigation("DataSet"); + + b.Navigation("MetaSummary"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionImport", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("Imports") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionMapping", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "SourceDataSetVersion") + .WithMany() + .HasForeignKey("SourceDataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "TargetDataSetVersion") + .WithMany() + .HasForeignKey("TargetDataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Filters", "Filters", b1 => + { + b1.Property("DataSetVersionMappingId") + .HasColumnType("uuid"); + + b1.HasKey("DataSetVersionMappingId"); + + b1.ToTable("DataSetVersionMappings"); + + b1.ToJson("Filters"); + + b1.WithOwner() + .HasForeignKey("DataSetVersionMappingId"); + + b1.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMapping", "Mappings", b2 => + { + b2.Property("FiltersDataSetVersionMappingId") + .HasColumnType("uuid"); + + b2.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b2.Property("TargetId") + .HasColumnType("integer"); + + b2.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("FiltersDataSetVersionMappingId", "Id"); + + b2.ToTable("DataSetVersionMappings"); + + b2.WithOwner() + .HasForeignKey("FiltersDataSetVersionMappingId"); + + b2.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Filter", "Source", b3 => + { + b3.Property("FilterMappingFiltersDataSetVersionMappingId") + .HasColumnType("uuid"); + + b3.Property("FilterMappingId") + .HasColumnType("integer"); + + b3.Property("Id") + .HasColumnType("integer"); + + b3.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("FilterMappingFiltersDataSetVersionMappingId", "FilterMappingId"); + + b3.ToTable("DataSetVersionMappings"); + + b3.WithOwner() + .HasForeignKey("FilterMappingFiltersDataSetVersionMappingId", "FilterMappingId"); + }); + + b2.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMapping", "Options", b3 => + { + b3.Property("FilterMappingFiltersDataSetVersionMappingId") + .HasColumnType("uuid"); + + b3.Property("FilterMappingId") + .HasColumnType("integer"); + + b3.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b3.Property("TargetId") + .HasColumnType("integer"); + + b3.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("FilterMappingFiltersDataSetVersionMappingId", "FilterMappingId", "Id"); + + b3.ToTable("DataSetVersionMappings"); + + b3.WithOwner() + .HasForeignKey("FilterMappingFiltersDataSetVersionMappingId", "FilterMappingId"); + + b3.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOption", "Source", b4 => + { + b4.Property("FilterOptionMappingFilterMappingFiltersDataSetVersionMappingId") + .HasColumnType("uuid"); + + b4.Property("FilterOptionMappingFilterMappingId") + .HasColumnType("integer"); + + b4.Property("FilterOptionMappingId") + .HasColumnType("integer"); + + b4.Property("Id") + .HasColumnType("integer"); + + b4.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b4.HasKey("FilterOptionMappingFilterMappingFiltersDataSetVersionMappingId", "FilterOptionMappingFilterMappingId", "FilterOptionMappingId"); + + b4.ToTable("DataSetVersionMappings"); + + b4.WithOwner() + .HasForeignKey("FilterOptionMappingFilterMappingFiltersDataSetVersionMappingId", "FilterOptionMappingFilterMappingId", "FilterOptionMappingId"); + }); + + b3.Navigation("Source") + .IsRequired(); + }); + + b2.Navigation("Options"); + + b2.Navigation("Source") + .IsRequired(); + }); + + b1.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterTarget", "Targets", b2 => + { + b2.Property("FiltersDataSetVersionMappingId") + .HasColumnType("uuid"); + + b2.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("FiltersDataSetVersionMappingId", "Id"); + + b2.ToTable("DataSetVersionMappings"); + + // b2.HasDiscriminator().HasValue("FilterTarget"); + + b2.WithOwner() + .HasForeignKey("FiltersDataSetVersionMappingId"); + + b2.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOption", "Options", b3 => + { + b3.Property("FilterTargetFiltersDataSetVersionMappingId") + .HasColumnType("uuid"); + + b3.Property("FilterTargetId") + .HasColumnType("integer"); + + b3.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b3.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("FilterTargetFiltersDataSetVersionMappingId", "FilterTargetId", "Id"); + + b3.ToTable("DataSetVersionMappings"); + + b3.WithOwner() + .HasForeignKey("FilterTargetFiltersDataSetVersionMappingId", "FilterTargetId"); + }); + + b2.Navigation("Options"); + }); + + b1.Navigation("Mappings"); + + b1.Navigation("Targets"); + }); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Locations", "Locations", b1 => + { + b1.Property("DataSetVersionMappingId") + .HasColumnType("uuid"); + + b1.HasKey("DataSetVersionMappingId"); + + b1.ToTable("DataSetVersionMappings"); + + b1.ToJson("Locations"); + + b1.WithOwner() + .HasForeignKey("DataSetVersionMappingId"); + + b1.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMappings", "Mappings", b2 => + { + b2.Property("LocationsDataSetVersionMappingId") + .HasColumnType("uuid"); + + b2.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b2.Property("Level") + .HasColumnType("integer"); + + b2.HasKey("LocationsDataSetVersionMappingId", "Id"); + + b2.ToTable("DataSetVersionMappings"); + + b2.WithOwner() + .HasForeignKey("LocationsDataSetVersionMappingId"); + + b2.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMapping", "Mappings", b3 => + { + b3.Property("LocationMappingsLocationsDataSetVersionMappingId") + .HasColumnType("uuid"); + + b3.Property("LocationMappingsId") + .HasColumnType("integer"); + + b3.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b3.Property("TargetId") + .HasColumnType("integer"); + + b3.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("LocationMappingsLocationsDataSetVersionMappingId", "LocationMappingsId", "Id"); + + b3.ToTable("DataSetVersionMappings"); + + b3.WithOwner() + .HasForeignKey("LocationMappingsLocationsDataSetVersionMappingId", "LocationMappingsId"); + + b3.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOption", "Source", b4 => + { + b4.Property("LocationMappingsLocationsDataSetVersionMappingId") + .HasColumnType("uuid"); + + b4.Property("LocationMappingsId") + .HasColumnType("integer"); + + b4.Property("LocationMappingId") + .HasColumnType("integer"); + + b4.Property("Id") + .HasColumnType("integer"); + + b4.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b4.HasKey("LocationMappingsLocationsDataSetVersionMappingId", "LocationMappingsId", "LocationMappingId"); + + b4.ToTable("DataSetVersionMappings"); + + b4.WithOwner() + .HasForeignKey("LocationMappingsLocationsDataSetVersionMappingId", "LocationMappingsId", "LocationMappingId"); + }); + + b3.Navigation("Source") + .IsRequired(); + }); + + b2.Navigation("Mappings"); + }); + + b1.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationTargets", "Targets", b2 => + { + b2.Property("LocationsDataSetVersionMappingId") + .HasColumnType("uuid"); + + b2.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b2.Property("Level") + .HasColumnType("integer"); + + b2.HasKey("LocationsDataSetVersionMappingId", "Id"); + + b2.ToTable("DataSetVersionMappings"); + + b2.WithOwner() + .HasForeignKey("LocationsDataSetVersionMappingId"); + + b2.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOption", "Options", b3 => + { + b3.Property("LocationTargetsLocationsDataSetVersionMappingId") + .HasColumnType("uuid"); + + b3.Property("LocationTargetsId") + .HasColumnType("integer"); + + b3.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b3.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("LocationTargetsLocationsDataSetVersionMappingId", "LocationTargetsId", "Id"); + + b3.ToTable("DataSetVersionMappings"); + + b3.WithOwner() + .HasForeignKey("LocationTargetsLocationsDataSetVersionMappingId", "LocationTargetsId"); + }); + + b2.Navigation("Options"); + }); + + b1.Navigation("Mappings"); + + b1.Navigation("Targets"); + }); + + b.Navigation("Filters") + .IsRequired(); + + b.Navigation("Locations") + .IsRequired(); + + b.Navigation("SourceDataSetVersion"); + + b.Navigation("TargetDataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("FilterMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMetaLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", "Meta") + .WithMany("OptionLinks") + .HasForeignKey("MetaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", "Option") + .WithMany("MetaLinks") + .HasForeignKey("OptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Meta"); + + b.Navigation("Option"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithOne("GeographicLevelMeta") + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", "DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("IndicatorMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("LocationMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMetaLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", "Meta") + .WithMany("OptionLinks") + .HasForeignKey("MetaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", "Option") + .WithMany("MetaLinks") + .HasForeignKey("OptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Meta"); + + b.Navigation("Option"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("TimePeriodMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", b => + { + b.Navigation("FilterChanges"); + + b.Navigation("FilterMetas"); + + b.Navigation("FilterOptionChanges"); + + b.Navigation("GeographicLevelMeta"); + + b.Navigation("Imports"); + + b.Navigation("IndicatorChanges"); + + b.Navigation("IndicatorMetas"); + + b.Navigation("LocationChanges"); + + b.Navigation("LocationMetas"); + + b.Navigation("TimePeriodChanges"); + + b.Navigation("TimePeriodMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => + { + b.Navigation("OptionLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", b => + { + b.Navigation("MetaLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", b => + { + b.Navigation("OptionLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", b => + { + b.Navigation("MetaLinks"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240618080650_EES4945_AddDataSetVersionMappingsTable.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240618080650_EES4945_AddDataSetVersionMappingsTable.cs new file mode 100644 index 00000000000..8cc71db1a2d --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240618080650_EES4945_AddDataSetVersionMappingsTable.cs @@ -0,0 +1,65 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Migrations +{ + /// + public partial class EES4945_AddDataSetVersionMappingsTable : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "DataSetVersionMappings", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + SourceDataSetVersionId = table.Column(type: "uuid", nullable: false), + TargetDataSetVersionId = table.Column(type: "uuid", nullable: false), + Created = table.Column(type: "timestamp with time zone", nullable: false), + Updated = table.Column(type: "timestamp with time zone", nullable: true), + FilterMappingPlan = table.Column(type: "jsonb", nullable: false), + LocationMappingPlan = table.Column(type: "jsonb", nullable: false), + FilterMappingsComplete = table.Column(type: "boolean", nullable: false, defaultValue: false), + LocationMappingsComplete = table.Column(type: "boolean", nullable: false, defaultValue: false) + }, + constraints: table => + { + table.PrimaryKey("PK_DataSetVersionMappings", x => x.Id); + table.ForeignKey( + name: "FK_DataSetVersionMappings_DataSetVersions_SourceDataSetVersion~", + column: x => x.SourceDataSetVersionId, + principalTable: "DataSetVersions", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_DataSetVersionMappings_DataSetVersions_TargetDataSetVersion~", + column: x => x.TargetDataSetVersionId, + principalTable: "DataSetVersions", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_DataSetVersionMappings_SourceDataSetVersionId", + table: "DataSetVersionMappings", + column: "SourceDataSetVersionId", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_DataSetVersionMappings_TargetDataSetVersionId", + table: "DataSetVersionMappings", + column: "TargetDataSetVersionId", + unique: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "DataSetVersionMappings"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240619085713_EES5235_AddFilterOptionMetaLinkSequence.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240619085713_EES5235_AddFilterOptionMetaLinkSequence.Designer.cs new file mode 100644 index 00000000000..e276ee5782a --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240619085713_EES5235_AddFilterOptionMetaLinkSequence.Designer.cs @@ -0,0 +1,1403 @@ +// +using System; +using System.Collections.Generic; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Migrations; + +[DbContext(typeof(PublicDataDbContext))] +[Migration("20240619085713_EES5235_AddFilterOptionMetaLinkSequence")] +partial class EES5235_AddFilterOptionMetaLinkSequence +{ + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.4") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.HasSequence("FilterOptionMetaLink_seq"); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetFilterOptions", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("ChangeSetFilterOptions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetFilters", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("ChangeSetFilters"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetIndicators", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("ChangeSetIndicators"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetLocations", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("ChangeSetLocations"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetTimePeriods", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("ChangeSetTimePeriods"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("LatestDraftVersionId") + .HasColumnType("uuid"); + + b.Property("LatestLiveVersionId") + .HasColumnType("uuid"); + + b.Property("PublicationId") + .HasColumnType("uuid"); + + b.Property("Published") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text"); + + b.Property("Summary") + .IsRequired() + .HasColumnType("text"); + + b.Property("SupersedingDataSetId") + .HasColumnType("uuid"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.Property("Withdrawn") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("LatestDraftVersionId") + .IsUnique(); + + b.HasIndex("LatestLiveVersionId") + .IsUnique(); + + b.HasIndex("SupersedingDataSetId"); + + b.ToTable("DataSets"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetId") + .HasColumnType("uuid"); + + b.Property("Notes") + .IsRequired() + .HasColumnType("text"); + + b.Property("Published") + .HasColumnType("timestamp with time zone"); + + b.Property("ReleaseFileId") + .HasColumnType("uuid"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text"); + + b.Property("TotalResults") + .HasColumnType("bigint"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.Property("VersionMajor") + .HasColumnType("integer"); + + b.Property("VersionMinor") + .HasColumnType("integer"); + + b.Property("VersionPatch") + .HasColumnType("integer"); + + b.Property("Withdrawn") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseFileId"); + + b.HasIndex("DataSetId", "VersionMajor", "VersionMinor", "VersionPatch") + .IsUnique() + .HasDatabaseName("IX_DataSetVersions_DataSetId_VersionNumber"); + + b.ToTable("DataSetVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionImport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Completed") + .HasColumnType("timestamp with time zone"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("InstanceId") + .HasColumnType("uuid"); + + b.Property("Stage") + .IsRequired() + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DataSetVersionImports"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Hint") + .IsRequired() + .HasColumnType("text"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b.Property("PublicId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "PublicId") + .IsUnique(); + + b.ToTable("FilterMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("IsAggregate") + .HasColumnType("boolean"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("FilterOptionMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMetaLink", b => + { + b.Property("MetaId") + .HasColumnType("integer"); + + b.Property("OptionId") + .HasColumnType("integer"); + + b.Property("PublicId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("MetaId", "OptionId"); + + b.HasIndex("OptionId"); + + b.HasIndex("PublicId"); + + b.HasIndex("MetaId", "PublicId") + .IsUnique(); + + b.ToTable("FilterOptionMetaLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property>("Levels") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId") + .IsUnique(); + + b.ToTable("GeographicLevelMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("DecimalPlaces") + .HasColumnType("smallint"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b.Property("PublicId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Unit") + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "PublicId") + .IsUnique(); + + b.ToTable("IndicatorMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Level") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("character varying(5)"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "Level") + .IsUnique(); + + b.ToTable("LocationMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Code") + .HasColumnType("text"); + + b.Property("LaEstab") + .HasColumnType("text"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b.Property("OldCode") + .HasColumnType("text"); + + b.Property("PublicId") + .IsRequired() + .HasColumnType("text"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Ukprn") + .HasColumnType("text"); + + b.Property("Urn") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Code"); + + b.HasIndex("LaEstab"); + + b.HasIndex("OldCode"); + + b.HasIndex("PublicId") + .IsUnique(); + + b.HasIndex("Type"); + + b.HasIndex("Ukprn"); + + b.HasIndex("Urn"); + + b.HasIndex(new[] { "Type", "Label", "Code", "OldCode", "Urn", "LaEstab", "Ukprn" }, "IX_LocationOptionMetas_All") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.AreNullsDistinct(b.HasIndex(new[] { "Type", "Label", "Code", "OldCode", "Urn", "LaEstab", "Ukprn" }, "IX_LocationOptionMetas_All"), false); + + b.ToTable("LocationOptionMetas"); + + b.HasDiscriminator("Type").HasValue("LocationOptionMeta"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMetaLink", b => + { + b.Property("MetaId") + .HasColumnType("integer"); + + b.Property("OptionId") + .HasColumnType("integer"); + + b.HasKey("MetaId", "OptionId"); + + b.HasIndex("OptionId"); + + b.ToTable("LocationOptionMetaLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Code") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Period") + .IsRequired() + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "Code", "Period") + .IsUnique(); + + b.ToTable("TimePeriodMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationCodedOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("CODE"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationLocalAuthorityOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("LA"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationProviderOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("PROV"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationRscRegionOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("RSC"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationSchoolOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("SCH"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetFilterOptions", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("FilterOptionChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Change", "Changes", b1 => + { + b1.Property("ChangeSetFilterOptionsId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Identifier") + .HasColumnType("uuid") + .HasAnnotation("Relational:JsonPropertyName", "Id"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("ChangeSetFilterOptionsId", "Id"); + + b1.ToTable("ChangeSetFilterOptions"); + + b1.ToJson("Changes"); + + b1.WithOwner() + .HasForeignKey("ChangeSetFilterOptionsId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionChangeState", "CurrentState", b2 => + { + b2.Property("ChangeSetFilterOptionsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("FilterId") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("IsAggregate") + .HasColumnType("boolean"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("ChangeSetFilterOptionsId", "ChangeId"); + + b2.ToTable("ChangeSetFilterOptions"); + + b2.WithOwner() + .HasForeignKey("ChangeSetFilterOptionsId", "ChangeId"); + }); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionChangeState", "PreviousState", b2 => + { + b2.Property("ChangeSetFilterOptionsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("FilterId") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("IsAggregate") + .HasColumnType("boolean"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("ChangeSetFilterOptionsId", "ChangeId"); + + b2.ToTable("ChangeSetFilterOptions"); + + b2.WithOwner() + .HasForeignKey("ChangeSetFilterOptionsId", "ChangeId"); + }); + + b1.Navigation("CurrentState"); + + b1.Navigation("PreviousState"); + }); + + b.Navigation("Changes"); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetFilters", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("FilterChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Change", "Changes", b1 => + { + b1.Property("ChangeSetFiltersId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Identifier") + .HasColumnType("uuid") + .HasAnnotation("Relational:JsonPropertyName", "Id"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("ChangeSetFiltersId", "Id"); + + b1.ToTable("ChangeSetFilters"); + + b1.ToJson("Changes"); + + b1.WithOwner() + .HasForeignKey("ChangeSetFiltersId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterChangeState", "CurrentState", b2 => + { + b2.Property("ChangeSetFiltersId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Hint") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("ChangeSetFiltersId", "ChangeId"); + + b2.ToTable("ChangeSetFilters"); + + b2.WithOwner() + .HasForeignKey("ChangeSetFiltersId", "ChangeId"); + }); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterChangeState", "PreviousState", b2 => + { + b2.Property("ChangeSetFiltersId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Hint") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("ChangeSetFiltersId", "ChangeId"); + + b2.ToTable("ChangeSetFilters"); + + b2.WithOwner() + .HasForeignKey("ChangeSetFiltersId", "ChangeId"); + }); + + b1.Navigation("CurrentState"); + + b1.Navigation("PreviousState"); + }); + + b.Navigation("Changes"); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetIndicators", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("IndicatorChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Change", "Changes", b1 => + { + b1.Property("ChangeSetIndicatorsId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Identifier") + .HasColumnType("uuid") + .HasAnnotation("Relational:JsonPropertyName", "Id"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("ChangeSetIndicatorsId", "Id"); + + b1.ToTable("ChangeSetIndicators"); + + b1.ToJson("Changes"); + + b1.WithOwner() + .HasForeignKey("ChangeSetIndicatorsId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorChangeState", "CurrentState", b2 => + { + b2.Property("ChangeSetIndicatorsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("DecimalPlaces") + .HasColumnType("smallint"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Unit") + .HasColumnType("text"); + + b2.HasKey("ChangeSetIndicatorsId", "ChangeId"); + + b2.ToTable("ChangeSetIndicators"); + + b2.WithOwner() + .HasForeignKey("ChangeSetIndicatorsId", "ChangeId"); + }); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorChangeState", "PreviousState", b2 => + { + b2.Property("ChangeSetIndicatorsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("DecimalPlaces") + .HasColumnType("smallint"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Unit") + .HasColumnType("text"); + + b2.HasKey("ChangeSetIndicatorsId", "ChangeId"); + + b2.ToTable("ChangeSetIndicators"); + + b2.WithOwner() + .HasForeignKey("ChangeSetIndicatorsId", "ChangeId"); + }); + + b1.Navigation("CurrentState"); + + b1.Navigation("PreviousState"); + }); + + b.Navigation("Changes"); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetLocations", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("LocationChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Change", "Changes", b1 => + { + b1.Property("ChangeSetLocationsId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Identifier") + .HasColumnType("uuid") + .HasAnnotation("Relational:JsonPropertyName", "Id"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("ChangeSetLocationsId", "Id"); + + b1.ToTable("ChangeSetLocations"); + + b1.ToJson("Changes"); + + b1.WithOwner() + .HasForeignKey("ChangeSetLocationsId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationChangeState", "CurrentState", b2 => + { + b2.Property("ChangeSetLocationsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Level") + .IsRequired() + .HasColumnType("text"); + + b2.HasKey("ChangeSetLocationsId", "ChangeId"); + + b2.ToTable("ChangeSetLocations"); + + b2.WithOwner() + .HasForeignKey("ChangeSetLocationsId", "ChangeId"); + }); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationChangeState", "PreviousState", b2 => + { + b2.Property("ChangeSetLocationsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Id") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Label") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Level") + .HasColumnType("integer"); + + b2.HasKey("ChangeSetLocationsId", "ChangeId"); + + b2.ToTable("ChangeSetLocations"); + + b2.WithOwner() + .HasForeignKey("ChangeSetLocationsId", "ChangeId"); + }); + + b1.Navigation("CurrentState"); + + b1.Navigation("PreviousState"); + }); + + b.Navigation("Changes"); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetTimePeriods", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("TimePeriodChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsMany("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Change", "Changes", b1 => + { + b1.Property("ChangeSetTimePeriodsId") + .HasColumnType("uuid"); + + b1.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + b1.Property("Identifier") + .HasColumnType("uuid") + .HasAnnotation("Relational:JsonPropertyName", "Id"); + + b1.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("ChangeSetTimePeriodsId", "Id"); + + b1.ToTable("ChangeSetTimePeriods"); + + b1.ToJson("Changes"); + + b1.WithOwner() + .HasForeignKey("ChangeSetTimePeriodsId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodChangeState", "CurrentState", b2 => + { + b2.Property("ChangeSetTimePeriodsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Year") + .HasColumnType("integer"); + + b2.HasKey("ChangeSetTimePeriodsId", "ChangeId"); + + b2.ToTable("ChangeSetTimePeriods"); + + b2.WithOwner() + .HasForeignKey("ChangeSetTimePeriodsId", "ChangeId"); + }); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodChangeState", "PreviousState", b2 => + { + b2.Property("ChangeSetTimePeriodsId") + .HasColumnType("uuid"); + + b2.Property("ChangeId") + .HasColumnType("integer"); + + b2.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b2.Property("Year") + .HasColumnType("integer"); + + b2.HasKey("ChangeSetTimePeriodsId", "ChangeId"); + + b2.ToTable("ChangeSetTimePeriods"); + + b2.WithOwner() + .HasForeignKey("ChangeSetTimePeriodsId", "ChangeId"); + }); + + b1.Navigation("CurrentState"); + + b1.Navigation("PreviousState"); + }); + + b.Navigation("Changes"); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "LatestDraftVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "LatestDraftVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "LatestLiveVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "LatestLiveVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "SupersedingDataSet") + .WithMany() + .HasForeignKey("SupersedingDataSetId"); + + b.Navigation("LatestDraftVersion"); + + b.Navigation("LatestLiveVersion"); + + b.Navigation("SupersedingDataSet"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "DataSet") + .WithMany("Versions") + .HasForeignKey("DataSetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionMetaSummary", "MetaSummary", b1 => + { + b1.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b1.Property>("Filters") + .IsRequired() + .HasColumnType("text[]"); + + b1.Property>("GeographicLevels") + .IsRequired() + .HasColumnType("text[]"); + + b1.Property>("Indicators") + .IsRequired() + .HasColumnType("text[]"); + + b1.HasKey("DataSetVersionId"); + + b1.ToTable("DataSetVersions"); + + b1.ToJson("MetaSummary"); + + b1.WithOwner() + .HasForeignKey("DataSetVersionId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodRange", "TimePeriodRange", b2 => + { + b2.Property("DataSetVersionMetaSummaryDataSetVersionId") + .HasColumnType("uuid"); + + b2.HasKey("DataSetVersionMetaSummaryDataSetVersionId"); + + b2.ToTable("DataSetVersions"); + + b2.WithOwner() + .HasForeignKey("DataSetVersionMetaSummaryDataSetVersionId"); + + b2.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodRangeBound", "End", b3 => + { + b3.Property("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId") + .HasColumnType("uuid"); + + b3.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b3.Property("Period") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + + b3.ToTable("DataSetVersions"); + + b3.WithOwner() + .HasForeignKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + }); + + b2.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodRangeBound", "Start", b3 => + { + b3.Property("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId") + .HasColumnType("uuid"); + + b3.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b3.Property("Period") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + + b3.ToTable("DataSetVersions"); + + b3.WithOwner() + .HasForeignKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + }); + + b2.Navigation("End") + .IsRequired(); + + b2.Navigation("Start") + .IsRequired(); + }); + + b1.Navigation("TimePeriodRange") + .IsRequired(); + }); + + b.Navigation("DataSet"); + + b.Navigation("MetaSummary"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionImport", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("Imports") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("FilterMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMetaLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", "Meta") + .WithMany("OptionLinks") + .HasForeignKey("MetaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", "Option") + .WithMany("MetaLinks") + .HasForeignKey("OptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Meta"); + + b.Navigation("Option"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithOne("GeographicLevelMeta") + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", "DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("IndicatorMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("LocationMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMetaLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", "Meta") + .WithMany("OptionLinks") + .HasForeignKey("MetaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", "Option") + .WithMany("MetaLinks") + .HasForeignKey("OptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Meta"); + + b.Navigation("Option"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("TimePeriodMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", b => + { + b.Navigation("FilterChanges"); + + b.Navigation("FilterMetas"); + + b.Navigation("FilterOptionChanges"); + + b.Navigation("GeographicLevelMeta"); + + b.Navigation("Imports"); + + b.Navigation("IndicatorChanges"); + + b.Navigation("IndicatorMetas"); + + b.Navigation("LocationChanges"); + + b.Navigation("LocationMetas"); + + b.Navigation("TimePeriodChanges"); + + b.Navigation("TimePeriodMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => + { + b.Navigation("OptionLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", b => + { + b.Navigation("MetaLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", b => + { + b.Navigation("OptionLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", b => + { + b.Navigation("MetaLinks"); + }); +#pragma warning restore 612, 618 + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240619085713_EES5235_AddFilterOptionMetaLinkSequence.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240619085713_EES5235_AddFilterOptionMetaLinkSequence.cs new file mode 100644 index 00000000000..2191c485acc --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20240619085713_EES5235_AddFilterOptionMetaLinkSequence.cs @@ -0,0 +1,25 @@ +using System.Diagnostics.CodeAnalysis; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Migrations; + +/// +[ExcludeFromCodeCoverage] +public partial class EES5235_AddFilterOptionMetaLinkSequence : Migration +{ + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateSequence( + name: "FilterOptionMetaLink_seq"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropSequence( + name: "FilterOptionMetaLink_seq"); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/PublicDataDbContextModelSnapshot.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/PublicDataDbContextModelSnapshot.cs index 8853a44b33a..7bb26f5452d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/PublicDataDbContextModelSnapshot.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/PublicDataDbContextModelSnapshot.cs @@ -18,11 +18,13 @@ protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "8.0.3") + .HasAnnotation("ProductVersion", "8.0.4") .HasAnnotation("Relational:MaxIdentifierLength", 63); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + modelBuilder.HasSequence("FilterOptionMetaLink_seq"); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetFilterOptions", b => { b.Property("Id") @@ -278,6 +280,51 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("DataSetVersionImports"); }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("FilterMappingPlan") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("FilterMappingsComplete") + .HasColumnType("boolean"); + + b.Property("LocationMappingPlan") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("LocationMappingsComplete") + .HasColumnType("boolean"); + + b.Property("SourceDataSetVersionId") + .HasColumnType("uuid"); + + b.Property("TargetDataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("SourceDataSetVersionId") + .IsUnique() + .HasDatabaseName("IX_DataSetVersionMappings_SourceDataSetVersionId"); + + b.HasIndex("TargetDataSetVersionId") + .IsUnique() + .HasDatabaseName("IX_DataSetVersionMappings_TargetDataSetVersionId"); + + b.ToTable("DataSetVersionMappings"); + }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => { b.Property("Id") @@ -1251,6 +1298,25 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("DataSetVersion"); }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionMapping", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "SourceDataSetVersion") + .WithMany() + .HasForeignKey("SourceDataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "TargetDataSetVersion") + .WithMany() + .HasForeignKey("TargetDataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("SourceDataSetVersion"); + + b.Navigation("TargetDataSetVersion"); + }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => { b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/TimePeriodRange.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/TimePeriodRange.cs index 1e58e95fc0a..a72ce5578c4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/TimePeriodRange.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/TimePeriodRange.cs @@ -2,14 +2,14 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model; -public class TimePeriodRange +public record TimePeriodRange { public required TimePeriodRangeBound Start { get; set; } public required TimePeriodRangeBound End { get; set; } } -public class TimePeriodRangeBound +public record TimePeriodRangeBound { public required TimeIdentifier Code { get; set; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests/NextDataSetVersionCreateRequest.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests/NextDataSetVersionCreateRequest.cs new file mode 100644 index 00000000000..d37e08bf36c --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests/NextDataSetVersionCreateRequest.cs @@ -0,0 +1,22 @@ +using FluentValidation; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests; + +public record NextDataSetVersionCreateRequest +{ + public required Guid DataSetId { get; init; } + + public required Guid ReleaseFileId { get; init; } + + public class Validator : AbstractValidator + { + public Validator() + { + RuleFor(request => request.DataSetId) + .NotEmpty(); + + RuleFor(request => request.ReleaseFileId) + .NotEmpty(); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests/Validators/ValidationMessages.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests/Validators/ValidationMessages.cs index a9b094d627d..020721184b8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests/Validators/ValidationMessages.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests/Validators/ValidationMessages.cs @@ -1,37 +1,66 @@ using GovUk.Education.ExploreEducationStatistics.Common.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests.Validators; public static class ValidationMessages { public static readonly LocalizableMessage FileNotFound = new( - Code: "FileNotFound", + Code: nameof(FileNotFound), Message: "The file could not be found." ); public static readonly LocalizableMessage FileHasApiDataSetVersion = new( - Code: "FileHasApiDataSetVersion", + Code: nameof(FileHasApiDataSetVersion), Message: "The file has already been used for an API data set version." ); public static readonly LocalizableMessage FileReleaseVersionNotDraft = new( - Code: "FileReleaseVersionNotDraft", + Code: nameof(FileReleaseVersionNotDraft), Message: "The file must belong to a release in 'Draft' approval status." ); public static readonly LocalizableMessage FileTypeNotData = new( - Code: "FileTypeNotData", + Code: nameof(FileTypeNotData), Message: "The file type must be 'Data'." ); public static readonly LocalizableMessage NoMetadataFile = new( - Code: "NoMetadataFile", + Code: nameof(NoMetadataFile), Message: "The data file must have a corresponding metadata file." ); + public static readonly LocalizableMessage FileNotInDataSetPublication = new( + Code: nameof(FileNotInDataSetPublication), + Message: "The file must belong to the same publication as the data set." + ); + + public static readonly LocalizableMessage FileMustBeInDifferentRelease = new( + Code: nameof(FileMustBeInDifferentRelease), + Message: "The file must be in a different release to previous data set versions." + ); + public static readonly LocalizableMessage DataSetVersionCanNotBeDeleted = new( - Code: "DataSetVersionCanNotBeDeleted", - Message: $"The data set version is not in a '{DataSetVersionStatus.Draft}' status, so cannot be deleted." + Code: nameof(DataSetVersionCanNotBeDeleted), + Message: $"The data set version is not in a draft status, or is currently being processed, so cannot be deleted." + ); + + public static readonly LocalizableMessage DataSetNotFound = new( + Code: nameof(DataSetNotFound), + Message: "The data set could not be found." ); + + public static readonly LocalizableMessage DataSetMustHaveNoExistingVersions = new( + Code: nameof(DataSetMustHaveNoExistingVersions), + Message: "The data set must have no existing versions when creating the initial version." + ); + + public static readonly LocalizableMessage DataSetNoLiveVersion = new( + Code: nameof(DataSetNoLiveVersion), + Message: "The data set must have a live version." + ); + + public static readonly LocalizableMessage MultipleDataSetVersionsCanNotBeDeleted = new( + Code: nameof(MultipleDataSetVersionsCanNotBeDeleted), + Message: $"One or more data set versions are not in a draft status, or are currently being processed, so cannot be deleted." +); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Extensions/LocationOptionMetaTestExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Extensions/LocationOptionMetaTestExtensions.cs new file mode 100644 index 00000000000..0f7677d9d0a --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Extensions/LocationOptionMetaTestExtensions.cs @@ -0,0 +1,56 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; + +public static class LocationOptionMetaTestExtensions +{ + public static void AssertEqual(this LocationOptionMeta expectedOption, ParquetLocationOption actualOption) + { + Assert.Equal(expectedOption.Label, actualOption.Label); + Assert.Equal(expectedOption.PublicId, actualOption.PublicId); + + switch (expectedOption) + { + case LocationCodedOptionMeta codedOption: + Assert.Equal(codedOption.Code, actualOption.Code); + Assert.Null(actualOption.OldCode); + Assert.Null(actualOption.Ukprn); + Assert.Null(actualOption.Urn); + Assert.Null(actualOption.LaEstab); + break; + case LocationLocalAuthorityOptionMeta laOption: + Assert.Equal(laOption.Code, actualOption.Code); + Assert.Equal(laOption.OldCode, actualOption.OldCode); + Assert.Null(actualOption.Ukprn); + Assert.Null(actualOption.Urn); + Assert.Null(actualOption.LaEstab); + break; + case LocationProviderOptionMeta providerOption: + Assert.Null(actualOption.Code); + Assert.Null(actualOption.OldCode); + Assert.Equal(providerOption.Ukprn, actualOption.Ukprn); + Assert.Null(actualOption.Urn); + Assert.Null(actualOption.LaEstab); + break; + case LocationRscRegionOptionMeta: + Assert.Null(actualOption.Code); + Assert.Null(actualOption.OldCode); + Assert.Null(actualOption.Ukprn); + Assert.Null(actualOption.Urn); + Assert.Null(actualOption.LaEstab); + break; + case LocationSchoolOptionMeta schoolOption: + Assert.Null(actualOption.Code); + Assert.Null(actualOption.OldCode); + Assert.Null(actualOption.Ukprn); + Assert.Equal(schoolOption.Urn, actualOption.Urn); + Assert.Equal(schoolOption.LaEstab, actualOption.LaEstab); + break; + default: + throw new ArgumentOutOfRangeException( + paramName: nameof(expectedOption), + $"Unsupported {expectedOption.GetType().Name} type"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/BulkDeleteDataSetVersionsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/BulkDeleteDataSetVersionsFunctionTests.cs new file mode 100644 index 00000000000..f36a5fc6433 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/BulkDeleteDataSetVersionsFunctionTests.cs @@ -0,0 +1,588 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Content.Model; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests.Validators; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using LinqToDB; +using Microsoft.AspNetCore.Mvc; +using System.Text.Json; +using File = GovUk.Education.ExploreEducationStatistics.Content.Model.File; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class BulkDeleteDataSetVersionsFunctionTests(ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessorFunctionsIntegrationTest(fixture) +{ + public class BulkDeleteDataSetVersionsTests : BulkDeleteDataSetVersionsFunctionTests + { + private readonly IDataSetVersionPathResolver _dataSetVersionPathResolver; + + public BulkDeleteDataSetVersionsTests(ProcessorFunctionsIntegrationTestFixture fixture) : base(fixture) + { + _dataSetVersionPathResolver = GetRequiredService(); + } + + [Theory] + [InlineData(DataSetVersionStatus.Failed)] + [InlineData(DataSetVersionStatus.Mapping)] + [InlineData(DataSetVersionStatus.Draft)] + [InlineData(DataSetVersionStatus.Cancelled)] + public async Task ReleaseVersionIsLinkedToOldFileWithApiDataSet(DataSetVersionStatus dataSetVersionStatus) + { + Publication publication = DataFixture.DefaultPublication(); + + Release release = DataFixture.DefaultRelease(); + + ReleaseVersion previousReleaseVersion = DataFixture.DefaultReleaseVersion() + .WithPublication(publication) + .WithRelease(release) + .WithApprovalStatus(ReleaseApprovalStatus.Approved) + .WithPublished(DateTime.UtcNow) + .WithVersion(0); + + ReleaseVersion targetReleaseVersion = DataFixture.DefaultReleaseVersion() + .WithPublication(publication) + .WithRelease(release) + .WithVersion(1); + + File oldFile = DataFixture.DefaultFile(FileType.Data); + + ReleaseFile previousReleaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(previousReleaseVersion) + .WithFile(oldFile); + + ReleaseFile targetReleaseFileWithOldFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(targetReleaseVersion) + .WithFile(oldFile); + + var targetReleaseFilesWithNewFiles = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(targetReleaseVersion) + .WithFile(() => DataFixture.DefaultFile(FileType.Data)) + .GenerateList(3); + + await AddTestData(context => + { + context.ReleaseFiles.AddRange([previousReleaseFile, targetReleaseFileWithOldFile, .. targetReleaseFilesWithNewFiles]); + }); + + DataSet otherDataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished() + .WithPublicationId(publication.Id); + + var targetDataSets = DataFixture + .DefaultDataSet() + .WithStatusDraft() + .WithPublicationId(publication.Id) + .GenerateList(3); + + await AddTestData(context => context.DataSets.AddRange([otherDataSet, .. targetDataSets])); + + DataSetVersion otherDataSetVersion = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) + .WithVersionNumber(1, 0, 0) + .WithStatusPublished() + .WithDataSet(otherDataSet) + .WithImports(() => DataFixture + .DefaultDataSetVersionImport() + .WithStage(DataSetVersionImportStage.Completing) + .Generate(1)) + .WithReleaseFileId(previousReleaseFile.Id) + .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); + + var targetDataSetVersions = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) + .WithVersionNumber(1, 0, 0) + .WithStatus(dataSetVersionStatus) + .WithImports(() => DataFixture + .DefaultDataSetVersionImport() + .Generate(1)) + .ForIndex(0, dsv => dsv + .SetReleaseFileId(targetReleaseFilesWithNewFiles[0].Id) + .SetDataSet(targetDataSets[0])) + .ForIndex(1, dsv => dsv + .SetReleaseFileId(targetReleaseFilesWithNewFiles[1].Id) + .SetDataSet(targetDataSets[1])) + .ForIndex(2, dsv => dsv + .SetReleaseFileId(targetReleaseFilesWithNewFiles[2].Id) + .SetDataSet(targetDataSets[2])) + .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv) + .GenerateList(); + + await AddTestData(context => + { + context.DataSetVersions.AddRange([otherDataSetVersion, .. targetDataSetVersions]); + context.DataSets.UpdateRange([otherDataSet, .. targetDataSets]); + }); + + previousReleaseFile.PublicApiDataSetId = otherDataSet.Id; + previousReleaseFile.PublicApiDataSetVersion = otherDataSetVersion.FullSemanticVersion(); + targetReleaseFileWithOldFile.PublicApiDataSetId = previousReleaseFile.PublicApiDataSetId; + targetReleaseFileWithOldFile.PublicApiDataSetVersion = previousReleaseFile.PublicApiDataSetVersion; + + foreach (var (targetReleaseFileWithNewFile, index) in targetReleaseFilesWithNewFiles.WithIndex()) + { + targetReleaseFileWithNewFile.PublicApiDataSetId = targetDataSets[index].Id; + targetReleaseFileWithNewFile.PublicApiDataSetVersion = targetDataSetVersions[index].FullSemanticVersion(); + } + + await AddTestData(context => context.ReleaseFiles.UpdateRange([ + previousReleaseFile, + targetReleaseFileWithOldFile, + ..targetReleaseFilesWithNewFiles])); + + foreach (var targetDataSetVersion in targetDataSetVersions) + { + var targetDataSetVersionDirectory = _dataSetVersionPathResolver.DirectoryPath(targetDataSetVersion); + Directory.CreateDirectory(targetDataSetVersionDirectory); + await System.IO.File.Create(Path.Combine(targetDataSetVersionDirectory, "version1.txt")).DisposeAsync(); + } + + var otherDataSetVersionDirectory = _dataSetVersionPathResolver.DirectoryPath(otherDataSetVersion); + Directory.CreateDirectory(otherDataSetVersionDirectory); + await System.IO.File.Create(Path.Combine(otherDataSetVersionDirectory, "version1.txt")).DisposeAsync(); + + var response = await BulkDeleteDataSetVersions(targetReleaseVersion.Id); + + response.AssertNoContent(); + + await using var publicDataDbContext = GetDbContext(); + await using var contentDataDbContext = GetDbContext(); + + // Assertions for the TARGET data sets linked to the TARGET release version being deleted + for (var i = 0; i < 3; i++) + { + var targetDataSet = targetDataSets[i]; + var targetDataSetVersion = targetDataSetVersions[i]; + var targetReleaseFileWithNewFile = targetReleaseFilesWithNewFiles[i]; + + // Assert that the TARGET parquet data set folder, linked to the release version being deleted, is now removed + var targetDataSetDirectory = Directory.GetParent(_dataSetVersionPathResolver.DirectoryPath(targetDataSetVersion)); + Assert.False(Directory.Exists(targetDataSetDirectory!.FullName)); + + // Assert that the TARGET Data Set has been deleted + Assert.Null(await publicDataDbContext.DataSets.SingleOrDefaultAsync(ds => ds.Id == targetDataSet.Id)); + + // Assert that the TARGET Data Set Version has been deleted + Assert.Null(await publicDataDbContext.DataSetVersions.SingleOrDefaultAsync(dsv => dsv.Id == targetDataSetVersion.Id)); + + // Assert that the TARGET Data Set Version metadata is deleted + await AssertMetadataIsDeleted(publicDataDbContext, targetDataSetVersion); + + // Assert that the TARGET Data Set Version Import has been deleted + Assert.Null(await publicDataDbContext.DataSetVersionImports.SingleOrDefaultAsync(dsvi => dsvi.Id == targetDataSetVersion.Imports.Single().Id)); + + // Assert that the TARGET Release File has been unassociated with its Public API DataSet + var targetReleaseFile = await contentDataDbContext.ReleaseFiles.SingleAsync(f => f.Id == targetReleaseFileWithNewFile.Id); + + Assert.Null(targetReleaseFile.PublicApiDataSetId); + Assert.Null(targetReleaseFile.PublicApiDataSetVersion); + } + + // Below are the Assertions for the OTHER data set linked to the NON-TARGET release version. NOTHING should be deleted + + // Assert that the OTHER data set folder, and its contents, linked to the NON-TARGET release version remains + // as it was + var otherDataSetFolder = Directory.GetParent(_dataSetVersionPathResolver.DirectoryPath(otherDataSetVersion)); + Assert.True(Directory.Exists(otherDataSetFolder!.FullName)); + + var otherDataSetFolderEntries = Directory.GetFileSystemEntries(otherDataSetFolder!.FullName); + var otherDataSetVersionFolder = Assert.Single(otherDataSetFolderEntries, + entry => new DirectoryInfo(entry).Name == $"v{otherDataSetVersion.Version}"); + + var otherDataSetVersionFolderEntries = Directory.GetFileSystemEntries(otherDataSetVersionFolder); + Assert.Single(otherDataSetVersionFolderEntries, entry => new System.IO.FileInfo(entry).Name == "version1.txt"); + + // Assert that the OTHER Data Set has NOT been deleted + Assert.NotNull(await publicDataDbContext.DataSets.SingleAsync(ds => ds.Id == otherDataSet.Id)); + + // Assert that the OTHER Data Set Version has NOT been deleted + Assert.NotNull(await publicDataDbContext.DataSetVersions.SingleAsync(dsv => dsv.Id == otherDataSetVersion.Id)); + + // Assert that the OTHER Data Set Version metadata is NOT deleted + await AssertMetadataIsNotDeleted(publicDataDbContext, otherDataSetVersion); + + // Assert that the OTHER Data Set Version Import has NOT been deleted + Assert.NotNull(await publicDataDbContext.DataSetVersionImports.SingleAsync(dsvi => dsvi.Id == otherDataSetVersion.Imports.Single().Id)); + + // Assert that the PREVIOUS Release File is still associated with its Public API DataSet + var previousReleaseFilePostDelete = await contentDataDbContext.ReleaseFiles.SingleAsync(f => f.Id == previousReleaseFile.Id); + Assert.Equal(otherDataSet.Id, previousReleaseFilePostDelete.PublicApiDataSetId); + Assert.Equal(otherDataSetVersion.FullSemanticVersion(), previousReleaseFilePostDelete.PublicApiDataSetVersion); + + // Assert that the TARGET Release File, which points to the OLD File, is still associated with its Public API DataSet + var targetReleaseFileWithOldFilePostDelete = await contentDataDbContext.ReleaseFiles.SingleAsync(f => f.Id == targetReleaseFileWithOldFile.Id); + Assert.Equal(otherDataSet.Id, previousReleaseFilePostDelete.PublicApiDataSetId); + Assert.Equal(otherDataSetVersion.FullSemanticVersion(), previousReleaseFilePostDelete.PublicApiDataSetVersion); + } + + [Theory] + [InlineData(DataSetVersionStatus.Failed)] + [InlineData(DataSetVersionStatus.Mapping)] + [InlineData(DataSetVersionStatus.Draft)] + [InlineData(DataSetVersionStatus.Cancelled)] + public async Task ReleaseVersionIsLinkedToSubsequentDataSetVersion(DataSetVersionStatus dataSetVersionStatus) + { + Publication publication = DataFixture.DefaultPublication(); + + Release release1 = DataFixture.DefaultRelease(); + + Release release2 = DataFixture.DefaultRelease(); + + ReleaseVersion release1Version1 = DataFixture.DefaultReleaseVersion() + .WithPublication(publication) + .WithRelease(release1) + .WithApprovalStatus(ReleaseApprovalStatus.Approved) + .WithPublished(DateTime.UtcNow) + .WithVersion(0); + + ReleaseVersion release2Version1 = DataFixture.DefaultReleaseVersion() + .WithPublication(publication) + .WithRelease(release2) + .WithApprovalStatus(ReleaseApprovalStatus.Approved) + .WithPublished(DateTime.UtcNow) + .WithVersion(0); + + ReleaseVersion release2Version2 = DataFixture.DefaultReleaseVersion() + .WithPublication(publication) + .WithRelease(release2) + .WithVersion(1); + + ReleaseFile release1Version1ReleaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(release1Version1) + .WithFile(DataFixture.DefaultFile(FileType.Data)); + + ReleaseFile release2Version1ReleaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(release2Version1) + .WithFile(DataFixture.DefaultFile(FileType.Data)); + + ReleaseFile release2Version2ReleaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(release2Version2) + .WithFile(DataFixture.DefaultFile(FileType.Data)); + + await AddTestData(context => + { + context.ReleaseFiles.AddRange( + release1Version1ReleaseFile, + release2Version1ReleaseFile, + release2Version2ReleaseFile); + }); + + DataSet release1Version1DataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished() + .WithPublicationId(publication.Id); + + DataSet release2Version1DataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished() + .WithPublicationId(publication.Id); + + await AddTestData(context => context.DataSets.AddRange( + release1Version1DataSet, + release2Version1DataSet)); + + DataSetVersion release1Version1DataSetVersion = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) + .WithVersionNumber(1, 0, 0) + .WithStatusPublished() + .WithDataSet(release1Version1DataSet) + .WithImports(() => DataFixture + .DefaultDataSetVersionImport() + .WithStage(DataSetVersionImportStage.Completing) + .Generate(1)) + .WithReleaseFileId(release1Version1ReleaseFile.Id) + .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); + + DataSetVersion release2Version1DataSetVersion = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) + .WithVersionNumber(1, 0, 0) + .WithStatusPublished() + .WithDataSet(release2Version1DataSet) + .WithImports(() => DataFixture + .DefaultDataSetVersionImport() + .WithStage(DataSetVersionImportStage.Completing) + .Generate(1)) + .WithReleaseFileId(release2Version1ReleaseFile.Id) + .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); + + DataSetVersion release2Version2DataSetVersion = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) + .WithVersionNumber(2, 0, 0) + .WithStatus(dataSetVersionStatus) + .WithDataSet(release1Version1DataSet) // This is the 2nd Data Set Version of the series + .WithImports(() => DataFixture + .DefaultDataSetVersionImport() + .Generate(1)) + .WithReleaseFileId(release2Version2ReleaseFile.Id) + .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + + await AddTestData(context => + { + context.DataSetVersions.AddRange( + release1Version1DataSetVersion, + release2Version1DataSetVersion, + release2Version2DataSetVersion); + + context.DataSets.UpdateRange( + release1Version1DataSet, + release2Version1DataSet); + }); + + release1Version1ReleaseFile.PublicApiDataSetId = release1Version1DataSet.Id; + release1Version1ReleaseFile.PublicApiDataSetVersion = release1Version1DataSetVersion.FullSemanticVersion(); + release2Version1ReleaseFile.PublicApiDataSetId = release2Version1DataSet.Id; + release2Version1ReleaseFile.PublicApiDataSetVersion = release2Version1DataSetVersion.FullSemanticVersion(); + release2Version2ReleaseFile.PublicApiDataSetId = release1Version1DataSet.Id; // Same Data Set series as release1version1 + release2Version2ReleaseFile.PublicApiDataSetVersion = release2Version2DataSetVersion.FullSemanticVersion(); + + await AddTestData(context => context.ReleaseFiles.UpdateRange( + release1Version1ReleaseFile, + release2Version1ReleaseFile, + release2Version2ReleaseFile)); + + var allDataSetVersions = new List() { + release1Version1DataSetVersion, + release2Version1DataSetVersion, + release2Version2DataSetVersion }; + + foreach (var dataSetVersion in allDataSetVersions) + { + var dataSetVersionDirectory = _dataSetVersionPathResolver.DirectoryPath(dataSetVersion); + Directory.CreateDirectory(dataSetVersionDirectory); + await System.IO.File.Create(Path.Combine(dataSetVersionDirectory, $"{dataSetVersion.Version}.txt")).DisposeAsync(); + } + + var response = await BulkDeleteDataSetVersions(release2Version2.Id); + + response.AssertNoContent(); + + await using var publicDataDbContext = GetDbContext(); + await using var contentDataDbContext = GetDbContext(); + + // Assert that the parquet folder for the new, DRAFT, API Data Set Version linked to the release version being deleted, is now deleted + var release2Version2DataSetVersionDirectory = _dataSetVersionPathResolver.DirectoryPath(release2Version2DataSetVersion); + Assert.False(Directory.Exists(release2Version2DataSetVersionDirectory)); + + // Assert that the parent parquet folder for the API Data Set, linked to the release version being deleted, has NOT been deleted + var release1Version1DataSetDirectory = Directory.GetParent(_dataSetVersionPathResolver.DirectoryPath(release2Version2DataSetVersion)); + Assert.True(Directory.Exists(release1Version1DataSetDirectory!.FullName)); + + // Assert that the parquet folder, and its contents, for the API Data Set Version linked to Release 1 Version 1, has NOT been deleted + var release1Version1DataSetVersionDirectory = _dataSetVersionPathResolver.DirectoryPath(release1Version1DataSetVersion); + Assert.True(Directory.Exists(release1Version1DataSetVersionDirectory)); + + var release1Version1DataSetVersionDirectoryEntries = Directory.GetFileSystemEntries(release1Version1DataSetVersionDirectory); + Assert.Single(release1Version1DataSetVersionDirectoryEntries, entry => new System.IO.FileInfo(entry).Name == $"{release1Version1DataSetVersion.Version}.txt"); + + // Assert that the parquet folder, and its contents, for the API Data Set Version linked to Release 2 Version 1, has NOT been deleted + var release2Version1DataSetVersionDirectory = _dataSetVersionPathResolver.DirectoryPath(release2Version1DataSetVersion); + Assert.True(Directory.Exists(release2Version1DataSetVersionDirectory)); + + var release2Version1DataSetVersionDirectoryEntries = Directory.GetFileSystemEntries(release2Version1DataSetVersionDirectory); + Assert.Single(release2Version1DataSetVersionDirectoryEntries, entry => new System.IO.FileInfo(entry).Name == $"{release2Version1DataSetVersion.Version}.txt"); + + // Assert that the TARGET API Data Set, linked to the release version being deleted, has NOT been deleted + Assert.NotNull(await publicDataDbContext.DataSets.SingleOrDefaultAsync(ds => ds.Id == release1Version1DataSet.Id)); + + // Assert that the OTHER Data Set has NOT been deleted + Assert.NotNull(await publicDataDbContext.DataSets.SingleAsync(ds => ds.Id == release2Version1DataSet.Id)); + + // Assert that the TARGET Data Set Version has been deleted + Assert.Null(await publicDataDbContext.DataSetVersions.SingleOrDefaultAsync(dsv => dsv.Id == release2Version2DataSetVersion.Id)); + + // Assert that the OTHER Data Set Versions have NOT been deleted + Assert.NotNull(await publicDataDbContext.DataSetVersions.SingleAsync(dsv => dsv.Id == release1Version1DataSetVersion.Id)); + Assert.NotNull(await publicDataDbContext.DataSetVersions.SingleAsync(dsv => dsv.Id == release2Version1DataSetVersion.Id)); + + // Assert that the TARGET Data Set Version metadata is deleted + await AssertMetadataIsDeleted(publicDataDbContext, release2Version2DataSetVersion); + + // Assert that the OTHER Data Set Versions metadata has NOT been deleted + await AssertMetadataIsNotDeleted(publicDataDbContext, release1Version1DataSetVersion); + await AssertMetadataIsNotDeleted(publicDataDbContext, release2Version1DataSetVersion); + + // Assert that the TARGET Data Set Version Import has been deleted + Assert.Null(await publicDataDbContext.DataSetVersionImports.SingleOrDefaultAsync(dsvi => dsvi.Id == release2Version2DataSetVersion.Imports.Single().Id)); + + // Assert that the OTHER Data Set Version Imports have NOT been deleted + Assert.NotNull(await publicDataDbContext.DataSetVersionImports.SingleAsync(dsvi => dsvi.Id == release1Version1DataSetVersion.Imports.Single().Id)); + Assert.NotNull(await publicDataDbContext.DataSetVersionImports.SingleAsync(dsvi => dsvi.Id == release2Version1DataSetVersion.Imports.Single().Id)); + + // Assert that the TARGET Release File has been unassociated with its Public API DataSet + var targetReleaseFile = await contentDataDbContext.ReleaseFiles.SingleAsync(f => f.Id == release2Version2ReleaseFile.Id); + + Assert.Null(targetReleaseFile.PublicApiDataSetId); + Assert.Null(targetReleaseFile.PublicApiDataSetVersion); + + // Assert that the OTHER Release Files are still associated with their Public API DataSets + var release1Version1ReleaseFilePostDelete = await contentDataDbContext.ReleaseFiles.SingleAsync(f => f.Id == release1Version1ReleaseFile.Id); + var release2Version1ReleaseFilePostDelete = await contentDataDbContext.ReleaseFiles.SingleAsync(f => f.Id == release2Version1ReleaseFile.Id); + + Assert.Equal(release1Version1DataSet.Id, release1Version1ReleaseFilePostDelete.PublicApiDataSetId); + Assert.Equal(release1Version1DataSetVersion.FullSemanticVersion(), release1Version1ReleaseFilePostDelete.PublicApiDataSetVersion); + Assert.Equal(release2Version1DataSet.Id, release2Version1ReleaseFilePostDelete.PublicApiDataSetId); + Assert.Equal(release2Version1DataSetVersion.FullSemanticVersion(), release2Version1ReleaseFilePostDelete.PublicApiDataSetVersion); + } + + private static async Task AssertMetadataIsDeleted(PublicDataDbContext publicDataDbContext, DataSetVersion dataSetVersion) + { + Assert.False(await publicDataDbContext.FilterMetas + .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); + Assert.False(await publicDataDbContext.FilterOptionMetaLinks + .AnyAsync(foml => dataSetVersion.FilterMetas.Contains(foml.Meta))); + Assert.False(await publicDataDbContext.LocationMetas + .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); + Assert.False(await publicDataDbContext.LocationOptionMetaLinks + .AnyAsync(loml => dataSetVersion.LocationMetas.Contains(loml.Meta))); + Assert.False(await publicDataDbContext.IndicatorMetas + .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); + Assert.False(await publicDataDbContext.GeographicLevelMetas + .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); + Assert.False(await publicDataDbContext.TimePeriodMetas + .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); + } + + private static async Task AssertMetadataIsNotDeleted(PublicDataDbContext publicDataDbContext, DataSetVersion dataSetVersion) + { + Assert.True(await publicDataDbContext.FilterMetas + .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); + Assert.True(await publicDataDbContext.FilterOptionMetaLinks + .AnyAsync(foml => dataSetVersion.FilterMetas.Contains(foml.Meta))); + Assert.True(await publicDataDbContext.LocationMetas + .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); + Assert.True(await publicDataDbContext.LocationOptionMetaLinks + .AnyAsync(loml => dataSetVersion.LocationMetas.Contains(loml.Meta))); + Assert.True(await publicDataDbContext.IndicatorMetas + .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); + Assert.True(await publicDataDbContext.GeographicLevelMetas + .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); + Assert.True(await publicDataDbContext.TimePeriodMetas + .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); + } + + [Theory] + [InlineData(DataSetVersionStatus.Processing)] + [InlineData(DataSetVersionStatus.Published)] + [InlineData(DataSetVersionStatus.Deprecated)] + [InlineData(DataSetVersionStatus.Withdrawn)] + public async Task DataSetVersionCanNotBeDeleted_FirstVersion_Returns400(DataSetVersionStatus dataSetVersionStatus) + { + ReleaseVersion releaseVersion = DataFixture.DefaultReleaseVersion() + .WithPublication(DataFixture.DefaultPublication()); + + ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(DataFixture.DefaultFile(FileType.Data)); + + await AddTestData(context => + { + context.ReleaseFiles.Add(releaseFile); + }); + + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithPublicationId(releaseVersion.Publication.Id) + .WithStatusDraft(); + + await AddTestData(context => context.DataSets.Add(dataSet)); + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) + .WithVersionNumber(1, 0, 0) + .WithStatus(dataSetVersionStatus) + .WithReleaseFileId(releaseFile.Id) + .WithDataSet(dataSet) + .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); + + await AddTestData(context => + { + context.DataSetVersions.Add(dataSetVersion); + context.DataSets.Update(dataSet); + }); + + var response = await BulkDeleteDataSetVersions(releaseVersion.Id); + + var validationProblem = response.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: "releaseVersionId", + expectedCode: ValidationMessages.MultipleDataSetVersionsCanNotBeDeleted.Code); + } + + [Theory] + [InlineData(DataSetVersionStatus.Processing)] + [InlineData(DataSetVersionStatus.Published)] + [InlineData(DataSetVersionStatus.Deprecated)] + [InlineData(DataSetVersionStatus.Withdrawn)] + public async Task DataSetVersionCanNotBeDeleted_SubsequentVersion_Returns400(DataSetVersionStatus dataSetVersionStatus) + { + ReleaseVersion releaseVersion = DataFixture.DefaultReleaseVersion() + .WithPublication(DataFixture.DefaultPublication()); + + ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(releaseVersion) + .WithFile(DataFixture.DefaultFile(FileType.Data)); + + await AddTestData(context => + { + context.ReleaseFiles.Add(releaseFile); + }); + + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithPublicationId(releaseVersion.Publication.Id) + .WithStatusPublished(); + + await AddTestData(context => context.DataSets.Add(dataSet)); + + DataSetVersion liveDataSetVersion = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) + .WithReleaseFileId(releaseFile.Id) + .WithDataSet(dataSet) + .WithVersionNumber(1, 0, 0) + .WithStatus(dataSetVersionStatus) + .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); + + DataSetVersion draftDataSetVersion = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) + .WithReleaseFileId(releaseFile.Id) + .WithDataSet(dataSet) + .WithVersionNumber(2, 0, 0) + .WithStatusDraft() + .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + + await AddTestData(context => + { + context.DataSetVersions.AddRange(liveDataSetVersion, draftDataSetVersion); + context.DataSets.Update(dataSet); + }); + + var response = await BulkDeleteDataSetVersions(releaseVersion.Id); + + var validationProblem = response.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: "releaseVersionId", + expectedCode: ValidationMessages.MultipleDataSetVersionsCanNotBeDeleted.Code); + } + + private async Task BulkDeleteDataSetVersions(Guid releaseVersionId) + { + var function = GetRequiredService(); + + return await function.BulkDeleteDataSetVersions( + null!, + releaseVersionId, + CancellationToken.None); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs index e70e2081ac6..4010d0d8e63 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs @@ -1,6 +1,5 @@ using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; -using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; @@ -22,23 +21,21 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; -public abstract class CreateDataSetFunctionTests(ProcessorFunctionsIntegrationTestFixture fixture) +public abstract class CreateDataSetFunctionTests( + ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { - private readonly DataFixture _fixture = new(); - - public class CreateDataSetTests(ProcessorFunctionsIntegrationTestFixture fixture) + public class CreateDataSetTests( + ProcessorFunctionsIntegrationTestFixture fixture) : CreateDataSetFunctionTests(fixture) { - private const string DurableTaskClientName = "TestClient"; - [Fact] public async Task Success() { - var (releaseFile, releaseMetaFile) = _fixture.DefaultReleaseFile() - .WithReleaseVersion(_fixture.DefaultReleaseVersion() - .WithPublication(_fixture.DefaultPublication())) - .WithFiles(_fixture.DefaultFile() + var (releaseFile, releaseMetaFile) = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(DataFixture.DefaultReleaseVersion() + .WithPublication(DataFixture.DefaultPublication())) + .WithFiles(DataFixture.DefaultFile() .ForIndex(0, s => s.SetType(FileType.Data)) .ForIndex(1, s => s.SetType(FileType.Metadata)) .WithSubjectId(Guid.NewGuid()) @@ -51,14 +48,14 @@ await AddTestData(context => context.ReleaseFiles.AddRange(releaseFile, releaseMetaFile); }); - var durableTaskClientMock = new Mock(DurableTaskClientName); + var durableTaskClientMock = new Mock(MockBehavior.Strict, "TestClient"); - ProcessInitialDataSetVersionContext? processInitialDataSetVersionContext = null; + ProcessDataSetVersionContext? processInitialDataSetVersionContext = null; StartOrchestrationOptions? startOrchestrationOptions = null; durableTaskClientMock.Setup(client => client.ScheduleNewOrchestrationInstanceAsync( nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion), - It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((TaskName _, object _, StartOrchestrationOptions? options, CancellationToken _) => @@ -67,7 +64,7 @@ await AddTestData(context => (_, input, options, _) => { processInitialDataSetVersionContext = - Assert.IsAssignableFrom(input); + Assert.IsAssignableFrom(input); startOrchestrationOptions = options; }); @@ -119,28 +116,16 @@ await AddTestData(context => // Assert the processing orchestrator was scheduled with the correct arguments Assert.NotNull(processInitialDataSetVersionContext); Assert.NotNull(startOrchestrationOptions); - Assert.Equal(new ProcessInitialDataSetVersionContext - { - DataSetVersionId = dataSetVersion.Id - }, + Assert.Equal(new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersion.Id }, processInitialDataSetVersionContext); - Assert.Equal(new StartOrchestrationOptions - { - InstanceId = dataSetVersionImport.InstanceId.ToString() - }, + Assert.Equal(new StartOrchestrationOptions { InstanceId = dataSetVersionImport.InstanceId.ToString() }, startOrchestrationOptions); } [Fact] public async Task ReleaseFileIdIsEmpty_ReturnsValidationProblem() { - var durableTaskClientMock = new Mock(DurableTaskClientName); - - var result = await CreateDataSet( - releaseFileId: Guid.Empty, - durableTaskClientMock.Object); - - VerifyAllMocks(durableTaskClientMock); + var result = await CreateDataSet(releaseFileId: Guid.Empty); var validationProblem = result.AssertBadRequestWithValidationProblem(); @@ -151,13 +136,7 @@ public async Task ReleaseFileIdIsEmpty_ReturnsValidationProblem() [Fact] public async Task ReleaseFileIdIsNotFound_ReturnsValidationProblem() { - var durableTaskClientMock = new Mock(DurableTaskClientName); - - var result = await CreateDataSet( - releaseFileId: Guid.NewGuid(), - durableTaskClientMock.Object); - - VerifyAllMocks(durableTaskClientMock); + var result = await CreateDataSet(releaseFileId: Guid.NewGuid()); var validationProblem = result.AssertBadRequestWithValidationProblem(); @@ -169,13 +148,13 @@ public async Task ReleaseFileIdIsNotFound_ReturnsValidationProblem() [Fact] public async Task ReleaseFileIdHasDataSetVersion_ReturnsValidationProblem() { - ReleaseFile releaseFile = _fixture.DefaultReleaseFile() - .WithReleaseVersion(_fixture.DefaultReleaseVersion()) - .WithFile(_fixture.DefaultFile()); + ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) + .WithFile(DataFixture.DefaultFile(FileType.Data)); - DataSet dataSet = _fixture.DefaultDataSet(); + DataSet dataSet = DataFixture.DefaultDataSet(); - DataSetVersion dataSetVersion = _fixture.DefaultDataSetVersion() + DataSetVersion dataSetVersion = DataFixture.DefaultDataSetVersion() .WithReleaseFileId(releaseFile.Id) .WithDataSet(dataSet); @@ -191,13 +170,7 @@ await AddTestData(context => context.DataSetVersions.Add(dataSetVersion); }); - var durableTaskClientMock = new Mock(DurableTaskClientName); - - var result = await CreateDataSet( - releaseFileId: releaseFile.Id, - durableTaskClientMock.Object); - - VerifyAllMocks(durableTaskClientMock); + var result = await CreateDataSet(releaseFileId: releaseFile.Id); var validationProblem = result.AssertBadRequestWithValidationProblem(); @@ -209,10 +182,10 @@ await AddTestData(context => [Fact] public async Task ReleaseVersionNotDraft_ReturnsValidationProblem() { - var (releaseFile, releaseMetaFile) = _fixture.DefaultReleaseFile() - .WithReleaseVersion(_fixture.DefaultReleaseVersion() + var (releaseFile, releaseMetaFile) = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(DataFixture.DefaultReleaseVersion() .WithApprovalStatus(ReleaseApprovalStatus.Approved)) - .WithFiles(_fixture.DefaultFile() + .WithFiles(DataFixture.DefaultFile() .ForIndex(0, s => s.SetType(FileType.Data)) .ForIndex(1, s => s.SetType(FileType.Metadata)) .Generate(2)) @@ -224,13 +197,7 @@ await AddTestData(context => context.ReleaseFiles.AddRange(releaseFile, releaseMetaFile); }); - var durableTaskClientMock = new Mock(DurableTaskClientName); - - var result = await CreateDataSet( - releaseFileId: releaseFile.Id, - durableTaskClientMock.Object); - - VerifyAllMocks(durableTaskClientMock); + var result = await CreateDataSet(releaseFileId: releaseFile.Id); var validationProblem = result.AssertBadRequestWithValidationProblem(); @@ -243,23 +210,16 @@ await AddTestData(context => [Fact] public async Task ReleaseFileTypeNotData_ReturnsValidationProblem() { - ReleaseFile releaseFile = _fixture.DefaultReleaseFile() - .WithReleaseVersion(_fixture.DefaultReleaseVersion()) - .WithFile(_fixture.DefaultFile() - .WithType(FileType.Ancillary)); + ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) + .WithFile(DataFixture.DefaultFile(FileType.Ancillary)); await AddTestData(context => { context.ReleaseFiles.Add(releaseFile); }); - var durableTaskClientMock = new Mock(DurableTaskClientName); - - var result = await CreateDataSet( - releaseFileId: releaseFile.Id, - durableTaskClientMock.Object); - - VerifyAllMocks(durableTaskClientMock); + var result = await CreateDataSet(releaseFileId: releaseFile.Id); var validationProblem = result.AssertBadRequestWithValidationProblem(); @@ -272,22 +232,16 @@ await AddTestData(context => [Fact] public async Task ReleaseFileHasNoMetaFile_ReturnsValidationProblem() { - ReleaseFile releaseFile = _fixture.DefaultReleaseFile() - .WithReleaseVersion(_fixture.DefaultReleaseVersion()) - .WithFile(_fixture.DefaultFile()); + ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) + .WithFile(DataFixture.DefaultFile(FileType.Data)); await AddTestData(context => { context.ReleaseFiles.Add(releaseFile); }); - var durableTaskClientMock = new Mock(DurableTaskClientName); - - var result = await CreateDataSet( - releaseFileId: releaseFile.Id, - durableTaskClientMock.Object); - - VerifyAllMocks(durableTaskClientMock); + var result = await CreateDataSet(releaseFileId: releaseFile.Id); var validationProblem = result.AssertBadRequestWithValidationProblem(); @@ -299,14 +253,11 @@ await AddTestData(context => private async Task CreateDataSet( Guid releaseFileId, - DurableTaskClient durableTaskClient) + DurableTaskClient? durableTaskClient = null) { var function = GetRequiredService(); - return await function.CreateDataSet(new DataSetCreateRequest - { - ReleaseFileId = releaseFileId - }, - durableTaskClient, + return await function.CreateDataSet(new DataSetCreateRequest { ReleaseFileId = releaseFileId }, + durableTaskClient ?? new Mock(MockBehavior.Strict, "TestClient").Object, CancellationToken.None); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionFunctionTests.cs new file mode 100644 index 00000000000..ccd540e9e23 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionFunctionTests.cs @@ -0,0 +1,482 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Content.Model; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests.Validators; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.ViewModels; +using Microsoft.AspNetCore.Mvc; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.EntityFrameworkCore; +using Moq; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; +using FileType = GovUk.Education.ExploreEducationStatistics.Common.Model.FileType; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class CreateNextDataSetVersionFunctionTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessorFunctionsIntegrationTest(fixture) +{ + public class CreateNextDataSetVersionTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : CreateNextDataSetVersionFunctionTests(fixture) + { + [Fact] + public async Task Success() + { + var (dataSet, liveDataSetVersion) = await AddDataSetAndLatestLiveVersion(); + var (nextReleaseFile, _) = await AddDataAndMetadataFiles(dataSet.PublicationId); + + var durableTaskClientMock = new Mock(MockBehavior.Strict, "TestClient"); + + ProcessDataSetVersionContext? processNextDataSetVersionContext = null; + StartOrchestrationOptions? startOrchestrationOptions = null; + durableTaskClientMock.Setup(client => + client.ScheduleNewOrchestrationInstanceAsync( + nameof(ProcessNextDataSetVersionFunction.ProcessNextDataSetVersion), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync((TaskName _, object _, StartOrchestrationOptions? options, CancellationToken _) => + options?.InstanceId ?? Guid.NewGuid().ToString()) + .Callback( + (_, input, options, _) => + { + processNextDataSetVersionContext = + Assert.IsAssignableFrom(input); + startOrchestrationOptions = options; + }); + + var result = await CreateNextDataSetVersion( + dataSetId: dataSet.Id, + releaseFileId: nextReleaseFile.Id, + durableTaskClientMock.Object); + + VerifyAllMocks(durableTaskClientMock); + + var responseViewModel = result.AssertOkObjectResult(); + + await using var publicDataDbContext = GetDbContext(); + + // Assert only the original data set exists. + var updatedDataSet = Assert.Single(await publicDataDbContext.DataSets + .Include(ds => ds.Versions) + .ThenInclude(dsv => dsv.Imports) + .ToListAsync()); + + // Assert that the existing data set is left in its Published state and that its + // name, summary and other properties that were created during its first creation + // are untouched. + Assert.Equal(DataSetStatus.Published, updatedDataSet.Status); + Assert.Equal(dataSet.Title, updatedDataSet.Title); + Assert.Equal(dataSet.Summary, updatedDataSet.Summary); + Assert.Equal(dataSet.PublicationId, updatedDataSet.PublicationId); + + // Assert that the new DataSetVersion is set as the latest draft version, and that the + // prior DataSetVersion is still set as the latest live version. + Assert.NotNull(updatedDataSet.LatestDraftVersion); + Assert.NotEqual(liveDataSetVersion.Id, updatedDataSet.LatestDraftVersion!.Id); + Assert.Equal(liveDataSetVersion.Id, updatedDataSet.LatestLiveVersion!.Id); + + // Assert the data set has a new version and that it is the latest draft version. + Assert.Equal(2, updatedDataSet.Versions.Count); + var nextDataSetVersion = updatedDataSet + .Versions + .OrderBy(v => v.FullSemanticVersion()) + .Last(); + Assert.Equal(nextDataSetVersion, updatedDataSet.LatestDraftVersion); + + Assert.Equal(nextReleaseFile.Id, nextDataSetVersion.ReleaseFileId); + Assert.Equal(updatedDataSet.Id, nextDataSetVersion.DataSetId); + Assert.Equal(DataSetVersionStatus.Processing, nextDataSetVersion.Status); + Assert.Empty(nextDataSetVersion.Notes); + Assert.Equal(1, nextDataSetVersion.VersionMajor); + Assert.Equal(1, nextDataSetVersion.VersionMinor); + + // Assert a single import was created. + var dataSetVersionImport = Assert.Single(nextDataSetVersion.Imports); + Assert.Equal(nextDataSetVersion.Id, dataSetVersionImport.DataSetVersionId); + Assert.NotEqual(Guid.Empty, dataSetVersionImport.InstanceId); + Assert.Equal(DataSetVersionImportStage.Pending, dataSetVersionImport.Stage); + + // Assert the response view model values match the created data set version and import. + Assert.Equal(updatedDataSet.Id, responseViewModel.DataSetId); + Assert.Equal(nextDataSetVersion.Id, responseViewModel.DataSetVersionId); + Assert.Equal(dataSetVersionImport.InstanceId, responseViewModel.InstanceId); + + // Assert the processing orchestrator was scheduled with the correct arguments + Assert.NotNull(processNextDataSetVersionContext); + Assert.NotNull(startOrchestrationOptions); + Assert.Equal(new ProcessDataSetVersionContext { DataSetVersionId = nextDataSetVersion.Id }, + processNextDataSetVersionContext); + Assert.Equal(new StartOrchestrationOptions { InstanceId = dataSetVersionImport.InstanceId.ToString() }, + startOrchestrationOptions); + } + + [Fact] + public async Task ReleaseFileIdIsEmpty_ReturnsValidationProblem() + { + var result = await CreateNextDataSetVersion( + dataSetId: Guid.NewGuid(), + releaseFileId: Guid.Empty); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasNotEmptyError( + nameof(NextDataSetVersionCreateRequest.ReleaseFileId).ToLowerFirst()); + } + + [Fact] + public async Task DataSetIdIsEmpty_ReturnsValidationProblem() + { + var result = await CreateNextDataSetVersion( + dataSetId: Guid.Empty, + releaseFileId: Guid.NewGuid()); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasNotEmptyError( + nameof(NextDataSetVersionCreateRequest.DataSetId).ToLowerFirst()); + } + + [Fact] + public async Task DataSetIdIsNotFound_ReturnsValidationProblem() + { + var (releaseFile, _) = await AddDataAndMetadataFiles(Guid.NewGuid()); + + var result = await CreateNextDataSetVersion( + dataSetId: Guid.NewGuid(), + releaseFileId: releaseFile.Id); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: nameof(NextDataSetVersionCreateRequest.DataSetId).ToLowerFirst(), + expectedCode: ValidationMessages.DataSetNotFound.Code); + } + + [Fact] + public async Task ReleaseFileIdIsNotFound_ReturnsValidationProblem() + { + var (dataSet, _) = await AddDataSetAndLatestLiveVersion(); + + var result = await CreateNextDataSetVersion( + dataSetId: dataSet.Id, + releaseFileId: Guid.NewGuid()); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: nameof(DataSetCreateRequest.ReleaseFileId).ToLowerFirst(), + expectedCode: ValidationMessages.FileNotFound.Code); + } + + [Fact] + public async Task ReleaseFileIdHasDataSetVersion_ReturnsValidationProblem() + { + var (dataSet, _) = await AddDataSetAndLatestLiveVersion(); + + ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) + .WithFile(DataFixture.DefaultFile(FileType.Data)); + + await AddTestData(context => + { + context.ReleaseFiles.Add(releaseFile); + }); + + // Create another DataSet and DataSetVersion which already references the ReleaseFile's Id. + DataSet otherDataSet = DataFixture.DefaultDataSet(); + + DataSetVersion otherDataSetVersion = DataFixture.DefaultDataSetVersion() + .WithReleaseFileId(releaseFile.Id) + .WithDataSet(otherDataSet); + + await AddTestData(context => + { + context.DataSets.Add(otherDataSet); + context.SaveChanges(); + context.DataSetVersions.Add(otherDataSetVersion); + }); + + var result = await CreateNextDataSetVersion( + dataSetId: dataSet.Id, + releaseFileId: releaseFile.Id); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: nameof(NextDataSetVersionCreateRequest.ReleaseFileId).ToLowerFirst(), + expectedCode: ValidationMessages.FileHasApiDataSetVersion.Code); + } + + [Fact] + public async Task ReleaseVersionNotDraft_ReturnsValidationProblem() + { + var (dataSet, _) = await AddDataSetAndLatestLiveVersion(); + + var subjectId = Guid.NewGuid(); + + var (releaseFile, releaseMetaFile) = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(DataFixture.DefaultReleaseVersion() + .WithApprovalStatus(ReleaseApprovalStatus.Approved)) + .WithFiles([ + DataFixture + .DefaultFile(FileType.Data) + .WithSubjectId(subjectId), + DataFixture + .DefaultFile(FileType.Metadata) + .WithSubjectId(subjectId) + ]) + .GenerateList() + .ToTuple2(); + + await AddTestData(context => + { + context.ReleaseFiles.AddRange(releaseFile, releaseMetaFile); + }); + + var result = await CreateNextDataSetVersion( + dataSetId: dataSet.Id, + releaseFileId: releaseFile.Id); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: nameof(DataSetCreateRequest.ReleaseFileId).ToLowerFirst(), + expectedCode: ValidationMessages.FileReleaseVersionNotDraft.Code + ); + } + + [Fact] + public async Task ReleaseFileTypeNotData_ReturnsValidationProblem() + { + var (dataSet, _) = await AddDataSetAndLatestLiveVersion(); + + ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) + .WithFile(DataFixture.DefaultFile(FileType.Ancillary)); + + await AddTestData(context => + { + context.ReleaseFiles.Add(releaseFile); + }); + + var result = await CreateNextDataSetVersion( + dataSetId: dataSet.Id, + releaseFileId: releaseFile.Id); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: nameof(DataSetCreateRequest.ReleaseFileId).ToLowerFirst(), + expectedCode: ValidationMessages.FileTypeNotData.Code + ); + } + + [Fact] + public async Task ReleaseFileHasNoMetaFile_ReturnsValidationProblem() + { + var (dataSet, _) = await AddDataSetAndLatestLiveVersion(); + + ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() + .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) + .WithFile(DataFixture.DefaultFile(FileType.Data)); + + await AddTestData(context => + { + context.ReleaseFiles.Add(releaseFile); + }); + + var result = await CreateNextDataSetVersion( + dataSetId: dataSet.Id, + releaseFileId: releaseFile.Id); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: nameof(DataSetCreateRequest.ReleaseFileId).ToLowerFirst(), + expectedCode: ValidationMessages.NoMetadataFile.Code + ); + } + + [Fact] + public async Task DataSetAndReleaseFileFromDifferentPublications_ReturnsValidationProblem() + { + var (dataSet, _) = await AddDataSetAndLatestLiveVersion(); + + // Add ReleaseFiles for a different Publication. + var (releaseFile, _) = await AddDataAndMetadataFiles(publicationId: Guid.NewGuid()); + + var result = await CreateNextDataSetVersion( + dataSetId: dataSet.Id, + releaseFileId: releaseFile.Id); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: nameof(NextDataSetVersionCreateRequest.ReleaseFileId).ToLowerFirst(), + expectedCode: ValidationMessages.FileNotInDataSetPublication.Code + ); + } + + [Fact] + public async Task DataSetWithoutLiveVersion_ReturnsValidationProblem() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await AddTestData(context => context.DataSets.Add(dataSet)); + + var (releaseFile, _) = await AddDataAndMetadataFiles(dataSet.PublicationId); + + var result = await CreateNextDataSetVersion( + dataSetId: dataSet.Id, + releaseFileId: releaseFile.Id); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: nameof(NextDataSetVersionCreateRequest.DataSetId).ToLowerFirst(), + expectedCode: ValidationMessages.DataSetNoLiveVersion.Code + ); + } + + [Fact] + public async Task ReleaseFileInSameReleaseSeriesAsCurrentLiveVersion_ReturnsValidationProblem() + { + var (dataSet, liveDataSetVersion) = await AddDataSetAndLatestLiveVersion(); + + var currentReleaseFile = GetDbContext() + .ReleaseFiles + .Single(releaseFile => releaseFile.Id == liveDataSetVersion.ReleaseFileId); + + var releaseVersion = await GetDbContext() + .ReleaseVersions + .SingleAsync(releaseVersion => releaseVersion.Id == currentReleaseFile.ReleaseVersionId); + + ReleaseVersion releaseAmendment = DataFixture + .DefaultReleaseVersion() + .WithReleaseId(releaseVersion.ReleaseId) + .WithPublicationId(dataSet.PublicationId); + + var subjectId = Guid.NewGuid(); + + var (nextDataFile, nextMetaFile) = DataFixture + .DefaultReleaseFile() + .WithReleaseVersion(releaseAmendment) + .WithFiles([ + DataFixture + .DefaultFile(FileType.Data) + .WithSubjectId(subjectId), + DataFixture + .DefaultFile(FileType.Metadata) + .WithSubjectId(subjectId) + ]) + .GenerateList() + .ToTuple2(); + + await AddTestData(context => + { + context.ReleaseFiles.AddRange(nextDataFile, nextMetaFile); + }); + + var result = await CreateNextDataSetVersion( + dataSetId: dataSet.Id, + releaseFileId: nextDataFile.Id); + + var validationProblem = result.AssertBadRequestWithValidationProblem(); + + validationProblem.AssertHasError( + expectedPath: nameof(NextDataSetVersionCreateRequest.ReleaseFileId).ToLowerFirst(), + expectedCode: ValidationMessages.FileMustBeInDifferentRelease.Code + ); + } + + private async Task<(DataSet, DataSetVersion)> AddDataSetAndLatestLiveVersion() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await AddTestData(context => context.DataSets.Add(dataSet)); + + var dataSetVersion = await AddLatestLiveDataSetVersion(dataSet); + return (dataSet, dataSetVersion); + } + + private async Task AddLatestLiveDataSetVersion(DataSet dataSet) + { + var (dataFile, _) = await AddDataAndMetadataFiles(dataSet.PublicationId); + + DataSetVersion liveDataSetVersion = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) + .WithVersionNumber(1, 0) + .WithStatusPublished() + .WithDataSet(dataSet) + .WithReleaseFileId(dataFile.Id) + .WithImports(() => DataFixture + .DefaultDataSetVersionImport() + .Generate(1)) + .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); + + await AddTestData(context => + { + context.DataSetVersions.Add(liveDataSetVersion); + context.DataSets.Update(dataSet); + }); + + return liveDataSetVersion; + } + + private async Task<(ReleaseFile, ReleaseFile)> AddDataAndMetadataFiles(Guid publicationId) + { + var subjectId = Guid.NewGuid(); + + var (dataFile, metaFile) = DataFixture + .DefaultReleaseFile() + .WithReleaseVersion(DataFixture + .DefaultReleaseVersion() + .WithPublicationId(publicationId)) + .WithFiles([ + DataFixture + .DefaultFile(FileType.Data) + .WithSubjectId(subjectId), + DataFixture + .DefaultFile(FileType.Metadata) + .WithSubjectId(subjectId) + ]) + .GenerateList() + .ToTuple2(); + + await AddTestData(context => + context.ReleaseFiles.AddRange(dataFile, metaFile)); + + return (dataFile, metaFile); + } + + private async Task CreateNextDataSetVersion( + Guid dataSetId, + Guid releaseFileId, + DurableTaskClient? durableTaskClient = null) + { + var function = GetRequiredService(); + return await function.CreateNextDataSetVersion(new NextDataSetVersionCreateRequest + { + DataSetId = dataSetId, + ReleaseFileId = releaseFileId + }, + durableTaskClient ?? new Mock(MockBehavior.Strict, "TestClient").Object, + CancellationToken.None); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/DeleteDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/DeleteDataSetVersionFunctionTests.cs index c20e745667d..2c67533844e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/DeleteDataSetVersionFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/DeleteDataSetVersionFunctionTests.cs @@ -1,5 +1,5 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; -using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; @@ -11,8 +11,6 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using LinqToDB; using Microsoft.AspNetCore.Mvc; -using File = GovUk.Education.ExploreEducationStatistics.Content.Model.File; - namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; public abstract class DeleteDataSetVersionFunctionTests(ProcessorFunctionsIntegrationTestFixture fixture) @@ -38,7 +36,7 @@ public async Task Success_FirstDataSetVersion(DataSetVersionStatus dataSetVersio .WithReleaseVersion(DataFixture.DefaultReleaseVersion() .WithPublication(DataFixture .DefaultPublication())) - .WithFile(DataFixture.DefaultFile()); + .WithFile(DataFixture.DefaultFile(FileType.Data)); await AddTestData(context => { @@ -69,10 +67,10 @@ await AddTestData(context => context.DataSets.Update(dataSet); }); - releaseFile.File.PublicApiDataSetId = dataSet.Id; - releaseFile.File.PublicApiDataSetVersion = dataSetVersion.FullSemanticVersion(); + releaseFile.PublicApiDataSetId = dataSet.Id; + releaseFile.PublicApiDataSetVersion = dataSetVersion.FullSemanticVersion(); - await AddTestData(context => context.Files.Update(releaseFile.File)); + await AddTestData(context => context.ReleaseFiles.Update(releaseFile)); var dataSetVersionDirectory = _dataSetVersionPathResolver.DirectoryPath(dataSetVersion); @@ -98,11 +96,11 @@ await AddTestData(context => Assert.False(await publicDataDbContext.FilterMetas .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); Assert.False(await publicDataDbContext.FilterOptionMetaLinks - .AnyAsync(foml => dataSetVersion.FilterMetas.Contains(foml.Meta))); + .AnyAsync(ml => dataSetVersion.FilterMetas.Contains(ml.Meta))); Assert.False(await publicDataDbContext.LocationMetas .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); Assert.False(await publicDataDbContext.LocationOptionMetaLinks - .AnyAsync(loml => dataSetVersion.LocationMetas.Contains(loml.Meta))); + .AnyAsync(ml => dataSetVersion.LocationMetas.Contains(ml.Meta))); Assert.False(await publicDataDbContext.IndicatorMetas .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); Assert.False(await publicDataDbContext.GeographicLevelMetas @@ -111,12 +109,15 @@ await AddTestData(context => .AnyAsync(fm => fm.DataSetVersionId == dataSetVersion.Id)); // Assert that the Data Set Version Import has been deleted - Assert.Null(await publicDataDbContext.DataSetVersionImports.SingleOrDefaultAsync(dsvi => dsvi.Id == dataSetVersion.Imports.Single().Id)); + Assert.Null(await publicDataDbContext.DataSetVersionImports + .SingleOrDefaultAsync(v => v.Id == dataSetVersion.Imports.Single().Id)); + + // Assert that the ReleaseFile has been unassociated with the Public API DataSet + var updatedReleaseFile = await contentDataDbContext.ReleaseFiles + .SingleAsync(rf => rf.Id == releaseFile.Id); - // Assert that the Release File has been unassociated with the Public API DataSet - var file = await contentDataDbContext.Files.SingleAsync(f => f.Id == releaseFile.FileId); - Assert.Null(file.PublicApiDataSetId); - Assert.Null(file.PublicApiDataSetVersion); + Assert.Null(updatedReleaseFile.PublicApiDataSetId); + Assert.Null(updatedReleaseFile.PublicApiDataSetVersion); } [Theory] @@ -126,15 +127,11 @@ await AddTestData(context => [InlineData(DataSetVersionStatus.Cancelled)] public async Task Success_SubsequentDataSetVersion(DataSetVersionStatus dataSetVersionStatus) { - File liveFile = DataFixture.DefaultFile(); - File draftFile = DataFixture.DefaultFile(); - var releaseFiles = DataFixture.DefaultReleaseFile() .WithReleaseVersion(DataFixture.DefaultReleaseVersion() .WithPublication(DataFixture .DefaultPublication())) - .ForIndex(0, rf => rf.SetFile(liveFile)) - .ForIndex(1, rf => rf.SetFile(draftFile)) + .WithFile(() => DataFixture.DefaultFile(FileType.Data)) .GenerateList(2); var liveReleaseFile = releaseFiles[0]; @@ -181,20 +178,28 @@ await AddTestData(context => context.DataSets.Update(dataSet); }); - liveFile.PublicApiDataSetId = dataSet.Id; - liveFile.PublicApiDataSetVersion = liveDataSetVersion.FullSemanticVersion(); - draftFile.PublicApiDataSetId = dataSet.Id; - draftFile.PublicApiDataSetVersion = draftDataSetVersion.FullSemanticVersion(); + liveReleaseFile.PublicApiDataSetId = dataSet.Id; + liveReleaseFile.PublicApiDataSetVersion = liveDataSetVersion.FullSemanticVersion(); + draftReleaseFile.PublicApiDataSetId = dataSet.Id; + draftReleaseFile.PublicApiDataSetVersion = draftDataSetVersion.FullSemanticVersion(); - await AddTestData(context => context.Files.UpdateRange(liveFile, draftFile)); + await AddTestData(context => + context.ReleaseFiles.UpdateRange(liveReleaseFile, draftReleaseFile)); var liveDataSetVersionDirectory = _dataSetVersionPathResolver.DirectoryPath(liveDataSetVersion); var draftDataSetVersionDirectory = _dataSetVersionPathResolver.DirectoryPath(draftDataSetVersion); Directory.CreateDirectory(liveDataSetVersionDirectory); Directory.CreateDirectory(draftDataSetVersionDirectory); - System.IO.File.WriteAllText(Path.Combine(liveDataSetVersionDirectory, "version1.txt"), "dummy file text"); - System.IO.File.WriteAllText(Path.Combine(draftDataSetVersionDirectory, "version2.txt"), "dummy file text"); + + await System.IO.File.WriteAllTextAsync( + Path.Combine(liveDataSetVersionDirectory, "version1.txt"), + "dummy file text" + ); + await System.IO.File.WriteAllTextAsync( + Path.Combine(draftDataSetVersionDirectory, "version2.txt"), + "dummy file text" + ); await DeleteDataSetVersion(draftDataSetVersion.Id); @@ -205,7 +210,9 @@ await AddTestData(context => Assert.False(Directory.Exists(draftDataSetVersionDirectory)); // Assert that the LIVE Data Set Version directory has been untouched - var liveDataSetVersionFileName = Assert.Single(Directory.GetFileSystemEntries(liveDataSetVersionDirectory)); + var liveDataSetVersionFileName = Assert.Single( + Directory.GetFileSystemEntries(liveDataSetVersionDirectory) + ); Assert.Contains("version1.txt", liveDataSetVersionFileName); // Assert that the Data Set has NOT been deleted @@ -220,7 +227,8 @@ await AddTestData(context => Assert.Equal(liveDataSetVersion.Id, updatedDataSet.LatestLiveVersionId); // Assert that the DRAFT Data Set Version has been deleted - Assert.Null(await publicDataDbContext.DataSetVersions.SingleOrDefaultAsync(dsv => dsv.Id == draftDataSetVersion.Id)); + Assert.Null(await publicDataDbContext.DataSetVersions + .SingleOrDefaultAsync(dsv => dsv.Id == draftDataSetVersion.Id)); // Assert that the LIVE Data Set Version has NOT been deleted Assert.Single(await publicDataDbContext.DataSetVersions @@ -234,7 +242,7 @@ await publicDataDbContext.FilterMetas .CountAsync()); Assert.Equal(0, await publicDataDbContext.FilterOptionMetaLinks - .Where(foml => draftDataSetVersion.FilterMetas.Contains(foml.Meta)) + .Where(ml => draftDataSetVersion.FilterMetas.Contains(ml.Meta)) .CountAsync()); Assert.Equal(0, await publicDataDbContext.LocationMetas @@ -242,7 +250,7 @@ await publicDataDbContext.LocationMetas .CountAsync()); Assert.Equal(0, await publicDataDbContext.LocationOptionMetaLinks - .Where(loml => draftDataSetVersion.LocationMetas.Contains(loml.Meta)) + .Where(ml => draftDataSetVersion.LocationMetas.Contains(ml.Meta)) .CountAsync()); Assert.Equal(0, await publicDataDbContext.IndicatorMetas @@ -262,13 +270,13 @@ await publicDataDbContext.TimePeriodMetas .Where(fm => fm.DataSetVersionId == liveDataSetVersion.Id) .ToListAsync()); Assert.NotEmpty(await publicDataDbContext.FilterOptionMetaLinks - .Where(foml => liveDataSetVersion.FilterMetas.Contains(foml.Meta)) + .Where(ml => liveDataSetVersion.FilterMetas.Contains(ml.Meta)) .ToListAsync()); Assert.NotEmpty(await publicDataDbContext.LocationMetas .Where(fm => fm.DataSetVersionId == liveDataSetVersion.Id) .ToListAsync()); Assert.NotEmpty(await publicDataDbContext.LocationOptionMetaLinks - .Where(loml => liveDataSetVersion.LocationMetas.Contains(loml.Meta)) + .Where(ml => liveDataSetVersion.LocationMetas.Contains(ml.Meta)) .ToListAsync()); Assert.NotEmpty(await publicDataDbContext.IndicatorMetas .Where(fm => fm.DataSetVersionId == liveDataSetVersion.Id) @@ -281,22 +289,27 @@ await publicDataDbContext.TimePeriodMetas .ToListAsync()); // Assert that the DRAFT Data Set Version Import has been deleted - Assert.Null(await publicDataDbContext.DataSetVersionImports.SingleOrDefaultAsync(dsvi => dsvi.Id == draftDataSetVersion.Imports.Single().Id)); + Assert.Null(await publicDataDbContext.DataSetVersionImports + .SingleOrDefaultAsync(i => i .Id == draftDataSetVersion.Imports.Single().Id)); // Assert that the LIVE Data Set Version Import has NOT been deleted Assert.Single(await publicDataDbContext.DataSetVersionImports - .Where(dsvi => dsvi.Id == liveDataSetVersion.Imports.Single().Id) + .Where(i => i.Id == liveDataSetVersion.Imports.Single().Id) .ToListAsync()); - // Assert that the Release File has been unassociated with the DRAFT Public API Data Set Version - var updatedDraftFile = await contentDataDbContext.Files.SingleAsync(f => f.Id == draftFile.Id); - Assert.Null(updatedDraftFile.PublicApiDataSetId); - Assert.Null(updatedDraftFile.PublicApiDataSetVersion); + // Assert that the DRAFT ReleaseFile has been unassociated with the DRAFT Public API Data Set Version + var updatedDraftReleaseFile = await contentDataDbContext.ReleaseFiles + .SingleAsync(f => f.Id == draftReleaseFile.Id); - // Assert that the Release File is still associated with the LIVE Public API Data Set Version - var updatedLiveFile = await contentDataDbContext.Files.SingleAsync(f => f.Id == liveFile.Id); - Assert.Equal(dataSet.Id, updatedLiveFile.PublicApiDataSetId); - Assert.Equal(liveDataSetVersion.FullSemanticVersion(), updatedLiveFile.PublicApiDataSetVersion); + Assert.Null(updatedDraftReleaseFile.PublicApiDataSetId); + Assert.Null(updatedDraftReleaseFile.PublicApiDataSetVersion); + + // Assert that the LIVE ReleaseFile is still associated with the LIVE Public API Data Set Version + var updatedLiveReleaseFile = await contentDataDbContext.ReleaseFiles + .SingleAsync(f => f.Id == liveReleaseFile.Id); + + Assert.Equal(dataSet.Id, updatedLiveReleaseFile.PublicApiDataSetId); + Assert.Equal(liveDataSetVersion.FullSemanticVersion(), updatedLiveReleaseFile.PublicApiDataSetVersion); } [Theory] @@ -333,36 +346,6 @@ await AddTestData(context => expectedCode: ValidationMessages.DataSetVersionCanNotBeDeleted.Code); } - [Fact] - public async Task ReleaseFileDoesNotExist_Returns500() - { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusDraft(); - - await AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion dataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(1, 0, 0) - .WithStatusDraft() - .WithDataSet(dataSet) - .WithImports(() => DataFixture - .DefaultDataSetVersionImport() - .Generate(1)) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); - - await AddTestData(context => - { - context.DataSetVersions.Add(dataSetVersion); - context.DataSets.Update(dataSet); - }); - - var response = await DeleteDataSetVersion(dataSetVersion.Id); - - response.AssertInternalServerError(); - } - [Fact] public async Task DataSetVersionDoesNotExist_Returns404() { @@ -370,7 +353,7 @@ public async Task DataSetVersionDoesNotExist_Returns404() .WithReleaseVersion(DataFixture.DefaultReleaseVersion() .WithPublication(DataFixture .DefaultPublication())) - .WithFile(DataFixture.DefaultFile()); + .WithFile(DataFixture.DefaultFile(FileType.Data)); await AddTestData(context => { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs index 128bfccb138..f5bbb13f792 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs @@ -1,7 +1,21 @@ +using Dapper; +using GovUk.Education.ExploreEducationStatistics.Common.Converters; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Options; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; +using InterpolatedSql.Dapper; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -13,24 +27,35 @@ public class ImportMetadataTests(ProcessorFunctionsIntegrationTestFixture fixtur { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ImportingMetadata; - [Fact] - public async Task Success() + public static readonly TheoryData TestDataFiles = new() { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + ProcessorTestData.AbsenceSchool, + }; - SetupCsvDataFilesForDataSetVersion(ProcessorTestData.AbsenceSchool, dataSetVersion); + public static readonly TheoryData TestDataFilesWithMetaInsertBatchSize = + new() + { + { ProcessorTestData.AbsenceSchool, 1 }, + { ProcessorTestData.AbsenceSchool, 1000 }, + }; - var function = GetRequiredService(); - await function.ImportMetadata(instanceId, CancellationToken.None); + [Theory] + [MemberData(nameof(TestDataFiles))] + public async Task Success(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportMetadata(testData, dataSetVersion, instanceId); await using var publicDataDbContext = GetDbContext(); var savedImport = await publicDataDbContext.DataSetVersionImports - .Include(dataSetVersionImport => dataSetVersionImport.DataSetVersion) + .Include(i => i.DataSetVersion) .SingleAsync(i => i.InstanceId == instanceId); + var savedDataSetVersion = savedImport.DataSetVersion; Assert.Equal(Stage, savedImport.Stage); - Assert.Equal(DataSetVersionStatus.Processing, savedImport.DataSetVersion.Status); + Assert.Equal(DataSetVersionStatus.Processing, savedDataSetVersion.Status); AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, [ @@ -38,6 +63,378 @@ public async Task Success() DataSetFilenames.CsvMetadataFile, DataSetFilenames.DuckDbDatabaseFile ]); + + Assert.Equal(testData.ExpectedTotalResults, savedDataSetVersion.TotalResults); + + var firstExpectedTimePeriod = testData.ExpectedTimePeriods.First(); + var lastExpectedTimePeriod = testData.ExpectedTimePeriods.Last(); + + var expectedMetaSummary = new DataSetVersionMetaSummary + { + Filters = testData.ExpectedFilters.Select(fm => fm.Label).ToList(), + Indicators = testData.ExpectedIndicators.Select(fm => fm.Label).ToList(), + GeographicLevels = testData.ExpectedGeographicLevels, + TimePeriodRange = new TimePeriodRange + { + Start = new TimePeriodRangeBound + { + Code = firstExpectedTimePeriod.Code, + Period = firstExpectedTimePeriod.Period + }, + End = new TimePeriodRangeBound + { + Code = firstExpectedTimePeriod.Code, + Period = lastExpectedTimePeriod.Period + } + } + }; + + savedDataSetVersion.MetaSummary.AssertDeepEqualTo(expectedMetaSummary); + } + + [Theory] + [MemberData(nameof(TestDataFiles))] + public async Task DataSetVersionMeta_CorrectGeographicLevels(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportMetadata(testData, dataSetVersion, instanceId); + + await using var publicDataDbContext = GetDbContext(); + + var actualGeographicLevelMeta = await publicDataDbContext.GeographicLevelMetas + .SingleAsync(glm => glm.DataSetVersionId == dataSetVersion.Id); + + var actualGeographicLevels = actualGeographicLevelMeta.Levels + .OrderBy(EnumToEnumLabelConverter.ToProvider) + .ToList(); + + Assert.Equal(testData.ExpectedGeographicLevels.Order(), actualGeographicLevels.Order()); + } + + [Theory] + [MemberData(nameof(TestDataFilesWithMetaInsertBatchSize))] + public async Task DataSetVersionMeta_CorrectLocations(ProcessorTestData testData, int metaInsertBatchSize) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportMetadata(testData, dataSetVersion, instanceId, metaInsertBatchSize); + + await using var publicDataDbContext = GetDbContext(); + + var actualLocations = await publicDataDbContext.LocationMetas + .Include(lm => lm.Options) + .Where(lm => lm.DataSetVersionId == dataSetVersion.Id) + .OrderBy(lm => lm.Level) + .ToListAsync(); + + // Locations are expected in order of level + // Location options are expected in order of code(s) and then by label + Assert.Equal(testData.ExpectedLocations.Count, actualLocations.Count); + Assert.All(testData.ExpectedLocations, + (expectedLocation, index) => + { + var actualLocation = actualLocations[index]; + actualLocation.AssertDeepEqualTo(expectedLocation, + notEqualProperties: AssertExtensions.Except( + l => l.Id, + l => l.DataSetVersionId, + l => l.Options, + l => l.OptionLinks, + l => l.Created + )); + + Assert.Equal(expectedLocation.Options.Count, actualLocation.Options.Count); + Assert.All(expectedLocation.Options, + (expectedOption, optionIndex) => + { + var actualOption = actualLocation.Options[optionIndex]; + actualOption.AssertDeepEqualTo(expectedOption, + notEqualProperties: AssertExtensions.Except( + o => o.Metas, + o => o.MetaLinks + )); + }); + }); + } + + [Theory] + [MemberData(nameof(TestDataFiles))] + public async Task DataSetVersionMeta_CorrectTimePeriods(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportMetadata(testData, dataSetVersion, instanceId); + + await using var publicDataDbContext = GetDbContext(); + + var actualTimePeriods = await publicDataDbContext.TimePeriodMetas + .Where(tpm => tpm.DataSetVersionId == dataSetVersion.Id) + .OrderBy(tpm => tpm.Period) + .ToListAsync(); + + Assert.Equal(testData.ExpectedTimePeriods.Count, actualTimePeriods.Count); + Assert.All(testData.ExpectedTimePeriods, + (expectedTimePeriod, index) => + { + var actualTimePeriod = actualTimePeriods[index]; + actualTimePeriod.AssertDeepEqualTo(expectedTimePeriod, + notEqualProperties: AssertExtensions.Except( + tpm => tpm.Id, + tpm => tpm.DataSetVersionId, + tpm => tpm.Created + )); + }); + } + + [Theory] + [MemberData(nameof(TestDataFilesWithMetaInsertBatchSize))] + public async Task DataSetVersionMeta_CorrectFilters(ProcessorTestData testData, int metaInsertBatchSize) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportMetadata(testData, dataSetVersion, instanceId, metaInsertBatchSize); + + await using var publicDataDbContext = GetDbContext(); + + var actualFilters = await publicDataDbContext.FilterMetas + .Include(fm => fm.Options.OrderBy(o => o.Label)) + .ThenInclude(fom => fom.MetaLinks) + .Where(fm => fm.DataSetVersionId == dataSetVersion.Id) + .OrderBy(fm => fm.Label) + .ToListAsync(); + + var globalOptionIndex = 0; + + Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); + Assert.All(testData.ExpectedFilters, + (expectedFilter, index) => + { + var actualFilter = actualFilters[index]; + actualFilter.AssertDeepEqualTo(expectedFilter, + notEqualProperties: AssertExtensions.Except( + fm => fm.Id, + fm => fm.DataSetVersionId, + fm => fm.Created, + fm => fm.Options, + fm => fm.OptionLinks + )); + + Assert.Equal(expectedFilter.Options.Count, actualFilter.Options.Count); + Assert.All(expectedFilter.Options, + (expectedOption, optionIndex) => + { + var actualOption = actualFilter.Options[optionIndex]; + actualOption.AssertDeepEqualTo(expectedOption, + notEqualProperties: AssertExtensions.Except( + o => o.Id, + o => o.Metas, + o => o.MetaLinks + )); + + var actualOptionLink = actualOption.MetaLinks + .Single(link => link.MetaId == actualFilter.Id); + + // Expect the PublicId to be encoded based on the sequence of option links inserted across all filters + Assert.Equal(SqidEncoder.Encode(++globalOptionIndex), actualOptionLink.PublicId); + }); + }); + } + + [Theory] + [MemberData(nameof(TestDataFiles))] + public async Task DataSetVersionMeta_CorrectIndicators(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportMetadata(testData, dataSetVersion, instanceId); + + await using var publicDataDbContext = GetDbContext(); + + var actualIndicators = await publicDataDbContext.IndicatorMetas + .Where(im => im.DataSetVersionId == dataSetVersion.Id) + .OrderBy(im => im.Label) + .ToListAsync(); + + Assert.Equal(testData.ExpectedIndicators.Count, actualIndicators.Count); + Assert.All(testData.ExpectedIndicators, + (expectedIndicator, index) => + { + var actualIndicator = actualIndicators[index]; + actualIndicator.AssertDeepEqualTo(expectedIndicator, + notEqualProperties: AssertExtensions.Except( + im => im.Id, + im => im.DataSetVersionId, + im => im.Created + )); + }); + } + + [Theory] + [MemberData(nameof(TestDataFiles))] + public async Task DuckDbMeta_CorrectLocationOptions(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportMetadata(testData, dataSetVersion, instanceId); + + await using var duckDbConnection = GetDuckDbConnection(dataSetVersion); + + var actualLocationOptions = (await duckDbConnection.SqlBuilder( + $""" + SELECT * + FROM {LocationOptionsTable.TableName:raw} + ORDER BY {LocationOptionsTable.Cols.Label:raw} + """ + ).QueryAsync()).AsList(); + + var actualOptionsByLevel = actualLocationOptions + .GroupBy(o => o.Level) + .ToDictionary(g => EnumUtil.GetFromEnumValue(g.Key), + g => g.ToList()); + + Assert.Equal(testData.ExpectedLocations.Count, actualOptionsByLevel.Count); + Assert.All(testData.ExpectedLocations, + expectedLocation => + { + Assert.True( + actualOptionsByLevel.TryGetValue(expectedLocation.Level, out var actualOptions)); + Assert.Equal(expectedLocation.Options.Count, actualOptions.Count); + Assert.All(expectedLocation.Options.OrderBy(o => o.Label), + (expectedOption, index) => expectedOption.AssertEqual(actualOptions[index])); + }); + } + + [Theory] + [MemberData(nameof(TestDataFiles))] + public async Task DuckDbMeta_CorrectTimePeriods(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportMetadata(testData, dataSetVersion, instanceId); + + await using var duckDbConnection = GetDuckDbConnection(dataSetVersion); + + var actualTimePeriods = (await duckDbConnection.SqlBuilder( + $""" + SELECT * + FROM {TimePeriodsTable.TableName:raw} + ORDER BY {TimePeriodsTable.Cols.Period:raw} + """ + ).QueryAsync()).AsList(); + + Assert.Equal(testData.ExpectedTimePeriods.Count, actualTimePeriods.Count); + Assert.All(testData.ExpectedTimePeriods, + (expectedTimePeriod, index) => + { + var actualTimePeriod = actualTimePeriods[index]; + Assert.Equal(expectedTimePeriod.Code, + EnumUtil.GetFromEnumLabel(actualTimePeriod.Identifier)); + Assert.Equal(expectedTimePeriod.Period, TimePeriodFormatter.FormatFromCsv(actualTimePeriod.Period)); + }); + } + + [Theory] + [MemberData(nameof(TestDataFiles))] + public async Task DuckDbMeta_CorrectFilterOptions(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportMetadata(testData, dataSetVersion, instanceId); + + await using var duckDbConnection = GetDuckDbConnection(dataSetVersion); + + var actualFilterOptions = (await duckDbConnection.SqlBuilder( + $""" + SELECT * + FROM {FilterOptionsTable.TableName:raw} + ORDER BY {FilterOptionsTable.Cols.Label:raw} + """ + ).QueryAsync()).AsList(); + + var actualOptionsByFilterId = actualFilterOptions + .GroupBy(o => o.FilterId) + .ToDictionary(g => g.Key, g => g.ToList()); + + var globalOptionIndex = 0; + + Assert.Equal(testData.ExpectedFilters.Count, actualOptionsByFilterId.Count); + Assert.All(testData.ExpectedFilters, + expectedFilter => + { + Assert.True( + actualOptionsByFilterId.TryGetValue(expectedFilter.PublicId, out var actualOptions)); + Assert.Equal(expectedFilter.Options.Count, actualOptions.Count); + Assert.All(expectedFilter.Options, + (expectedOption, index) => + { + var actualOption = actualOptions[index]; + Assert.Equal(expectedOption.Label, actualOption.Label); + Assert.Equal(expectedFilter.PublicId, actualOption.FilterId); + + // Expect the PublicId to be that of the option link which in turn is expected to be encoded + // based on the sequence of option links inserted across all filters + Assert.Equal(SqidEncoder.Encode(++globalOptionIndex), actualOption.PublicId); + }); + }); + } + + [Theory] + [MemberData(nameof(TestDataFiles))] + public async Task DuckDbMeta_CorrectIndicators(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportMetadata(testData, dataSetVersion, instanceId); + + await using var duckDbConnection = GetDuckDbConnection(dataSetVersion); + + var actualIndicators = (await duckDbConnection.SqlBuilder( + $""" + SELECT * + FROM {IndicatorsTable.TableName:raw} + ORDER BY {IndicatorsTable.Cols.Label:raw} + """ + ).QueryAsync()).AsList(); + + Assert.Equal(testData.ExpectedIndicators.Count, actualIndicators.Count); + Assert.All(testData.ExpectedIndicators, + (expectedIndicator, index) => + { + var actualIndicator = actualIndicators[index]; + Assert.Equal(expectedIndicator.PublicId, actualIndicator.Id); + Assert.Equal(expectedIndicator.Label, actualIndicator.Label); + Assert.Equal(expectedIndicator.DecimalPlaces, actualIndicator.DecimalPlaces); + + if (expectedIndicator.Unit != null) + { + Assert.Equal(expectedIndicator.Unit, + EnumUtil.GetFromEnumLabel(actualIndicator.Unit)); + } + else + { + Assert.Empty(actualIndicator.Unit); + } + }); + } + + private async Task ImportMetadata( + ProcessorTestData testData, + DataSetVersion dataSetVersion, + Guid instanceId, + int? metaInsertBatchSize = null) + { + SetupCsvDataFilesForDataSetVersion(testData, dataSetVersion); + + // Override default app settings if provided + if (metaInsertBatchSize.HasValue) + { + var appSettingsOptions = GetRequiredService>(); + appSettingsOptions.Value.MetaInsertBatchSize = metaInsertBatchSize.Value; + } + + var function = GetRequiredService(); + await function.ImportMetadata(instanceId, CancellationToken.None); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs index 563032429ec..94438dd2e08 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs @@ -1,11 +1,18 @@ +using Dapper; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; +using InterpolatedSql.Dapper; using Microsoft.DurableTask; using Microsoft.DurableTask.Entities; using Microsoft.EntityFrameworkCore; @@ -17,7 +24,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; -public abstract class ProcessInitialDataSetVersionFunctionTests(ProcessorFunctionsIntegrationTestFixture fixture) +public abstract class ProcessInitialDataSetVersionFunctionTests( + ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { private readonly string[] _allDataSetVersionFiles = @@ -34,7 +42,8 @@ public abstract class ProcessInitialDataSetVersionFunctionTests(ProcessorFunctio TimePeriodsTable.ParquetFile ]; - public class ProcessInitialDataSetVersionTests(ProcessorFunctionsIntegrationTestFixture fixture) + public class ProcessInitialDataSetVersionTests( + ProcessorFunctionsIntegrationTestFixture fixture) : ProcessInitialDataSetVersionFunctionTests(fixture) { [Fact] @@ -56,7 +65,7 @@ public async Task Success() ActivityNames.ImportMetadata, ActivityNames.ImportData, ActivityNames.WriteDataFiles, - ActivityNames.CompleteProcessing, + ActivityNames.CompleteInitialDataSetVersionProcessing, ]; foreach (var activityName in expectedActivitySequence) @@ -107,10 +116,7 @@ private async Task ProcessInitialDataSetVersion(TaskOrchestrationContext orchest var function = GetRequiredService(); await function.ProcessInitialDataSetVersion( orchestrationContext, - new ProcessInitialDataSetVersionContext - { - DataSetVersionId = Guid.NewGuid() - }); + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); } private static Mock DefaultMockOrchestrationContext( @@ -136,24 +142,24 @@ private static Mock DefaultMockOrchestrationContext( } } - public class ImportDataTests(ProcessorFunctionsIntegrationTestFixture fixture) + public class ImportDataTests( + ProcessorFunctionsIntegrationTestFixture fixture) : ProcessInitialDataSetVersionFunctionTests(fixture) { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ImportingData; - [Fact] - public async Task Success() + public static readonly TheoryData Data = new() { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); - - SetupCsvDataFilesForDataSetVersion(ProcessorTestData.AbsenceSchool, dataSetVersion); + ProcessorTestData.AbsenceSchool, + }; - // Prepare the metadata before calling the ImportData function - var importMetadataFunction = GetRequiredService(); - await importMetadataFunction.ImportMetadata(instanceId, CancellationToken.None); + [Theory] + [MemberData(nameof(Data))] + public async Task Success(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); - var processInitialDataSetVersionFunction = GetRequiredService(); - await processInitialDataSetVersionFunction.ImportData(instanceId, CancellationToken.None); + await ImportData(testData, dataSetVersion, instanceId); await using var publicDataDbContext = GetDbContext(); @@ -171,29 +177,214 @@ public async Task Success() DataSetFilenames.DuckDbDatabaseFile ]); } - } - public class WriteDataFilesTests(ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessInitialDataSetVersionFunctionTests(fixture) - { - private const DataSetVersionImportStage Stage = DataSetVersionImportStage.WritingDataFiles; + [Theory] + [MemberData(nameof(Data))] + public async Task DuckDbDataTable_CorrectRowCount(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); - [Fact] - public async Task Success() + await ImportData(testData, dataSetVersion, instanceId); + + await using var duckDbConnection = GetDuckDbConnection(dataSetVersion); + + var actualRowCount = await duckDbConnection.SqlBuilder( + $""" + SELECT COUNT(*) + FROM '{DataTable.TableName:raw}' + """ + ).QuerySingleAsync(); + + Assert.Equal(testData.ExpectedTotalResults, actualRowCount); + } + + [Theory] + [MemberData(nameof(Data))] + public async Task DuckDbDataTable_CorrectColumns(ProcessorTestData testData) { var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); - SetupCsvDataFilesForDataSetVersion(ProcessorTestData.AbsenceSchool, dataSetVersion); + await ImportData(testData, dataSetVersion, instanceId); - // Prepare the metadata and data before calling the WriteDataFiles function + await using var duckDbConnection = GetDuckDbConnection(dataSetVersion); + + var actualColumns = (await duckDbConnection.SqlBuilder( + $"DESCRIBE SELECT * FROM '{DataTable.TableName:raw}' LIMIT 1") + .QueryAsync()) + .Select(col => col.ColumnName) + .ToList(); + + string[] expectedColumns = + [ + DataTable.Cols.Id, + DataTable.Cols.GeographicLevel, + DataTable.Cols.TimePeriodId, + ..testData.ExpectedGeographicLevels.Select(DataTable.Cols.LocationId), + ..testData.ExpectedFilters.Select(fm => DataTable.Cols.Filter(fm).Trim('"')), + ..testData.ExpectedIndicators.Select(im => DataTable.Cols.Indicator(im).Trim('"')), + ]; + + Assert.Equal(expectedColumns.Order(), actualColumns.Order()); + } + + [Theory] + [MemberData(nameof(Data))] + public async Task DuckDbDataTable_CorrectDistinctGeographicLevels(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportData(testData, dataSetVersion, instanceId); + + await using var duckDbConnection = GetDuckDbConnection(dataSetVersion); + + var geographicLevelsCommand = duckDbConnection.SqlBuilder( + $""" + SELECT DISTINCT {DataTable.Ref().GeographicLevel:raw} + FROM '{DataTable.TableName:raw}' + ORDER BY {DataTable.Ref().GeographicLevel:raw} + """ + ); + + var actualGeographicLevels = (await geographicLevelsCommand.QueryAsync()) + .Select(EnumUtil.GetFromEnumLabel) + .AsList(); + + Assert.Equal(testData.ExpectedGeographicLevels.Order(), actualGeographicLevels.Order()); + } + + [Theory] + [MemberData(nameof(Data))] + public async Task DuckDbDataTable_CorrectDistinctLocationOptions(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportData(testData, dataSetVersion, instanceId); + + await using var duckDbConnection = GetDuckDbConnection(dataSetVersion); + + await Assert.AllAsync(testData.ExpectedLocations, + async expectedLocation => + { + var optionsCommand = duckDbConnection.SqlBuilder( + $""" + SELECT DISTINCT {LocationOptionsTable.TableName:raw}.* + FROM '{DataTable.TableName:raw}' JOIN '{LocationOptionsTable.TableName:raw}' + ON {DataTable.Ref().LocationId(expectedLocation.Level):raw} = {LocationOptionsTable.Ref().Id:raw} + ORDER BY {LocationOptionsTable.Ref().Label:raw} + """ + ); + + var actualOptions = (await optionsCommand.QueryAsync()).AsList(); + + Assert.Equal(expectedLocation.Options.Count, actualOptions.Count); + Assert.All(expectedLocation.Options.OrderBy(o => o.Label), + (expectedOption, index) => expectedOption.AssertEqual(actualOptions[index])); + }); + } + + [Theory] + [MemberData(nameof(Data))] + public async Task DuckDbDataTable_CorrectDistinctFilterOptions(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportData(testData, dataSetVersion, instanceId); + + await using var duckDbConnection = GetDuckDbConnection(dataSetVersion); + + await Assert.AllAsync(testData.ExpectedFilters, + async expectedFilter => + { + var optionsCommand = duckDbConnection.SqlBuilder( + $""" + SELECT DISTINCT {FilterOptionsTable.TableName:raw}.* + FROM '{DataTable.TableName:raw}' JOIN '{FilterOptionsTable.TableName:raw}' + ON {DataTable.Ref().Col(expectedFilter.PublicId):raw} = {FilterOptionsTable.Ref().Id:raw} + ORDER BY {FilterOptionsTable.Ref().Label:raw} + """ + ); + + var actualOptions = (await optionsCommand.QueryAsync()).AsList(); + + Assert.Equal(expectedFilter.Options.Count, actualOptions.Count); + Assert.All(expectedFilter.Options, + (expectedOption, index) => + { + var actualOption = actualOptions[index]; + Assert.Equal(expectedOption.Label, actualOption.Label); + Assert.Equal(expectedFilter.PublicId, actualOption.FilterId); + }); + }); + } + + [Theory] + [MemberData(nameof(Data))] + public async Task DuckDbDataTable_CorrectDistinctTimePeriods(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await ImportData(testData, dataSetVersion, instanceId); + + await using var duckDbConnection = GetDuckDbConnection(dataSetVersion); + + var timePeriodsCommand = duckDbConnection.SqlBuilder( + $""" + SELECT DISTINCT {TimePeriodsTable.TableName:raw}.* + FROM '{DataTable.TableName:raw}' JOIN '{TimePeriodsTable.TableName:raw}' + ON {DataTable.Ref().TimePeriodId:raw} = {TimePeriodsTable.Ref().Id:raw} + ORDER BY {TimePeriodsTable.Ref().Period:raw} + """ + ); + + var actualTimePeriods = ( + await timePeriodsCommand.QueryAsync() + ).AsList(); + + Assert.Equal(testData.ExpectedTimePeriods.Count, actualTimePeriods.Count); + Assert.All(testData.ExpectedTimePeriods, + (expectedTimePeriod, index) => + { + var actualTimePeriod = actualTimePeriods[index]; + Assert.Equal(expectedTimePeriod.Code, + EnumUtil.GetFromEnumLabel(actualTimePeriod.Identifier)); + Assert.Equal(expectedTimePeriod.Period, TimePeriodFormatter.FormatFromCsv(actualTimePeriod.Period)); + }); + } + + private async Task ImportData( + ProcessorTestData testData, + DataSetVersion dataSetVersion, + Guid instanceId) + { + SetupCsvDataFilesForDataSetVersion(testData, dataSetVersion); + + // Prepare the metadata before calling the ImportData function var importMetadataFunction = GetRequiredService(); await importMetadataFunction.ImportMetadata(instanceId, CancellationToken.None); - var importDataFunction = GetRequiredService(); - await importDataFunction.ImportData(instanceId, CancellationToken.None); - var processInitialDataSetVersionFunction = GetRequiredService(); - await processInitialDataSetVersionFunction.WriteDataFiles(instanceId, CancellationToken.None); + await processInitialDataSetVersionFunction.ImportData(instanceId, CancellationToken.None); + } + } + + public class WriteDataFilesTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessInitialDataSetVersionFunctionTests(fixture) + { + private const DataSetVersionImportStage Stage = DataSetVersionImportStage.WritingDataFiles; + + public static readonly TheoryData Data = new() + { + ProcessorTestData.AbsenceSchool, + }; + + [Theory] + [MemberData(nameof(Data))] + public async Task Success(ProcessorTestData testData) + { + var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + + await WriteData(testData, dataSetVersion, instanceId); await using var publicDataDbContext = GetDbContext(); @@ -206,9 +397,28 @@ public async Task Success() AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, _allDataSetVersionFiles); } + + private async Task WriteData( + ProcessorTestData testData, + DataSetVersion dataSetVersion, + Guid instanceId) + { + SetupCsvDataFilesForDataSetVersion(testData, dataSetVersion); + + // Prepare the metadata and data before calling the WriteDataFiles function + var importMetadataFunction = GetRequiredService(); + await importMetadataFunction.ImportMetadata(instanceId, CancellationToken.None); + + var importDataFunction = GetRequiredService(); + await importDataFunction.ImportData(instanceId, CancellationToken.None); + + var processInitialDataSetVersionFunction = GetRequiredService(); + await processInitialDataSetVersionFunction.WriteDataFiles(instanceId, CancellationToken.None); + } } - public class CompleteProcessingTests(ProcessorFunctionsIntegrationTestFixture fixture) + public class CompleteInitialDataSetVersionProcessingTests( + ProcessorFunctionsIntegrationTestFixture fixture) : ProcessInitialDataSetVersionFunctionTests(fixture) { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.Completing; @@ -261,7 +471,7 @@ public async Task DuckDbFileIsDeleted() private async Task CompleteProcessing(Guid instanceId) { var function = GetRequiredService(); - await function.CompleteProcessing(instanceId, CancellationToken.None); + await function.CompleteInitialDataSetVersionProcessing(instanceId, CancellationToken.None); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionFunctionTests.cs new file mode 100644 index 00000000000..63a94e038ee --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionFunctionTests.cs @@ -0,0 +1,1305 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using Microsoft.DurableTask; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Moq.Protected; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class ProcessNextDataSetVersionFunctionTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessorFunctionsIntegrationTest(fixture) +{ + public class ProcessNextDataSetVersionTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessNextDataSetVersionFunctionTests(fixture) + { + [Fact] + public async Task Success() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + var activitySequence = new MockSequence(); + + string[] expectedActivitySequence = + [ + ActivityNames.CopyCsvFiles, + ActivityNames.CreateMappings, + ActivityNames.ApplyAutoMappings, + ActivityNames.CompleteNextDataSetVersionMappingProcessing, + ]; + + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + } + + await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + [Fact] + public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + var activitySequence = new MockSequence(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.CopyCsvFiles, + mockOrchestrationContext.Object.InstanceId, + null)) + .Throws(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.HandleProcessingFailure, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + + await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + private async Task ProcessNextDataSetVersion(TaskOrchestrationContext orchestrationContext) + { + var function = GetRequiredService(); + await function.ProcessNextDataSetVersion( + orchestrationContext, + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); + } + + private static Mock DefaultMockOrchestrationContext( + Guid? instanceId = null, + bool isReplaying = false) + { + var mock = new Mock(MockBehavior.Strict); + + mock + .Protected() + .Setup("LoggerFactory") + .Returns(NullLoggerFactory.Instance); + + mock + .SetupGet(context => context.InstanceId) + .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); + + mock + .SetupGet(context => context.IsReplaying) + .Returns(isReplaying); + + return mock; + } + } + + public abstract class CreateMappingsTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessNextDataSetVersionFunctionTests(fixture) + { + protected const DataSetVersionImportStage Stage = DataSetVersionImportStage.CreatingMappings; + + protected async Task CreateMappings(Guid instanceId) + { + var function = GetRequiredService(); + await function.CreateMappings(instanceId, CancellationToken.None); + } + } + + public class CreateMappingMiscTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : CreateMappingsTests(fixture) + { + [Fact] + public async Task Success_ImportStatus() + { + var (instanceId, _, _) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + await CreateMappings(instanceId); + + var savedImport = await GetDbContext() + .DataSetVersionImports + .Include(dataSetVersionImport => dataSetVersionImport.DataSetVersion) + .SingleAsync(i => i.InstanceId == instanceId); + + Assert.Equal(Stage, savedImport.Stage); + Assert.Equal(DataSetVersionStatus.Processing, savedImport.DataSetVersion.Status); + } + + [Fact] + public async Task Success_MetaSummary() + { + var (instanceId, _, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + await CreateMappings(instanceId); + + var updatedDataSetVersion = GetDataSetVersion(nextVersion); + + // Assert that the MetaSummary has been generated correctly from the CSV. + var metaSummary = updatedDataSetVersion.MetaSummary; + Assert.NotNull(metaSummary); + + Assert.Equal( + ProcessorTestData + .AbsenceSchool + .ExpectedLocations + .Select(level => level.Level) + .Order(), + metaSummary + .GeographicLevels + .Order()); + + Assert.Equal( + ProcessorTestData + .AbsenceSchool + .ExpectedFilters + .Select(filterAndOptions => filterAndOptions.Label) + .Order(), + metaSummary + .Filters + .Order()); + + Assert.Equal( + ProcessorTestData + .AbsenceSchool + .ExpectedIndicators + .Select(l => l.Label) + .Order(), + metaSummary + .Indicators.Order()); + + Assert.Equal( + new TimePeriodRange + { + Start = TimePeriodRangeBound.Create( + ProcessorTestData + .AbsenceSchool + .ExpectedTimePeriods[0]), + End = TimePeriodRangeBound.Create( + ProcessorTestData + .AbsenceSchool + .ExpectedTimePeriods[^1]) + }, + metaSummary.TimePeriodRange); + } + } + + public class CreateMappingsLocationsTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : CreateMappingsTests(fixture) + { + [Fact] + public async Task Success_Mappings() + { + var (instanceId, initialVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + // Select a set of LocationOptions that rely on all the available Location + // Code fields. + var initialLocationMeta = DataFixture + .DefaultLocationMeta() + .WithDataSetVersionId(initialVersion.Id) + .ForIndex(0, s => s + .SetLevel(GeographicLevel.LocalAuthority) + .SetOptions(DataFixture + .DefaultLocationLocalAuthorityOptionMeta() + .GenerateList(2) + .Select(meta => meta as LocationOptionMeta) + .ToList())) + .ForIndex(1, s => s + .SetLevel(GeographicLevel.School) + .SetOptions(DataFixture + .DefaultLocationSchoolOptionMeta() + .GenerateList(2) + .Select(meta => meta as LocationOptionMeta) + .ToList())) + .ForIndex(2, s => s + .SetLevel(GeographicLevel.Provider) + .SetOptions(DataFixture + .DefaultLocationProviderOptionMeta() + .GenerateList(2) + .Select(meta => meta as LocationOptionMeta) + .ToList())) + .GenerateList(); + + await AddTestData(context => + context.LocationMetas.AddRange(initialLocationMeta)); + + await CreateMappings(instanceId); + + var mappings = GetDataSetVersionMapping(nextVersion); + + Assert.Equal(initialVersion.Id, mappings.SourceDataSetVersionId); + Assert.Equal(nextVersion.Id, mappings.TargetDataSetVersionId); + Assert.False(mappings.LocationMappingsComplete); + + var expectedLocationMappingsFromSource = initialLocationMeta + .ToDictionary( + levelMeta => levelMeta.Level, + levelMeta => new LocationLevelMappings + { + Mappings = levelMeta + .Options + .ToDictionary( + keySelector: option => $"{option.Label} :: {option.ToRow().GetRowKey()}", + elementSelector: option => new LocationOptionMapping + { + CandidateKey = null, + Type = MappingType.None, + Source = new MappableLocationOption(option.Label) + { + Code = option.ToRow().Code, + OldCode = option.ToRow().OldCode, + Urn = option.ToRow().Urn, + LaEstab = option.ToRow().LaEstab, + Ukprn = option.ToRow().Ukprn + } + }) + }); + + // There should be 5 levels of mappings when combining all the source and target levels. + Assert.Equal(ProcessorTestData + .AbsenceSchool + .ExpectedGeographicLevels + .Concat([GeographicLevel.Provider]) + .Order(), + mappings.LocationMappingPlan + .Levels + .Select(level => level.Key) + .Order()); + + mappings.LocationMappingPlan.Levels.ForEach(level => + { + var matchingLevelFromSource = expectedLocationMappingsFromSource.GetValueOrDefault(level.Key); + + if (matchingLevelFromSource != null) + { + level.Value.Mappings.AssertDeepEqualTo( + matchingLevelFromSource.Mappings, + ignoreCollectionOrders: true); + } + else + { + Assert.Empty(level.Value.Mappings); + } + }); + } + + [Fact] + public async Task Success_Candidates() + { + var (instanceId, initialVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + await CreateMappings(instanceId); + + var mappings = GetDataSetVersionMapping(nextVersion); + + Assert.Equal(initialVersion.Id, mappings.SourceDataSetVersionId); + Assert.Equal(nextVersion.Id, mappings.TargetDataSetVersionId); + + var expectedLocationLevels = ProcessorTestData + .AbsenceSchool + .ExpectedLocations + .Select(levelMeta => (level: levelMeta.Level, options: levelMeta.Options)) + .ToDictionary( + keySelector: levelMeta => levelMeta.level, + elementSelector: levelMeta => + new LocationLevelMappings + { + Candidates = levelMeta + .options + .ToDictionary( + keySelector: option => $"{option.Label} :: {option.ToRow().GetRowKey()}", + elementSelector: option => new MappableLocationOption(option.Label) + { + Code = option.ToRow().Code, + OldCode = option.ToRow().OldCode, + Urn = option.ToRow().Urn, + LaEstab = option.ToRow().LaEstab, + Ukprn = option.ToRow().Ukprn + }) + }); + + mappings.LocationMappingPlan.Levels.AssertDeepEqualTo( + expectedLocationLevels, + ignoreCollectionOrders: true); + } + } + + public class CreateMappingsFiltersTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : CreateMappingsTests(fixture) + { + [Fact] + public async Task Success_Mappings() + { + var (instanceId, initialVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + var initialFilterMeta = DataFixture + .DefaultFilterMeta() + .WithDataSetVersionId(initialVersion.Id) + .WithOptions(() => DataFixture + .DefaultFilterOptionMeta() + .GenerateList(2)) + .GenerateList(2); + + await AddTestData(context => + context.FilterMetas.AddRange(initialFilterMeta)); + + await CreateMappings(instanceId); + + var mappings = GetDataSetVersionMapping(nextVersion); + + Assert.Equal(initialVersion.Id, mappings.SourceDataSetVersionId); + Assert.Equal(nextVersion.Id, mappings.TargetDataSetVersionId); + Assert.False(mappings.FilterMappingsComplete); + + var expectedFilterMappings = initialFilterMeta + .ToDictionary( + keySelector: filter => filter.PublicId, + elementSelector: filter => + new FilterMapping + { + CandidateKey = null, + Type = MappingType.None, + Source = new MappableFilter(filter.Label), + OptionMappings = filter + .Options + .ToDictionary( + keySelector: option => option.Label, + elementSelector: option => + new FilterOptionMapping + { + CandidateKey = null, + Type = MappingType.None, + Source = new MappableFilterOption(option.Label) + }) + }); + + mappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( + expectedFilterMappings, + ignoreCollectionOrders: true); + } + + [Fact] + public async Task Success_Candidates() + { + var (instanceId, initialVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + await CreateMappings(instanceId); + + var mappings = GetDataSetVersionMapping(nextVersion); + + Assert.Equal(initialVersion.Id, mappings.SourceDataSetVersionId); + Assert.Equal(nextVersion.Id, mappings.TargetDataSetVersionId); + + var expectedFilterTargets = ProcessorTestData + .AbsenceSchool + .ExpectedFilters + .ToDictionary( + keySelector: filterAndOptions => filterAndOptions.PublicId, + elementSelector: filterAndOptions => + new FilterMappingCandidate(filterAndOptions.Label) + { + Options = filterAndOptions + .Options + .ToDictionary( + keySelector: optionMeta => optionMeta.Label, + elementSelector: optionMeta => + new MappableFilterOption(optionMeta.Label)) + }); + + mappings.FilterMappingPlan.Candidates.AssertDeepEqualTo( + expectedFilterTargets, + ignoreCollectionOrders: true); + } + } + + public abstract class ApplyAutoMappingsTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessNextDataSetVersionFunctionTests(fixture) + { + protected const DataSetVersionImportStage Stage = DataSetVersionImportStage.AutoMapping; + + protected async Task ApplyAutoMappings(Guid instanceId) + { + var function = GetRequiredService(); + await function.ApplyAutoMappings(instanceId, CancellationToken.None); + } + } + + public class ApplyAutoMappingsMiscTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ApplyAutoMappingsTests(fixture) + { + [Fact] + public async Task Success_ImportStatus() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + await AddTestData(context => + context.DataSetVersionMappings.Add(new DataSetVersionMapping + { + SourceDataSetVersionId = originalVersion.Id, + TargetDataSetVersionId = nextVersion.Id, + LocationMappingPlan = new LocationMappingPlan(), + FilterMappingPlan = new FilterMappingPlan() + })); + + await ApplyAutoMappings(instanceId); + + var savedImport = await GetDbContext() + .DataSetVersionImports + .Include(dataSetVersionImport => dataSetVersionImport.DataSetVersion) + .SingleAsync(i => i.InstanceId == instanceId); + + Assert.Equal(Stage, savedImport.Stage); + Assert.Equal(DataSetVersionStatus.Processing, savedImport.DataSetVersion.Status); + } + } + + public class ApplyAutoMappingsLocationsTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ApplyAutoMappingsTests(fixture) + { + [Fact] + public async Task PartiallyComplete() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + // Create a mapping plan based on 2 data set versions with partially overlapping locations and levels. + // Both have the "Local Authority" level and the "LA location 1" option, but the source has an additional + // "LA location 2" option that the target does not, and the target has an additional "LA location 3" option + // that the source does not. + // + // In addition to this, the source has a "RSC Region" level that the target does not have, and the target + // has a "Country" level that the source does not have. + var mappings = new DataSetVersionMapping + { + SourceDataSetVersionId = originalVersion.Id, + TargetDataSetVersionId = nextVersion.Id, + FilterMappingPlan = new FilterMappingPlan(), + LocationMappingPlan = new LocationMappingPlan + { + Levels = + { + { + GeographicLevel.LocalAuthority, new LocationLevelMappings + { + Mappings = + { + { + "LA location 1 key", + new LocationOptionMapping + { + Source = new MappableLocationOption("LA location 1 label") + } + }, + { + "LA location 2 key", + new LocationOptionMapping + { + Source = new MappableLocationOption("LA location 2 label") + } + } + }, + Candidates = + { + { "LA location 1 key", new MappableLocationOption("LA location 1 label") }, + { "LA location 3 key", new MappableLocationOption("LA location 3 label") }, + } + } + }, + { + GeographicLevel.RscRegion, + new LocationLevelMappings + { + Mappings = + { + { + "RSC location 1 key", + new LocationOptionMapping + { + Source = new MappableLocationOption("RSC location 1 label") + } + } + }, + Candidates = [] + } + }, + { + GeographicLevel.Country, new LocationLevelMappings + { + Mappings = [], + Candidates = + { + { "Country location 1 key", new MappableLocationOption("Country location 1 label") } + } + } + } + } + } + }; + + await AddTestData(context => + context.DataSetVersionMappings.Add(mappings)); + + await ApplyAutoMappings(instanceId); + + var updatedMappings = GetDataSetVersionMapping(nextVersion); + + Assert.False(updatedMappings.LocationMappingsComplete); + + Dictionary expectedLevelMappings = new() + { + { + GeographicLevel.LocalAuthority, new LocationLevelMappings + { + Mappings = + { + { + "LA location 1 key", + new LocationOptionMapping + { + Source = new MappableLocationOption("LA location 1 label"), + Type = MappingType.AutoMapped, + CandidateKey = "LA location 1 key" + } + }, + { + "LA location 2 key", + new LocationOptionMapping + { + Source = new MappableLocationOption("LA location 2 label"), + Type = MappingType.AutoNone, + CandidateKey = null + } + } + }, + Candidates = + { + { "LA location 1 key", new MappableLocationOption("LA location 1 label") }, + { "LA location 3 key", new MappableLocationOption("LA location 3 label") }, + } + } + }, + { + GeographicLevel.RscRegion, + new LocationLevelMappings + { + Mappings = + { + { + "RSC location 1 key", + new LocationOptionMapping + { + Source = new MappableLocationOption("RSC location 1 label"), + Type = MappingType.AutoNone, + CandidateKey = null + } + } + }, + Candidates = [] + } + }, + { + GeographicLevel.Country, new LocationLevelMappings + { + Mappings = [], + Candidates = + { + { "Country location 1 key", new MappableLocationOption("Country location 1 label") } + } + } + } + }; + + updatedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo( + expectedLevelMappings, + ignoreCollectionOrders: true); + } + + [Fact] + public async Task Complete_ExactMatch() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + // Create a mapping plan based on 2 data set versions with perfectly overlapping + // locations and levels. + var mappings = new DataSetVersionMapping + { + SourceDataSetVersionId = originalVersion.Id, + TargetDataSetVersionId = nextVersion.Id, + FilterMappingPlan = new FilterMappingPlan(), + LocationMappingPlan = new LocationMappingPlan + { + Levels = + { + { + GeographicLevel.LocalAuthority, + new LocationLevelMappings + { + Mappings = + { + { + "LA location 1 key", + new LocationOptionMapping + { + Source = new MappableLocationOption("LA location 1 label") + } + } + }, + Candidates = + { + { "LA location 1 key", new MappableLocationOption("LA location 1 label") } + } + } + } + } + } + }; + + await AddTestData(context => + context.DataSetVersionMappings.Add(mappings)); + + await ApplyAutoMappings(instanceId); + + var updatedMappings = GetDataSetVersionMapping(nextVersion); + + Assert.True(updatedMappings.LocationMappingsComplete); + + Dictionary expectedLevelMappings = new() + { + { + GeographicLevel.LocalAuthority, + new LocationLevelMappings + { + Mappings = + { + { + "LA location 1 key", + new LocationOptionMapping + { + Source = new MappableLocationOption("LA location 1 label"), + Type = MappingType.AutoMapped, + CandidateKey = "LA location 1 key" + } + } + }, + Candidates = { { "LA location 1 key", new MappableLocationOption("LA location 1 label") } } + } + } + }; + + updatedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo(expectedLevelMappings); + } + + [Fact] + public async Task Complete_AllSourcesMapped_OtherUnmappedCandidatesExist() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + // Create a mapping plan based on 2 data set versions with the same levels + // and location options, but additional options exist in the new version. + // Each source location option can be auto-mapped exactly to one in + // the target version, leaving some candidates and new levels unused but + // essentially the mapping is complete unless the user manually intervenes + // at this point. + var mappings = new DataSetVersionMapping + { + SourceDataSetVersionId = originalVersion.Id, + TargetDataSetVersionId = nextVersion.Id, + FilterMappingPlan = new FilterMappingPlan(), + LocationMappingPlan = new LocationMappingPlan + { + Levels = + { + { + GeographicLevel.LocalAuthority, + new LocationLevelMappings + { + Mappings = + { + { + "LA location 1 key", + new LocationOptionMapping + { + Source = new MappableLocationOption("LA location 1 label") + } + } + }, + Candidates = + { + { "LA location 1 key", new MappableLocationOption("LA location 1 label") }, + { "LA location 2 key", new MappableLocationOption("LA location 2 label") } + } + } + }, + { + GeographicLevel.RscRegion, new LocationLevelMappings + { + Mappings = [], + Candidates = + { + { "RSC location 1 key", new MappableLocationOption("RSC location 1 label") } + } + } + } + } + } + }; + + await AddTestData(context => + context.DataSetVersionMappings.Add(mappings)); + + await ApplyAutoMappings(instanceId); + + var updatedMappings = GetDataSetVersionMapping(nextVersion); + + Assert.True(updatedMappings.LocationMappingsComplete); + + Dictionary expectedLevelMappings = new() + { + { + GeographicLevel.LocalAuthority, + new LocationLevelMappings + { + Mappings = + { + { + "LA location 1 key", + new LocationOptionMapping + { + Source = new MappableLocationOption("LA location 1 label"), + Type = MappingType.AutoMapped, + CandidateKey = "LA location 1 key" + } + } + }, + Candidates = + { + { "LA location 1 key", new MappableLocationOption("LA location 1 label") }, + { "LA location 2 key", new MappableLocationOption("LA location 2 label") } + } + } + }, + { + GeographicLevel.RscRegion, new LocationLevelMappings + { + Mappings = [], + Candidates = { { "RSC location 1 key", new MappableLocationOption("RSC location 1 label") } } + } + } + }; + + updatedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo( + expectedLevelMappings, + ignoreCollectionOrders: true); + } + } + + public class ApplyAutoMappingsFiltersTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ApplyAutoMappingsTests(fixture) + { + [Fact] + public async Task PartiallyComplete() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + // Create a mapping plan based on 2 data set versions with partially overlapping filters. + // Both have "Filter 1" and both have "Filter 1 option 1", but then each also contains Filter 1 + // options that the other do not, and each also contains filters that the other does not. + var mappings = new DataSetVersionMapping + { + SourceDataSetVersionId = originalVersion.Id, + TargetDataSetVersionId = nextVersion.Id, + FilterMappingPlan = new FilterMappingPlan + { + Mappings = + { + { + "Filter 1 key", new FilterMapping + { + Source = new MappableFilter("Filter 1 label"), + OptionMappings = + { + { + "Filter 1 option 1 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 1 option 1 label") + } + }, + { + "Filter 1 option 2 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 1 option 2 label") + } + } + } + } + }, + { + "Filter 2 key", new FilterMapping + { + Source = new MappableFilter("Filter 2 label"), + OptionMappings = + { + { + "Filter 2 option 1 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 2 option 1 label") + } + } + } + } + } + }, + Candidates = + { + { + "Filter 1 key", + new FilterMappingCandidate("Filter 1 label") + { + Options = + { + { + "Filter 1 option 1 key", new MappableFilterOption("Filter 1 option 1 label") + }, + { "Filter 1 option 3 key", new MappableFilterOption("Filter 1 option 3 label") } + } + } + } + } + }, + LocationMappingPlan = new LocationMappingPlan() + }; + + await AddTestData(context => + context.DataSetVersionMappings.Add(mappings)); + + await ApplyAutoMappings(instanceId); + + var updatedMappings = GetDataSetVersionMapping(nextVersion); + + Assert.False(updatedMappings.FilterMappingsComplete); + + Dictionary expectedFilterMappings = new() + { + { + "Filter 1 key", new FilterMapping + { + Source = new MappableFilter("Filter 1 label"), + Type = MappingType.AutoMapped, + CandidateKey = "Filter 1 key", + OptionMappings = + { + { + "Filter 1 option 1 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 1 option 1 label"), + Type = MappingType.AutoMapped, + CandidateKey = "Filter 1 option 1 key" + } + }, + { + "Filter 1 option 2 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 1 option 2 label"), + Type = MappingType.AutoNone, + CandidateKey = null + } + } + } + } + }, + { + "Filter 2 key", new FilterMapping + { + Source = new MappableFilter("Filter 2 label"), + Type = MappingType.AutoNone, + CandidateKey = null, + OptionMappings = + { + { + "Filter 2 option 1 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 2 option 1 label"), + Type = MappingType.AutoNone, + CandidateKey = null + } + } + } + } + } + }; + + updatedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( + expectedFilterMappings, + ignoreCollectionOrders: true); + } + + [Fact] + public async Task Complete_ExactMatch() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + // Create a mapping plan based on 2 data set versions with exactly the same filters + // and filter options. + + var mappings = new DataSetVersionMapping + { + SourceDataSetVersionId = originalVersion.Id, + TargetDataSetVersionId = nextVersion.Id, + FilterMappingPlan = new FilterMappingPlan + { + Mappings = + { + { + "Filter 1 key", new FilterMapping + { + Source = new MappableFilter("Filter 1 label"), + OptionMappings = + { + { + "Filter 1 option 1 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 1 option 1 label") + } + }, + { + "Filter 1 option 2 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 1 option 2 label") + } + } + } + } + }, + { + "Filter 2 key", new FilterMapping + { + Source = new MappableFilter("Filter 2 label"), + OptionMappings = + { + { + "Filter 2 option 1 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 2 option 1 label") + } + } + } + } + } + }, + Candidates = + { + { + "Filter 1 key", + new FilterMappingCandidate("Filter 1 label") + { + Options = + { + { + "Filter 1 option 1 key", new MappableFilterOption("Filter 1 option 1 label") + }, + { "Filter 1 option 2 key", new MappableFilterOption("Filter 1 option 2 label") } + } + } + }, + { + "Filter 2 key", + new FilterMappingCandidate("Filter 2 label") + { + Options = + { + { + "Filter 2 option 1 key", new MappableFilterOption("Filter 2 option 1 label") + } + } + } + } + } + }, + LocationMappingPlan = new LocationMappingPlan() + }; + await AddTestData(context => + context.DataSetVersionMappings.Add(mappings)); + + await ApplyAutoMappings(instanceId); + + var updatedMappings = GetDataSetVersionMapping(nextVersion); + + Assert.True(updatedMappings.FilterMappingsComplete); + + Dictionary expectedFilterMappings = new() + { + { + "Filter 1 key", new FilterMapping + { + Source = new MappableFilter("Filter 1 label"), + Type = MappingType.AutoMapped, + CandidateKey = "Filter 1 key", + OptionMappings = + { + { + "Filter 1 option 1 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 1 option 1 label"), + Type = MappingType.AutoMapped, + CandidateKey = "Filter 1 option 1 key" + } + }, + { + "Filter 1 option 2 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 1 option 2 label"), + Type = MappingType.AutoMapped, + CandidateKey = "Filter 1 option 2 key" + } + } + } + } + }, + { + "Filter 2 key", new FilterMapping + { + Source = new MappableFilter("Filter 2 label"), + Type = MappingType.AutoMapped, + CandidateKey = "Filter 2 key", + OptionMappings = + { + { + "Filter 2 option 1 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 2 option 1 label"), + Type = MappingType.AutoMapped, + CandidateKey = "Filter 2 option 1 key" + } + } + } + } + } + }; + + updatedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo(expectedFilterMappings); + } + + [Fact] + public async Task Complete_AllSourcesMapped_OtherUnmappedCandidatesExist() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + // Create a mapping plan based on 2 data set versions with the same filters + // and filter options, but additional options exist in the new version. + // Each source filter and filter option can be auto-mapped exactly to one in + // the target version, leaving some candidates unused but essentially the mapping + // is complete unless the user manually intervenes at this point. + var mappings = new DataSetVersionMapping + { + SourceDataSetVersionId = originalVersion.Id, + TargetDataSetVersionId = nextVersion.Id, + FilterMappingPlan = new FilterMappingPlan + { + Mappings = + { + { + "Filter 1 key", new FilterMapping + { + Source = new MappableFilter("Filter 1 label"), + OptionMappings = + { + { + "Filter 1 option 1 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 1 option 1 label") + } + } + } + } + } + }, + Candidates = + { + { + "Filter 1 key", + new FilterMappingCandidate("Filter 1 label") + { + Options = + { + { + "Filter 1 option 1 key", new MappableFilterOption("Filter 1 option 1 label") + }, + { "Filter 1 option 2 key", new MappableFilterOption("Filter 1 option 2 label") } + } + } + }, + { + "Filter 2 key", + new FilterMappingCandidate("Filter 2 label") + { + Options = + { + { + "Filter 2 option 1 key", new MappableFilterOption("Filter 2 option 1 label") + } + } + } + } + } + }, + LocationMappingPlan = new LocationMappingPlan() + }; + + await AddTestData(context => + context.DataSetVersionMappings.Add(mappings)); + + await ApplyAutoMappings(instanceId); + + var updatedMappings = GetDataSetVersionMapping(nextVersion); + + Assert.True(updatedMappings.FilterMappingsComplete); + + Dictionary expectedFilterMappings = new() + { + { + "Filter 1 key", new FilterMapping + { + Source = new MappableFilter("Filter 1 label"), + Type = MappingType.AutoMapped, + CandidateKey = "Filter 1 key", + OptionMappings = + { + { + "Filter 1 option 1 key", + new FilterOptionMapping + { + Source = new MappableFilterOption("Filter 1 option 1 label"), + Type = MappingType.AutoMapped, + CandidateKey = "Filter 1 option 1 key" + } + } + } + } + } + }; + + updatedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( + expectedFilterMappings, + ignoreCollectionOrders: true); + } + } + + public class CompleteNextDataSetVersionMappingProcessingTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessNextDataSetVersionFunctionTests(fixture) + { + private const DataSetVersionImportStage Stage = DataSetVersionImportStage.Completing; + + [Fact] + public async Task Success() + { + var (instanceId, _, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + var dataSetVersionPathResolver = GetRequiredService(); + Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(nextVersion)); + + await CompleteProcessing(instanceId); + + await using var publicDataDbContext = GetDbContext(); + + var savedImport = await publicDataDbContext.DataSetVersionImports + .Include(i => i.DataSetVersion) + .SingleAsync(i => i.InstanceId == instanceId); + + Assert.Equal(Stage, savedImport.Stage); + savedImport.Completed.AssertUtcNow(); + + Assert.Equal(DataSetVersionStatus.Mapping, savedImport.DataSetVersion.Status); + } + + private async Task CompleteProcessing(Guid instanceId) + { + var function = GetRequiredService(); + await function.CompleteNextDataSetVersionMappingProcessing(instanceId, CancellationToken.None); + } + } + + private async Task<(Guid instanceId, DataSetVersion initialVersion, DataSetVersion nextVersion)> + CreateNextDataSetVersionAndDataFiles(DataSetVersionImportStage importStage) + { + var (initialDataSetVersion, _) = await CreateDataSet( + importStage: DataSetVersionImportStage.Completing, + status: DataSetVersionStatus.Published); + + var dataSet = await GetDbContext().DataSets.SingleAsync(dataSet => + dataSet.Id == initialDataSetVersion.DataSet.Id); + + var (nextDataSetVersion, instanceId) = await CreateDataSetVersionAndImport( + dataSet: dataSet, + importStage: importStage, + versionMajor: 1, + versionMinor: 1); + + SetupCsvDataFilesForDataSetVersion(ProcessorTestData.AbsenceSchool, nextDataSetVersion); + return (instanceId, initialDataSetVersion, nextDataSetVersion); + } + + private DataSetVersion GetDataSetVersion(DataSetVersion nextVersion) + { + return GetDbContext() + .DataSetVersions + .Single(dsv => dsv.Id == nextVersion.Id); + } + + private DataSetVersionMapping GetDataSetVersionMapping(DataSetVersion nextVersion) + { + return GetDbContext() + .DataSetVersionMappings + .Single(mapping => mapping.TargetDataSetVersionId == nextVersion.Id); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs index 2aab19df5bb..7af979819da 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs @@ -3,6 +3,7 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; @@ -10,21 +11,23 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Testcontainers.Azurite; using Testcontainers.PostgreSql; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests; -public abstract class ProcessorFunctionsIntegrationTest - : FunctionsIntegrationTest, IDisposable +public abstract class ProcessorFunctionsIntegrationTest( + FunctionsIntegrationTestFixture fixture) + : FunctionsIntegrationTest(fixture), IAsyncLifetime { - protected ProcessorFunctionsIntegrationTest(FunctionsIntegrationTestFixture fixture) : base(fixture) + public async Task InitializeAsync() { ResetDbContext(); - ClearTestData(); + await ClearTestData(); } - public void Dispose() + public Task DisposeAsync() { var dataSetVersionPathResolver = GetRequiredService(); @@ -33,9 +36,12 @@ public void Dispose() { Directory.Delete(testInstanceDataFilesDirectory, recursive: true); } + + return Task.CompletedTask; } - protected void SetupCsvDataFilesForDataSetVersion(ProcessorTestData processorTestData, DataSetVersion dataSetVersion) + protected void SetupCsvDataFilesForDataSetVersion(ProcessorTestData processorTestData, + DataSetVersion dataSetVersion) { var dataSetVersionPathResolver = GetRequiredService(); @@ -61,6 +67,17 @@ protected void SetupCsvDataFilesForDataSetVersion(ProcessorTestData processorTes await AddTestData(context => context.DataSets.Add(dataSet)); + return await CreateDataSetVersionAndImport(dataSet, importStage, status, releaseFileId); + } + + protected async Task<(DataSetVersion dataSetVersion, Guid instanceId)> CreateDataSetVersionAndImport( + DataSet dataSet, + DataSetVersionImportStage importStage, + DataSetVersionStatus? status = null, + Guid? releaseFileId = null, + int versionMajor = 1, + int versionMinor = 0) + { DataSetVersionImport dataSetVersionImport = DataFixture .DefaultDataSetVersionImport() .WithStage(importStage); @@ -71,7 +88,18 @@ protected void SetupCsvDataFilesForDataSetVersion(ProcessorTestData processorTes .WithReleaseFileId(releaseFileId ?? Guid.NewGuid()) .WithStatus(status ?? DataSetVersionStatus.Processing) .WithImports(() => [dataSetVersionImport]) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + .WithVersionNumber(major: versionMajor, minor: versionMinor) + .FinishWith(dsv => + { + if (status == DataSetVersionStatus.Published) + { + dsv.DataSet.LatestLiveVersion = dsv; + } + else + { + dsv.DataSet.LatestDraftVersion = dsv; + } + }); await AddTestData(context => { @@ -82,6 +110,12 @@ await AddTestData(context => return (dataSetVersion, dataSetVersionImport.InstanceId); } + protected DuckDbConnection GetDuckDbConnection(DataSetVersion dataSetVersion) + { + var dataSetVersionPathResolver = GetRequiredService(); + return DuckDbConnection.CreateFileConnectionReadOnly(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); + } + protected void AssertDataSetVersionDirectoryContainsOnlyFiles( DataSetVersion dataSetVersion, params string[] expectedFiles) @@ -128,9 +162,7 @@ public override IHostBuilder ConfigureTestHostBuilder() { config.AddInMemoryCollection(new Dictionary { - { - "CoreStorage", _azuriteContainer.GetConnectionString() - } + { "CoreStorage", _azuriteContainer.GetConnectionString() } }); }) .ConfigureServices(services => @@ -138,7 +170,12 @@ public override IHostBuilder ConfigureTestHostBuilder() services.UseInMemoryDbContext(databaseName: Guid.NewGuid().ToString()); services.AddDbContext( - options => options.UseNpgsql(_postgreSqlContainer.GetConnectionString())); + options => + { + options.UseNpgsql( + _postgreSqlContainer.GetConnectionString(), + psqlOptions => psqlOptions.EnableRetryOnFailure()); + }); using var serviceScope = services.BuildServiceProvider() .GetRequiredService() @@ -155,11 +192,14 @@ protected override IEnumerable GetFunctionTypes() [ typeof(CreateDataSetFunction), typeof(ProcessInitialDataSetVersionFunction), + typeof(CreateNextDataSetVersionFunction), + typeof(ProcessNextDataSetVersionFunction), typeof(DeleteDataSetVersionFunction), typeof(CopyCsvFilesFunction), typeof(ImportMetadataFunction), typeof(HandleProcessingFailureFunction), typeof(HealthCheckFunctions), + typeof(BulkDeleteDataSetVersionsFunction), ]; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs index 659420a54dc..ad4fea2a2a0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs @@ -1,21 +1,337 @@ using System.Reflection; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests; -public record ProcessorTestData(string Name) +public record ProcessorTestData { - private static string DataFilesDirectoryPath => Path.Combine( - Assembly.GetExecutingAssembly().GetDirectoryPath(), - "Resources", - "DataFiles" - ); + public required string Name { get; init; } - private string DirectoryPath => Path.Combine(DataFilesDirectoryPath, Name); + public required int ExpectedTotalResults { get; init; } + + public required List ExpectedGeographicLevels; + + public required List ExpectedFilters { get; init; } + + public required List ExpectedIndicators { get; init; } + + public required List ExpectedLocations { get; init; } + + public required List ExpectedTimePeriods { get; init; } public string CsvDataFilePath => Path.Combine(DirectoryPath, "data.csv"); + public string CsvDataGzipFilePath => Path.Combine(DirectoryPath, "data.csv.gz"); + public string CsvMetadataFilePath => Path.Combine(DirectoryPath, "metadata.csv"); - public static ProcessorTestData AbsenceSchool => new(nameof(AbsenceSchool)); + private string DirectoryPath => Path.Combine(DataFilesDirectoryPath, Name); + + private static string DataFilesDirectoryPath => Path.Combine( + Assembly.GetExecutingAssembly().GetDirectoryPath(), + "Resources", + "DataFiles" + ); + + public static ProcessorTestData AbsenceSchool => new() + { + Name = nameof(AbsenceSchool), + ExpectedTotalResults = 216, + ExpectedTimePeriods = + [ + new TimePeriodMeta + { + Code = TimeIdentifier.AcademicYear, + Period = "2020/2021", + DataSetVersionId = Guid.Empty, + }, + new TimePeriodMeta + { + Code = TimeIdentifier.AcademicYear, + Period = "2021/2022", + DataSetVersionId = Guid.Empty, + }, + new TimePeriodMeta + { + Code = TimeIdentifier.AcademicYear, + Period = "2022/2023", + DataSetVersionId = Guid.Empty, + } + ], + ExpectedGeographicLevels = + [ + GeographicLevel.LocalAuthority, + GeographicLevel.Country, + GeographicLevel.Region, + GeographicLevel.School, + ], + ExpectedLocations = + [ + new LocationMeta + { + Level = GeographicLevel.LocalAuthority, + DataSetVersionId = Guid.Empty, + Options = + [ + new LocationLocalAuthorityOptionMeta + { + Id = 1, + PublicId = SqidEncoder.Encode(1), + OldCode = "302", + Code = "E09000003", + Label = "Barnet", + }, + new LocationLocalAuthorityOptionMeta + { + Id = 2, + PublicId = SqidEncoder.Encode(2), + OldCode = "314", + Code = "E09000021 / E09000027", + Label = "Kingston upon Thames / Richmond upon Thames", + }, + new LocationLocalAuthorityOptionMeta + { + Id = 3, + PublicId = SqidEncoder.Encode(3), + OldCode = "370", + Code = "E08000016", + Label = "Barnsley", + }, + new LocationLocalAuthorityOptionMeta + { + Id = 4, + PublicId = SqidEncoder.Encode(4), + OldCode = "373", + Code = "E08000019", + Label = "Sheffield", + }, + ] + }, + new LocationMeta + { + Level = GeographicLevel.Country, + DataSetVersionId = Guid.Empty, + Options = + [ + new LocationCodedOptionMeta + { + Id = 5, + PublicId = SqidEncoder.Encode(5), + Code = "E92000001", + Label = "England", + }, + ] + }, + new LocationMeta + { + Level = GeographicLevel.Region, + DataSetVersionId = Guid.Empty, + Options = + [ + new LocationCodedOptionMeta + { + Id = 6, + PublicId = SqidEncoder.Encode(6), + Code = "E12000003", + Label = "Yorkshire and The Humber", + }, + new LocationCodedOptionMeta + { + Id = 7, + PublicId = SqidEncoder.Encode(7), + Code = "E13000002", + Label = "Outer London", + }, + ] + }, + new LocationMeta + { + Level = GeographicLevel.School, + DataSetVersionId = Guid.Empty, + Options = + [ + new LocationSchoolOptionMeta + { + Id = 8, + PublicId = SqidEncoder.Encode(8), + Urn = "101269", + LaEstab = "3022014", + Label = "Colindale Primary School", + }, + new LocationSchoolOptionMeta + { + Id = 9, + PublicId = SqidEncoder.Encode(9), + Urn = "102579", + LaEstab = "3142032", + Label = "King Athelstan Primary School", + }, + new LocationSchoolOptionMeta + { + Id = 10, + PublicId = SqidEncoder.Encode(10), + Urn = "106653", + LaEstab = "3704027", + Label = "Penistone Grammar School", + }, + new LocationSchoolOptionMeta + { + Id = 11, + PublicId = SqidEncoder.Encode(11), + Urn = "135507", + LaEstab = "3026906", + Label = "Wren Academy Finchley", + }, + new LocationSchoolOptionMeta + { + Id = 12, + PublicId = SqidEncoder.Encode(12), + Urn = "140821", + LaEstab = "3734008", + Label = "Newfield Secondary School", + }, + new LocationSchoolOptionMeta + { + Id = 13, + PublicId = SqidEncoder.Encode(13), + Urn = "141862", + LaEstab = "3144001", + Label = "The Kingston Academy", + }, + new LocationSchoolOptionMeta + { + Id = 14, + PublicId = SqidEncoder.Encode(14), + Urn = "141973", + LaEstab = "3702039", + Label = "Hoyland Springwood Primary School", + }, + new LocationSchoolOptionMeta + { + Id = 15, + PublicId = SqidEncoder.Encode(15), + Urn = "145374", + LaEstab = "3732341", + Label = "Greenhill Primary School", + }, + ] + }, + ], + ExpectedFilters = + [ + new FilterMeta + { + PublicId = "academy_type", + Label = "Academy type", + Hint = "Only applicable for academies, otherwise no value", + DataSetVersionId = Guid.Empty, + Options = + [ + new FilterOptionMeta + { + Label = "Primary sponsor led academy", + }, + new FilterOptionMeta + { + Label = "Secondary free school", + }, + new FilterOptionMeta + { + Label = "Secondary sponsor led academy", + }, + ], + }, + new FilterMeta + { + PublicId = "ncyear", + Label = "National Curriculum year", + Hint = "Ranges from years 1 to 11", + DataSetVersionId = Guid.Empty, + Options = + [ + new FilterOptionMeta + { + Label = "Year 10", + }, + new FilterOptionMeta + { + Label = "Year 4", + }, + new FilterOptionMeta + { + Label = "Year 6", + }, + new FilterOptionMeta + { + Label = "Year 8", + }, + ], + }, + new FilterMeta + { + PublicId = "school_type", + Label = "School type", + Hint = "", + DataSetVersionId = Guid.Empty, + Options = + [ + new FilterOptionMeta + { + Label = "State-funded primary", + }, + new FilterOptionMeta + { + Label = "State-funded secondary", + }, + new FilterOptionMeta + { + Label = "Total", + IsAggregate = true + }, + ], + }, + ], + ExpectedIndicators = + [ + new IndicatorMeta + { + PublicId = "enrolments", + Label = "Enrolments", + DecimalPlaces = 0, + DataSetVersionId = Guid.Empty, + }, + new IndicatorMeta + { + PublicId = "sess_authorised", + Label = "Number of authorised sessions", + DecimalPlaces = 0, + DataSetVersionId = Guid.Empty, + }, + new IndicatorMeta + { + PublicId = "sess_possible", + Label = "Number of possible sessions", + DecimalPlaces = 0, + DataSetVersionId = Guid.Empty, + }, + new IndicatorMeta + { + PublicId = "sess_unauthorised", + Label = "Number of unauthorised sessions", + DecimalPlaces = 0, + DataSetVersionId = Guid.Empty, + }, + new IndicatorMeta + { + PublicId = "sess_unauthorised_percent", + Label = "Percentage of unauthorised sessions", + DecimalPlaces = 2, + DataSetVersionId = Guid.Empty + }, + ], + }; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs index 723886762c1..80b60c33a97 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs @@ -8,5 +8,11 @@ internal static class ActivityNames public const string ImportData = nameof(ProcessInitialDataSetVersionFunction.ImportData); public const string WriteDataFiles = nameof(ProcessInitialDataSetVersionFunction.WriteDataFiles); - public const string CompleteProcessing = nameof(ProcessInitialDataSetVersionFunction.CompleteProcessing); + public const string CompleteInitialDataSetVersionProcessing = + nameof(ProcessInitialDataSetVersionFunction.CompleteInitialDataSetVersionProcessing); + + public const string CreateMappings = nameof(ProcessNextDataSetVersionFunction.CreateMappings); + public const string ApplyAutoMappings = nameof(ProcessNextDataSetVersionFunction.ApplyAutoMappings); + public const string CompleteNextDataSetVersionMappingProcessing = + nameof(ProcessNextDataSetVersionFunction.CompleteNextDataSetVersionMappingProcessing); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/BulkDeleteDataSetVersionsFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/BulkDeleteDataSetVersionsFunction.cs new file mode 100644 index 00000000000..ea8262645a4 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/BulkDeleteDataSetVersionsFunction.cs @@ -0,0 +1,35 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class BulkDeleteDataSetVersionsFunction( + IDataSetVersionService dataSetVersionService, + ILogger logger) +{ + [Function(nameof(BulkDeleteDataSetVersions))] + public async Task BulkDeleteDataSetVersions( + [HttpTrigger(AuthorizationLevel.Anonymous, "delete", + Route = $"{nameof(BulkDeleteDataSetVersions)}/{{releaseVersionId}}")] + HttpRequest httpRequest, + Guid releaseVersionId, + CancellationToken cancellationToken) + { + try + { + return await dataSetVersionService.BulkDeleteVersions( + releaseVersionId, + cancellationToken: cancellationToken) + .HandleFailuresOrNoContent(convertNotFoundToNoContent: false); + } + catch (Exception ex) + { + logger.LogError(exception: ex, "Exception occured while executing '{FunctionName}'", nameof(BulkDeleteDataSetVersionsFunction)); + return new StatusCodeResult(StatusCodes.Status500InternalServerError); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs index 1f012762cb7..d96a297ae79 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs @@ -61,15 +61,9 @@ private async Task ProcessInitialDataSetVersion( { const string orchestratorName = nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion); - var input = new ProcessInitialDataSetVersionContext - { - DataSetVersionId = dataSetVersionId - }; + var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; - var options = new StartOrchestrationOptions - { - InstanceId = instanceId.ToString() - }; + var options = new StartOrchestrationOptions { InstanceId = instanceId.ToString() }; logger.LogInformation( "Scheduling '{OrchestratorName}' (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionFunction.cs new file mode 100644 index 00000000000..44681eb7b1e --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionFunction.cs @@ -0,0 +1,81 @@ +using FluentValidation; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.ViewModels; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; +using FromBodyAttribute = Microsoft.Azure.Functions.Worker.Http.FromBodyAttribute; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class CreateNextDataSetVersionFunction( + ILogger logger, + IDataSetVersionService dataSetVersionService, + IValidator requestValidator) +{ + [Function(nameof(CreateNextDataSetVersion))] + public async Task CreateNextDataSetVersion( + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = nameof(CreateNextDataSetVersion))] [FromBody] + NextDataSetVersionCreateRequest request, + [DurableClient] DurableTaskClient client, + CancellationToken cancellationToken) + { + // Identifier of the scheduled processing orchestration instance + var instanceId = Guid.NewGuid(); + + return await requestValidator.Validate(request, cancellationToken) + .OnSuccess(() => dataSetVersionService.CreateNextVersion( + dataSetId: request.DataSetId, + releaseFileId: request.ReleaseFileId, + instanceId, + cancellationToken: cancellationToken + )) + .OnSuccess(async dataSetVersionId => + { + await ProcessNextDataSetVersion( + client, + dataSetVersionId: dataSetVersionId, + instanceId: instanceId, + cancellationToken); + + return new CreateDataSetResponseViewModel + { + DataSetId = request.DataSetId, + DataSetVersionId = dataSetVersionId, + InstanceId = instanceId + }; + }) + .HandleFailuresOr(result => new OkObjectResult(result)); + } + + private async Task ProcessNextDataSetVersion( + DurableTaskClient client, + Guid dataSetVersionId, + Guid instanceId, + CancellationToken cancellationToken) + { + const string orchestratorName = nameof(ProcessNextDataSetVersionFunction.ProcessNextDataSetVersion); + + var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; + + var options = new StartOrchestrationOptions { InstanceId = instanceId.ToString() }; + + logger.LogInformation( + "Scheduling '{OrchestratorName}' (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + orchestratorName, + instanceId, + dataSetVersionId); + + await client.ScheduleNewOrchestrationInstanceAsync( + orchestratorName, + input, + options, + cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HandleProcesssingFailureFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HandleProcessingFailureFunction.cs similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HandleProcesssingFailureFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/HandleProcessingFailureFunction.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs index c77798e5654..a59caf2b7b6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs @@ -14,15 +14,13 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Funct public class ProcessInitialDataSetVersionFunction( PublicDataDbContext publicDataDbContext, IDataDuckDbRepository dataDuckDbRepository, - IParquetService parquetService, - IDataSetVersionPathResolver dataSetVersionPathResolver) : BaseProcessDataSetVersionFunction(publicDataDbContext) + IDataSetVersionPathResolver dataSetVersionPathResolver, + IParquetService parquetService) : BaseProcessDataSetVersionFunction(publicDataDbContext) { - private readonly PublicDataDbContext _publicDataDbContext = publicDataDbContext; - [Function(nameof(ProcessInitialDataSetVersion))] public async Task ProcessInitialDataSetVersion( [OrchestrationTrigger] TaskOrchestrationContext context, - ProcessInitialDataSetVersionContext input) + ProcessDataSetVersionContext input) { var logger = context.CreateReplaySafeLogger(nameof(ProcessInitialDataSetVersion)); @@ -37,7 +35,8 @@ public async Task ProcessInitialDataSetVersion( await context.CallActivityExclusively(ActivityNames.ImportMetadata, logger, context.InstanceId); await context.CallActivity(ActivityNames.ImportData, logger, context.InstanceId); await context.CallActivity(ActivityNames.WriteDataFiles, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CompleteProcessing, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CompleteInitialDataSetVersionProcessing, logger, + context.InstanceId); } catch (Exception e) { @@ -70,8 +69,8 @@ public async Task WriteDataFiles( await parquetService.WriteDataFiles(dataSetVersionImport.DataSetVersionId, cancellationToken); } - [Function(ActivityNames.CompleteProcessing)] - public async Task CompleteProcessing( + [Function(ActivityNames.CompleteInitialDataSetVersionProcessing)] + public async Task CompleteInitialDataSetVersionProcessing( [ActivityTrigger] Guid instanceId, CancellationToken cancellationToken) { @@ -84,7 +83,8 @@ public async Task CompleteProcessing( File.Delete(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); dataSetVersion.Status = DataSetVersionStatus.Draft; + dataSetVersionImport.Completed = DateTimeOffset.UtcNow; - await _publicDataDbContext.SaveChangesAsync(cancellationToken); + await publicDataDbContext.SaveChangesAsync(cancellationToken); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionFunction.cs new file mode 100644 index 00000000000..8d558843573 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionFunction.cs @@ -0,0 +1,81 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class ProcessNextDataSetVersionFunction( + PublicDataDbContext publicDataDbContext, + IDataSetVersionMappingService mappingService) : BaseProcessDataSetVersionFunction(publicDataDbContext) +{ + [Function(nameof(ProcessNextDataSetVersion))] + public async Task ProcessNextDataSetVersion( + [OrchestrationTrigger] TaskOrchestrationContext context, + ProcessDataSetVersionContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessNextDataSetVersion)); + + logger.LogInformation( + "Processing next data set version (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + try + { + await context.CallActivity(ActivityNames.CopyCsvFiles, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CreateMappings, logger, context.InstanceId); + await context.CallActivity(ActivityNames.ApplyAutoMappings, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CompleteNextDataSetVersionMappingProcessing, logger, + context.InstanceId); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } + + [Function(ActivityNames.CreateMappings)] + public async Task CreateMappings( + [ActivityTrigger] Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.CreatingMappings, cancellationToken); + await mappingService.CreateMappings(dataSetVersionImport.DataSetVersionId, cancellationToken); + } + + [Function(ActivityNames.ApplyAutoMappings)] + public async Task ApplyAutoMappings( + [ActivityTrigger] Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.AutoMapping, cancellationToken); + await mappingService.ApplyAutoMappings(dataSetVersionImport.DataSetVersionId, cancellationToken); + } + + [Function(ActivityNames.CompleteNextDataSetVersionMappingProcessing)] + public async Task CompleteNextDataSetVersionMappingProcessing( + [ActivityTrigger] Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.Completing, cancellationToken); + + var dataSetVersion = dataSetVersionImport.DataSetVersion; + dataSetVersion.Status = DataSetVersionStatus.Mapping; + + dataSetVersionImport.Completed = DateTimeOffset.UtcNow; + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/DataSertVersionMappingMeta.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/DataSertVersionMappingMeta.cs new file mode 100644 index 00000000000..a5fe594f124 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/DataSertVersionMappingMeta.cs @@ -0,0 +1,8 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; + +public record DataSetVersionMappingMeta( + IDictionary> Filters, + IDictionary> Locations, + DataSetVersionMetaSummary MetaSummary); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/ProcessInitialDataSetVersionContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/ProcessDataSetVersionContext.cs similarity index 74% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/ProcessInitialDataSetVersionContext.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/ProcessDataSetVersionContext.cs index c234a3f09ec..264f019debb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/ProcessInitialDataSetVersionContext.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/ProcessDataSetVersionContext.cs @@ -1,6 +1,6 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -public record ProcessInitialDataSetVersionContext +public record ProcessDataSetVersionContext { public required Guid DataSetVersionId { get; init; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Options/AppSettingsOptions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Options/AppSettingsOptions.cs new file mode 100644 index 00000000000..9b26b9e7750 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Options/AppSettingsOptions.cs @@ -0,0 +1,12 @@ +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Options; + +public class AppSettingsOptions +{ + public static readonly string Section = "AppSettings"; + + /// + /// Batch size to use when inserting location option meta rows, location option meta link rows, + /// and filter option meta link rows into the public data db. + /// + public required int MetaInsertBatchSize { get; set; } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs index 7d339d502bc..9d7ba20908c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs @@ -6,7 +6,10 @@ using GovUk.Education.ExploreEducationStatistics.Common.Services; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Options; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests; @@ -69,8 +72,10 @@ public static IHostBuilder ConfigureProcessorHostBuilder(this IHostBuilder hostB .AddFluentValidation() .AddScoped() .AddScoped() + .AddScoped() .AddScoped() .AddScoped() + .AddScoped() .AddScoped() .AddScoped() .AddScoped() @@ -83,10 +88,12 @@ public static IHostBuilder ConfigureProcessorHostBuilder(this IHostBuilder hostB .AddScoped() .AddScoped() .AddScoped() - .AddScoped, - DataSetCreateRequest.Validator>() + .Configure( + hostBuilderContext.Configuration.GetSection(AppSettingsOptions.Section)) .Configure( hostBuilderContext.Configuration.GetSection(DataFilesOptions.Section)); + + services.AddValidatorsFromAssemblyContaining(); }); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/DataDuckDbRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/DataDuckDbRepository.cs index 2f8cca6e565..92b6c6e5ba8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/DataDuckDbRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/DataDuckDbRepository.cs @@ -30,7 +30,6 @@ public async Task CreateDataTable( await using var duckDbConnection = DuckDbConnection.CreateFileConnection(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); - duckDbConnection.Open(); await duckDbConnection.SqlBuilder("CREATE SEQUENCE data_seq START 1") .ExecuteAsync(cancellationToken: cancellationToken); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs index 2e0c23e53c0..75d8e637906 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs @@ -2,6 +2,7 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Models; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Options; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; @@ -9,61 +10,64 @@ using LinqToDB; using LinqToDB.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; public class FilterMetaRepository( PublicDataDbContext publicDataDbContext, + IOptions appSettingsOptions, IDataSetVersionPathResolver dataSetVersionPathResolver) : IFilterMetaRepository { + private readonly AppSettingsOptions _appSettingsOptions = appSettingsOptions.Value; + + public async Task>> ReadFilterMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default) + { + var metas = await GetFilterMetas( + duckDbConnection, + dataSetVersion, + allowedColumns, + cancellationToken); + + return await metas + .ToAsyncEnumerable() + .ToDictionaryAwaitAsync( + keySelector: ValueTask.FromResult, + elementSelector: async meta => + await GetFilterOptionMeta( + duckDbConnection, + dataSetVersion, + meta, + cancellationToken), + cancellationToken); + } + public async Task CreateFilterMetas( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, IReadOnlySet allowedColumns, CancellationToken cancellationToken = default) { - var metas = (await duckDbConnection.SqlBuilder( - $""" - SELECT * - FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}' - WHERE "col_type" = {MetaFileRow.ColumnType.Filter.ToString()} - AND "col_name" IN ({allowedColumns}) - """) - .QueryAsync(cancellationToken: cancellationToken) - ) - .OrderBy(row => row.Label) - .Select( - row => new FilterMeta - { - PublicId = row.ColName, - DataSetVersionId = dataSetVersion.Id, - Label = row.Label, - Hint = row.FilterHint ?? string.Empty, - } - ) - .ToList(); + var metas = await GetFilterMetas( + duckDbConnection, + dataSetVersion, + allowedColumns, + cancellationToken); publicDataDbContext.FilterMetas.AddRange(metas); await publicDataDbContext.SaveChangesAsync(cancellationToken); foreach (var meta in metas) { - var options = (await duckDbConnection.SqlBuilder( - $""" - SELECT DISTINCT "{meta.PublicId:raw}" - FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', ALL_VARCHAR = true) AS data - WHERE "{meta.PublicId:raw}" != '' - ORDER BY "{meta.PublicId:raw}" - """ - ).QueryAsync(cancellationToken: cancellationToken)) - .Select( - label => new FilterOptionMeta - { - Label = label, - IsAggregate = label == "Total" ? true : null - } - ) - .ToList(); + var options = await GetFilterOptionMeta( + duckDbConnection, + dataSetVersion, + meta, + cancellationToken); var optionTable = publicDataDbContext.GetTable(); @@ -73,23 +77,32 @@ await optionTable .Merge() .Using(options) .On( - o => new { o.Label, o.IsAggregate }, - o => new { o.Label, o.IsAggregate } + o => new + { + o.Label, + o.IsAggregate + }, + o => new + { + o.Label, + o.IsAggregate + } ) .InsertWhenNotMatched() .MergeAsync(cancellationToken); - var startIndex = await publicDataDbContext.FilterOptionMetaLinks.CountAsync(token: cancellationToken); + var startIndex = await publicDataDbContext.NextSequenceValue( + PublicDataDbContext.FilterOptionMetaLinkSequence, + cancellationToken); var current = 0; - const int batchSize = 1000; while (current < options.Count) { var batchStartIndex = startIndex + current; var batch = options .Skip(current) - .Take(batchSize) + .Take(_appSettingsOptions.MetaInsertBatchSize) .ToList(); // Although not necessary for filter options, we've adopted the 'row key' @@ -102,6 +115,7 @@ await optionTable var links = await optionTable .Where(o => batchRowKeys.Contains(o.Label + ',' + (o.IsAggregate == true ? "True" : ""))) + .OrderBy(o => o.Label) .Select((option, index) => new FilterOptionMetaLink { PublicId = SqidEncoder.Encode(batchStartIndex + index), @@ -113,7 +127,7 @@ await optionTable publicDataDbContext.FilterOptionMetaLinks.AddRange(links); await publicDataDbContext.SaveChangesAsync(cancellationToken); - current += batchSize; + current += _appSettingsOptions.MetaInsertBatchSize; } var insertedLinks = await publicDataDbContext.FilterOptionMetaLinks @@ -126,6 +140,63 @@ await optionTable $"Inserted incorrect number of filter option meta links for {meta.PublicId}. " + $"Inserted: {insertedLinks}, expected: {options.Count}"); } + + await publicDataDbContext.SetSequenceValue( + PublicDataDbContext.FilterOptionMetaLinkSequence, + startIndex + insertedLinks - 1, + cancellationToken); } } + + private async Task> GetFilterMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken) + { + return (await duckDbConnection.SqlBuilder( + $""" + SELECT * + FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}' + WHERE "col_type" = {MetaFileRow.ColumnType.Filter.ToString()} + AND "col_name" IN ({allowedColumns}) + """) + .QueryAsync(cancellationToken: cancellationToken) + ) + .OrderBy(row => row.Label) + .Select( + row => new FilterMeta + { + PublicId = row.ColName, + DataSetVersionId = dataSetVersion.Id, + Label = row.Label, + Hint = row.FilterHint ?? string.Empty + } + ) + .ToList(); + } + + private async Task> GetFilterOptionMeta( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + FilterMeta meta, + CancellationToken cancellationToken) + { + return (await duckDbConnection.SqlBuilder( + $""" + SELECT DISTINCT "{meta.PublicId:raw}" + FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', ALL_VARCHAR = true) AS data + WHERE "{meta.PublicId:raw}" != '' + ORDER BY "{meta.PublicId:raw}" + """ + ).QueryAsync(cancellationToken: cancellationToken)) + .Select( + label => new FilterOptionMeta + { + Label = label, + IsAggregate = label == "Total" ? true : null + } + ) + .ToList(); + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs index e2a61829b78..4dace1eb67d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs @@ -13,10 +13,31 @@ public class GeographicLevelMetaRepository( PublicDataDbContext publicDataDbContext, IDataSetVersionPathResolver dataSetVersionPathResolver) : IGeographicLevelMetaRepository { + public Task ReadGeographicLevelMeta( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default) + { + return GetGeographicLevelMeta(duckDbConnection, dataSetVersion, cancellationToken); + } + public async Task CreateGeographicLevelMeta( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, CancellationToken cancellationToken = default) + { + var meta = await GetGeographicLevelMeta(duckDbConnection, dataSetVersion, cancellationToken); + + publicDataDbContext.GeographicLevelMetas.Add(meta); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + return meta; + } + + private async Task GetGeographicLevelMeta( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken) { var geographicLevels = (await duckDbConnection.SqlBuilder( @@ -26,6 +47,7 @@ FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', AL """ ).QueryAsync(cancellationToken: cancellationToken)) .Select(EnumToEnumLabelConverter.FromProvider) + .OrderBy(EnumToEnumLabelConverter.ToProvider) .ToList(); var meta = new GeographicLevelMeta @@ -33,10 +55,6 @@ FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', AL DataSetVersionId = dataSetVersion.Id, Levels = geographicLevels }; - - publicDataDbContext.GeographicLevelMetas.Add(meta); - await publicDataDbContext.SaveChangesAsync(cancellationToken); - return meta; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterMetaRepository.cs index 19a7200adc6..8ef5d3e21dd 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IFilterMetaRepository.cs @@ -5,6 +5,12 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repos public interface IFilterMetaRepository { + Task>> ReadFilterMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default); + Task CreateFilterMetas( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IGeographicLevelMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IGeographicLevelMetaRepository.cs index f1a73550a84..4e051c8208e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IGeographicLevelMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IGeographicLevelMetaRepository.cs @@ -5,6 +5,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repos public interface IGeographicLevelMetaRepository { + Task ReadGeographicLevelMeta( + IDuckDbConnection duckDb, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default); + Task CreateGeographicLevelMeta( IDuckDbConnection duckDb, DataSetVersion dataSetVersion, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationMetaRepository.cs index 3f0fc89a868..bc36724ffa2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ILocationMetaRepository.cs @@ -5,6 +5,12 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repos public interface ILocationMetaRepository { + Task>> ReadLocationMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default); + Task CreateLocationMetas( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodMetaRepository.cs index 7e631d7a573..f33ecee52ce 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/ITimePeriodMetaRepository.cs @@ -5,6 +5,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repos public interface ITimePeriodMetaRepository { + Task> ReadTimePeriodMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default); + Task> CreateTimePeriodMetas( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationMetaRepository.cs index 17a3784d4d7..6026ca27921 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationMetaRepository.cs @@ -2,10 +2,10 @@ using GovUk.Education.ExploreEducationStatistics.Common.Converters; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; -using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Options; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; @@ -14,56 +14,57 @@ using LinqToDB.Data; using LinqToDB.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; using static GovUk.Education.ExploreEducationStatistics.Common.Utils.GeographicLevelUtils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; public class LocationMetaRepository( PublicDataDbContext publicDataDbContext, + IOptions appSettingsOptions, IDataSetVersionPathResolver dataSetVersionPathResolver) : ILocationMetaRepository { + private readonly AppSettingsOptions _appSettingsOptions = appSettingsOptions.Value; + + public async Task>> ReadLocationMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken) + { + var metas = GetLocationMetas(dataSetVersion, allowedColumns); + + return await metas + .ToAsyncEnumerable() + .ToDictionaryAwaitAsync( + keySelector: ValueTask.FromResult, + elementSelector: async meta => + await GetLocationOptionMetas( + duckDbConnection, + dataSetVersion, + meta, + cancellationToken), + cancellationToken); + } + public async Task CreateLocationMetas( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, IReadOnlySet allowedColumns, CancellationToken cancellationToken = default) { - var levels = ListLocationLevels(allowedColumns); - - var metas = levels - .Select(level => new LocationMeta - { - DataSetVersionId = dataSetVersion.Id, - Level = level, - }) - .ToList(); + var metas = GetLocationMetas(dataSetVersion, allowedColumns); publicDataDbContext.LocationMetas.AddRange(metas); await publicDataDbContext.SaveChangesAsync(cancellationToken); foreach (var meta in metas) { - var nameCol = meta.Level.CsvNameColumn(); - var codeCols = meta.Level.CsvCodeColumns(); - string[] cols = [..codeCols, nameCol]; - - var options = (await duckDbConnection.SqlBuilder( - $""" - SELECT {cols.JoinToString(", "):raw} - FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', ALL_VARCHAR = true) - WHERE {cols.Select(col => $"{col} != ''").JoinToString(" AND "):raw} - GROUP BY {cols.JoinToString(", "):raw} - ORDER BY {cols.JoinToString(", "):raw} - """ - ).QueryAsync(cancellationToken: cancellationToken) - ) - .Cast>() - .Select(row => row.ToDictionary( - kv => kv.Key, - kv => (string)kv.Value! - )) - .Select(row => MapLocationOptionMeta(row, meta.Level).ToRow()) - .ToList(); + var options = await GetLocationOptionMetas( + duckDbConnection, + dataSetVersion, + meta, + cancellationToken); var optionTable = publicDataDbContext .GetTable() @@ -71,13 +72,11 @@ FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', AL var current = 0; - const int batchSize = 1000; - while (current < options.Count) { var batch = options .Skip(current) - .Take(batchSize) + .Take(_appSettingsOptions.MetaInsertBatchSize) .ToList(); // We create a 'row key' for each option that allows us to quickly @@ -113,7 +112,9 @@ FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', AL .Where(o => !existingRowKeys.Contains(o.GetRowKey())) .ToList(); - var startIndex = await publicDataDbContext.LocationOptionMetas.CountAsync(token: cancellationToken); + var startIndex = await publicDataDbContext.NextSequenceValue( + PublicDataDbContext.LocationOptionMetasIdSequence, + cancellationToken); foreach (var option in newOptions) { @@ -121,7 +122,15 @@ FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', AL option.PublicId = SqidEncoder.Encode(option.Id); } - await optionTable.BulkCopyAsync(newOptions, cancellationToken); + await optionTable.BulkCopyAsync( + new BulkCopyOptions { KeepIdentity = true }, + newOptions, + cancellationToken); + + await publicDataDbContext.SetSequenceValue( + PublicDataDbContext.LocationOptionMetasIdSequence, + startIndex - 1, + cancellationToken); } var links = await optionTable @@ -136,7 +145,7 @@ FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', AL publicDataDbContext.LocationOptionMetaLinks.AddRange(links); await publicDataDbContext.SaveChangesAsync(cancellationToken); - current += batchSize; + current += _appSettingsOptions.MetaInsertBatchSize; } var insertedLinks = await publicDataDbContext.LocationOptionMetaLinks @@ -152,16 +161,58 @@ FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', AL } } + private async Task> GetLocationOptionMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + LocationMeta meta, + CancellationToken cancellationToken) + { + var nameCol = meta.Level.CsvNameColumn(); + var codeCols = meta.Level.CsvCodeColumns(); + string[] cols = [..codeCols, nameCol]; + + return (await duckDbConnection.SqlBuilder( + $""" + SELECT {cols.JoinToString(", "):raw} + FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', ALL_VARCHAR = true) + WHERE {cols.Select(col => $"{col} != ''").JoinToString(" AND "):raw} + GROUP BY {cols.JoinToString(", "):raw} + ORDER BY {cols.JoinToString(", "):raw} + """ + ).QueryAsync(cancellationToken: cancellationToken) + ) + .Cast>() + .Select(row => row.ToDictionary( + kv => kv.Key, + kv => (string)kv.Value! + )) + .Select(row => MapLocationOptionMeta(row, meta.Level).ToRow()) + .ToList(); + } + + private static List GetLocationMetas( + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns) + { + var levels = ListLocationLevels(allowedColumns); + + return levels + .Select(level => new LocationMeta + { + DataSetVersionId = dataSetVersion.Id, + Level = level + }) + .ToList(); + } + private static List ListLocationLevels(IReadOnlySet allowedColumns) { - return - [ - .. allowedColumns - .Where(CsvColumnsToGeographicLevel.ContainsKey) - .Select(col => CsvColumnsToGeographicLevel[col]) - .Distinct() - .OrderBy(EnumToEnumLabelConverter.ToProvider) - ]; + return allowedColumns + .Where(CsvColumnsToGeographicLevel.ContainsKey) + .Select(col => CsvColumnsToGeographicLevel[col]) + .Distinct() + .OrderBy(EnumToEnumLabelConverter.ToProvider) + .ToList(); } private static LocationOptionMeta MapLocationOptionMeta( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodMetaRepository.cs index c3553534203..ce42f78d124 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/TimePeriodMetaRepository.cs @@ -14,12 +14,33 @@ public class TimePeriodMetaRepository( PublicDataDbContext publicDataDbContext, IDataSetVersionPathResolver dataSetVersionPathResolver) : ITimePeriodMetaRepository { + public Task> ReadTimePeriodMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default) + { + return GetTimePeriodMetas(duckDbConnection, dataSetVersion, cancellationToken); + } + public async Task> CreateTimePeriodMetas( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, CancellationToken cancellationToken = default) { - var metas = (await duckDbConnection.SqlBuilder( + var metas = await GetTimePeriodMetas(duckDbConnection, dataSetVersion, cancellationToken); + + publicDataDbContext.TimePeriodMetas.AddRange(metas); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + return metas; + } + + private async Task> GetTimePeriodMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken) + { + return (await duckDbConnection.SqlBuilder( $""" SELECT DISTINCT time_period, time_identifier FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', ALL_VARCHAR = true) @@ -36,10 +57,5 @@ ORDER BY time_period .OrderBy(meta => meta.Period) .ThenBy(meta => meta.Code) .ToList(); - - publicDataDbContext.TimePeriodMetas.AddRange(metas); - await publicDataDbContext.SaveChangesAsync(cancellationToken); - - return metas; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs index f895d59e269..a205af7ae0b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs @@ -1,7 +1,9 @@ using Dapper; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Models; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; @@ -25,90 +27,185 @@ public class DataSetMetaService( ITimePeriodsDuckDbRepository timePeriodsDuckDbRepository ) : IDataSetMetaService { - public async Task CreateDataSetVersionMeta( + public async Task ReadDataSetVersionMetaForMappings( Guid dataSetVersionId, CancellationToken cancellationToken = default) { - var dataSetVersion = await publicDataDbContext.DataSetVersions - .SingleAsync(dsv => dsv.Id == dataSetVersionId, cancellationToken: cancellationToken); + var dataSetVersion = await GetDataSetVersion(dataSetVersionId, cancellationToken); await using var duckDbConnection = DuckDbConnection.CreateFileConnection(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); duckDbConnection.Open(); - var columns = (await duckDbConnection.SqlBuilder( - $"DESCRIBE SELECT * FROM '{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}'") - .QueryAsync<(string ColumnName, string ColumnType)>(cancellationToken: cancellationToken)) - .Select(row => row.ColumnName) - .ToList(); - - var allowedColumns = columns.ToHashSet(); - - var metaFileRows = (await duckDbConnection.SqlBuilder( - $"SELECT * FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}'") - .QueryAsync(cancellationToken: cancellationToken)).AsList(); + var allowedColumns = await GetAllowedColumns( + duckDbConnection, + dataSetVersion, + cancellationToken); - await filterMetaRepository.CreateFilterMetas( + var metaFileRows = await GetMetaFileRows( duckDbConnection, dataSetVersion, - allowedColumns, cancellationToken); - await indicatorMetaRepository.CreateIndicatorMetas( + var filterMetas = await filterMetaRepository.ReadFilterMetas( duckDbConnection, dataSetVersion, allowedColumns, cancellationToken); - var geographicLevelMeta = await geographicLevelMetaRepository.CreateGeographicLevelMeta( + var geographicLevelMeta = await geographicLevelMetaRepository.ReadGeographicLevelMeta( duckDbConnection, dataSetVersion, cancellationToken); - await locationMetaRepository.CreateLocationMetas( + var locationMetas = await locationMetaRepository.ReadLocationMetas( duckDbConnection, dataSetVersion, allowedColumns, cancellationToken); - var timePeriodMetas = await timePeriodMetaRepository.CreateTimePeriodMetas( + var timePeriodMetas = await timePeriodMetaRepository.ReadTimePeriodMetas( duckDbConnection, dataSetVersion, cancellationToken); - dataSetVersion.MetaSummary = - BuildMetaSummary(timePeriodMetas, metaFileRows, allowedColumns, geographicLevelMeta); - dataSetVersion.TotalResults = await CountCsvRows(duckDbConnection, dataSetVersion, cancellationToken); + var metaSummary = BuildMetaSummary( + timePeriodMetas, + metaFileRows, + allowedColumns, + geographicLevelMeta); - await publicDataDbContext.SaveChangesAsync(cancellationToken); + return new DataSetVersionMappingMeta(filterMetas, locationMetas, metaSummary); + } - await indicatorsDuckDbRepository.CreateIndicatorsTable( - duckDbConnection, - dataSetVersion, - cancellationToken); + public async Task CreateDataSetVersionMeta( + Guid dataSetVersionId, + CancellationToken cancellationToken = default) + { + var dataSetVersion = await GetDataSetVersion(dataSetVersionId, cancellationToken); - await locationsDuckDbRepository.CreateLocationsTable( - duckDbConnection, - dataSetVersion, - cancellationToken); + await using var duckDbConnection = + DuckDbConnection.CreateFileConnection(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); + duckDbConnection.Open(); - await filterOptionsDuckDbRepository.CreateFilterOptionsTable( + var allowedColumns = await GetAllowedColumns( duckDbConnection, dataSetVersion, cancellationToken); - await timePeriodsDuckDbRepository.CreateTimePeriodsTable( + var metaFileRows = await GetMetaFileRows( duckDbConnection, dataSetVersion, cancellationToken); + + await publicDataDbContext.RequireTransaction(async () => + { + await filterMetaRepository.CreateFilterMetas( + duckDbConnection, + dataSetVersion, + allowedColumns, + cancellationToken); + + await indicatorMetaRepository.CreateIndicatorMetas( + duckDbConnection, + dataSetVersion, + allowedColumns, + cancellationToken); + + var geographicLevelMeta = await geographicLevelMetaRepository.CreateGeographicLevelMeta( + duckDbConnection, + dataSetVersion, + cancellationToken); + + await locationMetaRepository.CreateLocationMetas( + duckDbConnection, + dataSetVersion, + allowedColumns, + cancellationToken); + + var timePeriodMetas = await timePeriodMetaRepository.CreateTimePeriodMetas( + duckDbConnection, + dataSetVersion, + cancellationToken); + + dataSetVersion.MetaSummary = BuildMetaSummary( + timePeriodMetas, + metaFileRows, + allowedColumns, + geographicLevelMeta); + + dataSetVersion.TotalResults = await CountCsvRows( + duckDbConnection, + dataSetVersion, + cancellationToken); + + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + await indicatorsDuckDbRepository.CreateIndicatorsTable( + duckDbConnection, + dataSetVersion, + cancellationToken); + + await locationsDuckDbRepository.CreateLocationsTable( + duckDbConnection, + dataSetVersion, + cancellationToken); + + await filterOptionsDuckDbRepository.CreateFilterOptionsTable( + duckDbConnection, + dataSetVersion, + cancellationToken); + + await timePeriodsDuckDbRepository.CreateTimePeriodsTable( + duckDbConnection, + dataSetVersion, + cancellationToken); + }); + } + + private async Task> GetMetaFileRows( + DuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default) + { + return (await duckDbConnection + .SqlBuilder($"SELECT * FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}'") + .QueryAsync(cancellationToken: cancellationToken)) + .AsList(); + } + + private async Task> GetAllowedColumns( + DuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken = default) + { + var columns = (await duckDbConnection.SqlBuilder( + $"DESCRIBE SELECT * FROM '{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}'") + .QueryAsync<(string ColumnName, string ColumnType)>(cancellationToken: cancellationToken)) + .Select(row => row.ColumnName) + .ToList(); + + return [.. columns]; + } + + private async Task GetDataSetVersion( + Guid dataSetVersionId, + CancellationToken cancellationToken = default) + { + return await publicDataDbContext + .DataSetVersions + .SingleAsync( + dsv => dsv.Id == dataSetVersionId, + cancellationToken); } private static DataSetVersionMetaSummary BuildMetaSummary( IList timePeriodMetas, IList metaFileRows, HashSet allowedColumns, - GeographicLevelMeta geographicLevelMeta) => - new() + GeographicLevelMeta geographicLevelMeta) + { + return new() { TimePeriodRange = new TimePeriodRange { @@ -121,7 +218,7 @@ private static DataSetVersionMetaSummary BuildMetaSummary( { Period = timePeriodMetas[^1].Period, Code = timePeriodMetas[^1].Code - }, + } }, Filters = metaFileRows .Where(row => row.ColType == MetaFileRow.ColumnType.Filter @@ -139,6 +236,7 @@ private static DataSetVersionMetaSummary BuildMetaSummary( .ToList(), GeographicLevels = geographicLevelMeta.Levels }; + } private async Task CountCsvRows( IDuckDbConnection duckDbConnection, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs index 9bc9a54bb0a..3fd049f6737 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetService.cs @@ -1,6 +1,6 @@ -using System.Transactions; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Validators; using GovUk.Education.ExploreEducationStatistics.Common.Validators.ErrorDetails; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; @@ -19,7 +19,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Servi public class DataSetService( ContentDbContext contentDbContext, - PublicDataDbContext publicDataDbContext + PublicDataDbContext publicDataDbContext, + IDataSetVersionService dataSetVersionService ) : IDataSetService { public async Task> CreateDataSet( @@ -27,101 +28,16 @@ PublicDataDbContext publicDataDbContext Guid instanceId, CancellationToken cancellationToken = default) { - var strategy = contentDbContext.Database.CreateExecutionStrategy(); - - return await strategy.ExecuteAsync(async () => - { - using var transactionScope = new TransactionScope( - TransactionScopeOption.Required, - new TransactionOptions - { - IsolationLevel = IsolationLevel.ReadCommitted - }, - TransactionScopeAsyncFlowOption.Enabled); - - return await GetReleaseFile(request.ReleaseFileId, cancellationToken) - .OnSuccess(async releaseFile => await ValidateReleaseFile(releaseFile, cancellationToken) - .OnSuccess(async () => await CreateDataSet(releaseFile, cancellationToken)) - .OnSuccess(async dataSet => - await CreateDataSetVersion(dataSet, releaseFile, cancellationToken)) - .OnSuccessDo(async dataSetVersion => - await CreateDataSetVersionImport(dataSetVersion, instanceId, cancellationToken)) - .OnSuccessDo(async dataSetVersion => - await UpdateFilePublicDataSetVersionId(releaseFile, dataSetVersion, cancellationToken)) - .OnSuccessDo(transactionScope.Complete) - .OnSuccess(dataSetVersion => - (dataSetId: dataSetVersion.DataSetId, dataSetVersionId: dataSetVersion.Id))); - }); - } - - private async Task> GetReleaseFile( - Guid releaseFileId, - CancellationToken cancellationToken) - { - var releaseFile = await contentDbContext.ReleaseFiles - .Include(rf => rf.File) - .Include(rf => rf.ReleaseVersion) - .FirstOrDefaultAsync(rf => rf.Id == releaseFileId, cancellationToken); - - return releaseFile is null - ? ValidationUtils.ValidationResult(CreateReleaseFileIdError( - message: ValidationMessages.FileNotFound, - releaseFileId: releaseFileId - )) - : releaseFile; - } - - private async Task> ValidateReleaseFile( - ReleaseFile releaseFile, - CancellationToken cancellationToken) - { - // ReleaseFile must not already have a DataSetVersion - if (await publicDataDbContext.DataSetVersions.AnyAsync( - dsv => dsv.ReleaseFileId == releaseFile.Id, - cancellationToken: cancellationToken)) - { - return ValidationUtils.ValidationResult( - [ - CreateReleaseFileIdError( - message: ValidationMessages.FileHasApiDataSetVersion, - releaseFileId: releaseFile.Id) - ]); - } - - // ReleaseFile must relate to a ReleaseVersion in Draft approval status - if (releaseFile.ReleaseVersion.ApprovalStatus != ReleaseApprovalStatus.Draft) - { - return ValidationUtils.ValidationResult( - [ - CreateReleaseFileIdError( - message: ValidationMessages.FileReleaseVersionNotDraft, - releaseFileId: releaseFile.Id) - ]); - } - - List errors = []; - - // ReleaseFile must relate to a File of type Data - if (releaseFile.File.Type != FileType.Data) - { - errors.Add(CreateReleaseFileIdError( - message: ValidationMessages.FileTypeNotData, - releaseFileId: releaseFile.Id)); - } - - // There must be a ReleaseFile related to the same ReleaseVersion and Subject with File of type Metadata - if (!await contentDbContext.ReleaseFiles - .Where(rf => rf.ReleaseVersionId == releaseFile.ReleaseVersionId) - .Where(rf => rf.File.SubjectId == releaseFile.File.SubjectId) - .Where(rf => rf.File.Type == FileType.Metadata) - .AnyAsync(cancellationToken: cancellationToken)) - { - errors.Add(CreateReleaseFileIdError( - message: ValidationMessages.NoMetadataFile, - releaseFileId: releaseFile.Id)); - } - - return errors.Count == 0 ? Unit.Instance : ValidationUtils.ValidationResult(errors); + return await publicDataDbContext.RequireTransaction(async () => + await GetReleaseFile(request.ReleaseFileId, cancellationToken) + .OnSuccess(releaseFile => CreateDataSet(releaseFile, cancellationToken)) + .OnSuccess(dataSet => dataSetVersionService + .CreateInitialVersion( + dataSetId: dataSet.Id, + releaseFileId: request.ReleaseFileId, + instanceId: instanceId, + cancellationToken) + .OnSuccess(dataSetVersionId => (dataSet.Id, dataSetVersionId)))); } private async Task CreateDataSet( @@ -142,66 +58,23 @@ private async Task CreateDataSet( return dataSet; } - private async Task CreateDataSetVersion( - DataSet dataSet, - ReleaseFile releaseFile, - CancellationToken cancellationToken) - { - var dataSetVersion = new DataSetVersion - { - ReleaseFileId = releaseFile.Id, - DataSetId = dataSet.Id, - Status = DataSetVersionStatus.Processing, - Notes = "", - VersionMajor = dataSet.LatestLiveVersion?.VersionMajor ?? 1, - VersionMinor = dataSet.LatestLiveVersion?.VersionMinor + 1 ?? 0 - }; - - dataSet.Versions.Add(dataSetVersion); - dataSet.LatestDraftVersion = dataSetVersion; - - publicDataDbContext.DataSets.Update(dataSet); - await publicDataDbContext.SaveChangesAsync(cancellationToken); - - return dataSetVersion; - } - - private async Task CreateDataSetVersionImport( - DataSetVersion dataSetVersion, - Guid instanceId, - CancellationToken cancellationToken) - { - var dataSetVersionImport = new DataSetVersionImport - { - DataSetVersionId = dataSetVersion.Id, - InstanceId = instanceId, - Stage = DataSetVersionImportStage.Pending - }; - - publicDataDbContext.DataSetVersionImports.Add(dataSetVersionImport); - await publicDataDbContext.SaveChangesAsync(cancellationToken); - } - - private async Task UpdateFilePublicDataSetVersionId( - ReleaseFile releaseFile, - DataSetVersion dataSetVersion, + private async Task> GetReleaseFile( + Guid releaseFileId, CancellationToken cancellationToken) { - releaseFile.File.PublicApiDataSetId = dataSetVersion.DataSetId; - releaseFile.File.PublicApiDataSetVersion = dataSetVersion.FullSemanticVersion(); - await contentDbContext.SaveChangesAsync(cancellationToken); - } + var releaseFile = await contentDbContext.ReleaseFiles + .Include(rf => rf.File) + .Include(rf => rf.ReleaseVersion) + .FirstOrDefaultAsync(rf => rf.Id == releaseFileId, cancellationToken); - private static ErrorViewModel CreateReleaseFileIdError( - LocalizableMessage message, - Guid releaseFileId) - { - return new ErrorViewModel - { - Code = message.Code, - Message = message.Message, - Path = nameof(DataSetCreateRequest.ReleaseFileId).ToLowerFirst(), - Detail = new InvalidErrorDetail(releaseFileId) - }; + return releaseFile is null + ? ValidationUtils.ValidationResult(new ErrorViewModel + { + Code = ValidationMessages.FileNotFound.Code, + Message = ValidationMessages.FileNotFound.Message, + Path = nameof(DataSetCreateRequest.ReleaseFileId).ToLowerFirst(), + Detail = new InvalidErrorDetail(releaseFileId) + }) + : releaseFile; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs new file mode 100644 index 00000000000..8e28a6c0f08 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs @@ -0,0 +1,352 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services; + +internal class DataSetVersionMappingService( + IDataSetMetaService dataSetMetaService, + PublicDataDbContext publicDataDbContext) + : IDataSetVersionMappingService +{ + private static readonly MappingType[] IncompleteMappingTypes = + [ + MappingType.None, + MappingType.AutoNone + ]; + + private static Func LocationOptionKeyGenerator => + option => $"{option.Label} :: {option.GetRowKey()}"; + + private static Func FilterKeyGenerator => + filter => filter.PublicId; + + private static Func FilterOptionKeyGenerator => + option => option.Label; + + public async Task> CreateMappings( + Guid nextDataSetVersionId, + CancellationToken cancellationToken = default) + { + var nextVersion = await publicDataDbContext + .DataSetVersions + .Include(dsv => dsv.DataSet) + .ThenInclude(ds => ds.LatestLiveVersion) + .SingleAsync(dsv => dsv.Id == nextDataSetVersionId, cancellationToken); + + var liveVersion = nextVersion.DataSet.LatestLiveVersion!; + + var nextVersionMeta = await dataSetMetaService.ReadDataSetVersionMetaForMappings( + dataSetVersionId: nextDataSetVersionId, + cancellationToken); + + var sourceLocationMeta = + await GetLocationMeta(liveVersion.Id, cancellationToken); + + var locationMappings = CreateLocationMappings( + sourceLocationMeta, + nextVersionMeta.Locations); + + var sourceFilterMeta = + await GetFilterMeta(liveVersion.Id, cancellationToken); + + var filterMappings = CreateFilterMappings( + sourceFilterMeta, + nextVersionMeta.Filters); + + nextVersion.MetaSummary = nextVersionMeta.MetaSummary; + + publicDataDbContext + .DataSetVersionMappings + .Add(new DataSetVersionMapping + { + SourceDataSetVersionId = liveVersion.Id, + TargetDataSetVersionId = nextDataSetVersionId, + LocationMappingPlan = locationMappings, + FilterMappingPlan = filterMappings + }); + + await publicDataDbContext.SaveChangesAsync(cancellationToken); + return Unit.Instance; + } + + public async Task ApplyAutoMappings( + Guid nextDataSetVersionId, + CancellationToken cancellationToken = default) + { + var mappings = await publicDataDbContext + .DataSetVersionMappings + .SingleAsync( + mapping => mapping.TargetDataSetVersionId == nextDataSetVersionId, + cancellationToken); + + AutoMapLocations(mappings.LocationMappingPlan); + AutoMapFilters(mappings.FilterMappingPlan); + + mappings.LocationMappingsComplete = !mappings + .LocationMappingPlan + .Levels + .Any(level => level + .Value + .Mappings + .Any(optionMapping => + IncompleteMappingTypes.Contains(optionMapping.Value.Type))); + + mappings.FilterMappingsComplete = !mappings + .FilterMappingPlan + .Mappings + .Any(filterMapping => + IncompleteMappingTypes.Contains(filterMapping.Value.Type) + || filterMapping + .Value + .OptionMappings + .Any(optionMapping => IncompleteMappingTypes.Contains(optionMapping.Value.Type))); + + publicDataDbContext.Update(mappings); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } + + private static void AutoMapLocations(LocationMappingPlan locationPlan) + { + locationPlan + .Levels + .ForEach(level => level.Value.Mappings + .ForEach(locationMapping => AutoMapElement( + sourceKey: locationMapping.Key, + mapping: locationMapping.Value, + candidates: level + .Value + .Candidates))); + } + + private static void AutoMapFilters(FilterMappingPlan filtersPlan) + { + filtersPlan + .Mappings + .ForEach(filterMapping => AutoMapParentAndOptions( + sourceParentKey: filterMapping.Key, + parentMapping: filterMapping.Value, + parentCandidates: filtersPlan.Candidates, + candidateOptionsSupplier: autoMappedCandidate => autoMappedCandidate.Options)); + } + + private static TCandidate? AutoMapElement( + string sourceKey, + Mapping mapping, + Dictionary candidates) + where TMappableElement : MappableElement + where TCandidate : MappableElement + { + if (candidates.Count == 0) + { + mapping.Type = MappingType.AutoNone; + return null; + } + + var matchingCandidate = candidates.GetValueOrDefault(sourceKey); + + if (matchingCandidate is not null) + { + mapping.CandidateKey = sourceKey; + mapping.Type = MappingType.AutoMapped; + } + else + { + mapping.CandidateKey = null; + mapping.Type = MappingType.AutoNone; + } + + return matchingCandidate; + } + + private static void AutoMapParentAndOptions( + string sourceParentKey, + ParentMapping parentMapping, + Dictionary parentCandidates, + Func> candidateOptionsSupplier) + where TMappableParent : MappableElement + where TParentCandidate : MappableElement + where TMappableOption : MappableElement + where TOptionMapping : Mapping + { + var autoMapCandidate = AutoMapElement( + sourceKey: sourceParentKey, + mapping: parentMapping, + candidates: parentCandidates); + + if (autoMapCandidate is not null) + { + var candidateOptions = candidateOptionsSupplier.Invoke(autoMapCandidate); + + parentMapping + .OptionMappings + .ForEach(optionMapping => AutoMapElement( + sourceKey: optionMapping.Key, + mapping: optionMapping.Value, + candidates: candidateOptions)); + } + else + { + parentMapping + .OptionMappings + .Select(optionMapping => optionMapping.Value) + .ForEach(optionMapping => + { + optionMapping.CandidateKey = null; + optionMapping.Type = parentMapping.Type; + }); + } + } + + private LocationMappingPlan CreateLocationMappings( + List sourceLocationMeta, + IDictionary> targetLocationMeta) + { + // Create mappings by level for each Geographic Level that appeared in the source data set version. + var sourceMappingsByLevel = sourceLocationMeta + .ToDictionary( + keySelector: level => level.Level, + elementSelector: level => + { + var candidatesForLevel = targetLocationMeta + .Any(entry => entry.Key.Level == level.Level) + ? targetLocationMeta + .Single(entry => entry.Key.Level == level.Level) + .Value + : []; + + return new LocationLevelMappings + { + Mappings = level + .Options + .Select(option => option.ToRow()) + .ToDictionary( + keySelector: LocationOptionKeyGenerator, + elementSelector: option => new LocationOptionMapping + { + Source = CreateLocationOptionFromMetaRow(option) + }), + Candidates = candidatesForLevel + .ToDictionary( + keySelector: LocationOptionKeyGenerator, + elementSelector: CreateLocationOptionFromMetaRow) + }; + }); + + var sourceLevels = sourceMappingsByLevel.Select(level => level.Key); + + // Additionally find any Geographic Levels that appear in the target data set version but not in the source, + // and create mappings by level for them. + var onlyTargetMappingsByLevel = targetLocationMeta + .Where(entry => !sourceLevels.Contains(entry.Key)) + .Select(meta => ( + levelMeta: meta.Key, + optionsMeta: meta.Value)) + .ToDictionary( + keySelector: meta => meta.levelMeta.Level, + elementSelector: meta => new LocationLevelMappings + { + Mappings = [], + Candidates = meta + .optionsMeta + .ToDictionary( + keySelector: LocationOptionKeyGenerator, + elementSelector: CreateLocationOptionFromMetaRow) + }); + + return new LocationMappingPlan + { + Levels = sourceMappingsByLevel + .Concat(onlyTargetMappingsByLevel) + .ToDictionary( + e => e.Key, + e => e.Value) + }; + } + + private FilterMappingPlan CreateFilterMappings( + List sourceFilterMeta, + IDictionary> targetFilterMeta) + { + var filterMappings = sourceFilterMeta + .ToDictionary( + keySelector: FilterKeyGenerator, + elementSelector: filter => + new FilterMapping + { + Source = new MappableFilter(filter.Label), + OptionMappings = filter + .Options + .ToDictionary( + keySelector: FilterOptionKeyGenerator, + elementSelector: option => + new FilterOptionMapping { Source = CreateFilterOptionFromMetaRow(option) }) + }); + + var filterTargets = targetFilterMeta + .Select(meta => ( + filterMeta: meta.Key, + optionsMeta: meta.Value)) + .ToDictionary( + keySelector: meta => FilterKeyGenerator(meta.filterMeta), + elementSelector: meta => + new FilterMappingCandidate(meta.filterMeta.Label) + { + Options = meta.optionsMeta + .ToDictionary( + keySelector: FilterOptionKeyGenerator, + elementSelector: CreateFilterOptionFromMetaRow) + }); + + var filters = new FilterMappingPlan + { + Mappings = filterMappings, + Candidates = filterTargets + }; + + return filters; + } + + private static MappableLocationOption CreateLocationOptionFromMetaRow(LocationOptionMetaRow option) + { + return new MappableLocationOption(option.Label) + { + Code = option.Code, + OldCode = option.OldCode, + Ukprn = option.Ukprn, + Urn = option.Urn, + LaEstab = option.LaEstab + }; + } + + private static MappableFilterOption CreateFilterOptionFromMetaRow(FilterOptionMeta option) + { + return new MappableFilterOption(option.Label); + } + + private async Task> GetLocationMeta( + Guid dataSetVersionId, + CancellationToken cancellationToken) + { + return await publicDataDbContext + .LocationMetas + .AsNoTracking() + .Include(levelMeta => levelMeta.Options) + .Where(meta => meta.DataSetVersionId == dataSetVersionId) + .ToListAsync(cancellationToken); + } + + private async Task> GetFilterMeta(Guid sourceVersionId, CancellationToken cancellationToken) + { + return await publicDataDbContext + .FilterMetas + .AsNoTracking() + .Include(filterMeta => filterMeta.Options) + .Where(meta => meta.DataSetVersionId == sourceVersionId) + .ToListAsync(cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionService.cs index ad755598861..5b502825941 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionService.cs @@ -5,13 +5,14 @@ using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using System.Transactions; using ValidationMessages = GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Requests.Validators.ValidationMessages; @@ -20,37 +21,88 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Servi internal class DataSetVersionService( ContentDbContext contentDbContext, PublicDataDbContext publicDataDbContext, + IReleaseFileRepository releaseFileRepository, IDataSetVersionPathResolver dataSetVersionPathResolver ) : IDataSetVersionService { - public async Task> DeleteVersion(Guid dataSetVersionId, + public async Task> CreateInitialVersion( + Guid dataSetId, + Guid releaseFileId, + Guid instanceId, CancellationToken cancellationToken = default) { - var strategy = contentDbContext.Database.CreateExecutionStrategy(); - - return await strategy.ExecuteAsync( - async () => - { - using var transactionScope = new TransactionScope( - TransactionScopeOption.Required, - new TransactionOptions { IsolationLevel = IsolationLevel.ReadCommitted }, - TransactionScopeAsyncFlowOption.Enabled); - - return await GetDataSetVersion(dataSetVersionId, cancellationToken) - .OnSuccessDo(CheckCanDeleteDataSetVersion) - .OnSuccessDo(async dataSetVersion => await GetReleaseFile(dataSetVersion, cancellationToken) - .OnSuccessVoid(async releaseFile => - await UpdateFilePublicApiDataSetId(releaseFile, cancellationToken)) - .OnFailureDo(_ => - throw new KeyNotFoundException( - $"The expected 'ReleaseFile', with ID '{dataSetVersion.ReleaseFileId}', was not found."))) - .OnSuccessDo(async dataSetVersion => await DeleteDataSetVersion(dataSetVersion, cancellationToken)) - .OnSuccessVoid(DeleteParquetFiles) - .OnSuccessVoid(transactionScope.Complete); - }); - } - - private async Task> GetDataSetVersion(Guid dataSetVersionId, + return await GetDataSet(dataSetId, cancellationToken) + .OnSuccess(ValidateInitialDataSet) + .OnSuccess(dataSet => CreateDataSetVersion( + releaseFileId: releaseFileId, + instanceId: instanceId, + dataSet: dataSet, + cancellationToken: cancellationToken)) + .OnSuccess(dataSetVersion => dataSetVersion.Id); + } + + public async Task> CreateNextVersion( + Guid dataSetId, + Guid releaseFileId, + Guid instanceId, + CancellationToken cancellationToken = default) + { + return await GetDataSet(dataSetId, cancellationToken) + .OnSuccess(ValidateNextDataSet) + .OnSuccess(dataSet => CreateDataSetVersion( + releaseFileId: releaseFileId, + instanceId: instanceId, + dataSet: dataSet, + cancellationToken: cancellationToken)) + .OnSuccess(dataSetVersion => dataSetVersion.Id); + } + + public async Task> BulkDeleteVersions(Guid releaseVersionId, CancellationToken cancellationToken = default) + { + return await publicDataDbContext.RequireTransaction(() => + GetReleaseFiles(releaseVersionId, cancellationToken) + .OnSuccessCombineWith(async releaseFiles => await GetDataSetVersions(releaseFiles, cancellationToken)) + .OnSuccess(releaseFilesAndDataSetVersions => + { + var (releaseFiles, dataSetVersions) = releaseFilesAndDataSetVersions; + + return CheckCanDeleteDataSetVersions(dataSetVersions) + .OnSuccess(() => (releaseFiles, dataSetVersions)); + }) + .OnSuccessDo(async releaseFilesAndDataSetVersions => await UnlinkReleaseFilesFromApiDataSets(releaseFilesAndDataSetVersions.releaseFiles, cancellationToken)) + .OnSuccessDo(async releaseFilesAndDataSetVersions => await DeleteDataSetVersions(releaseFilesAndDataSetVersions.dataSetVersions, cancellationToken)) + .OnSuccessVoid(releaseFilesAndDataSetVersions => DeleteDuckDbFiles(releaseFilesAndDataSetVersions.dataSetVersions))); + } + + public async Task> DeleteVersion( + Guid dataSetVersionId, + CancellationToken cancellationToken = default) + { + return await publicDataDbContext.RequireTransaction(() => + GetDataSetVersion(dataSetVersionId, cancellationToken) + .OnSuccessDo(CheckCanDeleteDataSetVersion) + .OnSuccessDo(async dataSetVersion => await UpdateReleaseFiles(dataSetVersion, cancellationToken)) + .OnSuccessDo(async dataSetVersion => await DeleteDataSetVersion(dataSetVersion, cancellationToken)) + .OnSuccessVoid(DeleteDuckDbFiles)); + } + + private async Task>> GetDataSetVersions( + IReadOnlyList releaseFiles, + CancellationToken cancellationToken) + { + var releaseFileIds = releaseFiles + .Select(rf => rf.Id) + .ToList(); + + return await publicDataDbContext.DataSetVersions + .AsNoTracking() + .Include(dsv => dsv.DataSet) + .Where(dsv => releaseFileIds.Contains(dsv.ReleaseFileId)) + .ToListAsync(cancellationToken); + } + + private async Task> GetDataSetVersion( + Guid dataSetVersionId, CancellationToken cancellationToken) { return await publicDataDbContext.DataSetVersions @@ -60,7 +112,28 @@ private async Task> GetDataSetVersion(Guid .SingleOrNotFoundAsync(cancellationToken); } - private Either CheckCanDeleteDataSetVersion(DataSetVersion dataSetVersion) + private static Either CheckCanDeleteDataSetVersions(IReadOnlyList dataSetVersions) + { + var versionsWhichCanNotBeDeleted = dataSetVersions + .Where(dsv => !dsv.CanBeDeleted) + .Select(dsv => dsv.Id) + .ToList(); + + if (!versionsWhichCanNotBeDeleted.Any()) + { + return Unit.Instance; + } + + return ValidationUtils.ValidationResult(new ErrorViewModel + { + Code = ValidationMessages.MultipleDataSetVersionsCanNotBeDeleted.Code, + Message = ValidationMessages.MultipleDataSetVersionsCanNotBeDeleted.Message, + Detail = new InvalidErrorDetail>(versionsWhichCanNotBeDeleted), + Path = "releaseVersionId" + }); + } + + private static Either CheckCanDeleteDataSetVersion(DataSetVersion dataSetVersion) { if (dataSetVersion.CanBeDeleted) { @@ -76,6 +149,31 @@ private Either CheckCanDeleteDataSetVersion(DataSetVersion d }); } + private async Task UpdateReleaseFiles( + DataSetVersion dataSetVersion, + CancellationToken cancellationToken) + { + var releaseFiles = await contentDbContext.ReleaseFiles + .Where(rf => rf.PublicApiDataSetId == dataSetVersion.DataSetId) + .Where(rf => rf.PublicApiDataSetVersion == dataSetVersion.Version) + .ToListAsync(cancellationToken); + + await UnlinkReleaseFilesFromApiDataSets(releaseFiles, cancellationToken); + } + + private async Task DeleteDataSetVersions(IReadOnlyList dataSetVersions, CancellationToken cancellationToken) + { + var dataSetsWithNoOtherVersions = dataSetVersions + .Where(dsv => dsv.IsFirstVersion) + .Select(dsv => dsv.DataSet); + + publicDataDbContext.DataSetVersions.RemoveRange(dataSetVersions); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + publicDataDbContext.DataSets.RemoveRange(dataSetsWithNoOtherVersions); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } + private async Task DeleteDataSetVersion(DataSetVersion dataSetVersion, CancellationToken cancellationToken) { publicDataDbContext.DataSetVersions.Remove(dataSetVersion); @@ -88,39 +186,338 @@ private async Task DeleteDataSetVersion(DataSetVersion dataSetVersion, Cancellat } } - private async Task> GetReleaseFile(DataSetVersion dataSetVersion, + private async Task>> GetReleaseFiles( + Guid releaseVersionId, CancellationToken cancellationToken) { - return await contentDbContext.ReleaseFiles - .Include(rf => rf.File) - .SingleOrNotFoundAsync(rf => rf.Id == dataSetVersion.ReleaseFileId, cancellationToken); + return await releaseFileRepository.GetByFileType(releaseVersionId, cancellationToken, FileType.Data); } - private async Task UpdateFilePublicApiDataSetId(ReleaseFile releaseFile, CancellationToken cancellationToken) + private async Task UnlinkReleaseFilesFromApiDataSets(IReadOnlyList releaseFiles, CancellationToken cancellationToken) { - releaseFile.File.PublicApiDataSetId = null; - releaseFile.File.PublicApiDataSetVersion = null; + foreach (var releaseFile in releaseFiles) + { + UnlinkReleaseFileFromApiDataSet(releaseFile); + } + await contentDbContext.SaveChangesAsync(cancellationToken); } - private void DeleteParquetFiles(DataSetVersion dataSetVersion) + private static void UnlinkReleaseFileFromApiDataSet(ReleaseFile releaseFile) { - var directory = dataSetVersionPathResolver.DirectoryPath(dataSetVersion); + releaseFile.PublicApiDataSetId = null; + releaseFile.PublicApiDataSetVersion = null; + } - if (!Directory.Exists(directory)) + private void DeleteDuckDbFiles(IReadOnlyList dataSetVersions) + { + foreach (var dataSetVersion in dataSetVersions) { - return; + DeleteDuckDbFiles(dataSetVersion); } + } + private void DeleteDuckDbFiles(DataSetVersion dataSetVersion) + { if (dataSetVersion.IsFirstVersion) { - var dataSetDirectory = Directory.GetParent(directory)!.FullName; + DeleteDataSetDirectory(dataSetVersion); + + return; + } + + DeleteDataSetVersionDirectory(dataSetVersion); + } - Directory.Delete(dataSetDirectory, true); + private void DeleteDataSetDirectory(DataSetVersion dataSetVersion) + { + var dataSetDirectory = GetDataSetDirectory(dataSetVersion); + + DeleteDirectoryIfExists(dataSetDirectory); + } + + private void DeleteDataSetVersionDirectory(DataSetVersion dataSetVersion) + { + var directory = dataSetVersionPathResolver.DirectoryPath(dataSetVersion); + + DeleteDirectoryIfExists(directory); + } + + private string GetDataSetDirectory(DataSetVersion dataSetVersion) + { + var dataSetVersionDirectory = dataSetVersionPathResolver.DirectoryPath(dataSetVersion); + return Directory.GetParent(dataSetVersionDirectory)!.FullName; + } + private static void DeleteDirectoryIfExists(string directory) + { + if (!Directory.Exists(directory)) + { return; } Directory.Delete(directory, true); } + + private async Task> GetDataSet( + Guid dataSetId, + CancellationToken cancellationToken = default) + { + var dataSet = await publicDataDbContext + .DataSets + .Include(dataSet => dataSet.LatestLiveVersion) + .Include(dataSet => dataSet.Versions) + .FirstOrDefaultAsync(dataSet => dataSet.Id == dataSetId, cancellationToken); + + return dataSet is null + ? ValidationUtils.ValidationResult(CreateDataSetIdError( + message: ValidationMessages.DataSetNotFound, + dataSetId: dataSetId + )) + : dataSet; + } + + private Either ValidateInitialDataSet(DataSet dataSet) + { + if (dataSet.Versions.Count > 0) + { + return ValidationUtils.ValidationResult(CreateDataSetIdError( + message: ValidationMessages.DataSetMustHaveNoExistingVersions, + dataSetId: dataSet.Id)); + } + + return dataSet; + } + + private Either ValidateNextDataSet(DataSet dataSet) + { + if (dataSet.LatestLiveVersionId is null) + { + return ValidationUtils.ValidationResult(CreateDataSetIdError( + message: ValidationMessages.DataSetNoLiveVersion, + dataSetId: dataSet.Id)); + } + + return dataSet; + } + + private async Task> CreateDataSetVersion( + Guid releaseFileId, + Guid instanceId, + DataSet dataSet, + CancellationToken cancellationToken = default) + { + return await publicDataDbContext.RequireTransaction(async () => + await GetReleaseFile(releaseFileId, cancellationToken) + .OnSuccess(async releaseFile => + await ValidateReleaseFileAndDataSet( + releaseFile, + dataSet, + cancellationToken) + .OnSuccess(async () => + await CreateDataSetVersion( + dataSet, + releaseFile, + cancellationToken)) + .OnSuccessDo(async dataSetVersion => + await CreateDataSetVersionImport( + dataSetVersion, + instanceId, + cancellationToken)) + .OnSuccessDo(async dataSetVersion => + await UpdateReleaseFilePublicDataSetVersionId( + releaseFile, + dataSetVersion, + cancellationToken)))); + } + + private async Task> GetReleaseFile( + Guid releaseFileId, + CancellationToken cancellationToken) + { + var releaseFile = await contentDbContext.ReleaseFiles + .Include(rf => rf.File) + .Include(rf => rf.ReleaseVersion) + .FirstOrDefaultAsync(rf => rf.Id == releaseFileId, cancellationToken); + + return releaseFile is null + ? ValidationUtils.ValidationResult(CreateReleaseFileIdError( + message: ValidationMessages.FileNotFound, + releaseFileId: releaseFileId + )) + : releaseFile; + } + + private async Task> ValidateReleaseFileAndDataSet( + ReleaseFile releaseFile, + DataSet dataSet, + CancellationToken cancellationToken) + { + // ReleaseFile must not already have a DataSetVersion + if (await publicDataDbContext.DataSetVersions.AnyAsync( + dsv => dsv.ReleaseFileId == releaseFile.Id, + cancellationToken: cancellationToken)) + { + return ValidationUtils.ValidationResult( + [ + CreateReleaseFileIdError( + message: ValidationMessages.FileHasApiDataSetVersion, + releaseFileId: releaseFile.Id) + ]); + } + + // ReleaseFile must relate to a ReleaseVersion in Draft approval status + if (releaseFile.ReleaseVersion.ApprovalStatus != ReleaseApprovalStatus.Draft) + { + return ValidationUtils.ValidationResult( + [ + CreateReleaseFileIdError( + message: ValidationMessages.FileReleaseVersionNotDraft, + releaseFileId: releaseFile.Id) + ]); + } + + List errors = []; + + // ReleaseFile must relate to a File of type Data + if (releaseFile.File.Type != FileType.Data) + { + errors.Add(CreateReleaseFileIdError( + message: ValidationMessages.FileTypeNotData, + releaseFileId: releaseFile.Id)); + } + + // There must be a ReleaseFile related to the same ReleaseVersion and Subject with File of type Metadata + if (!await contentDbContext.ReleaseFiles + .Where(rf => rf.ReleaseVersionId == releaseFile.ReleaseVersionId) + .Where(rf => rf.File.SubjectId == releaseFile.File.SubjectId) + .Where(rf => rf.File.Type == FileType.Metadata) + .AnyAsync(cancellationToken: cancellationToken)) + { + errors.Add(CreateReleaseFileIdError( + message: ValidationMessages.NoMetadataFile, + releaseFileId: releaseFile.Id)); + } + + if (releaseFile.ReleaseVersion.PublicationId != dataSet.PublicationId) + { + errors.Add(CreateReleaseFileIdError( + message: ValidationMessages.FileNotInDataSetPublication, + releaseFileId: releaseFile.Id)); + } + + var previousReleaseFileIds = dataSet + .Versions + .Select(version => version.ReleaseFileId) + .ToList(); + + var previousReleaseIds = await GetReleaseIdsForReleaseFiles( + contentDbContext, + previousReleaseFileIds, + cancellationToken); + + var selectedReleaseFileReleaseId = (await GetReleaseIdsForReleaseFiles( + contentDbContext, + [releaseFile.Id], + cancellationToken)) + .Single(); + + if (previousReleaseIds.Contains(selectedReleaseFileReleaseId)) + { + errors.Add(CreateReleaseFileIdError( + message: ValidationMessages.FileMustBeInDifferentRelease, + releaseFileId: releaseFile.Id)); + } + + return errors.Count == 0 ? Unit.Instance : ValidationUtils.ValidationResult(errors); + } + + private async Task CreateDataSetVersion( + DataSet dataSet, + ReleaseFile releaseFile, + CancellationToken cancellationToken) + { + var dataSetVersion = new DataSetVersion + { + ReleaseFileId = releaseFile.Id, + DataSetId = dataSet.Id, + Status = DataSetVersionStatus.Processing, + Notes = "", + VersionMajor = dataSet.LatestLiveVersion?.VersionMajor ?? 1, + VersionMinor = dataSet.LatestLiveVersion?.VersionMinor + 1 ?? 0 + }; + + dataSet.Versions.Add(dataSetVersion); + dataSet.LatestDraftVersion = dataSetVersion; + + publicDataDbContext.DataSets.Update(dataSet); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + return dataSetVersion; + } + + private async Task CreateDataSetVersionImport( + DataSetVersion dataSetVersion, + Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = new DataSetVersionImport + { + DataSetVersionId = dataSetVersion.Id, + InstanceId = instanceId, + Stage = DataSetVersionImportStage.Pending + }; + + publicDataDbContext.DataSetVersionImports.Add(dataSetVersionImport); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } + + private async Task UpdateReleaseFilePublicDataSetVersionId( + ReleaseFile releaseFile, + DataSetVersion dataSetVersion, + CancellationToken cancellationToken) + { + releaseFile.PublicApiDataSetId = dataSetVersion.DataSetId; + releaseFile.PublicApiDataSetVersion = dataSetVersion.FullSemanticVersion(); + await contentDbContext.SaveChangesAsync(cancellationToken); + } + + private static async Task> GetReleaseIdsForReleaseFiles( + ContentDbContext contentDbContext, + List releaseFileIds, + CancellationToken cancellationToken) + { + return await contentDbContext + .ReleaseFiles + .Include(releaseFile => releaseFile.ReleaseVersion) + .Where(releaseFile => releaseFileIds.Contains(releaseFile.Id)) + .Select(releaseFile => releaseFile.ReleaseVersion.ReleaseId) + .ToListAsync(cancellationToken); + } + + private static ErrorViewModel CreateReleaseFileIdError( + LocalizableMessage message, + Guid releaseFileId) + { + return new ErrorViewModel + { + Code = message.Code, + Message = message.Message, + Path = nameof(DataSetCreateRequest.ReleaseFileId).ToLowerFirst(), + Detail = new InvalidErrorDetail(releaseFileId) + }; + } + + private static ErrorViewModel CreateDataSetIdError( + LocalizableMessage message, + Guid dataSetId) + { + return new ErrorViewModel + { + Code = message.Code, + Message = message.Message, + Path = nameof(NextDataSetVersionCreateRequest.DataSetId).ToLowerFirst(), + Detail = new InvalidErrorDetail(dataSetId) + }; + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs index 7c92c998cc6..c6005a98904 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs @@ -1,7 +1,13 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; + namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; public interface IDataSetMetaService { + Task ReadDataSetVersionMetaForMappings( + Guid dataSetVersionId, + CancellationToken cancellationToken = default); + Task CreateDataSetVersionMeta( Guid dataSetVersionId, CancellationToken cancellationToken = default); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionMappingService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionMappingService.cs new file mode 100644 index 00000000000..7a6be3e8454 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionMappingService.cs @@ -0,0 +1,15 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using Microsoft.AspNetCore.Mvc; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; + +public interface IDataSetVersionMappingService +{ + Task> CreateMappings( + Guid nextDataSetVersionId, + CancellationToken cancellationToken = default); + + Task ApplyAutoMappings( + Guid nextDataSetVersionId, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionService.cs index 49e7c99f75d..ff83c88c3c4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionService.cs @@ -5,6 +5,22 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Servi public interface IDataSetVersionService { + Task> CreateInitialVersion( + Guid dataSetId, + Guid releaseFileId, + Guid instanceId, + CancellationToken cancellationToken = default); + + Task> CreateNextVersion( + Guid dataSetId, + Guid releaseFileId, + Guid instanceId, + CancellationToken cancellationToken = default); + + Task> BulkDeleteVersions( + Guid releaseVersionId, + CancellationToken cancellationToken = default); + Task> DeleteVersion( Guid dataSetVersionId, CancellationToken cancellationToken = default); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/ParquetService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/ParquetService.cs index 4b738b58e22..a0c5ceb2deb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/ParquetService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/ParquetService.cs @@ -26,8 +26,7 @@ public async Task WriteDataFiles( logger.LogDebug("Writing data files to data set version directory '{VersionDir}'", versionDir); await using var duckDbConnection = - DuckDbConnection.CreateFileConnection(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); - duckDbConnection.Open(); + DuckDbConnection.CreateFileConnectionReadOnly(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); await duckDbConnection.SqlBuilder( $"EXPORT DATABASE '{versionDir:raw}' (FORMAT PARQUET, CODEC ZSTD)") diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/appsettings.json b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/appsettings.json index de89935dc15..c5b9a323b73 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/appsettings.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/appsettings.json @@ -6,6 +6,9 @@ } }, "CoreStorage": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://data-storage:10000/devstoreaccount1;QueueEndpoint=http://data-storage:10001/devstoreaccount1;TableEndpoint=http://data-storage:10002/devstoreaccount1", + "AppSettings": { + "MetaInsertBatchSize": 1000 + }, "DataFiles": { "BasePath": "data/public-api-data" } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs index eff794aa535..f5fefcbf356 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs @@ -108,13 +108,13 @@ private static async Task SetUpPublicDataDbContext(Cancella LinqToDBForEFTools.Initialize(); + // Clear any tables and restart any sequences in case we're re-running the command var tables = dbContext.Model.GetEntityTypes() .Select(type => type.GetTableName()) .Distinct() - .Cast() + .OfType() .ToList(); - // Clear any tables in case we're re-running the command foreach (var table in tables) { #pragma warning disable EF1002 @@ -125,6 +125,18 @@ await dbContext.Database.ExecuteSqlRawAsync( ); } + var sequences = dbContext.Model.GetSequences(); + + foreach (var sequence in sequences) + { +#pragma warning disable EF1002 + await dbContext.Database.ExecuteSqlRawAsync( +#pragma warning restore EF1002 + $"""ALTER SEQUENCE "{sequence.Name}" RESTART WITH 1;""", + cancellationToken: cancellationToken + ); + } + return dbContext; } @@ -282,6 +294,7 @@ FROM read_csv('{_dataFilePath:raw}', ALL_VARCHAR = true) """ ).QueryAsync(cancellationToken: _cancellationToken)) .Select(EnumToEnumLabelConverter.FromProvider) + .OrderBy(EnumToEnumLabelConverter.ToProvider) .ToList(); var timePeriods = (await _duckDbConnection.SqlBuilder( @@ -449,7 +462,10 @@ await optionTable .InsertWhenNotMatched() .MergeAsync(_cancellationToken); - var startIndex = await _dbContext.FilterOptionMetaLinks.CountAsync(token: _cancellationToken); + var startIndex = await _dbContext.NextSequenceValue( + PublicDataDbContext.FilterOptionMetaLinkSequence, + _cancellationToken + ); var current = 0; const int batchSize = 1000; @@ -472,6 +488,7 @@ await optionTable var links = await optionTable .Where(o => batchRowKeys.Contains(o.Label + ',' + (o.IsAggregate == true ? "True" : ""))) + .OrderBy(o => o.Label) .Select((option, index) => new FilterOptionMetaLink { PublicId = SqidEncoder.Encode(batchStartIndex + index), @@ -496,6 +513,12 @@ await optionTable $"Inserted incorrect number of filter option meta links for {meta.PublicId}. " + $"Inserted: {insertedLinks}, expected: {options.Count}"); } + + await _dbContext.SetSequenceValue( + PublicDataDbContext.FilterOptionMetaLinkSequence, + startIndex + insertedLinks - 1, + _cancellationToken + ); } } @@ -585,7 +608,10 @@ FROM read_csv('{_dataFilePath:raw}', ALL_VARCHAR = true) .Where(o => !existingRowKeys.Contains(o.GetRowKey())) .ToList(); - var startIndex = await _dbContext.LocationOptionMetas.CountAsync(token: _cancellationToken); + var startIndex = await _dbContext.NextSequenceValue( + PublicDataDbContext.LocationOptionMetasIdSequence, + _cancellationToken + ); foreach (var option in newOptions) { @@ -593,7 +619,16 @@ FROM read_csv('{_dataFilePath:raw}', ALL_VARCHAR = true) option.PublicId = SqidEncoder.Encode(option.Id); } - await optionTable.BulkCopyAsync(newOptions, cancellationToken: _cancellationToken); + await optionTable.BulkCopyAsync( + new BulkCopyOptions { KeepIdentity = true }, + newOptions, + cancellationToken: _cancellationToken); + + await _dbContext.SetSequenceValue( + PublicDataDbContext.LocationOptionMetasIdSequence, + startIndex - 1, + _cancellationToken + ); } var links = await optionTable diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/PublisherFunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/PublisherFunctionsIntegrationTest.cs index 588b7c8d4ec..2b323a79844 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/PublisherFunctionsIntegrationTest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/PublisherFunctionsIntegrationTest.cs @@ -15,13 +15,19 @@ namespace GovUk.Education.ExploreEducationStatistics.Publisher.Tests; -public abstract class PublisherFunctionsIntegrationTest - : FunctionsIntegrationTest +public abstract class PublisherFunctionsIntegrationTest( + FunctionsIntegrationTestFixture fixture) + : FunctionsIntegrationTest(fixture), IAsyncLifetime { - protected PublisherFunctionsIntegrationTest(FunctionsIntegrationTestFixture fixture) : base(fixture) + public async Task InitializeAsync() { ResetDbContext(); - ClearTestData(); + await ClearTestData(); + } + + public Task DisposeAsync() + { + return Task.CompletedTask; } } @@ -55,7 +61,10 @@ public override IHostBuilder ConfigureTestHostBuilder() services.UseInMemoryDbContext(); services.AddDbContext( - options => options.UseNpgsql(_postgreSqlContainer.GetConnectionString())); + options => options + .UseNpgsql( + _postgreSqlContainer.GetConnectionString(), + psqlOptions => psqlOptions.EnableRetryOnFailure())); using var serviceScope = services.BuildServiceProvider() .GetRequiredService() @@ -68,7 +77,8 @@ public override IHostBuilder ConfigureTestHostBuilder() protected override IEnumerable GetFunctionTypes() { - return [ + return + [ typeof(NotifyChangeFunction), typeof(PublishImmediateReleaseContentFunction), typeof(PublishMethodologyFilesFunction), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/DataSetPublishingServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/DataSetPublishingServiceTests.cs index a1b4005b085..0c24c1df85b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/DataSetPublishingServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/DataSetPublishingServiceTests.cs @@ -29,9 +29,7 @@ public async Task FirstDataSetVersionPublished() ReleaseFile releaseDataFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture - .DefaultFile() - .WithType(FileType.Data)) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(releaseVersion); DataSet dataSet = DataFixture @@ -87,9 +85,7 @@ public async Task SecondDataSetVersionPublished() ReleaseFile releaseDataFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture - .DefaultFile() - .WithType(FileType.Data)) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(releaseVersion); DataSet dataSet = DataFixture @@ -158,9 +154,7 @@ public async Task NoDraftDataSetVersionsToPublish() ReleaseFile releaseDataFile = DataFixture .DefaultReleaseFile() - .WithFile(DataFixture - .DefaultFile() - .WithType(FileType.Data)) + .WithFile(DataFixture.DefaultFile(FileType.Data)) .WithReleaseVersion(releaseVersion); DataSet dataSet = DataFixture @@ -222,8 +216,7 @@ public async Task ReleaseAmendmentPublished_UpdatesReleaseFileId() .WithPreviousVersionId(originalReleaseVersion.Id); File dataFile = DataFixture - .DefaultFile() - .WithType(FileType.Data); + .DefaultFile(FileType.Data); var (originalReleaseDataFile, amendmentReleaseDataFile) = DataFixture .DefaultReleaseFile() @@ -294,8 +287,7 @@ public async Task ReleaseAmendmentPublishedWithNewDataSetVersion_CorrectReleaseF .WithPreviousVersionId(originalReleaseVersion.Id); var (dataFile, newDataFile) = DataFixture - .DefaultFile() - .WithType(FileType.Data) + .DefaultFile(FileType.Data) .Generate(2) .ToTuple2(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Configuration/AppSettingOptions.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Configuration/AppSettingsOptions.cs similarity index 89% rename from src/GovUk.Education.ExploreEducationStatistics.Publisher/Configuration/AppSettingOptions.cs rename to src/GovUk.Education.ExploreEducationStatistics.Publisher/Configuration/AppSettingsOptions.cs index 87c5d94c905..f030d991e64 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Configuration/AppSettingOptions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Configuration/AppSettingsOptions.cs @@ -1,6 +1,6 @@ namespace GovUk.Education.ExploreEducationStatistics.Publisher.Configuration; -public class AppSettingOptions +public class AppSettingsOptions { public const string AppSettings = "AppSettings"; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Functions/StageReleaseContentFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Functions/StageReleaseContentFunction.cs index 1678767a748..aa838c8faca 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Functions/StageReleaseContentFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Functions/StageReleaseContentFunction.cs @@ -16,11 +16,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Publisher.Functions { public class StageReleaseContentFunction( ILogger logger, - IOptions appSettingOptions, + IOptions appSettingsOptions, IContentService contentService, IReleasePublishingStatusService releasePublishingStatusService) { - private readonly AppSettingOptions _appSettingOptions = appSettingOptions.Value; + private readonly AppSettingsOptions _appSettingsOptions = appSettingsOptions.Value; /// /// Azure function which generates the content for a Release into a staging directory. @@ -30,8 +30,7 @@ public class StageReleaseContentFunction( /// [Function("StageReleaseContent")] public async Task StageReleaseContent( - [QueueTrigger(StageReleaseContentQueue)] - StageReleaseContentMessage message, + [QueueTrigger(StageReleaseContentQueue)] StageReleaseContentMessage message, FunctionContext context) { logger.LogInformation("{FunctionName} triggered: {Message}", @@ -40,7 +39,7 @@ public async Task StageReleaseContent( await UpdateContentStage(message, Started); try { - var publishStagedReleasesCronExpression = _appSettingOptions.PublishReleaseContentCronSchedule; + var publishStagedReleasesCronExpression = _appSettingsOptions.PublishReleaseContentCronSchedule; var nextScheduledPublishingTime = CrontabSchedule.Parse(publishStagedReleasesCronExpression, new CrontabSchedule.ParseOptions { @@ -54,7 +53,8 @@ await contentService.UpdateContentStaged(nextScheduledPublishingTime, { logger.LogError(e, "Exception occured while executing {FunctionName}", context.FunctionDefinition.Name); - await UpdateContentStage(message, Failed, + await UpdateContentStage(message, + Failed, new ReleasePublishingStatusLogMessage($"Exception in content stage: {e.Message}")); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs index d035691dc57..4eaa6827bd4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs @@ -30,7 +30,8 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using IMethodologyService = GovUk.Education.ExploreEducationStatistics.Publisher.Services.Interfaces.IMethodologyService; +using IMethodologyService = + GovUk.Education.ExploreEducationStatistics.Publisher.Services.Interfaces.IMethodologyService; using IReleaseService = GovUk.Education.ExploreEducationStatistics.Publisher.Services.Interfaces.IReleaseService; using MethodologyService = GovUk.Education.ExploreEducationStatistics.Publisher.Services.MethodologyService; using ReleaseService = GovUk.Education.ExploreEducationStatistics.Publisher.Services.ReleaseService; @@ -72,7 +73,8 @@ public static IHostBuilder ConfigurePublisherHostBuilder(this IHostBuilder hostB .AddDbContext(options => options.UseSqlServer( ConnectionUtils.GetAzureSqlConnectionString("ContentDb"), - providerOptions => SqlServerDbContextOptionsBuilderExtensions.EnableCustomRetryOnFailure(providerOptions))) + providerOptions => + SqlServerDbContextOptionsBuilderExtensions.EnableCustomRetryOnFailure(providerOptions))) .AddDbContext(options => options.UseSqlServer( ConnectionUtils.GetAzureSqlConnectionString("StatisticsDb"), @@ -125,8 +127,7 @@ public static IHostBuilder ConfigurePublisherHostBuilder(this IHostBuilder hostB .AddScoped() .AddScoped() .AddScoped() - - .Configure(configuration.GetSection(AppSettingOptions.AppSettings)); + .Configure(configuration.GetSection(AppSettingsOptions.AppSettings)); // TODO EES-5073 Remove this check when the Public Data db is available in all Azure environments. if (publicDataDbExists) @@ -137,7 +138,7 @@ public static IHostBuilder ConfigurePublisherHostBuilder(this IHostBuilder hostB { services.AddScoped(); } - + // TODO EES-3510 These services from the Content.Services namespace are used to update cached resources. // EES-3528 plans to send a request to the Content API to update its cached resources instead of this // being done from Publisher directly, and so these DI dependencies should eventually be removed. diff --git a/src/explore-education-statistics-admin/src/hooks/useCKEditorConfig.ts b/src/explore-education-statistics-admin/src/hooks/useCKEditorConfig.ts index caf3c6420c5..354c8b9cf00 100644 --- a/src/explore-education-statistics-admin/src/hooks/useCKEditorConfig.ts +++ b/src/explore-education-statistics-admin/src/hooks/useCKEditorConfig.ts @@ -134,15 +134,6 @@ const useCKEditorConfig = ({ url?.match(/\/data-tables\/fast-track\/[a-zA-Z-0-9-]/), attributes: { 'data-featured-table': '' }, }, - openInNewTab: { - mode: 'manual', - label: 'Open in a new tab', - defaultValue: false, - attributes: { - target: '_blank', - rel: 'noopener noreferrer', - }, - }, }, defaultProtocol: 'https://', }, diff --git a/src/explore-education-statistics-admin/src/pages/release/ReleaseDataGuidancePage.tsx b/src/explore-education-statistics-admin/src/pages/release/ReleaseDataGuidancePage.tsx index a917c8dec53..8c891d4d517 100644 --- a/src/explore-education-statistics-admin/src/pages/release/ReleaseDataGuidancePage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/ReleaseDataGuidancePage.tsx @@ -60,7 +60,7 @@ const ReleaseDataGuidancePage = ({ renderDataCatalogueLink={ model.release.published ? ( data catalogue diff --git a/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx b/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx index 04e9e851b65..0aee5ba3f39 100644 --- a/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/ReleasePageContainer.tsx @@ -46,7 +46,6 @@ const allNavRoutes = [ releaseContentRoute, releaseStatusRoute, releasePreReleaseAccessRoute, - releaseApiDataSetsRoute, ]; const routes = [ @@ -56,13 +55,14 @@ const routes = [ releaseDataFileRoute, releaseDataFileReplaceRoute, releaseDataFileReplacementCompleteRoute, + releaseApiDataSetsRoute, + releaseApiDataSetDetailsRoute, releaseSummaryEditRoute, releaseFootnotesCreateRoute, releaseFootnotesEditRoute, releaseTableToolRoute, releaseDataBlockCreateRoute, releaseDataBlockEditRoute, - releaseApiDataSetDetailsRoute, ]; interface MatchProps { diff --git a/src/explore-education-statistics-admin/src/pages/release/components/ReleaseStatusChecklist.tsx b/src/explore-education-statistics-admin/src/pages/release/components/ReleaseStatusChecklist.tsx index eed26b5a55d..89e5fc12478 100644 --- a/src/explore-education-statistics-admin/src/pages/release/components/ReleaseStatusChecklist.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/components/ReleaseStatusChecklist.tsx @@ -1,5 +1,5 @@ import Link from '@admin/components/Link'; -import { releaseDataPageTabIds } from '@admin/pages/release/data/ReleaseDataPage'; +import releaseDataPageTabIds from '@admin/pages/release/data/utils/releaseDataPageTabIds'; import { releasePreReleaseAccessPageTabs } from '@admin/pages/release/pre-release/ReleasePreReleaseAccessPage'; import { MethodologyRouteParams, diff --git a/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx index efd70fef9dd..9c3985adbc5 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx @@ -131,7 +131,7 @@ describe('ReleaseContentPage', () => { slug: 'publication-1', releases: [], releaseSeries: [], - topic: { theme: { title: 'Theme 1' } }, + topic: { theme: { id: 'theme-1', title: 'Theme 1' } }, contact: { contactName: 'John Smith', contactTelNo: '0777777777', diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetDetailsPage.tsx similarity index 89% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetDetailsPage.tsx index 7ad3f9f226f..a082aeb3de9 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetDetailsPage.tsx @@ -1,6 +1,7 @@ import Link from '@admin/components/Link'; import { useConfig } from '@admin/contexts/ConfigContext'; -import ApiDataSetVersionSummaryList from '@admin/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList'; +import ApiDataSetVersionSummaryList from '@admin/pages/release/data/components/ApiDataSetVersionSummaryList'; +import DeleteDraftVersionButton from '@admin/pages/release/data/components/DeleteDraftVersionButton'; import { useReleaseContext } from '@admin/pages/release/contexts/ReleaseContext'; import apiDataSetQueries from '@admin/queries/apiDataSetQueries'; import { @@ -9,7 +10,6 @@ import { ReleaseRouteParams, } from '@admin/routes/releaseRoutes'; import { DataSetStatus } from '@admin/services/apiDataSetService'; -import ButtonText from '@common/components/ButtonText'; import ContentHtml from '@common/components/ContentHtml'; import LoadingSpinner from '@common/components/LoadingSpinner'; import SummaryCard from '@common/components/SummaryCard'; @@ -20,10 +20,8 @@ import TaskList from '@common/components/TaskList'; import TaskListItem from '@common/components/TaskListItem'; import { useQuery } from '@tanstack/react-query'; import React from 'react'; -import { generatePath, useParams } from 'react-router-dom'; +import { generatePath, useHistory, useParams } from 'react-router-dom'; -// TODO: EES-4370 -const showRemoveDraft = false; // TODO: Version mapping const showDraftVersionTasks = false; @@ -34,6 +32,8 @@ const showVersionHistory = false; export default function ReleaseApiDataSetDetailsPage() { const { dataSetId } = useParams(); + const history = useHistory(); + const { publicAppUrl } = useConfig(); const { release } = useReleaseContext(); @@ -56,12 +56,21 @@ export default function ReleaseApiDataSetDetailsPage() { publicationId={release.publicationId} collapsibleButtonHiddenText="for draft version" actions={ - showRemoveDraft ? ( -
    -
  • - Remove draft version -
    • -
+ dataSet.draftVersion.status !== 'Processing' ? ( + + history.push( + generatePath(releaseApiDataSetsRoute.path, { + publicationId: release.publicationId, + releaseId: release.id, + }), + ) + } + > + Delete draft version + ) : undefined } /> diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataPage.tsx index 025bde59171..d9f1e24f830 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataPage.tsx @@ -1,8 +1,11 @@ +import { useAuthContext } from '@admin/contexts/AuthContext'; import { useReleaseContext } from '@admin/pages/release/contexts/ReleaseContext'; -import ReleaseDataUploadsSection from '@admin/pages/release/data/components/ReleaseDataUploadsSection'; -import ReleaseFileUploadsSection from '@admin/pages/release/data/components/ReleaseFileUploadsSection'; +import ReleaseApiDataSetsSection from '@admin/pages/release/data/components/ReleaseApiDataSetsSection'; import ReleaseDataGuidanceSection from '@admin/pages/release/data/components/ReleaseDataGuidanceSection'; import ReleaseDataReorderSection from '@admin/pages/release/data/components/ReleaseDataReorderSection'; +import ReleaseDataUploadsSection from '@admin/pages/release/data/components/ReleaseDataUploadsSection'; +import ReleaseFileUploadsSection from '@admin/pages/release/data/components/ReleaseFileUploadsSection'; +import releaseDataPageTabIds from '@admin/pages/release/data/utils/releaseDataPageTabIds'; import permissionService from '@admin/services/permissionService'; import { DataFile } from '@admin/services/releaseDataFileService'; import LoadingSpinner from '@common/components/LoadingSpinner'; @@ -11,15 +14,10 @@ import TabsSection from '@common/components/TabsSection'; import useAsyncHandledRetry from '@common/hooks/useAsyncHandledRetry'; import React, { useState } from 'react'; -export const releaseDataPageTabIds = { - dataUploads: 'data-uploads', - fileUploads: 'file-uploads', - dataGuidance: 'data-guidance', - reordering: 'reordering', -}; - const ReleaseDataPage = () => { const { release, releaseId } = useReleaseContext(); + const { user } = useAuthContext(); + const [dataFiles, setDataFiles] = useState([]); const { value: canUpdateRelease = false, isLoading } = useAsyncHandledRetry( @@ -29,7 +27,7 @@ const ReleaseDataPage = () => { return ( - + { canUpdateRelease={canUpdateRelease} /> + {user?.permissions.isBauUser && ( + + + + )} ); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetDetailsPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetDetailsPage.test.tsx similarity index 98% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetDetailsPage.test.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetDetailsPage.test.tsx index d4df2f347e6..9b1ce6d1876 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetDetailsPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetDetailsPage.test.tsx @@ -1,6 +1,6 @@ import { TestConfigContextProvider } from '@admin/contexts/ConfigContext'; import { testRelease } from '@admin/pages/release/__data__/testRelease'; -import ReleaseApiDataSetDetailsPage from '@admin/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage'; +import ReleaseApiDataSetDetailsPage from '@admin/pages/release/data/ReleaseApiDataSetDetailsPage'; import { ReleaseContextProvider } from '@admin/pages/release/contexts/ReleaseContext'; import { releaseApiDataSetDetailsRoute, diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateForm.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateForm.tsx similarity index 95% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateForm.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateForm.tsx index dc6c1f082fa..217bfa18015 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateForm.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateForm.tsx @@ -58,9 +58,12 @@ export default function ApiDataSetCreateForm({ Cancel diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx similarity index 94% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx index ab35b982e58..911a3dde342 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetCreateModal.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx @@ -1,7 +1,7 @@ import Link from '@admin/components/Link'; import ApiDataSetCreateForm, { ApiDataSetCreateFormValues, -} from '@admin/pages/release/api-data-sets/components/ApiDataSetCreateForm'; +} from '@admin/pages/release/data/components/ApiDataSetCreateForm'; import apiDataSetCandidateQueries from '@admin/queries/apiDataSetCandidateQueries'; import { releaseApiDataSetDetailsRoute, @@ -86,8 +86,8 @@ export default function ApiDataSetCreateModal({ ) : ( <> - No API data sets can be created as there are no candidates data - files available. New candidate data files can be uploaded in the{' '} + No API data sets can be created as there are no candidate data files + available. New candidate data files can be uploaded in the{' '} (releaseDataRoute.path, { publicationId, diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetVersionSummaryList.tsx similarity index 89% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetVersionSummaryList.tsx index 12692b319cd..ee34874b87e 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetVersionSummaryList.tsx @@ -1,6 +1,6 @@ import Link from '@admin/components/Link'; -import getVersionStatusTagColour from '@admin/pages/release/api-data-sets/utils/getVersionStatusColour'; -import getVersionStatusText from '@admin/pages/release/api-data-sets/utils/getVersionStatusText'; +import getDataSetVersionStatusTagColour from '@admin/pages/release/data/components/utils/getDataSetVersionStatusColour'; +import getDataSetVersionStatusText from '@admin/pages/release/data/components/utils/getDataSetVersionStatusText'; import { ReleaseRouteParams, releaseSummaryRoute, @@ -38,12 +38,12 @@ export default function ApiDataSetVersionSummaryList({ {`v${dataSetVersion.version}`} - - {getVersionStatusText(dataSetVersion.status)} + + {getDataSetVersionStatusText(dataSetVersion.status)} diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/DeleteDraftVersionButton.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/DeleteDraftVersionButton.tsx new file mode 100644 index 00000000000..f5fc738ee0b --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/DeleteDraftVersionButton.tsx @@ -0,0 +1,56 @@ +import apiDataSetQueries from '@admin/queries/apiDataSetQueries'; +import { + ApiDataSet, + ApiDataSetVersion, +} from '@admin/services/apiDataSetService'; +import apiDataSetVersionService from '@admin/services/apiDataSetVersionService'; +import ButtonText from '@common/components/ButtonText'; +import ModalConfirm from '@common/components/ModalConfirm'; +import { useQueryClient } from '@tanstack/react-query'; +import React, { ReactNode, useCallback } from 'react'; + +export interface DeleteDraftVersionButtonProps { + children: ReactNode; + dataSet: Pick; + dataSetVersion: Pick; + modalTitle?: string; + onDeleted?: () => void; +} + +export default function DeleteDraftVersionButton({ + children, + dataSet, + dataSetVersion, + modalTitle = 'Remove draft version', + onDeleted, +}: DeleteDraftVersionButtonProps) { + const queryClient = useQueryClient(); + + const handleConfirm = useCallback(async () => { + await apiDataSetVersionService.deleteVersion(dataSetVersion.id); + + onDeleted?.(); + + queryClient.removeQueries({ + queryKey: apiDataSetQueries.get(dataSet.id).queryKey, + }); + await queryClient.invalidateQueries({ + queryKey: apiDataSetQueries.list._def, + }); + }, [dataSet.id, dataSetVersion.id, onDeleted, queryClient]); + + return ( + {children}} + hiddenConfirmingText="Removing draft version" + onConfirm={handleConfirm} + > +

+ Confirm that you want to delete the draft version{' '} + {dataSetVersion.version} for API data set:
+ {dataSet.title} +

+ + ); +} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.module.scss b/src/explore-education-statistics-admin/src/pages/release/data/components/DraftApiDataSetsTable.module.scss similarity index 100% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.module.scss rename to src/explore-education-statistics-admin/src/pages/release/data/components/DraftApiDataSetsTable.module.scss diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/DraftApiDataSetsTable.tsx similarity index 79% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/DraftApiDataSetsTable.tsx index 319195d2fbe..9c2beaaba21 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/DraftApiDataSetsTable.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/DraftApiDataSetsTable.tsx @@ -1,6 +1,7 @@ import Link from '@admin/components/Link'; -import getVersionStatusTagColour from '@admin/pages/release/api-data-sets/utils/getVersionStatusColour'; -import getVersionStatusText from '@admin/pages/release/api-data-sets/utils/getVersionStatusText'; +import DeleteDraftVersionButton from '@admin/pages/release/data/components/DeleteDraftVersionButton'; +import getDataSetVersionStatusTagColour from '@admin/pages/release/data/components/utils/getDataSetVersionStatusColour'; +import getDataSetVersionStatusText from '@admin/pages/release/data/components/utils/getDataSetVersionStatusText'; import { releaseApiDataSetDetailsRoute, ReleaseDataSetRouteParams, @@ -10,7 +11,6 @@ import { ApiDataSetSummary, } from '@admin/services/apiDataSetService'; import ButtonGroup from '@common/components/ButtonGroup'; -import ButtonText from '@common/components/ButtonText'; import InsetText from '@common/components/InsetText'; import Tag from '@common/components/Tag'; import VisuallyHidden from '@common/components/VisuallyHidden'; @@ -70,7 +70,7 @@ export default function DraftApiDataSetsTable({ {`v${draftVersion.version}`} @@ -89,8 +89,10 @@ export default function DraftApiDataSetsTable({ ) : null} {dataSet.title} - - {getVersionStatusText(draftVersion.status)} + + {getDataSetVersionStatusText(draftVersion.status)} @@ -98,12 +100,6 @@ export default function DraftApiDataSetsTable({ className="govuk-!-margin-bottom-0" horizontalSpacing="m" > - {draftVersion.status !== 'Processing' && ( - - Remove draft - for {dataSet.title} - - )} ( releaseApiDataSetDetailsRoute.path, @@ -114,9 +110,20 @@ export default function DraftApiDataSetsTable({ }, )} > - View / edit draft + {draftVersion.version === '1.0' + ? 'View details' + : 'View details / edit draft'} for {dataSet.title} + {draftVersion.status !== 'Processing' && ( + + Delete draft + for {dataSet.title} + + )} diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.module.scss b/src/explore-education-statistics-admin/src/pages/release/data/components/LiveApiDataSetsTable.module.scss similarity index 100% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.module.scss rename to src/explore-education-statistics-admin/src/pages/release/data/components/LiveApiDataSetsTable.module.scss diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/LiveApiDataSetsTable.tsx similarity index 100% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/LiveApiDataSetsTable.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/LiveApiDataSetsTable.tsx diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetsPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseApiDataSetsSection.tsx similarity index 92% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetsPage.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseApiDataSetsSection.tsx index a25770ff836..fb4e530c2c3 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/ReleaseApiDataSetsPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseApiDataSetsSection.tsx @@ -1,12 +1,12 @@ import { useAuthContext } from '@admin/contexts/AuthContext'; -import ApiDataSetCreateModal from '@admin/pages/release/api-data-sets/components/ApiDataSetCreateModal'; +import { useReleaseContext } from '@admin/pages/release/contexts/ReleaseContext'; +import ApiDataSetCreateModal from '@admin/pages/release/data/components/ApiDataSetCreateModal'; import DraftApiDataSetsTable, { DraftApiDataSetSummary, -} from '@admin/pages/release/api-data-sets/components/DraftApiDataSetsTable'; +} from '@admin/pages/release/data/components/DraftApiDataSetsTable'; import LiveApiDataSetsTable, { LiveApiDataSetSummary, -} from '@admin/pages/release/api-data-sets/components/LiveApiDataSetsTable'; -import { useReleaseContext } from '@admin/pages/release/contexts/ReleaseContext'; +} from '@admin/pages/release/data/components/LiveApiDataSetsTable'; import apiDataSetQueries from '@admin/queries/apiDataSetQueries'; import InsetText from '@common/components/InsetText'; import LoadingSpinner from '@common/components/LoadingSpinner'; @@ -14,7 +14,7 @@ import WarningMessage from '@common/components/WarningMessage'; import { useQuery } from '@tanstack/react-query'; import React from 'react'; -export default function ReleaseApiDataSetsPage() { +export default function ReleaseApiDataSetsSection() { const { release } = useReleaseContext(); const { user } = useAuthContext(); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateForm.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateForm.test.tsx similarity index 98% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateForm.test.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateForm.test.tsx index 6eca941006d..8e3bda62bba 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateForm.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateForm.test.tsx @@ -1,6 +1,6 @@ import ApiDataSetCreateForm, { ApiDataSetCreateFormProps, -} from '@admin/pages/release/api-data-sets/components/ApiDataSetCreateForm'; +} from '@admin/pages/release/data/components/ApiDataSetCreateForm'; import { ApiDataSetCandidate } from '@admin/services/apiDataSetCandidateService'; import render from '@common-test/render'; import { screen, waitFor, within } from '@testing-library/react'; diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx similarity index 96% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx index db67af381bb..c01df585a04 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetCreateModal.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx @@ -1,4 +1,4 @@ -import ApiDataSetCreateModal from '@admin/pages/release/api-data-sets/components/ApiDataSetCreateModal'; +import ApiDataSetCreateModal from '@admin/pages/release/data/components/ApiDataSetCreateModal'; import _apiDataSetCandidateService, { ApiDataSetCandidate, } from '@admin/services/apiDataSetCandidateService'; @@ -43,7 +43,7 @@ describe('ApiDataSetCreateModal', () => { expect( await screen.findByText( - /No API data sets can be created as there are no candidates data files available/, + /No API data sets can be created as there are no candidate data files available/, ), ).toBeInTheDocument(); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetVersionSummaryList.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetVersionSummaryList.test.tsx similarity index 98% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetVersionSummaryList.test.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetVersionSummaryList.test.tsx index fc10dcfe73f..b7213aca4cd 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/ApiDataSetVersionSummaryList.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetVersionSummaryList.test.tsx @@ -1,4 +1,4 @@ -import ApiDataSetVersionSummaryList from '@admin/pages/release/api-data-sets/components/ApiDataSetVersionSummaryList'; +import ApiDataSetVersionSummaryList from '@admin/pages/release/data/components/ApiDataSetVersionSummaryList'; import { ApiDataSetDraftVersion } from '@admin/services/apiDataSetService'; import { render as baseRender, screen, within } from '@testing-library/react'; import { ReactElement } from 'react'; diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/DeleteDraftVersionButton.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/DeleteDraftVersionButton.test.tsx new file mode 100644 index 00000000000..6a8619496f9 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/DeleteDraftVersionButton.test.tsx @@ -0,0 +1,112 @@ +import DeleteDraftVersionButton, { + DeleteDraftVersionButtonProps, +} from '@admin/pages/release/data/components/DeleteDraftVersionButton'; +import _apiDataSetVersionService from '@admin/services/apiDataSetVersionService'; +import render from '@common-test/render'; +import { screen, within, waitFor } from '@testing-library/react'; + +jest.mock('@admin/services/apiDataSetVersionService'); + +const apiDataSetVersionService = jest.mocked(_apiDataSetVersionService); + +describe('DeleteDraftVersionButton', () => { + const testDataSet: DeleteDraftVersionButtonProps['dataSet'] = { + id: 'data-set-id', + title: 'Data set title', + }; + + const testDataSetVersion: DeleteDraftVersionButtonProps['dataSetVersion'] = { + id: 'data-set-version-id', + version: '1.0', + }; + + test('renders correctly with data set version details', () => { + render( + + Delete draft version + , + ); + + expect( + screen.getByRole('button', { name: 'Delete draft version' }), + ).toBeInTheDocument(); + }); + + test('clicking the `Confirm` button opens a confirmation modal', async () => { + const { user } = render( + + Delete draft version + , + ); + + await user.click( + screen.getByRole('button', { name: 'Delete draft version' }), + ); + + const modal = within(screen.getByRole('dialog')); + + expect( + modal.getByRole('heading', { name: 'Remove draft version' }), + ).toBeInTheDocument(); + + expect(modal.getByTestId('confirm-text')).toHaveTextContent( + 'Confirm that you want to delete the draft version 1.0 for API data set: Data set title', + ); + }); + + test('confirming the deletion calls the correct service', async () => { + const { user } = render( + + Delete draft version + , + ); + + await user.click( + screen.getByRole('button', { name: 'Delete draft version' }), + ); + + const modal = within(screen.getByRole('dialog')); + + expect(apiDataSetVersionService.deleteVersion).not.toHaveBeenCalled(); + + await user.click(modal.getByRole('button', { name: 'Confirm' })); + + expect(apiDataSetVersionService.deleteVersion).toHaveBeenCalledWith( + testDataSetVersion.id, + ); + }); + + test('confirming the deletion closes the modal', async () => { + const { user } = render( + + Delete draft version + , + ); + + await user.click( + screen.getByRole('button', { name: 'Delete draft version' }), + ); + + const modal = within(screen.getByRole('dialog')); + + await user.click(modal.getByRole('button', { name: 'Confirm' })); + + await waitFor(() => { + expect(screen.queryByText('Confirm')).not.toBeInTheDocument(); + }); + + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + }); +}); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/DraftApiDataSetsTable.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/DraftApiDataSetsTable.test.tsx similarity index 87% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/DraftApiDataSetsTable.test.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/DraftApiDataSetsTable.test.tsx index 2f262e3d523..0b6945a16e2 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/DraftApiDataSetsTable.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/DraftApiDataSetsTable.test.tsx @@ -1,7 +1,8 @@ import DraftApiDataSetsTable, { DraftApiDataSetSummary, -} from '@admin/pages/release/api-data-sets/components/DraftApiDataSetsTable'; -import { render as baseRender, screen, within } from '@testing-library/react'; +} from '@admin/pages/release/data/components/DraftApiDataSetsTable'; +import baseRender from '@common-test/render'; +import { screen, within } from '@testing-library/react'; import { ReactNode } from 'react'; import { MemoryRouter } from 'react-router-dom'; @@ -122,12 +123,12 @@ describe('DraftApiDataSetsTable', () => { expect( within(row1Cells[4]).getByRole('link', { - name: 'View / edit draft for Data set 1 title', + name: 'View details / edit draft for Data set 1 title', }), ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-1`); expect( within(row1Cells[4]).getByRole('button', { - name: 'Remove draft for Data set 1 title', + name: 'Delete draft for Data set 1 title', }), ).toBeInTheDocument(); @@ -142,13 +143,13 @@ describe('DraftApiDataSetsTable', () => { expect( within(row2Cells[4]).getByRole('link', { - name: 'View / edit draft for Data set 2 title', + name: 'View details / edit draft for Data set 2 title', }), ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-2`); expect( within(row2Cells[4]).getByRole('button', { - name: 'Remove draft for Data set 2 title', + name: 'Delete draft for Data set 2 title', }), ).toBeInTheDocument(); @@ -163,11 +164,13 @@ describe('DraftApiDataSetsTable', () => { expect( within(row3Cells[4]).getByRole('link', { - name: 'View / edit draft for Data set 3 title', + name: 'View details for Data set 3 title', }), ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-3`); expect( - within(row3Cells[4]).queryByRole('button', { name: /Remove draft/ }), + within(row3Cells[4]).queryByRole('button', { + name: /Delete draft/, + }), ).not.toBeInTheDocument(); // Row 4 @@ -181,13 +184,13 @@ describe('DraftApiDataSetsTable', () => { expect( within(row4Cells[4]).getByRole('link', { - name: 'View / edit draft for Data set 4 title', + name: 'View details for Data set 4 title', }), ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-4`); expect( within(row4Cells[4]).getByRole('button', { - name: 'Remove draft for Data set 4 title', + name: 'Delete draft for Data set 4 title', }), ).toBeInTheDocument(); @@ -202,13 +205,13 @@ describe('DraftApiDataSetsTable', () => { expect( within(row5Cells[4]).getByRole('link', { - name: 'View / edit draft for Data set 5 title', + name: 'View details for Data set 5 title', }), ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-5`); expect( within(row5Cells[4]).getByRole('button', { - name: 'Remove draft for Data set 5 title', + name: 'Delete draft for Data set 5 title', }), ).toBeInTheDocument(); @@ -223,13 +226,13 @@ describe('DraftApiDataSetsTable', () => { expect( within(row6Cells[4]).getByRole('link', { - name: 'View / edit draft for Data set 6 title', + name: 'View details for Data set 6 title', }), ).toHaveAttribute('href', `${baseDataSetUrl}/data-set-6`); expect( within(row6Cells[4]).getByRole('button', { - name: 'Remove draft for Data set 6 title', + name: 'Delete draft for Data set 6 title', }), ).toBeInTheDocument(); }); diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/LiveApiDataSetsTable.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/LiveApiDataSetsTable.test.tsx similarity index 98% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/LiveApiDataSetsTable.test.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/LiveApiDataSetsTable.test.tsx index cc862d37a25..d41acf1522d 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/components/__tests__/LiveApiDataSetsTable.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/LiveApiDataSetsTable.test.tsx @@ -1,6 +1,6 @@ import LiveApiDataSetsTable, { LiveApiDataSetSummary, -} from '@admin/pages/release/api-data-sets/components/LiveApiDataSetsTable'; +} from '@admin/pages/release/data/components/LiveApiDataSetsTable'; import { render as baseRender, screen, within } from '@testing-library/react'; import { ReactNode } from 'react'; import { MemoryRouter } from 'react-router-dom'; diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetsPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReleaseApiDataSetsSection.test.tsx similarity index 91% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetsPage.test.tsx rename to src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReleaseApiDataSetsSection.test.tsx index 3b55569546f..b41373f87fe 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/__tests__/ReleaseApiDataSetsPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReleaseApiDataSetsSection.test.tsx @@ -1,11 +1,7 @@ import { AuthContextTestProvider, User } from '@admin/contexts/AuthContext'; import { testRelease } from '@admin/pages/release/__data__/testRelease'; -import ReleaseApiDataSetsPage from '@admin/pages/release/api-data-sets/ReleaseApiDataSetsPage'; import { ReleaseContextProvider } from '@admin/pages/release/contexts/ReleaseContext'; -import { - releaseApiDataSetsRoute, - ReleaseRouteParams, -} from '@admin/routes/releaseRoutes'; +import ReleaseApiDataSetsSection from '@admin/pages/release/data/components/ReleaseApiDataSetsSection'; import _apiDataSetCandidateService, { ApiDataSetCandidate, } from '@admin/services/apiDataSetCandidateService'; @@ -16,7 +12,7 @@ import { GlobalPermissions } from '@admin/services/authService'; import { Release } from '@admin/services/releaseService'; import render, { CustomRenderResult } from '@common-test/render'; import { screen, within } from '@testing-library/react'; -import { generatePath, MemoryRouter, Route } from 'react-router-dom'; +import { MemoryRouter } from 'react-router-dom'; jest.mock('@admin/services/apiDataSetService'); jest.mock('@admin/services/apiDataSetCandidateService'); @@ -24,7 +20,7 @@ jest.mock('@admin/services/apiDataSetCandidateService'); const apiDataSetCandidateService = jest.mocked(_apiDataSetCandidateService); const apiDataSetService = jest.mocked(_apiDataSetService); -describe('ReleaseApiDataSetsPage', () => { +describe('ReleaseApiDataSetsSection', () => { const testBauUser: User = { id: 'user-id-1', name: 'BAU user', @@ -238,18 +234,8 @@ describe('ReleaseApiDataSetsPage', () => { return render( - (releaseApiDataSetsRoute.path, { - publicationId: testRelease.publicationId, - releaseId: testRelease.id, - }), - ]} - > - + + , diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusColour.ts b/src/explore-education-statistics-admin/src/pages/release/data/components/utils/getDataSetVersionStatusColour.ts similarity index 90% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusColour.ts rename to src/explore-education-statistics-admin/src/pages/release/data/components/utils/getDataSetVersionStatusColour.ts index 2250120c1b9..8646c44cc20 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusColour.ts +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/utils/getDataSetVersionStatusColour.ts @@ -1,7 +1,7 @@ import { DataSetVersionStatus } from '@admin/services/apiDataSetService'; import { TagProps } from '@common/components/Tag'; -export default function getVersionStatusTagColour( +export default function getDataSetVersionStatusTagColour( status: DataSetVersionStatus, ): TagProps['colour'] { switch (status) { diff --git a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusText.ts b/src/explore-education-statistics-admin/src/pages/release/data/components/utils/getDataSetVersionStatusText.ts similarity index 83% rename from src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusText.ts rename to src/explore-education-statistics-admin/src/pages/release/data/components/utils/getDataSetVersionStatusText.ts index 19e2866300d..49b2a7df6bc 100644 --- a/src/explore-education-statistics-admin/src/pages/release/api-data-sets/utils/getVersionStatusText.ts +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/utils/getDataSetVersionStatusText.ts @@ -1,6 +1,6 @@ import { DataSetVersionStatus } from '@admin/services/apiDataSetService'; -export default function getVersionStatusText( +export default function getDataSetVersionStatusText( status: DataSetVersionStatus, ): string { switch (status) { diff --git a/src/explore-education-statistics-admin/src/pages/release/data/utils/releaseDataPageTabIds.ts b/src/explore-education-statistics-admin/src/pages/release/data/utils/releaseDataPageTabIds.ts new file mode 100644 index 00000000000..22d7904cafc --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/data/utils/releaseDataPageTabIds.ts @@ -0,0 +1,9 @@ +const releaseDataPageTabIds = { + dataUploads: 'data-uploads', + fileUploads: 'file-uploads', + dataGuidance: 'data-guidance', + reordering: 'reordering', + apiDataSets: 'api-data-sets', +} as const; + +export default releaseDataPageTabIds; diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseTableToolPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseTableToolPage.test.tsx index e1b5b4dce69..0f8eb6ff14c 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseTableToolPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseTableToolPage.test.tsx @@ -113,9 +113,7 @@ describe('ReleaseTableToolPage', () => { const stepHeadings = screen.queryAllByRole('heading', { name: /Step/ }); expect(stepHeadings).toHaveLength(1); - expect(stepHeadings[0]).toHaveTextContent( - 'Step 1 (current) Select a data set', - ); + expect(stepHeadings[0]).toHaveTextContent('Step 1 Select a data set'); }); const step = within(screen.getByTestId('wizardStep-1')); diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/__tests__/DataBlockPageTabs.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/__tests__/DataBlockPageTabs.test.tsx index f4f4d1ca5b7..7b2bfc136e6 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/__tests__/DataBlockPageTabs.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/__tests__/DataBlockPageTabs.test.tsx @@ -95,9 +95,7 @@ describe('DataBlockPageTabs', () => { const stepHeadings = screen.queryAllByRole('heading', { name: /Step/ }); expect(stepHeadings).toHaveLength(1); - expect(stepHeadings[0]).toHaveTextContent( - 'Step 1 (current) Select a data set', - ); + expect(stepHeadings[0]).toHaveTextContent('Step 1 Select a data set'); expect(screen.getByTestId('wizardStep-1')).toBeVisible(); expect(screen.getByTestId('wizardStep-2')).not.toBeVisible(); @@ -147,9 +145,7 @@ describe('DataBlockPageTabs', () => { expect(stepHeadings[1]).toHaveTextContent('Step 2 Choose locations'); expect(stepHeadings[2]).toHaveTextContent('Step 3 Choose time period'); expect(stepHeadings[3]).toHaveTextContent('Step 4 Choose your filters'); - expect(stepHeadings[4]).toHaveTextContent( - 'Step 5 (current) Update data block', - ); + expect(stepHeadings[4]).toHaveTextContent('Step 5 Update data block'); expect(screen.getByLabelText('Name')).toHaveValue('Test data block'); expect(screen.getByLabelText('Table title')).toHaveValue('Test title'); @@ -207,9 +203,7 @@ describe('DataBlockPageTabs', () => { expect(stepHeadings).toHaveLength(2); expect(stepHeadings[0]).toHaveTextContent('Step 1 Select a data set'); - expect(stepHeadings[1]).toHaveTextContent( - 'Step 2 (current) Choose locations', - ); + expect(stepHeadings[1]).toHaveTextContent('Step 2 Choose locations'); }); }); @@ -249,9 +243,7 @@ describe('DataBlockPageTabs', () => { expect(stepHeadings).toHaveLength(2); expect(stepHeadings[0]).toHaveTextContent('Step 1 Select a data set'); - expect(stepHeadings[1]).toHaveTextContent( - 'Step 2 (current) Choose locations', - ); + expect(stepHeadings[1]).toHaveTextContent('Step 2 Choose locations'); }); }); diff --git a/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseContentPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseContentPage.test.tsx index 2d925fcc0d9..30d572ed90e 100644 --- a/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseContentPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseContentPage.test.tsx @@ -93,7 +93,7 @@ describe('PreReleaseContentPage', () => { slug: 'publication-1', releases: [], releaseSeries: [], - topic: { theme: { title: 'Theme 1' } }, + topic: { theme: { id: 'theme-1', title: 'Theme 1' } }, contact: { contactName: 'John Smith', contactTelNo: '0777777777', diff --git a/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseTableToolPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseTableToolPage.test.tsx index 2309beb01e2..b3f6bb13f94 100644 --- a/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseTableToolPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseTableToolPage.test.tsx @@ -17,7 +17,7 @@ import _tableBuilderService, { FeaturedTable, Subject, } from '@common/services/tableBuilderService'; -import { screen, waitFor, within } from '@testing-library/react'; +import { screen, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import React from 'react'; import { MemoryRouter, Route } from 'react-router'; @@ -306,12 +306,14 @@ describe('PreReleaseTableToolPage', () => { renderPage(); - await waitFor(() => { - expect(screen.getByTestId('wizardStep-1')).toHaveAttribute( - 'aria-current', - 'step', - ); - }); + expect(await screen.findByText('Step 1')).toBeInTheDocument(); + + expect( + screen.getByRole('heading', { + name: 'Step 1 Select a data set', + }), + ).toHaveAttribute('aria-current', 'step'); + const step1 = within(screen.getByTestId('wizardStep-1')); expect(step1.getByLabelText('Test subject')).toBeInTheDocument(); @@ -331,12 +333,11 @@ describe('PreReleaseTableToolPage', () => { renderPage(); - await waitFor(() => { - expect(screen.getByTestId('wizardStep-1')).toHaveAttribute( - 'aria-current', - 'step', - ); - }); + expect(await screen.findByText('Step 1')).toBeInTheDocument(); + + expect( + screen.getByRole('heading', { name: 'Step 1 Select a data set' }), + ).toHaveAttribute('aria-current', 'step'); const step1 = within(screen.getByTestId('wizardStep-1')); @@ -373,12 +374,11 @@ describe('PreReleaseTableToolPage', () => { ), ]); - await waitFor(() => { - expect(screen.getByTestId('wizardStep-5')).toHaveAttribute( - 'aria-current', - 'step', - ); - }); + expect(await screen.findByText('Step 5')).toBeInTheDocument(); + + expect( + screen.getByRole('heading', { name: 'Step 5 Explore data' }), + ).toHaveAttribute('aria-current', 'step'); expect(screen.getByTestId('dataTableCaption')).toHaveTextContent( /Number of authorised absence sessions for 'Absence by characteristic'/, diff --git a/src/explore-education-statistics-admin/src/prototypes/components/PrototypeTableToolWizard.tsx b/src/explore-education-statistics-admin/src/prototypes/components/PrototypeTableToolWizard.tsx index 487135ae30a..af04d36110e 100644 --- a/src/explore-education-statistics-admin/src/prototypes/components/PrototypeTableToolWizard.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/components/PrototypeTableToolWizard.tsx @@ -402,6 +402,7 @@ const PrototypeTableToolWizard = ({ subjects={state.subjects} subjectId={state.query.subjectId} renderFeaturedTableLink={renderFeaturedTableLink} + stepTitle="Select a data set" onSubmit={handleSubjectFormSubmit} loadingFastTrack={loadingFastTrack} /> @@ -423,6 +424,7 @@ const PrototypeTableToolWizard = ({ {...stepProps} initialValues={state.query.timePeriod} options={state.subjectMeta.timePeriod.options} + stepTitle="Choose a time period" onSubmit={handleTimePeriodFormSubmit} /> )} @@ -436,6 +438,7 @@ const PrototypeTableToolWizard = ({ filters: state.query.filters, }} selectedPublication={state.selectedPublication} + stepTitle="Choose filters" subject={ state.subjects.filter( subject => subject.id === state.query.subjectId, diff --git a/src/explore-education-statistics-admin/src/prototypes/data/releaseContentData.ts b/src/explore-education-statistics-admin/src/prototypes/data/releaseContentData.ts index 0ff21efbc20..4083574c3c1 100644 --- a/src/explore-education-statistics-admin/src/prototypes/data/releaseContentData.ts +++ b/src/explore-education-statistics-admin/src/prototypes/data/releaseContentData.ts @@ -140,6 +140,7 @@ const prototypeReleaseContent: ReleaseContent = { title: 'Initial Teacher Training Census', topic: { theme: { + id: 'test-theme', title: 'Test theme', }, }, diff --git a/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts b/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts index 10ea3f77f86..e999b1129ae 100644 --- a/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts +++ b/src/explore-education-statistics-admin/src/routes/releaseRoutes.ts @@ -1,12 +1,12 @@ import { ProtectedRouteProps } from '@admin/components/ProtectedRoute'; -import ReleaseApiDataSetDetailsPage from '@admin/pages/release/api-data-sets/ReleaseApiDataSetDetailsPage'; -import ReleaseApiDataSetsPage from '@admin/pages/release/api-data-sets/ReleaseApiDataSetsPage'; +import ReleaseApiDataSetDetailsPage from '@admin/pages/release/data/ReleaseApiDataSetDetailsPage'; import ReleaseContentPage from '@admin/pages/release/content/ReleaseContentPage'; import ReleaseDataFilePage from '@admin/pages/release/data/ReleaseDataFilePage'; import ReleaseAncillaryFilePage from '@admin/pages/release/data/ReleaseAncillaryFilePage'; import ReleaseDataFileReplacePage from '@admin/pages/release/data/ReleaseDataFileReplacePage'; import ReleaseDataFileReplacementCompletePage from '@admin/pages/release/data/ReleaseDataFileReplacementCompletePage'; import ReleaseDataPage from '@admin/pages/release/data/ReleaseDataPage'; +import releaseDataPageTabIds from '@admin/pages/release/data/utils/releaseDataPageTabIds'; import ReleaseDataBlockCreatePage from '@admin/pages/release/datablocks/ReleaseDataBlockCreatePage'; import ReleaseDataBlockEditPage from '@admin/pages/release/datablocks/ReleaseDataBlockEditPage'; import ReleaseDataBlocksPage from '@admin/pages/release/datablocks/ReleaseDataBlocksPage'; @@ -72,7 +72,7 @@ export const releaseDataRoute: ReleaseRouteProps = { }; export const releaseAncillaryFilesRoute: ReleaseRouteProps = { - path: '/publication/:publicationId/release/:releaseId/data#file-uploads', + path: `/publication/:publicationId/release/:releaseId/data#${releaseDataPageTabIds.fileUploads}`, title: 'Data and files', component: ReleaseDataPage, }; @@ -101,6 +101,20 @@ export const releaseDataFileReplacementCompleteRoute: ReleaseRouteProps = { component: ReleaseDataFileReplacementCompletePage, }; +export const releaseApiDataSetsRoute: ReleaseRouteProps = { + path: `/publication/:publicationId/release/:releaseId/data#${releaseDataPageTabIds.apiDataSets}`, + title: 'API data sets', + component: ReleaseDataPage, + protectionAction: permissions => permissions.isBauUser, +}; + +export const releaseApiDataSetDetailsRoute: ReleaseRouteProps = { + path: '/publication/:publicationId/release/:releaseId/api-data-sets/:dataSetId', + title: 'API data set details', + component: ReleaseApiDataSetDetailsPage, + protectionAction: permissions => permissions.isBauUser, +}; + export const releaseFootnotesRoute: ReleaseRouteProps = { path: '/publication/:publicationId/release/:releaseId/footnotes', title: 'Footnotes', @@ -160,17 +174,3 @@ export const releasePreReleaseAccessRoute: ReleaseRouteProps = { title: 'Pre-release access', component: ReleasePreReleaseAccessPage, }; - -export const releaseApiDataSetsRoute: ReleaseRouteProps = { - path: '/publication/:publicationId/release/:releaseId/api-data-sets', - title: 'API data sets', - component: ReleaseApiDataSetsPage, - protectionAction: permissions => permissions.isBauUser, -}; - -export const releaseApiDataSetDetailsRoute: ReleaseRouteProps = { - path: '/publication/:publicationId/release/:releaseId/api-data-sets/:dataSetId', - title: 'API data set details', - component: ReleaseApiDataSetDetailsPage, - protectionAction: permissions => permissions.isBauUser, -}; diff --git a/src/explore-education-statistics-admin/src/services/apiDataSetService.ts b/src/explore-education-statistics-admin/src/services/apiDataSetService.ts index 47646aaa87d..bea4c45dee2 100644 --- a/src/explore-education-statistics-admin/src/services/apiDataSetService.ts +++ b/src/explore-education-statistics-admin/src/services/apiDataSetService.ts @@ -107,6 +107,6 @@ const apiDataSetService = { getDataSet(dataSetId: string): Promise { return client.get(`/public-data/data-sets/${dataSetId}`); }, -}; +} as const; export default apiDataSetService; diff --git a/src/explore-education-statistics-admin/src/services/apiDataSetVersionService.ts b/src/explore-education-statistics-admin/src/services/apiDataSetVersionService.ts new file mode 100644 index 00000000000..a9ed2969f58 --- /dev/null +++ b/src/explore-education-statistics-admin/src/services/apiDataSetVersionService.ts @@ -0,0 +1,9 @@ +import client from '@admin/services/utils/service'; + +const apiDataSetVersionService = { + deleteVersion(versionId: string): Promise { + return client.delete(`/public-data/data-set-versions/${versionId}`); + }, +} as const; + +export default apiDataSetVersionService; diff --git a/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts b/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts index 37901477fb1..e17304ca7e5 100644 --- a/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts +++ b/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts @@ -103,7 +103,7 @@ const defaultPublication: Publication = { ], slug: 'publication-slug', title: 'Publication title', - topic: { theme: { title: 'Test theme' } }, + topic: { theme: { id: 'test-theme', title: 'Test theme' } }, }; const defaultKeyStatistics: KeyStatistic[] = [ diff --git a/src/explore-education-statistics-common/src/components/CollapsibleList.module.scss b/src/explore-education-statistics-common/src/components/CollapsibleList.module.scss new file mode 100644 index 00000000000..d63edb80f96 --- /dev/null +++ b/src/explore-education-statistics-common/src/components/CollapsibleList.module.scss @@ -0,0 +1,7 @@ +@import '~govuk-frontend/dist/govuk/base'; + +.focusableItem { + &:focus-visible { + @include govuk-focused-text; + } +} diff --git a/src/explore-education-statistics-common/src/components/CollapsibleList.tsx b/src/explore-education-statistics-common/src/components/CollapsibleList.tsx index 14829bab77a..c5d6c1bfebc 100644 --- a/src/explore-education-statistics-common/src/components/CollapsibleList.tsx +++ b/src/explore-education-statistics-common/src/components/CollapsibleList.tsx @@ -1,8 +1,16 @@ import ButtonText from '@common/components/ButtonText'; +import styles from '@common/components/CollapsibleList.module.scss'; +import VisuallyHidden from '@common/components/VisuallyHidden'; import useToggle from '@common/hooks/useToggle'; import classNames from 'classnames'; -import React, { Children, createElement, ReactNode } from 'react'; -import VisuallyHidden from './VisuallyHidden'; +import React, { + Children, + cloneElement, + createElement, + ReactElement, + ReactNode, + useRef, +} from 'react'; interface BaseProps { buttonClassName?: string; @@ -43,7 +51,30 @@ const CollapsibleList = ({ }: Props) => { const [collapsed, toggleCollapsed] = useToggle(isCollapsed); - const listItems = Children.toArray(children); + const firstItemRef = useRef(null); + const firstHiddenItemRef = useRef(null); + + const getItemProperties = (index: number) => { + if (index === 0) { + return { + className: styles.focusableItem, + ref: firstItemRef, + tabIndex: -1, + }; + } + if (collapseAfter === index) { + return { + className: styles.focusableItem, + ref: firstHiddenItemRef, + tabIndex: -1, + }; + } + return undefined; + }; + + const listItems = Children.toArray(children).map((item, index) => + cloneElement(item as ReactElement, getItemProperties(index)), + ); const renderedListItems = collapsed ? listItems.slice(0, collapseAfter > 0 ? collapseAfter : 0) @@ -76,7 +107,18 @@ const CollapsibleList = ({ className={classNames('govuk-!-display-none-print', buttonClassName, { 'govuk-!-margin-bottom-4': !buttonClassName, })} - onClick={toggleCollapsed} + onClick={() => { + if (collapsed) { + toggleCollapsed.off(); + // Timeout to make sure the list item is shown before focusing it. + setTimeout(() => { + firstHiddenItemRef.current?.focus(); + }, 0); + } else { + toggleCollapsed.on(); + firstItemRef.current?.focus(); + } + }} > {collapsed ? `Show ${collapsedCount} ${collapseAfter > 0 ? 'more' : ''} ${ diff --git a/src/explore-education-statistics-common/src/components/ContentHtml.tsx b/src/explore-education-statistics-common/src/components/ContentHtml.tsx index 78ece63a99a..d29edefb3a5 100644 --- a/src/explore-education-statistics-common/src/components/ContentHtml.tsx +++ b/src/explore-education-statistics-common/src/components/ContentHtml.tsx @@ -42,31 +42,17 @@ export default function ContentHtml({ const cleanHtml = useMemo(() => { const opts: SanitizeHtmlOptions = { ...sanitizeOptions, - transformTags: { - ...sanitizeOptions?.transformTags, - ...(formatLinks && { - a: (tagName, attribs) => { - return { - tagName, - attribs: { - ...attribs, - href: formatContentLink(attribs.href), - }, - }; - }, - }), - }, + transformTags: sanitizeOptions?.transformTags, }; return sanitizeHtml(html, opts); - }, [formatLinks, html, sanitizeOptions]); + }, [html, sanitizeOptions]); const parsedContent = parseHtmlString(cleanHtml, { replace: (node: DOMNode) => { if (!(node instanceof Element)) { return undefined; } - if ( getGlossaryEntry && node.name === 'a' && @@ -94,6 +80,21 @@ export default function ContentHtml({ : undefined; } + if (formatLinks && node.name === 'a') { + const url = formatContentLink(node.attribs.href); + const text = domToReact(node.children); + + return !node.attribs.href.includes( + 'explore-education-statistics.service.gov.uk', + ) && typeof node.attribs['data-featured-table'] === 'undefined' ? ( + + {text} (opens in a new tab) + + ) : ( + {text} + ); + } + if (node.name === 'figure' && node.attribs.class === 'table') { return renderTable(node); } diff --git a/src/explore-education-statistics-common/src/components/Details.tsx b/src/explore-education-statistics-common/src/components/Details.tsx index 79b9e03eeab..457d37e8fef 100644 --- a/src/explore-education-statistics-common/src/components/Details.tsx +++ b/src/explore-education-statistics-common/src/components/Details.tsx @@ -95,7 +95,6 @@ const Details = ({ className={classNames('govuk-details', className)} open={open} ref={ref} - role={onMounted('group')} data-testid={testId} > { + const [alertText, setAlertText] = useState(''); + + useEffect(() => { + let timeout: ReturnType; + + if (!alert || !text) { + return () => clearTimeout(timeout); + } + + if (loading) { + // Set alert text with a delay so that screen readers + // detect the change and announce it correctly. + timeout = setTimeout(() => setAlertText(text), 300); + } else { + setAlertText(''); + } + + return () => clearTimeout(timeout); + }, [alert, loading, text]); + return ( <> {loading ? ( @@ -47,7 +67,7 @@ const LoadingSpinner = ({ role={alert ? 'alert' : undefined} className={classNames({ 'govuk-visually-hidden': hideText })} > - {text} + {alert ? alertText : text} ) : ( diff --git a/src/explore-education-statistics-common/src/components/Modal.module.scss b/src/explore-education-statistics-common/src/components/Modal.module.scss index 8224985fc8c..b782da6125e 100644 --- a/src/explore-education-statistics-common/src/components/Modal.module.scss +++ b/src/explore-education-statistics-common/src/components/Modal.module.scss @@ -12,6 +12,10 @@ z-index: 1001; } +.noUnderlayClick { + cursor: default; +} + .dialog { align-items: center; background: #fff; diff --git a/src/explore-education-statistics-common/src/components/Modal.tsx b/src/explore-education-statistics-common/src/components/Modal.tsx index 9bc39dd5e6a..b30455e201d 100644 --- a/src/explore-education-statistics-common/src/components/Modal.tsx +++ b/src/explore-education-statistics-common/src/components/Modal.tsx @@ -1,10 +1,13 @@ import Button from '@common/components/Button'; import styles from '@common/components/Modal.module.scss'; +import VisuallyHidden from '@common/components/VisuallyHidden'; import useToggle from '@common/hooks/useToggle'; import * as Dialog from '@radix-ui/react-dialog'; import classNames from 'classnames'; import React, { ReactNode, useEffect, useRef } from 'react'; +const defaultCloseText = 'Close'; + export interface ModalProps { children: ReactNode; className?: string; @@ -29,7 +32,7 @@ const Modal = ({ className, closeOnEsc = true, closeOnOutsideClick = true, - closeText = 'Close', + closeText = defaultCloseText, description, fullScreen = false, hideTitle = false, @@ -66,7 +69,13 @@ const Modal = ({ )} - + )} diff --git a/src/explore-education-statistics-common/src/components/ModalConfirm.tsx b/src/explore-education-statistics-common/src/components/ModalConfirm.tsx index 3927538fdab..b4daa6d3cdd 100644 --- a/src/explore-education-statistics-common/src/components/ModalConfirm.tsx +++ b/src/explore-education-statistics-common/src/components/ModalConfirm.tsx @@ -1,15 +1,18 @@ import ButtonGroup from '@common/components/ButtonGroup'; import Button from '@common/components/Button'; +import LoadingSpinner from '@common/components/LoadingSpinner'; import Modal from '@common/components/Modal'; import useMountedRef from '@common/hooks/useMountedRef'; import useToggle from '@common/hooks/useToggle'; -import React, { ReactNode, useEffect } from 'react'; +import React, { ReactNode, useCallback, useEffect } from 'react'; interface Props { children?: ReactNode; className?: string; cancelText?: string; confirmText?: string; + hiddenCancellingText?: string; + hiddenConfirmingText?: string; open?: boolean; showCancel?: boolean; title: string; @@ -20,11 +23,13 @@ interface Props { onExit?(): void | Promise; } -const ModalConfirm = ({ +export default function ModalConfirm({ cancelText = 'Cancel', children, className, confirmText = 'Confirm', + hiddenCancellingText = 'Cancelling', + hiddenConfirmingText = 'Confirming', open: initialOpen = false, showCancel = true, title, @@ -33,42 +38,65 @@ const ModalConfirm = ({ onExit, onCancel = onExit, onConfirm, -}: Props) => { +}: Props) { const isMounted = useMountedRef(); - const [isDisabled, toggleDisabled] = useToggle(false); - const [open, toggleOpen] = useToggle(initialOpen); + + const [isOpen, toggleOpen] = useToggle(initialOpen); + const [isCancelling, toggleCancelling] = useToggle(false); + const [isConfirming, toggleConfirming] = useToggle(false); + + const isCompleting = isCancelling || isConfirming; useEffect(() => { toggleOpen(initialOpen); }, [initialOpen, toggleOpen]); - const handleAction = (callback?: () => void | Promise) => async () => { - if (!callback) { + const handleCancel = useCallback(async () => { + if (!onCancel) { + toggleOpen.off(); + return; + } + + if (isCompleting || !isMounted.current) { + return; + } + + toggleCancelling.on(); + + await onCancel(); + + if (isMounted.current) { + toggleCancelling.off(); + toggleOpen.off(); + } + }, [isCompleting, isMounted, onCancel, toggleCancelling, toggleOpen]); + + const handleConfirm = useCallback(async () => { + if (!onConfirm) { toggleOpen.off(); return; } - if (isDisabled || !isMounted.current) { + + if (isCompleting || !isMounted.current) { return; } - toggleDisabled.on(); + toggleConfirming.on(); - await callback(); + await onConfirm(); - // Callback may finish after - // component has been unmounted. if (isMounted.current) { - toggleDisabled.off(); + toggleConfirming.off(); toggleOpen.off(); } - }; + }, [isCompleting, isMounted, onConfirm, toggleConfirming, toggleOpen]); return ( {children} - + {showCancel && ( )} - + + ); -}; - -export default ModalConfirm; +} diff --git a/src/explore-education-statistics-common/src/components/Tabs.tsx b/src/explore-education-statistics-common/src/components/Tabs.tsx index cc04d4bc33d..c722594d61c 100644 --- a/src/explore-education-statistics-common/src/components/Tabs.tsx +++ b/src/explore-education-statistics-common/src/components/Tabs.tsx @@ -154,7 +154,7 @@ const Tabs = ({ children, id, modifyHash = true, testId, onToggle }: Props) => { data-testid={testId} ref={ref} > -
    +
      {sections.map(({ props }, index) => { const sectionId = props.id || `${id}-${index + 1}`; @@ -164,7 +164,7 @@ const Tabs = ({ children, id, modifyHash = true, testId, onToggle }: Props) => { 'govuk-tabs__list-item--selected': selectedTabIndex === index, })} key={sectionId} - role="presentation" + role={onMedia('presentation')} > { className="govuk-tabs__tab" href={`#${sectionId}`} id={`${sectionId}-tab`} - tabIndex={selectedTabIndex !== index ? -1 : undefined} + tabIndex={onMedia(selectedTabIndex !== index ? -1 : undefined)} onClick={event => { event.preventDefault(); selectTab(index); diff --git a/src/explore-education-statistics-common/src/components/__tests__/ContentHtml.test.tsx b/src/explore-education-statistics-common/src/components/__tests__/ContentHtml.test.tsx index 0fc9c9d9695..f2522aa9126 100644 --- a/src/explore-education-statistics-common/src/components/__tests__/ContentHtml.test.tsx +++ b/src/explore-education-statistics-common/src/components/__tests__/ContentHtml.test.tsx @@ -43,7 +43,7 @@ describe('ContentHtml', () => { modal.getByRole('heading', { name: 'Absence heading' }), ).toBeInTheDocument(); expect(modal.getByText('Absence body')).toBeInTheDocument(); - const closeButton = modal.getByRole('button', { name: 'Close' }); + const closeButton = modal.getByRole('button', { name: 'Close modal' }); expect(closeButton).toBeInTheDocument(); }); @@ -73,7 +73,7 @@ describe('ContentHtml', () => { expect(getEntry).toHaveBeenCalled(); const modal = within(screen.getByRole('dialog')); - const closeButton = modal.getByRole('button', { name: 'Close' }); + const closeButton = modal.getByRole('button', { name: 'Close modal' }); await userEvent.click(closeButton); await waitFor(() => { @@ -198,47 +198,112 @@ describe('ContentHtml', () => { }); describe('formatting links', () => { - const testContentWithLinks = `Test content - - EES link with uppercase characters - External link with whitespace, space to encode and uppercase characters`; - - test('formats links in content', () => { - render(); + test('encodes special characters in urls', () => { + render( + , + ); expect( screen.getByRole('link', { - name: 'EES link with uppercase characters', + name: 'External link (opens in a new tab)', }), - ).toHaveAttribute( - 'href', - 'https://explore-education-statistics.service.gov.uk/find-statistics/pupil-attendance-in-schools?testParam=Something', + ).toHaveAttribute('href', 'https://gov.uk/TEST%20something'); + }); + + test('trims whitespace in urls', () => { + render( + , ); expect( screen.getByRole('link', { - name: 'External link with whitespace, space to encode and uppercase characters', + name: 'External link (opens in a new tab)', }), - ).toHaveAttribute('href', 'https://gov.uk/TEST%20something'); + ).toHaveAttribute('href', 'https://gov.uk/TEST'); }); - test('does not format links in content when `formatLinks` is false', () => { - render(); + test('lower cases internal urls, excluding query params', () => { + render( + , + ); expect( screen.getByRole('link', { - name: 'EES link with uppercase characters', + name: 'Internal link', }), ).toHaveAttribute( 'href', - 'https://explore-education-statistics.service.gov.uk/find-statistics/Pupil-Attendance-In-Schools?testParam=Something', + 'https://explore-education-statistics.service.gov.uk/find-statistics?testParam=Something', + ); + }); + + test('does not lower case external urls', () => { + render( + , + ); + expect( + screen.getByRole('link', { + name: 'External link (opens in a new tab)', + }), + ).toHaveAttribute('href', 'https://gov.uk/TEST'); + }); + + test('opens external links in a new tab', () => { + render( + , + ); + const link = screen.getByRole('link', { + name: 'External link (opens in a new tab)', + }); + expect(link).toHaveAttribute('href', 'https://gov.uk/'); + expect(link).toHaveAttribute('target', '_blank'); + expect(link).toHaveAttribute('rel', 'noopener noreferrer'); + }); + + test('opens internal links in the same tab', () => { + render( + , + ); + const link = screen.getByRole('link', { + name: 'Internal link', + }); + expect(link).toHaveAttribute( + 'href', + 'https://explore-education-statistics.service.gov.uk/', + ); + expect(link).not.toHaveAttribute('target', '_blank'); + expect(link).not.toHaveAttribute('rel', 'noopener noreferrer'); + }); + + test('does not format links when `formatLinks` is false', () => { + render( + + Internal link + External link`} + />, ); expect( screen.getByRole('link', { - name: 'External link with whitespace, space to encode and uppercase characters', + name: 'Internal link', }), - ).toHaveAttribute('href', ' https://gov.uk/TEST something '); + ).toHaveAttribute( + 'href', + 'https://explore-education-statistics.service.gov.uk/Find-Statistics?testParam=Something', + ); + + const externalLink = screen.getByRole('link', { + name: 'External link', + }); + expect(externalLink).toHaveAttribute( + 'href', + ' https://gov.uk/TEST something ', + ); + expect(externalLink).not.toHaveAttribute('target', '_blank'); + expect(externalLink).not.toHaveAttribute('rel', 'noopener noreferrer'); }); }); }); diff --git a/src/explore-education-statistics-common/src/components/__tests__/LoadingSpinner.test.tsx b/src/explore-education-statistics-common/src/components/__tests__/LoadingSpinner.test.tsx new file mode 100644 index 00000000000..78638b220e3 --- /dev/null +++ b/src/explore-education-statistics-common/src/components/__tests__/LoadingSpinner.test.tsx @@ -0,0 +1,51 @@ +import LoadingSpinner from '@common/components/LoadingSpinner'; +import { render, screen } from '@testing-library/react'; +import React from 'react'; + +describe('LoadingSpinner', () => { + test('does not render if `loading` is false', () => { + render(); + + expect(screen.queryByTestId('loadingSpinner')).not.toBeInTheDocument(); + }); + + test('renders alert text with a slight delay for screen readers', async () => { + jest.useFakeTimers(); + + render(); + + expect(screen.queryByText('Test text')).not.toBeInTheDocument(); + + jest.runOnlyPendingTimers(); + + expect(await screen.findByText('Test text')).toBeInTheDocument(); + + jest.useRealTimers(); + }); + + test('re-renders new alert text with slight delay', async () => { + jest.useFakeTimers(); + + const { rerender } = render(); + + jest.runOnlyPendingTimers(); + + expect(await screen.findByText('Test text')).toBeInTheDocument(); + + rerender(); + + expect(screen.getByText('Test text')).toBeInTheDocument(); + + jest.runOnlyPendingTimers(); + + expect(await screen.findByText('Test text updated')).toBeInTheDocument(); + + jest.useRealTimers(); + }); + + test('renders non-alert text immediately', () => { + render(); + + expect(screen.getByText('Test text')).toBeInTheDocument(); + }); +}); diff --git a/src/explore-education-statistics-common/src/components/__tests__/Modal.test.tsx b/src/explore-education-statistics-common/src/components/__tests__/Modal.test.tsx index c3a4a421541..f0495937f73 100644 --- a/src/explore-education-statistics-common/src/components/__tests__/Modal.test.tsx +++ b/src/explore-education-statistics-common/src/components/__tests__/Modal.test.tsx @@ -1,11 +1,11 @@ +import render from '@common-test/render'; import Modal from '@common/components/Modal'; -import { fireEvent, render, screen, waitFor } from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; +import { fireEvent, screen, waitFor } from '@testing-library/react'; import React from 'react'; describe('Modal', () => { test('clicking the trigger button opens the modal', async () => { - render( + const { user } = render( Open} title="Test modal" @@ -19,7 +19,7 @@ describe('Modal', () => { screen.queryByRole('heading', { name: 'Test modal' }), ).not.toBeInTheDocument(); - await userEvent.click(screen.getByRole('button', { name: 'Open' })); + await user.click(screen.getByRole('button', { name: 'Open' })); expect(screen.getByRole('dialog')).toBeInTheDocument(); expect( @@ -28,7 +28,7 @@ describe('Modal', () => { }); test('clicking the close button closes the modal', async () => { - render( + const { user } = render( Open} @@ -44,7 +44,7 @@ describe('Modal', () => { screen.getByRole('heading', { name: 'Test modal' }), ).toBeInTheDocument(); - await userEvent.click(screen.getByRole('button', { name: 'Close' })); + await user.click(screen.getByRole('button', { name: 'Close modal' })); expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); expect( @@ -119,7 +119,7 @@ describe('Modal', () => { test('when opened the `onOpen` handler is called', async () => { const onOpen = jest.fn(); - render( + const { user } = render( Open} title="Test modal" @@ -132,7 +132,7 @@ describe('Modal', () => { expect(onOpen).not.toHaveBeenCalled(); - await userEvent.click(screen.getByRole('button', { name: 'Open' })); + await user.click(screen.getByRole('button', { name: 'Open' })); await waitFor(() => { expect(onOpen).toHaveBeenCalled(); @@ -142,7 +142,7 @@ describe('Modal', () => { test('closing the modal calls the `onExit` handler', async () => { const onExit = jest.fn(); - render( + const { user } = render( Open} title="Test modal" @@ -156,7 +156,7 @@ describe('Modal', () => { expect(onExit).not.toHaveBeenCalled(); - await userEvent.click(screen.getByRole('button', { name: 'Close' })); + await user.click(screen.getByRole('button', { name: 'Close modal' })); await waitFor(() => { expect(onExit).toHaveBeenCalled(); diff --git a/src/explore-education-statistics-common/src/components/__tests__/ModalConfirm.test.tsx b/src/explore-education-statistics-common/src/components/__tests__/ModalConfirm.test.tsx index 9c6684f5a31..9c84138d32c 100644 --- a/src/explore-education-statistics-common/src/components/__tests__/ModalConfirm.test.tsx +++ b/src/explore-education-statistics-common/src/components/__tests__/ModalConfirm.test.tsx @@ -1,4 +1,3 @@ -import flushPromises from '@common-test/flushPromises'; import ModalConfirm from '@common/components/ModalConfirm'; import delay from '@common/utils/delay'; import { render, screen, waitFor } from '@testing-library/react'; @@ -7,7 +6,7 @@ import React from 'react'; describe('ModalConfirm', () => { describe('confirming', () => { - test('clicking Confirm button disables all buttons', async () => { + test('clicking Confirm button disables all buttons and shows loading spinner', async () => { const handleExit = jest.fn(); const handleCancel = jest.fn(); const handleConfirm = jest.fn(async () => { @@ -31,8 +30,12 @@ describe('ModalConfirm', () => { await userEvent.click(screen.getByRole('button', { name: 'Confirm' })); - expect(screen.getByRole('button', { name: 'Confirm' })).toBeDisabled(); + expect( + screen.getByRole('button', { name: 'Confirm' }), + ).toBeAriaDisabled(); expect(screen.getByRole('button', { name: 'Cancel' })).toBeDisabled(); + + expect(screen.getByTestId('loadingSpinner')).toBeInTheDocument(); }); test('clicking Confirm button prevents closing modal using Esc', async () => { @@ -107,12 +110,16 @@ describe('ModalConfirm', () => { await userEvent.click(screen.getByRole('button', { name: 'Confirm' })); - expect(screen.getByRole('button', { name: 'Confirm' })).toBeDisabled(); + expect( + screen.getByRole('button', { name: 'Confirm' }), + ).toBeAriaDisabled(); expect(screen.getByRole('button', { name: 'Cancel' })).toBeDisabled(); await waitFor(() => { - expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + expect(screen.queryByText('Confirm')).not.toBeInTheDocument(); }); + + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); }); }); @@ -167,7 +174,7 @@ describe('ModalConfirm', () => { }); }); - test('clicking Cancel button disables all buttons', async () => { + test('clicking Cancel button disables all buttons and shows loading spinner', async () => { const handleExit = jest.fn(); const handleCancel = jest.fn(async () => { await delay(100); @@ -188,7 +195,9 @@ describe('ModalConfirm', () => { await userEvent.click(screen.getByRole('button', { name: 'Cancel' })); expect(screen.getByRole('button', { name: 'Confirm' })).toBeDisabled(); - expect(screen.getByRole('button', { name: 'Cancel' })).toBeDisabled(); + expect(screen.getByRole('button', { name: 'Cancel' })).toBeAriaDisabled(); + + expect(screen.getByTestId('loadingSpinner')).toBeInTheDocument(); }); test('clicking Cancel button prevents closing modal using Esc', async () => { @@ -263,13 +272,13 @@ describe('ModalConfirm', () => { await userEvent.click(screen.getByRole('button', { name: 'Cancel' })); expect(screen.getByRole('button', { name: 'Confirm' })).toBeDisabled(); - expect(screen.getByRole('button', { name: 'Cancel' })).toBeDisabled(); - - await flushPromises(); + expect(screen.getByRole('button', { name: 'Cancel' })).toBeAriaDisabled(); await waitFor(() => { - expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + expect(screen.queryByText('Confirm')).not.toBeInTheDocument(); }); + + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); }); }); }); diff --git a/src/explore-education-statistics-common/src/components/__tests__/__snapshots__/Details.test.tsx.snap b/src/explore-education-statistics-common/src/components/__tests__/__snapshots__/Details.test.tsx.snap index c7d69335f72..b5e4c72c66a 100644 --- a/src/explore-education-statistics-common/src/components/__tests__/__snapshots__/Details.test.tsx.snap +++ b/src/explore-education-statistics-common/src/components/__tests__/__snapshots__/Details.test.tsx.snap @@ -3,7 +3,6 @@ exports[`Details renders correctly 1`] = `
      ReactNode; @@ -37,6 +38,7 @@ export default function DataSetStep({ featuredTables = [], loadingFastTrack = false, release, + stepTitle, subjects, subjectId = '', renderFeaturedTableLink, @@ -50,9 +52,7 @@ export default function DataSetStep({ {...stepProps} fieldsetHeading={!renderFeaturedTableLink} > - {featuredTables.length > 0 - ? 'Select a data set or featured table' - : 'Select a data set'} + {stepTitle} ); diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/FiltersForm.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/FiltersForm.tsx index 60adfe0a986..a6bb786d5f9 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/FiltersForm.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/FiltersForm.tsx @@ -52,6 +52,7 @@ interface Props extends InjectedWizardProps { }; selectedPublication?: SelectedPublication; showTableQueryErrorDownload?: boolean; + stepTitle: string; subject: Subject; subjectMeta: SubjectMeta; onSubmit: FilterFormSubmitHandler; @@ -65,6 +66,7 @@ interface Props extends InjectedWizardProps { export default function FiltersForm({ initialValues, selectedPublication, + stepTitle, subject, subjectMeta, showTableQueryErrorDownload = true, @@ -114,7 +116,7 @@ export default function FiltersForm({ }, [initialValues, subjectMeta]); const stepHeading = ( - Choose your filters + {stepTitle} ); const handleSubmit = async (values: FiltersFormValues) => { diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/LocationFiltersForm.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/LocationFiltersForm.tsx index 55adee7c333..86912d8cccf 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/LocationFiltersForm.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/LocationFiltersForm.tsx @@ -16,7 +16,6 @@ import React, { ReactNode, useMemo } from 'react'; import { ObjectSchema } from 'yup'; import { InjectedWizardProps } from './Wizard'; import WizardStepFormActions from './WizardStepFormActions'; -import WizardStepHeading from './WizardStepHeading'; interface FormValues { locations: Dictionary; @@ -29,12 +28,14 @@ export type LocationFiltersFormSubmitHandler = (values: { export interface LocationFiltersFormProps extends InjectedWizardProps { options: SubjectMeta['locations']; initialValues?: string[]; + stepHeading?: ReactNode; onSubmit: LocationFiltersFormSubmitHandler; } const LocationFiltersForm = ({ initialValues = [], options, + stepHeading, onSubmit, ...stepProps }: LocationFiltersFormProps) => { @@ -164,7 +165,7 @@ const LocationFiltersForm = ({ > } + legend={stepHeading} hint="Select at least one" error={ showError ? getErrorMessage(formState.errors, 'locations') : '' @@ -242,21 +243,3 @@ const LocationFiltersForm = ({ }; export default LocationFiltersForm; - -export interface LocationStepHeadingProps extends InjectedWizardProps { - options: SubjectMeta['locations']; -} - -export function LocationStepHeading( - stepProps: LocationStepHeadingProps, -): ReactNode { - const { options } = stepProps; - const levelKeys = Object.keys(options); - return ( - - {levelKeys.length === 1 && locationLevelsMap[levelKeys[0]] - ? `Choose ${locationLevelsMap[levelKeys[0]].plural}` - : 'Choose locations'} - - ); -} diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/LocationStep.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/LocationStep.tsx index 70ed16d6ac0..2fdc7fecaba 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/LocationStep.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/LocationStep.tsx @@ -2,21 +2,26 @@ import CollapsibleList from '@common/components/CollapsibleList'; import SummaryList from '@common/components/SummaryList'; import SummaryListItem from '@common/components/SummaryListItem'; import WizardStepSummary from '@common/modules/table-tool/components/WizardStepSummary'; -import { InjectedWizardProps } from '@common/modules/table-tool/components/Wizard'; +import WizardStepHeading from '@common/modules/table-tool/components/WizardStepHeading'; import LocationFiltersForm, { LocationFiltersFormProps, - LocationStepHeading, -} from '@common/modules/table-tool/components//LocationFiltersForm'; - -import { - LocationOption, - SubjectMeta, -} from '@common/services/tableBuilderService'; +} from '@common/modules/table-tool/components/LocationFiltersForm'; +import { LocationOption } from '@common/services/tableBuilderService'; import sortBy from 'lodash/sortBy'; import React, { useMemo, useState } from 'react'; -export default function LocationStep(stepProps: LocationFiltersFormProps) { - const { initialValues = [], isActive, options, onSubmit } = stepProps; +interface Props extends LocationFiltersFormProps { + stepTitle: string; +} + +export default function LocationStep(stepProps: Props) { + const { + initialValues = [], + isActive, + options, + stepTitle, + onSubmit, + } = stepProps; const [selectedLocationIds, setSelectedLocationIds] = useState(initialValues); @@ -46,11 +51,16 @@ export default function LocationStep(stepProps: LocationFiltersFormProps) { : []; }, [options, selectedLocationIds]); + const stepHeading = ( + + {stepTitle} + + ); + if (!isActive) { return ( - - + {stepHeading} {selectedLocations.map(level => { return ( @@ -75,6 +85,7 @@ export default function LocationStep(stepProps: LocationFiltersFormProps) { return ( { setSelectedLocationIds(values.locationIds); await onSubmit(values); @@ -82,7 +93,3 @@ export default function LocationStep(stepProps: LocationFiltersFormProps) { /> ); } - -export interface LocationStepHeadingProps extends InjectedWizardProps { - options: SubjectMeta['locations']; -} diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/PublicationForm.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/PublicationForm.tsx index 0afe47f1770..b3cc03c0597 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/PublicationForm.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/PublicationForm.tsx @@ -1,8 +1,4 @@ -import { - FormFieldset, - FormGroup, - FormTextSearchInput, -} from '@common/components/form'; +import { FormGroup, FormTextSearchInput } from '@common/components/form'; import FormProvider from '@common/components/form/FormProvider'; import Form from '@common/components/form/Form'; import FormFieldRadioGroup from '@common/components/form/FormFieldRadioGroup'; @@ -37,6 +33,7 @@ const formId = 'publicationForm'; interface Props extends InjectedWizardProps { initialValues?: FormValues; showSupersededPublications?: boolean; + stepTitle: string; themes: Theme[]; onSubmit: PublicationFormSubmitHandler; renderSummaryAfter?: ReactNode; @@ -48,6 +45,7 @@ const PublicationForm = ({ themeId: '', }, showSupersededPublications = false, + stepTitle, themes, onSubmit, renderSummaryAfter, @@ -102,9 +100,7 @@ const PublicationForm = ({ .find(publication => publication.id === publicationId); const stepHeading = ( - - Choose a publication - + {stepTitle} ); const validationSchema = useMemo>(() => { @@ -143,110 +139,107 @@ const PublicationForm = ({ if (isActive) { return (
      - -

      Search or select a theme to find publications

      - - { - setSearchTerm(event.target.value); - setSelectedThemeId(''); - resetField('themeId'); - resetField('publicationId'); - }} - onKeyPress={event => { - if (event.key === 'Enter') { - event.preventDefault(); - } - }} - value={searchTerm} - width={20} - /> - {searchTerm && publications.length > 0 && ( -

      - - Skip to search results - -

      - )} -       - -

      or

      + {stepHeading} +

      Search or select a theme to find publications

      + + { + setSearchTerm(event.target.value); + setSelectedThemeId(''); + resetField('themeId'); + resetField('publicationId'); + }} + onKeyPress={event => { + if (event.key === 'Enter') { + event.preventDefault(); + } + }} + value={searchTerm} + width={20} + /> + {searchTerm && publications.length > 0 && ( +

      + + Skip to search results + +

      + )} +       +

      or

      +
      + + legend="Select a theme" + legendSize="s" + name="themeId" + small + options={themes.map(theme => ({ + label: theme.title, + value: theme.id, + }))} + onChange={e => { + setSelectedThemeId(e.target.value); + resetField('publicationId'); + setSearchTerm(''); + }} + /> -
      +
      - legend="Select a theme" + id="publications" + legend={ + <> + Select a publication + + {` ${publications.length} ${ + publications.length === 1 + ? `publication` + : `publications` + } found`} + + + } legendSize="s" - name="themeId" + name="publicationId" small - options={themes.map(theme => ({ - label: theme.title, - value: theme.id, - }))} - onChange={e => { - setSelectedThemeId(e.target.value); - resetField('publicationId'); - setSearchTerm(''); - }} + options={orderBy(publications, 'title').map( + publication => ({ + hint: searchTerm + ? getThemeForPublication(publication.id) + : '', + hintSmall: true, + label: publication.title, + value: publication.id, + }), + )} /> -
      - - id="publications" - legend={ - <> - Select a publication - - {` ${publications.length} ${ - publications.length === 1 - ? `publication` - : `publications` - } found`} - - - } - legendSize="s" - name="publicationId" - small - options={orderBy(publications, 'title').map( - publication => ({ - hint: searchTerm - ? getThemeForPublication(publication.id) - : '', - hintSmall: true, - label: publication.title, - value: publication.id, - }), + {!publications.length && ( + <> +

      Search or select a theme to view publications

      + {(searchTerm || selectedThemeId) && ( + No publications found )} - /> - - {!publications.length && ( - <> -

      Search or select a theme to view publications

      - {(searchTerm || selectedThemeId) && ( - No publications found - )} - - )} + + )} -
      - -
      +
      +
      - +
      ); } diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/TableToolWizard.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/TableToolWizard.tsx index ae0cdf14a6b..67e6d6bff71 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/TableToolWizard.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/TableToolWizard.tsx @@ -1,4 +1,6 @@ import SubmitError from '@common/components/form/util/SubmitError'; +import WarningMessage from '@common/components/WarningMessage'; +import locationLevelsMap from '@common/utils/locationLevelsMap'; import { ConfirmContextProvider } from '@common/contexts/ConfirmContext'; import FiltersForm, { FilterFormSubmitHandler, @@ -31,13 +33,17 @@ import publicationService, { } from '@common/services/publicationService'; import tableBuilderService, { FeaturedTable, + LocationOption, ReleaseTableDataQuery, Subject, SubjectMeta, } from '@common/services/tableBuilderService'; import React, { ReactElement, ReactNode, useMemo, useState } from 'react'; import { useImmer } from 'use-immer'; -import WarningMessage from '@common/components/WarningMessage'; +import { Dictionary } from 'lodash'; + +const defaultLocationStepTitle = 'Choose locations'; +const defaultDataSetStepTitle = 'Select a data set'; export interface InitialTableToolState { initialStep: number; @@ -62,12 +68,14 @@ interface TableToolState extends InitialTableToolState { export interface FinalStepRenderProps { query?: ReleaseTableDataQuery; selectedPublication?: SelectedPublication; + stepTitle: string; table?: FullTable; tableHeaders?: TableHeadersConfig; onReorder: (reorderedTableHeaders: TableHeadersConfig) => void; } export interface TableToolWizardProps { + currentStep?: number; finalStep?: (props: FinalStepRenderProps) => ReactElement; hidePublicationStep?: boolean; initialState?: Partial; @@ -76,10 +84,16 @@ export interface TableToolWizardProps { scrollOnMount?: boolean; showTableQueryErrorDownload?: boolean; themeMeta?: Theme[]; - currentStep?: number; onPublicationFormSubmit?: (publication: PublicationTreeSummary) => void; onPublicationStepBack?: () => void; onStepChange?: (nextStep: number, previousStep: number) => void; + onStepSubmit?: ({ + nextStepNumber, + nextStepTitle, + }: { + nextStepNumber: number; + nextStepTitle: string; + }) => void; onSubjectFormSubmit?(params: { publication: SelectedPublication; release: SelectedPublication['selectedRelease']; @@ -95,6 +109,7 @@ export interface TableToolWizardProps { } export default function TableToolWizard({ + currentStep, finalStep, hidePublicationStep, initialState = {}, @@ -103,10 +118,10 @@ export default function TableToolWizard({ scrollOnMount, showTableQueryErrorDownload = true, themeMeta = [], - currentStep, onPublicationFormSubmit, onPublicationStepBack, onStepChange, + onStepSubmit, onSubjectFormSubmit, onSubjectStepBack, onSubmit, @@ -136,6 +151,24 @@ export default function TableToolWizard({ }); const [reorderedTableHeaders, setReorderedTableHeaders] = useState(); + const locationStepNumber = hidePublicationStep ? 2 : 3; + const [locationStepTitle, setLocationStepTitle] = useState( + initialState.initialStep === locationStepNumber && initialState.subjectMeta + ? getLocationsStepTitle(initialState.subjectMeta?.locations) + : defaultLocationStepTitle, + ); + const [dataSetStepTitle, setDataSetStepTitle] = useState( + defaultDataSetStepTitle, + ); + + const stepTitles = { + publication: 'Choose a publication', + dataSet: dataSetStepTitle, + location: locationStepTitle, + timePeriod: 'Choose time period', + filter: 'Choose your filters', + final: 'Explore data', + }; const handlePublicationFormSubmit: PublicationFormSubmitHandler = async ({ publication, @@ -152,6 +185,11 @@ export default function TableToolWizard({ publication.slug, ); + const updatedDataSetTitle = + featuredTables.length > 0 + ? 'Select a data set or featured table' + : defaultDataSetStepTitle; + updateState(draft => { draft.subjects = subjects; draft.featuredTables = featuredTables; @@ -171,17 +209,29 @@ export default function TableToolWizard({ }, }; }); + + setDataSetStepTitle(updatedDataSetTitle); + onStepSubmit?.({ nextStepNumber: 2, nextStepTitle: updatedDataSetTitle }); }; - const handleSubjectStepBack = () => { + const handlePublicationStepBack = () => { + onPublicationStepBack?.(); + onStepSubmit?.({ + nextStepNumber: 1, + nextStepTitle: stepTitles.publication, + }); + }; + + const handleDataSetStepBack = () => { updateState(draft => { draft.query.subjectId = ''; }); onSubjectStepBack?.(state.selectedPublication); + onStepSubmit?.({ nextStepNumber: 2, nextStepTitle: dataSetStepTitle }); }; - const handleSubjectFormSubmit: DataSetFormSubmitHandler = async ({ + const handleDataSetFormSubmit: DataSetFormSubmitHandler = async ({ subjectId: selectedSubjectId, }) => { if (state.selectedPublication) { @@ -199,6 +249,11 @@ export default function TableToolWizard({ setReorderedTableHeaders(undefined); + const updatedLocationsTitle = getLocationsStepTitle( + nextSubjectMeta.locations, + ); + setLocationStepTitle(updatedLocationsTitle); + updateState(draft => { draft.subjectMeta = nextSubjectMeta; draft.query.subjectId = selectedSubjectId; @@ -207,6 +262,8 @@ export default function TableToolWizard({ draft.query.locationIds = []; draft.query.timePeriod = undefined; }); + + onStepSubmit?.({ nextStepNumber: 3, nextStepTitle: updatedLocationsTitle }); }; const handleLocationStepBack = async () => { @@ -220,6 +277,8 @@ export default function TableToolWizard({ updateState(draft => { draft.subjectMeta = nextSubjectMeta; }); + + onStepSubmit?.({ nextStepNumber: 3, nextStepTitle: locationStepTitle }); }; const handleLocationFiltersFormSubmit: LocationFiltersFormSubmitHandler = @@ -260,6 +319,11 @@ export default function TableToolWizard({ draft.query.timePeriod = undefined; } }); + + onStepSubmit?.({ + nextStepNumber: 4, + nextStepTitle: stepTitles.timePeriod, + }); }; const handleTimePeriodStepBack = async () => { @@ -274,6 +338,11 @@ export default function TableToolWizard({ updateState(draft => { draft.subjectMeta.timePeriod = nextSubjectMeta.timePeriod; }); + + onStepSubmit?.({ + nextStepNumber: 4, + nextStepTitle: stepTitles.timePeriod, + }); }; const handleTimePeriodFormSubmit: TimePeriodFormSubmitHandler = @@ -326,6 +395,8 @@ export default function TableToolWizard({ endCode, }; }); + + onStepSubmit?.({ nextStepNumber: 5, nextStepTitle: stepTitles.filter }); }; const handleFiltersStepBack = async () => { @@ -342,6 +413,8 @@ export default function TableToolWizard({ draft.subjectMeta.indicators = nextSubjectMeta.indicators; draft.subjectMeta.filters = nextSubjectMeta.filters; }); + + onStepSubmit?.({ nextStepNumber: 5, nextStepTitle: stepTitles.filter }); }; const handleFiltersFormSubmit: FilterFormSubmitHandler = async ({ @@ -380,6 +453,8 @@ export default function TableToolWizard({ tableHeaders, }; }); + + onStepSubmit?.({ nextStepNumber: 6, nextStepTitle: stepTitles.final }); }; const orderedTableHeaders: TableHeadersConfig | undefined = useMemo(() => { @@ -423,7 +498,7 @@ export default function TableToolWizard({ }} > {!hidePublicationStep && ( - + {stepProps => ( )} - + {stepProps => ( )} @@ -471,6 +548,7 @@ export default function TableToolWizard({ {...stepProps} initialValues={state.query.locationIds} options={state.subjectMeta.locations} + stepTitle={stepTitles.location} onSubmit={handleLocationFiltersFormSubmit} /> )} @@ -481,6 +559,7 @@ export default function TableToolWizard({ {...stepProps} initialValues={state.query.timePeriod} options={state.subjectMeta.timePeriod.options} + stepTitle={stepTitles.timePeriod} onSubmit={handleTimePeriodFormSubmit} /> )} @@ -494,6 +573,7 @@ export default function TableToolWizard({ filters: state.query.filters, }} selectedPublication={state.selectedPublication} + stepTitle={stepTitles.filter} subject={ state.subjects.filter( subject => subject.id === state.query.subjectId, @@ -510,6 +590,7 @@ export default function TableToolWizard({ finalStep({ query: state.query, selectedPublication: state.selectedPublication, + stepTitle: stepTitles.final, table: state.response?.table, tableHeaders: orderedTableHeaders, onReorder: reordered => setReorderedTableHeaders(reordered), @@ -522,3 +603,15 @@ export default function TableToolWizard({ ); } + +function getLocationsStepTitle( + locations: Dictionary<{ + legend: string; + options: LocationOption[]; + }>, +) { + const levelKeys = Object.keys(locations); + return levelKeys.length === 1 && locationLevelsMap[levelKeys[0]] + ? `Choose ${locationLevelsMap[levelKeys[0]].plural}` + : defaultLocationStepTitle; +} diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/TimePeriodForm.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/TimePeriodForm.tsx index 48bd1cdec74..84fec712600 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/TimePeriodForm.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/TimePeriodForm.tsx @@ -30,12 +30,14 @@ export type TimePeriodFormSubmitHandler = ( interface Props extends InjectedWizardProps { initialValues?: Partial; options: SubjectMeta['timePeriod']['options']; + stepTitle: string; onSubmit: TimePeriodFormSubmitHandler; } const TimePeriodForm = ({ initialValues = {}, options, + stepTitle, onSubmit, ...stepProps }: Props) => { @@ -93,7 +95,7 @@ const TimePeriodForm = ({ const stepHeading = ( - Choose time period + {stepTitle} ); diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/WizardStep.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/WizardStep.tsx index 2918cd4e0df..826e255e245 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/WizardStep.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/WizardStep.tsx @@ -42,7 +42,6 @@ const WizardStep = ({ return (
    • currentStep, diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/WizardStepHeading.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/WizardStepHeading.tsx index 443f6d0ef62..7ab087adf68 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/WizardStepHeading.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/WizardStepHeading.tsx @@ -17,6 +17,7 @@ const WizardStepHeading = ({ }: Props) => { return (

      {`Step ${stepNumber} `} - {isActive && (current) } {children} diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/DataSetStep.test.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/DataSetStep.test.tsx index 91566da4bee..d019583f62d 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/DataSetStep.test.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/DataSetStep.test.tsx @@ -109,18 +109,23 @@ describe('DataSetStep', () => { test('renders radios with details if no `renderFeaturedTableLink `', () => { render( - , + , ); expect( screen.getByRole('heading', { - name: 'Step 1 (current) Select a data set', + name: 'Step 1 Select a data set', }), ).toBeInTheDocument(); const radios = within( screen.getByRole('group', { - name: 'Step 1 (current) Select a data set', + name: 'Step 1 Select a data set', }), ).getAllByRole('radio'); expect(radios).toHaveLength(2); @@ -192,6 +197,7 @@ describe('DataSetStep', () => { render( , @@ -216,6 +222,7 @@ describe('DataSetStep', () => { render( {table.name}} subjects={testSubjects} @@ -256,6 +263,7 @@ describe('DataSetStep', () => { render( {table.name}} subjects={testSubjects} @@ -265,7 +273,7 @@ describe('DataSetStep', () => { expect( screen.getByRole('heading', { - name: 'Step 1 (current) Select a data set or featured table', + name: 'Step 1 Select a data set or featured table', }), ).toBeInTheDocument(); @@ -286,6 +294,7 @@ describe('DataSetStep', () => { render( {table.name}} subjects={testSubjects} onSubmit={noop} @@ -294,7 +303,7 @@ describe('DataSetStep', () => { expect( screen.getByRole('heading', { - name: 'Step 1 (current) Select a data set', + name: 'Step 1 Select a data set', }), ).toBeInTheDocument(); @@ -316,6 +325,7 @@ describe('DataSetStep', () => { render( {table.name}} subjects={testSubjects} @@ -371,6 +381,7 @@ describe('DataSetStep', () => { render( {table.name}} subjects={testSubjects} @@ -414,6 +425,7 @@ describe('DataSetStep', () => { render( {table.name}} subjects={testSubjects} @@ -456,6 +468,7 @@ describe('DataSetStep', () => { render( {table.name}} @@ -538,6 +551,7 @@ describe('DataSetStep', () => { render( {table.name}} release={testRelease} @@ -556,7 +570,14 @@ describe('DataSetStep', () => { }); test('renders empty message when there are no subjects', () => { - render(); + render( + , + ); expect(screen.getByText('No data sets available.')).toBeInTheDocument(); }); @@ -565,6 +586,7 @@ describe('DataSetStep', () => { render( {table.name}} @@ -583,6 +605,7 @@ describe('DataSetStep', () => { render( {table.name}} @@ -603,6 +626,7 @@ describe('DataSetStep', () => { render( {table.name}} release={testRelease} diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/FiltersForm.test.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/FiltersForm.test.tsx index 26e7a5181e8..ca1049ac5d7 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/FiltersForm.test.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/FiltersForm.test.tsx @@ -268,6 +268,7 @@ describe('FiltersForm', () => { render( { render( { render( { render( { render( { render( { render( { render( { render( { const { container, rerender } = render( { await rerender( { render( { render( { render( { render( { render( { render( { render( { render( { render( { render( { render( , @@ -96,6 +97,7 @@ describe('LocationFiltersForm', () => { render( , @@ -172,6 +174,7 @@ describe('LocationFiltersForm', () => { render( , @@ -191,6 +194,7 @@ describe('LocationFiltersForm', () => { render( , @@ -248,6 +252,7 @@ describe('LocationFiltersForm', () => { render( , @@ -288,6 +293,7 @@ describe('LocationFiltersForm', () => { render( , @@ -321,6 +327,7 @@ describe('LocationFiltersForm', () => { render( , @@ -345,6 +352,7 @@ describe('LocationFiltersForm', () => { render( { render( { render( , @@ -441,6 +451,7 @@ describe('LocationFiltersForm', () => { render( , @@ -472,6 +483,7 @@ describe('LocationFiltersForm', () => { render( , @@ -497,6 +509,7 @@ describe('LocationFiltersForm', () => { render( , @@ -517,88 +530,4 @@ describe('LocationFiltersForm', () => { expect(handleSubmit).toHaveBeenCalledWith(expected); }); }); - - test('sets the step heading to `Choose locations` if multiple location types', () => { - render( - , - ); - - expect( - screen.getByRole('heading', { - name: 'Step 1 (current) Choose locations', - }), - ).toBeInTheDocument(); - }); - - test('sets the step heading based on location type if only one type', () => { - const testSingleLocationType: SubjectMeta['locations'] = { - country: { - legend: 'Country', - options: [ - { - id: 'country-1', - label: 'Country 1', - value: 'country-1', - }, - { - id: 'country-2', - label: 'Country 2', - value: 'country-2', - }, - ], - }, - }; - - render( - , - ); - - expect( - screen.getByRole('heading', { - name: 'Step 1 (current) Choose Countries', - }), - ).toBeInTheDocument(); - }); - - test('sets the step heading to `Choose locations` if single location type is not in the locationLevelsMap', () => { - const testSingleLocationTypeUnknown: SubjectMeta['locations'] = { - unknownType: { - legend: 'Unknown', - options: [ - { - id: 'unknown-1', - label: 'Unknown 1', - value: 'unknown-1', - }, - { - id: 'unknown-2', - label: 'Unknown 2', - value: 'unknown-2', - }, - ], - }, - }; - - render( - , - ); - - expect( - screen.getByRole('heading', { - name: 'Step 1 (current) Choose locations', - }), - ).toBeInTheDocument(); - }); }); diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/LocationStep.test.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/LocationStep.test.tsx index df6a8ed6f1b..34415e7e19d 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/LocationStep.test.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/LocationStep.test.tsx @@ -19,13 +19,14 @@ describe('LocationStep', () => { , ); expect( screen.getByRole('group', { - name: 'Step 1 (current) Choose locations', + name: 'Step 1 Choose locations', }), ).toBeInTheDocument(); }); @@ -36,6 +37,7 @@ describe('LocationStep', () => { {...testWizardStepPropsInActive} options={testLocationsFlat} initialValues={['country-1', 'region-1', 'region-2']} + stepTitle="Choose locations" onSubmit={noop} />, ); @@ -64,6 +66,7 @@ describe('LocationStep', () => { {...testWizardStepPropsInActive} options={testLocationsNested} initialValues={['country-1', 'local-authority-1', 'local-authority-3']} + stepTitle="Choose locations" onSubmit={noop} />, ); diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/PublicationForm.test.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/PublicationForm.test.tsx index 7e56b1a8289..bf5acfe79bf 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/PublicationForm.test.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/PublicationForm.test.tsx @@ -218,7 +218,12 @@ describe('PublicationForm', () => { test('renders the form with the search form, themes list and empty publications list', () => { render( - , + , ); expect(screen.getByLabelText('Search publications')).toBeInTheDocument(); @@ -259,7 +264,12 @@ describe('PublicationForm', () => { test('renders publication options filtered by title when using search field', async () => { render( - , + , ); await userEvent.type( @@ -297,6 +307,7 @@ describe('PublicationForm', () => { render( , @@ -319,7 +330,12 @@ describe('PublicationForm', () => { test('renders the theme as a hint on the publication options when using search field', async () => { render( - , + , ); await userEvent.type( @@ -355,7 +371,12 @@ describe('PublicationForm', () => { test('renders publication options filtered by case-insensitive title', async () => { render( - , + , ); await userEvent.type( @@ -390,7 +411,12 @@ describe('PublicationForm', () => { test('renders the `no publications found` message when there are no search results', async () => { render( - , + , ); await userEvent.type(screen.getByLabelText('Search publications'), 'Nope'); @@ -412,7 +438,12 @@ describe('PublicationForm', () => { const user = userEvent.setup({ delay: null }); render( - , + , ); await user.type(screen.getByLabelText('Search publications'), '[['); @@ -425,7 +456,14 @@ describe('PublicationForm', () => { }); test('renders empty message when there are no themes', () => { - render(); + render( + , + ); expect( screen.queryByRole('group', { name: 'Select a theme' }), @@ -439,7 +477,12 @@ describe('PublicationForm', () => { test('renders the publication options when a theme is selected', async () => { render( - , + , ); await userEvent.click( @@ -481,6 +524,7 @@ describe('PublicationForm', () => { render( , @@ -517,6 +561,7 @@ describe('PublicationForm', () => { render( { test('renders read-only view with selected publication when step is not active', async () => { const { rerender } = render( - , + , ); await userEvent.click( @@ -564,6 +614,7 @@ describe('PublicationForm', () => { rerender( { render( , diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TableToolWizard.test.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TableToolWizard.test.tsx index da7c322367c..b14b9587283 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TableToolWizard.test.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TableToolWizard.test.tsx @@ -219,9 +219,7 @@ describe('TableToolWizard', () => { const stepHeadings = screen.queryAllByRole('heading', { name: /Step/ }); expect(stepHeadings).toHaveLength(1); - expect(stepHeadings[0]).toHaveTextContent( - 'Step 1 (current) Choose a publication', - ); + expect(stepHeadings[0]).toHaveTextContent('Step 1 Choose a publication'); expect(screen.getAllByRole('listitem')).toHaveLength(1); }); @@ -317,9 +315,7 @@ describe('TableToolWizard', () => { const stepHeadings = screen.queryAllByRole('heading', { name: /Step/ }); expect(stepHeadings).toHaveLength(1); - expect(stepHeadings[0]).toHaveTextContent( - 'Step 1 (current) Select a data set', - ); + expect(stepHeadings[0]).toHaveTextContent('Step 1 Select a data set'); }); }); @@ -357,9 +353,7 @@ describe('TableToolWizard', () => { expect(stepHeadings[1]).toHaveTextContent('Step 2 Select a data set'); expect(stepHeadings[2]).toHaveTextContent('Step 3 Choose locations'); expect(stepHeadings[3]).toHaveTextContent('Step 4 Choose time period'); - expect(stepHeadings[4]).toHaveTextContent( - 'Step 5 (current) Choose your filters', - ); + expect(stepHeadings[4]).toHaveTextContent('Step 5 Choose your filters'); // Step 1 @@ -656,4 +650,147 @@ describe('TableToolWizard', () => { ).toBeInTheDocument(); }); }); + + describe('location step heading', () => { + test('sets the step heading to `Choose locations` if multiple location types', () => { + render( + , + ); + + expect( + screen.getByRole('heading', { + name: 'Step 3 Choose locations', + }), + ).toBeInTheDocument(); + }); + + test('sets the step heading based on location type if only one type', () => { + const testSingleLocationType: SubjectMeta['locations'] = { + country: { + legend: 'Country', + options: [ + { + id: 'country-1', + label: 'Country 1', + value: 'country-1', + }, + { + id: 'country-2', + label: 'Country 2', + value: 'country-2', + }, + ], + }, + }; + + render( + , + ); + + expect( + screen.getByRole('heading', { + name: 'Step 3 Choose Countries', + }), + ).toBeInTheDocument(); + }); + + test('sets the step heading to `Choose locations` if single location type is not in the locationLevelsMap', () => { + const testSingleLocationTypeUnknown: SubjectMeta['locations'] = { + unknownType: { + legend: 'Unknown', + options: [ + { + id: 'unknown-1', + label: 'Unknown 1', + value: 'unknown-1', + }, + { + id: 'unknown-2', + label: 'Unknown 2', + value: 'unknown-2', + }, + ], + }, + }; + + render( + , + ); + + expect( + screen.getByRole('heading', { + name: 'Step 3 Choose locations', + }), + ).toBeInTheDocument(); + }); + }); }); diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/Wizard.test.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/Wizard.test.tsx index 23b4f8230e4..2b82389b9ef 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/Wizard.test.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/Wizard.test.tsx @@ -34,8 +34,6 @@ describe('Wizard', () => { const step3 = screen.getByTestId('wizardStep-3'); expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); - expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); }); @@ -95,7 +93,6 @@ describe('Wizard', () => { expect(step1).toBeVisible(); expect(step2).toBeVisible(); - expect(step2).toHaveAttribute('aria-current', 'step'); expect(step3).not.toBeVisible(); }); @@ -159,7 +156,6 @@ describe('Wizard', () => { const step3 = screen.getByTestId('wizardStep-3'); expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); @@ -168,7 +164,6 @@ describe('Wizard', () => { expect(step1).toBeVisible(); expect(step2).toBeVisible(); expect(step3).toBeVisible(); - expect(step3).toHaveAttribute('aria-current', 'step'); }); test('calling `setCurrentStep` with a step greater than the last step will not change the wizard', async () => { @@ -193,14 +188,12 @@ describe('Wizard', () => { const step3 = screen.getByTestId('wizardStep-3'); expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); await userEvent.click(screen.getByRole('button', { name: 'Go to step 4' })); expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); }); @@ -227,7 +220,6 @@ describe('Wizard', () => { const step3 = screen.getByTestId('wizardStep-3'); expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); @@ -236,7 +228,6 @@ describe('Wizard', () => { ); expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); }); @@ -268,7 +259,6 @@ describe('Wizard', () => { // Still on same step expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); @@ -278,7 +268,6 @@ describe('Wizard', () => { expect(step2).toBeVisible(); }); expect(step3).toBeVisible(); - expect(step3).toHaveAttribute('aria-current', 'step'); }); test('calling `goToPreviousStep` render prop moves wizard to previous step', async () => { @@ -302,7 +291,6 @@ describe('Wizard', () => { expect(step1).toBeVisible(); expect(step2).toBeVisible(); - expect(step2).toHaveAttribute('aria-current', 'step'); expect(step3).not.toBeVisible(); await userEvent.click( @@ -310,7 +298,6 @@ describe('Wizard', () => { ); expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); }); @@ -335,7 +322,6 @@ describe('Wizard', () => { const step3 = screen.getByTestId('wizardStep-3'); expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); @@ -344,7 +330,6 @@ describe('Wizard', () => { ); expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); }); @@ -379,16 +364,14 @@ describe('Wizard', () => { // Still on same step expect(step1).toBeVisible(); expect(step2).toBeVisible(); - expect(step2).toHaveAttribute('aria-current', 'step'); expect(step3).not.toBeVisible(); // Moved to next step await waitFor(() => { - expect(step1).toHaveAttribute('aria-current', 'step'); + expect(step1).toBeVisible(); + expect(step2).not.toBeVisible(); + expect(step3).not.toBeVisible(); }); - expect(step1).toBeVisible(); - expect(step2).not.toBeVisible(); - expect(step3).not.toBeVisible(); }); test('calling `goToPreviousStep` with a task sets correct loading render props', async () => { @@ -464,7 +447,6 @@ describe('Wizard', () => { expect(step1).toBeVisible(); expect(step2).toBeVisible(); - expect(step2).toHaveAttribute('aria-current', 'step'); expect(step3).not.toBeVisible(); await userEvent.click( @@ -474,7 +456,6 @@ describe('Wizard', () => { expect(step1).toBeVisible(); expect(step2).toBeVisible(); expect(step3).toBeVisible(); - expect(step3).toHaveAttribute('aria-current', 'step'); }); test('calling `goToNextStep` on last step does not change wizard', async () => { @@ -499,7 +480,6 @@ describe('Wizard', () => { expect(step1).toBeVisible(); expect(step2).toBeVisible(); expect(step3).toBeVisible(); - expect(step3).toHaveAttribute('aria-current', 'step'); await userEvent.click( screen.getByRole('button', { name: 'Go to next step' }), @@ -508,7 +488,6 @@ describe('Wizard', () => { expect(step1).toBeVisible(); expect(step2).toBeVisible(); expect(step3).toBeVisible(); - expect(step3).toHaveAttribute('aria-current', 'step'); }); test('calling `goToNextStep` with a task will not transition the wizard until it completes', async () => { @@ -540,14 +519,12 @@ describe('Wizard', () => { // Still on same step expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); // Moved to next step await waitFor(() => expect(step2).toBeVisible()); expect(step1).toBeVisible(); - expect(step2).toHaveAttribute('aria-current', 'step'); expect(step3).not.toBeVisible(); }); @@ -724,7 +701,6 @@ describe('Wizard', () => { const step3 = screen.getByTestId('wizardStep-3'); expect(step1).toBeVisible(); - expect(step1).toHaveAttribute('aria-current', 'step'); expect(step2).not.toBeVisible(); expect(step3).not.toBeVisible(); @@ -733,7 +709,6 @@ describe('Wizard', () => { await waitFor(() => { expect(step1).toBeVisible(); expect(step2).toBeVisible(); - expect(step2).toHaveAttribute('aria-current', 'step'); expect(step3).not.toBeVisible(); }); }); @@ -810,11 +785,9 @@ describe('Wizard', () => { const step1 = screen.getByTestId('wizardStep-1'); const step2 = screen.getByTestId('wizardStep-2'); - await waitFor(() => expect(step1).not.toHaveAttribute('aria-current')); - expect(step2).toHaveAttribute('aria-current', 'step'); + await waitFor(() => expect(step1).toBeVisible()); - await waitFor(() => expect(step1).toHaveAttribute('aria-current')); - expect(step2).not.toHaveAttribute('aria-current'); + await waitFor(() => expect(step2).not.toBeVisible()); expect(handleBack).toHaveBeenCalledTimes(1); }); diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/__snapshots__/FiltersForm.test.tsx.snap b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/__snapshots__/FiltersForm.test.tsx.snap index a29925a8829..7477896fea3 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/__snapshots__/FiltersForm.test.tsx.snap +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/__snapshots__/FiltersForm.test.tsx.snap @@ -22,7 +22,10 @@ exports[`FiltersForm renders a read-only view of selected options when no longer class="govuk-list" id="indicatorsList" > -
    • +
    • Number of excluded sessions

    @@ -46,7 +49,10 @@ exports[`FiltersForm renders a read-only view of selected options when no longer class="govuk-list" id="filtersList-SchoolType" > -
  • +
  • State-funded secondary
@@ -70,7 +76,10 @@ exports[`FiltersForm renders a read-only view of selected options when no longer class="govuk-list" id="filtersList-Characteristic" > -
  • +
  • Ethnicity Major Black Total
    • diff --git a/src/explore-education-statistics-common/src/services/publicationService.ts b/src/explore-education-statistics-common/src/services/publicationService.ts index 16d49aec33a..38f92de4506 100644 --- a/src/explore-education-statistics-common/src/services/publicationService.ts +++ b/src/explore-education-statistics-common/src/services/publicationService.ts @@ -28,6 +28,7 @@ export interface Publication { releaseSeries: ReleaseSeriesItem[]; topic: { theme: { + id: string; title: string; }; }; @@ -131,21 +132,8 @@ export interface ContentSection { content: BlockType[]; } -export const publicationSortOptions = [ - 'newest', - 'oldest', - 'relevance', - 'title', -] as const; - -export type PublicationSortOption = (typeof publicationSortOptions)[number]; - export type PublicationSortParam = 'published' | 'title' | 'relevance'; -export const publicationFilters = ['releaseType', 'search', 'themeId'] as const; - -export type PublicationFilter = (typeof publicationFilters)[number]; - export interface PublicationListRequest { page?: number; pageSize?: number; diff --git a/src/explore-education-statistics-common/src/styles/_form.scss b/src/explore-education-statistics-common/src/styles/_form.scss index 8105c5a34c4..093276f4282 100644 --- a/src/explore-education-statistics-common/src/styles/_form.scss +++ b/src/explore-education-statistics-common/src/styles/_form.scss @@ -6,3 +6,9 @@ .govuk-textarea[disabled] { cursor: not-allowed; } + +// Hide the clear button on search inputs as they +// are not keyboard accessible, see EES-5210. +[type='search']::-webkit-search-cancel-button { + appearance: none; +} diff --git a/src/explore-education-statistics-frontend/.env b/src/explore-education-statistics-frontend/.env index e97acc35c5b..c952c8c8636 100644 --- a/src/explore-education-statistics-frontend/.env +++ b/src/explore-education-statistics-frontend/.env @@ -5,4 +5,5 @@ PUBLIC_API_DOCS_URL=TODO-GUIDANCE-URL NOTIFICATION_API_BASE_URL=http://localhost:7073/api GA_TRACKING_ID= PUBLIC_URL=http://localhost:3000/ +PROD_PUBLIC_URL=https://explore-education-statistics.service.gov.uk APP_ENV=Local diff --git a/src/explore-education-statistics-frontend/.env.production b/src/explore-education-statistics-frontend/.env.production deleted file mode 100644 index 7b981a86798..00000000000 --- a/src/explore-education-statistics-frontend/.env.production +++ /dev/null @@ -1 +0,0 @@ -PUBLIC_URL=https://explore-education-statistics.service.gov.uk/ diff --git a/src/explore-education-statistics-frontend/next-sitemap.config.js b/src/explore-education-statistics-frontend/next-sitemap.config.js index 697249efc6e..2497341321d 100644 --- a/src/explore-education-statistics-frontend/next-sitemap.config.js +++ b/src/explore-education-statistics-frontend/next-sitemap.config.js @@ -1,6 +1,6 @@ /** @type {import('next-sitemap').IConfig} */ module.exports = { - siteUrl: process.env.PUBLIC_URL, + siteUrl: process.env.PROD_PUBLIC_URL, sitemapSize: 5000, exclude: ['/server-sitemap.xml'], generateRobotsTxt: true, @@ -16,7 +16,7 @@ module.exports = { ], }, ], - additionalSitemaps: [`${process.env.PUBLIC_URL}server-sitemap.xml`], + additionalSitemaps: [`${process.env.PROD_PUBLIC_URL}/server-sitemap.xml`], }, transform: async (config, path) => { if (path === '/') { diff --git a/src/explore-education-statistics-frontend/src/components/Breadcrumbs.tsx b/src/explore-education-statistics-frontend/src/components/Breadcrumbs.tsx index 3f9deb734eb..7c2da9a5239 100644 --- a/src/explore-education-statistics-frontend/src/components/Breadcrumbs.tsx +++ b/src/explore-education-statistics-frontend/src/components/Breadcrumbs.tsx @@ -10,7 +10,6 @@ export interface BreadcrumbsProps { const Breadcrumbs = ({ breadcrumbs = [] }: BreadcrumbsProps) => { const currentBreadcrumbIndex = breadcrumbs.length - 1; - return (