From a9b0df141f4a710bf9b002cf1227fc444b2d6bb9 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 15 Nov 2024 14:35:45 +0000 Subject: [PATCH 001/144] EES-5663 Remove minor versions from public API versioning Minor versions are not necessary as only major versions are important to end users. Simplifying the API versioning is also needed for future work to determine the latest API version during builds. --- infrastructure/templates/template.json | 2 +- .../Controllers/DataSetVersionsController.cs | 2 +- .../Controllers/DataSetsController.cs | 2 +- .../Controllers/PublicationsController.cs | 2 +- src/explore-education-statistics-frontend/.env | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index 2262edd6e97..001148a1e01 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -2114,7 +2114,7 @@ "NEXT_CONFIG_MODE": "server", "NODE_ENV": "production", "PUBLIC_URL": "[concat(variables('publicAppUrl'), '/')]", - "PUBLIC_API_BASE_URL": "[concat('https://', parameters('publicApiUrl'),'/api/v1.0')]", + "PUBLIC_API_BASE_URL": "[concat('https://', parameters('publicApiUrl'), '/api/v1')]", "PUBLIC_API_DOCS_URL": "[concat('https://', parameters('publicApiDocsUrl'))]", "WEBSITE_NODE_DEFAULT_VERSION": "20.16.0", "WEBSITES_PORT": 3000 diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs index dc46e2e74fb..ed5f01adad9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs @@ -10,7 +10,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Controllers; -[ApiVersion(1.0)] +[ApiVersion("1")] [ApiController] [Route("api/v{version:apiVersion}/data-sets/{dataSetId:guid}/versions")] public class DataSetVersionsController( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs index ecd16d7cf40..f76c8442638 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs @@ -11,7 +11,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Controllers; -[ApiVersion(1.0)] +[ApiVersion("1")] [ApiController] [Route("api/v{version:apiVersion}/data-sets")] public class DataSetsController( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs index 8ee6cdd0eb7..7f87b103164 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs @@ -10,7 +10,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Controllers; -[ApiVersion(1.0)] +[ApiVersion("1")] [ApiController] [Route("api/v{version:apiVersion}/publications")] public class PublicationsController(IPublicationService publicationService, IDataSetService dataSetService) diff --git a/src/explore-education-statistics-frontend/.env b/src/explore-education-statistics-frontend/.env index a2390fb9898..12c3f953ce4 100644 --- a/src/explore-education-statistics-frontend/.env +++ b/src/explore-education-statistics-frontend/.env @@ -1,6 +1,6 @@ CONTENT_API_BASE_URL=http://localhost:5010/api DATA_API_BASE_URL=http://localhost:5000/api -PUBLIC_API_BASE_URL=http://localhost:5050/api/v1.0 +PUBLIC_API_BASE_URL=http://localhost:5050/api/v1 PUBLIC_API_DOCS_URL=https://dev.statistics.api.education.gov.uk/docs NOTIFICATION_API_BASE_URL=http://localhost:7073/api GA_TRACKING_ID= From 5c64ab662b4b989b840cb500337b51f70cc4224b Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 15 Nov 2024 14:55:15 +0000 Subject: [PATCH 002/144] EES-5663 Remove `version` path parameters from OpenAPI documents --- .../VersionedPathsDocumentFilterTests.cs | 130 ++++++++++++++++++ .../Swagger/SwaggerConfig.cs | 3 + .../Swagger/VersionedPathsDocumentFilter.cs | 32 +++++ 3 files changed, 165 insertions(+) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/VersionedPathsDocumentFilterTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/VersionedPathsDocumentFilter.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/VersionedPathsDocumentFilterTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/VersionedPathsDocumentFilterTests.cs new file mode 100644 index 00000000000..6f8ae622251 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/VersionedPathsDocumentFilterTests.cs @@ -0,0 +1,130 @@ +using System.Text.Json; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Swagger; +using Microsoft.OpenApi.Models; +using Swashbuckle.AspNetCore.SwaggerGen; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Swagger; + +public class VersionedPathsDocumentFilterTests +{ + [Fact] + public void VersionsInlinedIntoPaths() + { + var document = new OpenApiDocument + { + Info = new OpenApiInfo { Version = "1" }, + Paths = new OpenApiPaths + { + {"/v{version}/endpoint-1", new OpenApiPathItem()}, + {"/v{version}/endpoint-2/{id}", new OpenApiPathItem()} + } + }; + + var filter = new VersionedPathsDocumentFilter(); + var context = CreateDocumentFilterContext(); + + filter.Apply(document, context); + + Assert.Equal(2, document.Paths.Count); + Assert.Contains("/v1/endpoint-1", document.Paths.Keys); + Assert.Contains("/v1/endpoint-2/{id}", document.Paths.Keys); + } + + [Fact] + public void VersionParametersRemoved() + { + var document = new OpenApiDocument + { + Info = new OpenApiInfo { Version = "1" }, + Paths = new OpenApiPaths + { + { + "/v{version}/endpoint-1", + new OpenApiPathItem + { + Parameters = + [ + new OpenApiParameter { Name = "version" } + ], + Operations = + { + { + OperationType.Get, + new OpenApiOperation + { + Parameters = + [ + new OpenApiParameter { Name = "version" } + ] + } + } + + } + } + }, + { + "/v{version}/endpoint-2/{id}", + new OpenApiPathItem + { + Parameters = + [ + new OpenApiParameter { Name = "version" }, + new OpenApiParameter { Name = "id" } + ], + Operations = + { + { + OperationType.Get, + new OpenApiOperation + { + Parameters = + [ + new OpenApiParameter { Name = "version" }, + new OpenApiParameter { Name = "id" } + ] + } + } + + } + } + } + } + }; + + var filter = new VersionedPathsDocumentFilter(); + var context = CreateDocumentFilterContext(); + + filter.Apply(document, context); + + Assert.Equal(2, document.Paths.Count); + + var endpoint1Paths = document.Paths["/v1/endpoint-1"]; + + Assert.Empty(endpoint1Paths.Parameters); + Assert.Empty(endpoint1Paths.Operations[OperationType.Get].Parameters); + + var endpoint2Paths = document.Paths["/v1/endpoint-2/{id}"]; + + Assert.Single(endpoint2Paths.Parameters); + Assert.Equal("id", endpoint2Paths.Parameters[0].Name); + + Assert.Single(endpoint2Paths.Operations[OperationType.Get].Parameters); + Assert.Equal("id", endpoint2Paths.Parameters[0].Name); + } + + private static DocumentFilterContext CreateDocumentFilterContext() + { + var schemaGenerator = new SchemaGenerator( + new SchemaGeneratorOptions(), + new JsonSerializerDataContractResolver( + new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + } + ) + ); + var schemaRepository = new SchemaRepository(); + + return new DocumentFilterContext([], schemaGenerator, schemaRepository); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs index 8d00439bc1e..babd1bdec18 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs @@ -16,8 +16,11 @@ public class SwaggerConfig( { public void Configure(SwaggerGenOptions options) { + options.DocumentFilter(); + options.OperationFilter(); options.OperationFilter(); + options.SchemaFilter(); options.SchemaFilter(); options.SchemaFilter(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/VersionedPathsDocumentFilter.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/VersionedPathsDocumentFilter.cs new file mode 100644 index 00000000000..3e0b85e334a --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/VersionedPathsDocumentFilter.cs @@ -0,0 +1,32 @@ +using Microsoft.OpenApi.Models; +using Swashbuckle.AspNetCore.SwaggerGen; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Swagger; + +public class VersionedPathsDocumentFilter : IDocumentFilter +{ + public void Apply(OpenApiDocument document, DocumentFilterContext context) + { + var newPaths = new OpenApiPaths(); + + foreach (var path in document.Paths) + { + var versionedPath = path.Key.Replace("{version}", document.Info.Version); + + newPaths[versionedPath] = path.Value; + + path.Value.Parameters = path.Value.Parameters + .Where(p => p.Name != "version") + .ToList(); + + foreach (var operation in path.Value.Operations.Values) + { + operation.Parameters = operation.Parameters + .Where(p => p.Name != "version") + .ToList(); + } + } + + document.Paths = newPaths; + } +} From 6c0e642cf06c467f33bf7e787fb8ebcb9bfcc752 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 15 Nov 2024 15:22:12 +0000 Subject: [PATCH 003/144] EES-5663 Prefix OpenAPI document version paths with `v` --- .../Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Program.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Program.cs index 214a54bb4fa..81854448f17 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Program.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Program.cs @@ -39,7 +39,7 @@ app.UseSwagger(options => { - options.RouteTemplate = "/swagger/{documentName}/openapi.json"; + options.RouteTemplate = "/swagger/v{documentName}/openapi.json"; }); app.UseSwaggerUI(options => { @@ -48,7 +48,7 @@ foreach (var description in app.DescribeApiVersions()) { options.SwaggerEndpoint( - url: $"/swagger/{description.GroupName}/openapi.json", + url: $"/swagger/v{description.GroupName}/openapi.json", name: $"v{description.GroupName}"); } }); From 0c204af8b8fe14ce4c6e978ba1c32a6995ab38ee Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 18 Nov 2024 02:58:54 +0000 Subject: [PATCH 004/144] EES-5663 Remove `/api` prefix from public API routes --- infrastructure/templates/template.json | 2 +- .../Services/Public.Data/PublicDataApiClient.cs | 2 +- .../Controllers/DataSetVersionsControllerTests.cs | 2 +- .../Controllers/DataSetsControllerGetQueryTests.cs | 2 +- .../Controllers/DataSetsControllerPostQueryTests.cs | 2 +- .../Controllers/DataSetsControllerTests.cs | 2 +- .../Controllers/PublicationsControllerTests.cs | 2 +- .../Controllers/DataSetVersionsController.cs | 2 +- .../Controllers/DataSetsController.cs | 2 +- .../Controllers/PublicationsController.cs | 2 +- src/explore-education-statistics-frontend/.env | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index 001148a1e01..615d5ebfcca 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -2114,7 +2114,7 @@ "NEXT_CONFIG_MODE": "server", "NODE_ENV": "production", "PUBLIC_URL": "[concat(variables('publicAppUrl'), '/')]", - "PUBLIC_API_BASE_URL": "[concat('https://', parameters('publicApiUrl'), '/api/v1')]", + "PUBLIC_API_BASE_URL": "[concat('https://', parameters('publicApiUrl'), '/v1')]", "PUBLIC_API_DOCS_URL": "[concat('https://', parameters('publicApiDocsUrl'))]", "WEBSITE_NODE_DEFAULT_VERSION": "20.16.0", "WEBSITES_PORT": 3000 diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/PublicDataApiClient.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/PublicDataApiClient.cs index f30928a4d95..7d98fb8687b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/PublicDataApiClient.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/PublicDataApiClient.cs @@ -35,7 +35,7 @@ public async Task> GetDataSetVersionCh { return await SendRequest( () => httpClient.GetAsync( - $"api/v1/data-sets/{dataSetId}/versions/{dataSetVersion}/changes", + $"v1/data-sets/{dataSetId}/versions/{dataSetVersion}/changes", cancellationToken ), cancellationToken diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs index 8e64f7b9d08..ba8fae60144 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs @@ -20,7 +20,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Contr public abstract class DataSetVersionsControllerTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { - private const string BaseUrl = "api/v1/data-sets"; + private const string BaseUrl = "v1/data-sets"; public class ListDataSetVersionsTests(TestApplicationFactory testApp) : DataSetVersionsControllerTests(testApp) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs index 0a1287e3c12..b3f4e040f08 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs @@ -23,7 +23,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Contr public abstract class DataSetsControllerGetQueryTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { - private const string BaseUrl = "api/v1/data-sets"; + private const string BaseUrl = "v1/data-sets"; private readonly TestDataSetVersionPathResolver _dataSetVersionPathResolver = new() { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerPostQueryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerPostQueryTests.cs index f1547e63fe4..15cc2c59ed3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerPostQueryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerPostQueryTests.cs @@ -25,7 +25,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Contr public abstract class DataSetsControllerPostQueryTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { - private const string BaseUrl = "api/v1/data-sets"; + private const string BaseUrl = "v1/data-sets"; private readonly TestDataSetVersionPathResolver _dataSetVersionPathResolver = new() { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerTests.cs index 974d7a84797..4b9cb85467a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerTests.cs @@ -25,7 +25,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Contr public abstract class DataSetsControllerTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { - private const string BaseUrl = "api/v1/data-sets"; + private const string BaseUrl = "v1/data-sets"; public class GetDataSetTests(TestApplicationFactory testApp) : DataSetsControllerTests(testApp) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/PublicationsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/PublicationsControllerTests.cs index bc99ad83d1a..0101cd1e8d7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/PublicationsControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/PublicationsControllerTests.cs @@ -21,7 +21,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Contr public abstract class PublicationsControllerTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { - private const string BaseUrl = "api/v1/publications"; + private const string BaseUrl = "v1/publications"; public class ListPublicationsTests(TestApplicationFactory testApp) : PublicationsControllerTests(testApp) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs index ed5f01adad9..dcd5a61e06f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs @@ -12,7 +12,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Controllers [ApiVersion("1")] [ApiController] -[Route("api/v{version:apiVersion}/data-sets/{dataSetId:guid}/versions")] +[Route("v{version:apiVersion}/data-sets/{dataSetId:guid}/versions")] public class DataSetVersionsController( IDataSetService dataSetService, IDataSetVersionChangeService dataSetVersionChangeService) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs index f76c8442638..de99e4dd527 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs @@ -13,7 +13,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Controllers [ApiVersion("1")] [ApiController] -[Route("api/v{version:apiVersion}/data-sets")] +[Route("v{version:apiVersion}/data-sets")] public class DataSetsController( IDataSetService dataSetService, IDataSetQueryService dataSetQueryService) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs index 7f87b103164..74d901d6163 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs @@ -12,7 +12,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Controllers [ApiVersion("1")] [ApiController] -[Route("api/v{version:apiVersion}/publications")] +[Route("v{version:apiVersion}/publications")] public class PublicationsController(IPublicationService publicationService, IDataSetService dataSetService) : ControllerBase { diff --git a/src/explore-education-statistics-frontend/.env b/src/explore-education-statistics-frontend/.env index 12c3f953ce4..95c495d7605 100644 --- a/src/explore-education-statistics-frontend/.env +++ b/src/explore-education-statistics-frontend/.env @@ -1,6 +1,6 @@ CONTENT_API_BASE_URL=http://localhost:5010/api DATA_API_BASE_URL=http://localhost:5000/api -PUBLIC_API_BASE_URL=http://localhost:5050/api/v1 +PUBLIC_API_BASE_URL=http://localhost:5050/v1 PUBLIC_API_DOCS_URL=https://dev.statistics.api.education.gov.uk/docs NOTIFICATION_API_BASE_URL=http://localhost:7073/api GA_TRACKING_ID= From fffc5a63462ed98894f5062d8df2e376e7014612 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 15 Nov 2024 15:34:37 +0000 Subject: [PATCH 005/144] EES-5660 - using special "draft" folder to hold the latest draft version of a data set prior to its publishing rather than including version in the folder name, as the version can change multiple times prior to publishing. --- .../Extensions/EnumerableExtensionsTests.cs | 90 +++++++++++++++++++ .../Extensions/EnumerableExtensions.cs | 21 +++++ .../DataSetVersionStatus.cs | 17 ++++ .../DataSetVersionAuthExtensions.cs | 10 +-- ...etionOfNextDataSetVersionFunctionsTests.cs | 55 ------------ ...nOfNextDataSetVersionOrchestrationTests.cs | 3 +- .../Functions/ActivityNames.cs | 2 - ...CompletionOfNextDataSetVersionFunctions.cs | 27 ------ ...letionOfNextDataSetVersionOrchestration.cs | 1 - .../DataSetVersionPathResolverTests.cs | 55 ++++++++++-- .../TestDataSetVersionPathResolver.cs | 7 +- .../DataSetVersionPathResolver.cs | 19 +++- .../Interfaces/IDataSetVersionPathResolver.cs | 4 +- .../Services/DataSetPublishingServiceTests.cs | 29 ++++++ .../appsettings.IntegrationTest.json | 3 + ...xploreEducationStatistics.Publisher.csproj | 1 + .../PublisherHostBuilderExtensions.cs | 6 ++ .../Services/DataSetPublishingService.cs | 19 ++-- 18 files changed, 256 insertions(+), 113 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs index 08917f36617..0ccf2a08f0a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs @@ -466,6 +466,96 @@ public void ContainsAll_SourceListContainsAllValues_ReturnsTrue() Assert.True(containsAll); } + public class CartesianTests + { + [Fact] + public void TwoLists_Cartesian() + { + List list1 = [1, 2]; + List list2 = ["3", "4"]; + + List> expected = + [ + new Tuple(1, "3"), + new Tuple(1, "4"), + new Tuple(2, "3"), + new Tuple(2, "4") + ]; + + var actual = list1.Cartesian(list2); + Assert.Equal(expected, actual); + } + + [Fact] + public void TwoLists_EmptyList() + { + List list1 = [1, 2]; + Assert.Empty(list1.Cartesian(new List())); + } + + [Fact] + public void TwoLists_NullList() + { + List list1 = [1, 2]; + Assert.Empty(list1.Cartesian((List?) null)); + } + + [Fact] + public void ThreeLists_Cartesian() + { + List list1 = [1, 2]; + List list2 = ["3", "4"]; + List list3 = ['5', '6']; + + List> expected = + [ + new Tuple(1, "3", '5'), + new Tuple(1, "3", '6'), + new Tuple(1, "4", '5'), + new Tuple(1, "4", '6'), + new Tuple(2, "3", '5'), + new Tuple(2, "3", '6'), + new Tuple(2, "4", '5'), + new Tuple(2, "4", '6'), + ]; + + var actual = list1.Cartesian(list2, list3); + Assert.Equal(expected, actual); + } + + [Fact] + public void ThreeLists_EmptyFirstList() + { + List list1 = [1, 2]; + List list3 = ['5', '6']; + Assert.Empty(list1.Cartesian(new List(), list3)); + } + + [Fact] + public void ThreeLists_EmptySecondList() + { + List list1 = [1, 2]; + List list2 = ["3", "4"]; + Assert.Empty(list1.Cartesian(list2, new List())); + } + + [Fact] + public void ThreeLists_NullFirstList() + { + List list1 = [1, 2]; + List list3 = ['5', '6']; + Assert.Empty(list1.Cartesian((List?) null, list3)); + } + + [Fact] + public void ThreeLists_NullSecondList() + { + List list1 = [1, 2]; + List list2 = ["3", "4"]; + Assert.Empty(list1.Cartesian(list2, (List?) null)); + } + } + private static async Task> GetSuccessfulEither(int value) { await Task.Delay(5); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs index 4a780e7f1d3..36b32889a16 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs @@ -319,5 +319,26 @@ public static IOrderedEnumerable NaturalThenBy( { return source.ThenBy(keySelector, comparison.WithNaturalSort()); } + + public static IEnumerable> Cartesian( + this IEnumerable list1, + IEnumerable? list2) + { + return list2 == null + ? [] + : list1.Join(list2, _ => true, _ => true,(t1, t2) => new Tuple(t1, t2)); + } + + public static IEnumerable> Cartesian( + this IEnumerable list1, + IEnumerable? list2, + IEnumerable? list3) + { + return list2 == null || list3 == null + ? [] + : list1 + .Join(list2, _ => true, _ => true, (t1, t2) => new Tuple(t1, t2)) + .Join(list3, _ => true, _ => true, (tuple, t3) => new Tuple(tuple.Item1, tuple.Item2, t3)); + } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs index f62f4ac3578..6124daf0eea 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; using Newtonsoft.Json.Converters; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model; @@ -16,3 +17,19 @@ public enum DataSetVersionStatus Withdrawn, Cancelled, } + +public class DataSetVersionStatusConstants +{ + public static readonly IReadOnlyList PublicStatuses = new List( + [ + DataSetVersionStatus.Published, + DataSetVersionStatus.Withdrawn, + DataSetVersionStatus.Deprecated + ] + ); + + public static readonly IReadOnlyList PrivateStatuses = EnumUtil + .GetEnums() + .Except(PublicStatuses) + .ToList(); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs index 40170ae2c23..44d01584a51 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs @@ -4,14 +4,6 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Extension public static class DataSetVersionAuthExtensions { - private static readonly IReadOnlyList PublicStatuses = new List( - [ - DataSetVersionStatus.Published, - DataSetVersionStatus.Withdrawn, - DataSetVersionStatus.Deprecated - ] - ); - public static bool IsPublicStatus(this DataSetVersion dataSetVersion) => IsPublicStatus().Compile()(dataSetVersion); @@ -19,5 +11,5 @@ public static IQueryable WherePublicStatus(this IQueryable queryable.Where(IsPublicStatus()); private static Expression> IsPublicStatus() - => dataSetVersion => PublicStatuses.Contains(dataSetVersion.Status); + => dataSetVersion => DataSetVersionStatusConstants.PublicStatuses.Contains(dataSetVersion.Status); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionFunctionsTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionFunctionsTests.cs index ad18ad3daa2..e28cb2e1847 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionFunctionsTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionFunctionsTests.cs @@ -2990,61 +2990,6 @@ .. DataFixture.DefaultTimePeriodMeta() } } - public class UpdateFileStoragePathTests( - ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture) - { - private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ManualMapping; - - [Fact] - public async Task Success_PathUpdated() - { - var (_, nextDataSetVersion, instanceId) = await CreateDataSetInitialAndNextVersion( - nextVersionStatus: DataSetVersionStatus.Mapping, - nextVersionImportStage: Stage); - - var dataSetVersionPathResolver = GetRequiredService(); - var originalStoragePath = dataSetVersionPathResolver.DirectoryPath(nextDataSetVersion); - Directory.CreateDirectory(originalStoragePath); - - await using var publicDataDbContext = GetDbContext(); - - nextDataSetVersion.VersionMajor++; - - publicDataDbContext.DataSetVersions.Update(nextDataSetVersion); - await publicDataDbContext.SaveChangesAsync(); - - var newStoragePath = dataSetVersionPathResolver.DirectoryPath(nextDataSetVersion); - - await UpdateFileStoragePath(instanceId); - - Assert.False(Directory.Exists(originalStoragePath)); - Assert.True(Directory.Exists(newStoragePath)); - } - - [Fact] - public async Task Success_PathNotUpdated() - { - var (_, nextDataSetVersion, instanceId) = await CreateDataSetInitialAndNextVersion( - nextVersionStatus: DataSetVersionStatus.Mapping, - nextVersionImportStage: Stage); - - var dataSetVersionPathResolver = GetRequiredService(); - var originalStoragePath = dataSetVersionPathResolver.DirectoryPath(nextDataSetVersion); - Directory.CreateDirectory(originalStoragePath); - - await UpdateFileStoragePath(instanceId); - - Assert.True(Directory.Exists(originalStoragePath)); - } - - private async Task UpdateFileStoragePath(Guid instanceId) - { - var function = GetRequiredService(); - await function.UpdateFileStoragePath(instanceId, CancellationToken.None); - } - } - public class CompleteNextDataSetVersionImportProcessingTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs index 4cf6c417a07..c4fc07e53de 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs @@ -27,7 +27,6 @@ public async Task Success() string[] expectedActivitySequence = [ - ActivityNames.UpdateFileStoragePath, ActivityNames.ImportMetadata, ActivityNames.CreateChanges, ActivityNames.ImportData, @@ -60,7 +59,7 @@ public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() mockOrchestrationContext .InSequence(activitySequence) .Setup(context => - context.CallActivityAsync(ActivityNames.UpdateFileStoragePath, + context.CallActivityAsync(ActivityNames.ImportMetadata, mockOrchestrationContext.Object.InstanceId, null)) .Throws(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs index f1f002e15c4..77b7a53dda9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs @@ -18,8 +18,6 @@ internal static class ActivityNames public const string CreateChanges = nameof(ProcessCompletionOfNextDataSetVersionFunctions.CreateChanges); - public const string UpdateFileStoragePath = - nameof(ProcessCompletionOfNextDataSetVersionFunctions.UpdateFileStoragePath); public const string CompleteNextDataSetVersionImportProcessing = nameof(ProcessCompletionOfNextDataSetVersionFunctions.CompleteNextDataSetVersionImportProcessing); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs index e9bae2c8a76..af874a9ba47 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs @@ -24,33 +24,6 @@ public async Task CreateChanges( await dataSetVersionChangeService.CreateChanges(dataSetVersionImport.DataSetVersionId, cancellationToken); } - [Function(ActivityNames.UpdateFileStoragePath)] - public async Task UpdateFileStoragePath( - [ActivityTrigger] Guid instanceId, - CancellationToken cancellationToken) - { - var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); - - var dataSetVersion = dataSetVersionImport.DataSetVersion; - - var currentLiveVersion = await publicDataDbContext - .DataSets - .Where(dataSet => dataSet.Id == dataSetVersion.DataSetId) - .Select(dataSet => dataSet.LatestLiveVersion!) - .SingleAsync(cancellationToken); - - var originalPath = dataSetVersionPathResolver.DirectoryPath( - dataSetVersion: dataSetVersion, - versionNumber: currentLiveVersion.DefaultNextVersion()); - - var newPath = dataSetVersionPathResolver.DirectoryPath(dataSetVersion); - - if (originalPath != newPath) - { - Directory.Move(sourceDirName: originalPath, destDirName: newPath); - } - } - [Function(ActivityNames.CompleteNextDataSetVersionImportProcessing)] public async Task CompleteNextDataSetVersionImportProcessing( [ActivityTrigger] Guid instanceId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs index 57db11c7180..299f3d1bcb1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs @@ -23,7 +23,6 @@ public static async Task ProcessCompletionOfNextDataSetVersionImport( try { - await context.CallActivity(ActivityNames.UpdateFileStoragePath, logger, context.InstanceId); await context.CallActivityExclusively(ActivityNames.ImportMetadata, logger, context.InstanceId); await context.CallActivity(ActivityNames.CreateChanges, logger, context.InstanceId); await context.CallActivity(ActivityNames.ImportData, logger, context.InstanceId); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs index 3e8efed3b8a..e988bf36e26 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs @@ -9,7 +9,6 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Options; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Options; using Moq; using Semver; @@ -39,12 +38,20 @@ public void EmptyBasePath_Throws(string basePath) public class PathTests : DataSetVersionPathResolverTests { - public static readonly TheoryData GetEnvironmentNames = new() - { + private static readonly string[] EnvironmentNames = + [ Environments.Development, HostEnvironmentExtensions.IntegrationTestEnvironment, Environments.Production - }; + ]; + + public static readonly TheoryData GetEnvironmentNames = new(EnvironmentNames); + + public static readonly TheoryData> GetEnvironmentNamesAndPublicStatuses = + new(EnvironmentNames.Cartesian(DataSetVersionStatusConstants.PublicStatuses)); + + public static readonly TheoryData> GetEnvironmentNamesAndPrivateStatuses = + new(EnvironmentNames.Cartesian(DataSetVersionStatusConstants.PrivateStatuses)); [Fact] public void DevelopmentEnv_ValidBasePath() @@ -116,10 +123,14 @@ public void ProductionEnv_ValidBasePath() } [Theory] - [MemberData(nameof(GetEnvironmentNames))] - public void ValidDirectoryPath(string environmentName) + [MemberData(nameof(GetEnvironmentNamesAndPublicStatuses))] + public void ValidDirectoryPath_PublicVersion(Tuple environmentNameAndStatus) { - DataSetVersion version = _dataFixture.DefaultDataSetVersion(); + var (environmentName, status) = environmentNameAndStatus; + + DataSetVersion version = _dataFixture + .DefaultDataSetVersion() + .WithStatus(status); _webHostEnvironmentMock .SetupGet(s => s.EnvironmentName) @@ -139,9 +150,37 @@ public void ValidDirectoryPath(string environmentName) resolver.DirectoryPath(version)); } + [Theory] + [MemberData(nameof(GetEnvironmentNamesAndPrivateStatuses))] + public void ValidDirectoryPath_PrivateVersion(Tuple environmentNameAndStatus) + { + var (environmentName, status) = environmentNameAndStatus; + + DataSetVersion version = _dataFixture + .DefaultDataSetVersion() + .WithStatus(status); + + _webHostEnvironmentMock + .SetupGet(s => s.EnvironmentName) + .Returns(environmentName); + + var resolver = BuildService(options: new DataFilesOptions + { + BasePath = Path.Combine("data", "data-files") + }); + + Assert.Equal( + Path.Combine( + resolver.DataSetsPath(), + version.DataSetId.ToString(), + "draft" + ), + resolver.DirectoryPath(version)); + } + [Theory] [MemberData(nameof(GetEnvironmentNames))] - public void ValidDirectoryPath_OptionalVersionArgument(string environmentName) + public void ValidDirectoryPath_VersionArgument(string environmentName) { DataSetVersion version = _dataFixture.DefaultDataSetVersion(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/TestDataSetVersionPathResolver.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/TestDataSetVersionPathResolver.cs index b94f8f2faeb..3b74f9bdc81 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/TestDataSetVersionPathResolver.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/TestDataSetVersionPathResolver.cs @@ -18,7 +18,12 @@ public class TestDataSetVersionPathResolver : IDataSetVersionPathResolver public string Directory { get; set; } = string.Empty; - public string DirectoryPath(DataSetVersion dataSetVersion, SemVersion? versionNumber = null) + public string DirectoryPath(DataSetVersion dataSetVersion) + { + return Path.Combine(_basePath, Directory); + } + + public string DirectoryPath(DataSetVersion dataSetVersion, SemVersion versionNumber) { return Path.Combine(_basePath, Directory); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs index 85c53c51d19..c8e16d37ce2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs @@ -1,6 +1,7 @@ using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Options; using Microsoft.AspNetCore.Hosting; @@ -12,6 +13,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Services; public class DataSetVersionPathResolver : IDataSetVersionPathResolver { + private const string DraftVersionFolderName = "draft"; + private readonly IOptions _options; private readonly IWebHostEnvironment _environment; private readonly string _basePath; @@ -34,12 +37,24 @@ public DataSetVersionPathResolver(IOptions options, IWebHostEn public string BasePath() => _basePath; - public string DirectoryPath(DataSetVersion dataSetVersion, SemVersion? versionNumber = null) + public string DirectoryPath(DataSetVersion dataSetVersion) { + var folderName = !dataSetVersion.IsPublicStatus() + ? DraftVersionFolderName + : $"v{dataSetVersion.SemVersion()}"; + return Path.Combine( ((IDataSetVersionPathResolver) this).DataSetsPath(), dataSetVersion.DataSetId.ToString(), - $"v{versionNumber ?? dataSetVersion.SemVersion()}"); + folderName); + } + + public string DirectoryPath(DataSetVersion dataSetVersion, SemVersion versionNumber) + { + return Path.Combine( + ((IDataSetVersionPathResolver) this).DataSetsPath(), + dataSetVersion.DataSetId.ToString(), + $"v{versionNumber}"); } private string GetBasePath() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Interfaces/IDataSetVersionPathResolver.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Interfaces/IDataSetVersionPathResolver.cs index 8bff9e4e50b..22c865671c1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Interfaces/IDataSetVersionPathResolver.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/Interfaces/IDataSetVersionPathResolver.cs @@ -10,7 +10,9 @@ public interface IDataSetVersionPathResolver string DataSetsPath() => Path.Combine(BasePath(), DataSetFilenames.DataSetsDirectory); - string DirectoryPath(DataSetVersion dataSetVersion, SemVersion? versionNumber = null); + string DirectoryPath(DataSetVersion dataSetVersion); + + string DirectoryPath(DataSetVersion dataSetVersion, SemVersion versionNumber); string CsvDataPath(DataSetVersion dataSetVersion) => Path.Combine(DirectoryPath(dataSetVersion), DataSetFilenames.CsvDataFile); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/DataSetPublishingServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/DataSetPublishingServiceTests.cs index 0b1eaea2cb9..e0239943b25 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/DataSetPublishingServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/DataSetPublishingServiceTests.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -9,9 +10,11 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Publisher.Services.Interfaces; using Microsoft.EntityFrameworkCore; using Xunit; +using File = GovUk.Education.ExploreEducationStatistics.Content.Model.File; namespace GovUk.Education.ExploreEducationStatistics.Publisher.Tests.Services; @@ -53,6 +56,10 @@ await AddTestData(context => dataSet.LatestDraftVersionId = dataSetVersion.Id; }); + + var dataSetVersionPathResolver = GetRequiredService(); + var draftFolderPath = dataSetVersionPathResolver.DirectoryPath(dataSetVersion); + Directory.CreateDirectory(draftFolderPath); var service = GetRequiredService(); await service.PublishDataSets([releaseVersion.Id]); @@ -76,6 +83,13 @@ await AddTestData(context => // Data set should be published at the same time as the latest live version Assert.Equal(publishedDataSet.LatestLiveVersion.Published, publishedDataSet.Published); + + // The Public API folder for the data set version should be updated to reflect its + // version number at the time of publishing. + var versionedFolderPath = dataSetVersionPathResolver.DirectoryPath(publishedDataSet.LatestLiveVersion); + Assert.False(Directory.Exists(draftFolderPath)); + Assert.True(Directory.Exists(versionedFolderPath)); + Assert.EndsWith($"v{publishedDataSet.LatestLiveVersion.SemVersion()}", versionedFolderPath); } [Fact] @@ -116,6 +130,10 @@ await AddTestData(context => dataSet.LatestLiveVersionId = dataSetVersions[0].Id; dataSet.LatestDraftVersionId = dataSetVersions[1].Id; }); + + var dataSetVersionPathResolver = GetRequiredService(); + var draftFolderPath = dataSetVersionPathResolver.DirectoryPath(dataSetVersions[1]); + Directory.CreateDirectory(draftFolderPath); var service = GetRequiredService(); await service.PublishDataSets([releaseVersion.Id]); @@ -144,6 +162,13 @@ await AddTestData(context => Assert.Equal(2, publishedDataSet.Versions.Count); publishedDataSet.Versions.ForEach(version => Assert.Equal(DataSetVersionStatus.Published, version.Status)); + + // The Public API folder for the data set version should be updated to reflect its + // version number at the time of publishing. + var versionedFolderPath = dataSetVersionPathResolver.DirectoryPath(publishedDataSet.LatestLiveVersion); + Assert.False(Directory.Exists(draftFolderPath)); + Assert.True(Directory.Exists(versionedFolderPath)); + Assert.EndsWith($"v{publishedDataSet.LatestLiveVersion.SemVersion()}", versionedFolderPath); } [Fact] @@ -336,6 +361,10 @@ await AddTestData(context => dataSet.LatestLiveVersionId = dataSetVersion1.Id; dataSet.LatestDraftVersionId = dataSetVersion2.Id; }); + + var dataSetVersionPathResolver = GetRequiredService(); + var draftFolderPath = dataSetVersionPathResolver.DirectoryPath(dataSetVersion2); + Directory.CreateDirectory(draftFolderPath); var service = GetRequiredService(); await service.PublishDataSets([amendmentReleaseVersion.Id]); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/appsettings.IntegrationTest.json b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/appsettings.IntegrationTest.json index 69bb2edb9b6..502829a54d9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/appsettings.IntegrationTest.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/appsettings.IntegrationTest.json @@ -5,5 +5,8 @@ "NotifierStorageConnectionString": "this-will-be-overridden-at-startup", "PublisherStorageConnectionString": "this-will-be-overridden-at-startup" }, + "DataFiles": { + "BasePath": "Resources/DataFiles" + }, "PublicDataDbExists": true } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/GovUk.Education.ExploreEducationStatistics.Publisher.csproj b/src/GovUk.Education.ExploreEducationStatistics.Publisher/GovUk.Education.ExploreEducationStatistics.Publisher.csproj index f16b100adeb..3534a088a69 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/GovUk.Education.ExploreEducationStatistics.Publisher.csproj +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/GovUk.Education.ExploreEducationStatistics.Publisher.csproj @@ -30,6 +30,7 @@ + diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs index 05a69e197fa..3fc0b16ebdd 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs @@ -21,6 +21,9 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Notifier.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Options; using GovUk.Education.ExploreEducationStatistics.Publisher.Model; using GovUk.Education.ExploreEducationStatistics.Publisher.Options; using GovUk.Education.ExploreEducationStatistics.Publisher.Services; @@ -132,6 +135,9 @@ public static IHostBuilder ConfigurePublisherHostBuilder(this IHostBuilder hostB // TODO EES-5073 Remove this check when the Public Data db is available in all Azure environments. if (publicDataDbExists) { + services.AddOptions() + .Bind(configuration.GetRequiredSection(DataFilesOptions.Section)); + services.AddScoped(); services.AddScoped(); } else diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/DataSetPublishingService.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/DataSetPublishingService.cs index 1e31d317953..e74d05a7390 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/DataSetPublishingService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/DataSetPublishingService.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -8,6 +9,7 @@ using GovUk.Education.ExploreEducationStatistics.Notifier.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Publisher.Services.Interfaces; using Microsoft.EntityFrameworkCore; @@ -16,7 +18,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Publisher.Services; public class DataSetPublishingService( ContentDbContext contentDbContext, PublicDataDbContext publicDataDbContext, - INotifierClient notifierClient + INotifierClient notifierClient, + IDataSetVersionPathResolver dataSetVersionPathResolver ) : IDataSetPublishingService { public async Task PublishDataSets(Guid[] releaseVersionIds) @@ -46,8 +49,11 @@ private async Task> PromoteDraftDataSetVersions(IE foreach (var dataSet in dataSets) { - var currentDraftVersion = dataSet.LatestDraftVersion; - currentDraftVersion!.Status = DataSetVersionStatus.Published; + var currentDraftVersion = dataSet.LatestDraftVersion!; + + var originalFilePath = dataSetVersionPathResolver.DirectoryPath(currentDraftVersion); + + currentDraftVersion.Status = DataSetVersionStatus.Published; currentDraftVersion.Published = DateTimeOffset.UtcNow; dataSet.Status = DataSetStatus.Published; @@ -58,9 +64,12 @@ private async Task> PromoteDraftDataSetVersions(IE dataSet.LatestDraftVersion = null; dataSet.LatestDraftVersionId = null; - } + + var newFilePath = dataSetVersionPathResolver.DirectoryPath(currentDraftVersion); + Directory.Move(sourceDirName: originalFilePath, destDirName: newFilePath); - await publicDataDbContext.SaveChangesAsync(); + await publicDataDbContext.SaveChangesAsync(); + } return dataSets .Select(ds => ds.LatestLiveVersion!) From 34db1b37dfec6550c52fc886395d039aec53e062 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 18 Nov 2024 14:26:28 +0000 Subject: [PATCH 006/144] EES-5560 - added custom migration to update draft DataSetVersion folders to be "draft" rather than version-based names. --- .../Migrations/Custom/ICustomMigration.cs | 7 - .../Startup.cs | 13 +- .../Database/ICustomMigration.cs | 6 + .../Extensions/EnumerableExtensions.cs | 32 ++-- ...rateDraftDataSetVersionFolderNamesTests.cs | 174 ++++++++++++++++++ ...0_MigrateDraftDataSetVersionFolderNames.cs | 48 +++++ .../Startup.cs | 31 +++- .../Fixtures/DataSetGeneratorExtensions.cs | 12 +- .../DataSetVersionGeneratorExtensions.cs | 10 + .../DataSetVersionAuthExtensions.cs | 9 + ...ucationStatistics.Public.Data.Model.csproj | 76 ++++---- ...nOfNextDataSetVersionOrchestrationTests.cs | 5 + 12 files changed, 351 insertions(+), 72 deletions(-) delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/Custom/ICustomMigration.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Common/Database/ICustomMigration.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Migrations/EES5660_MigrateDraftDataSetVersionFolderNames.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/Custom/ICustomMigration.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/Custom/ICustomMigration.cs deleted file mode 100644 index afe6a91620b..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/Custom/ICustomMigration.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.Custom -{ - public interface ICustomMigration - { - void Apply(); - } -} \ No newline at end of file diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs index 736cacf8089..cd6e7f0f16d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs @@ -3,7 +3,6 @@ using GovUk.Education.ExploreEducationStatistics.Admin.Database; using GovUk.Education.ExploreEducationStatistics.Admin.Hubs; using GovUk.Education.ExploreEducationStatistics.Admin.Hubs.Filters; -using GovUk.Education.ExploreEducationStatistics.Admin.Migrations.Custom; using GovUk.Education.ExploreEducationStatistics.Admin.Models; using GovUk.Education.ExploreEducationStatistics.Admin.Options; using GovUk.Education.ExploreEducationStatistics.Admin.Requests.Public.Data; @@ -782,9 +781,9 @@ private void UpdateDatabase(IApplicationBuilder app, IWebHostEnvironment env) { context.Database.SetCommandTimeout(int.MaxValue); context.Database.Migrate(); - - ApplyCustomMigrations(); } + + ApplyCustomMigrations(app); } if (env.IsDevelopment()) @@ -798,8 +797,14 @@ private void UpdateDatabase(IApplicationBuilder app, IWebHostEnvironment env) } } - private static void ApplyCustomMigrations(params ICustomMigration[] migrations) + private static void ApplyCustomMigrations(IApplicationBuilder app) { + using var serviceScope = app.ApplicationServices + .GetRequiredService() + .CreateScope(); + + var migrations = serviceScope.ServiceProvider.GetServices(); + foreach (var migration in migrations) { migration.Apply(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Database/ICustomMigration.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Database/ICustomMigration.cs new file mode 100644 index 00000000000..a899c9b3e16 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Database/ICustomMigration.cs @@ -0,0 +1,6 @@ +namespace GovUk.Education.ExploreEducationStatistics.Common.Database; + +public interface ICustomMigration +{ + void Apply(); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs index 36b32889a16..130bb844d3b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs @@ -221,7 +221,7 @@ public static IEnumerable WhereNotNull(this IEnumerable source) } public static IAsyncEnumerable WhereNotNull(this IAsyncEnumerable source) - where T: class + where T : class { return source.Where(item => item is not null)!; } @@ -264,7 +264,7 @@ public static bool IsSameAsIgnoringOrder(this IEnumerable first, IEnumerab return !(firstNotInSecond.Any() || secondNotInFirst.Any()); } - + public static Tuple ToTuple2(this IEnumerable collection) where T : class { @@ -275,10 +275,10 @@ public static Tuple ToTuple2(this IEnumerable collection) throw new ArgumentException( $"Expected 2 list items when constructing a 2-tuple, but found {list.Count}"); } - + return new Tuple(list[0], list[1]); } - + public static Tuple ToTuple3(this IEnumerable collection) where T : class { @@ -289,7 +289,7 @@ public static Tuple ToTuple3(this IEnumerable collection) throw new ArgumentException( $"Expected 3 list items when constructing a 3-tuple, but found {list.Count}"); } - + return new Tuple(list[0], list[1], list[2]); } @@ -297,7 +297,7 @@ public static bool ContainsAll(this IEnumerable source, IEnumerable val { return values.All(id => source.Contains(id)); } - + /// /// Order some objects, according to a string key, in natural order for humans to read. /// @@ -320,25 +320,29 @@ public static IOrderedEnumerable NaturalThenBy( return source.ThenBy(keySelector, comparison.WithNaturalSort()); } - public static IEnumerable> Cartesian( + public static List> Cartesian( this IEnumerable list1, IEnumerable? list2) { - return list2 == null - ? [] - : list1.Join(list2, _ => true, _ => true,(t1, t2) => new Tuple(t1, t2)); + return list2 == null + ? [] + : list1 + .Join(list2, _ => true, _ => true, (t1, t2) => new Tuple(t1, t2)) + .ToList(); } - - public static IEnumerable> Cartesian( + + public static List> Cartesian( this IEnumerable list1, IEnumerable? list2, IEnumerable? list3) { return list2 == null || list3 == null - ? [] + ? [] : list1 .Join(list2, _ => true, _ => true, (t1, t2) => new Tuple(t1, t2)) - .Join(list3, _ => true, _ => true, (tuple, t3) => new Tuple(tuple.Item1, tuple.Item2, t3)); + .Join(list3, _ => true, _ => true, + (tuple, t3) => new Tuple(tuple.Item1, tuple.Item2, t3)) + .ToList(); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs new file mode 100644 index 00000000000..450d88f0f11 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs @@ -0,0 +1,174 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Migrations; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Fixture; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.TheoryData; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using Microsoft.Extensions.DependencyInjection; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Migrations; + +public class EES5660_MigrateDraftDataSetVersionFolderNamesTests(TestApplicationFactory testApp) + : IntegrationTestFixture(testApp) +{ + public static readonly TheoryData> + PrivateDataSetVersionStatusAndTypes = new(DataSetVersionStatusConstants + .PrivateStatuses + .Cartesian(EnumUtil.GetEnums())); + + public static readonly TheoryData> + PublicDataSetVersionStatusAndTypes = new(DataSetVersionStatusConstants + .PublicStatuses + .Cartesian(EnumUtil.GetEnums())); + + [Theory] + [MemberData(nameof(PrivateDataSetVersionStatusAndTypes))] + public async Task Success_NonMigratedDraft(Tuple statusAndType) + { + var (status, type) = statusAndType; + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithId(Guid.NewGuid()) + .WithDataSet(DataFixture + .DefaultDataSet()) + .WithStatus(status) + .WithVersionNumber( + major: type == DataSetVersionType.Major ? 2 : 1, + minor: type == DataSetVersionType.Major ? 0 : 1); + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var pathResolver = TestApp.Services.GetRequiredService(); + + var versionedFolder = pathResolver.DirectoryPath(dataSetVersion, dataSetVersion.SemVersion()); + Directory.CreateDirectory(versionedFolder); + File.Create(Path.Combine(versionedFolder, "test.txt")); + + var migration = TestApp.Services.GetRequiredService(); + + migration.Apply(); + + // Assert that the original folder that used the draft's version in its name no longer exists. + Assert.False(Directory.Exists(versionedFolder)); + + // Assert that the original folder has moved to use the new special "draft" folder. + var expectedDraftFolder = pathResolver.DirectoryPath(dataSetVersion); + Assert.True(Directory.Exists(expectedDraftFolder)); + Assert.True(File.Exists(Path.Combine(expectedDraftFolder, "test.txt"))); + } + + [Theory] + [MemberData(nameof(PrivateDataSetVersionStatusAndTypes))] + public async Task Success_AlreadyMigratedDraft(Tuple statusAndType) + { + var (status, type) = statusAndType; + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithId(Guid.NewGuid()) + .WithDataSet(DataFixture + .DefaultDataSet() + .WithId(Guid.NewGuid())) + .WithStatus(status) + .WithVersionNumber( + major: type == DataSetVersionType.Major ? 2 : 1, + minor: type == DataSetVersionType.Major ? 0 : 1); + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var pathResolver = TestApp.Services.GetRequiredService(); + + var draftFolder = pathResolver.DirectoryPath(dataSetVersion); + Directory.CreateDirectory(draftFolder); + File.Create(Path.Combine(draftFolder, "test.txt")); + + var migration = TestApp.Services.GetRequiredService(); + + migration.Apply(); + + // Assert that the existing draft folder is left untouched. + Assert.True(Directory.Exists(draftFolder)); + Assert.True(File.Exists(Path.Combine(draftFolder, "test.txt"))); + } + + [Fact] + public async Task Failure_DraftFolderAndVersionedFolderExist() + { + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithDataSet(DataFixture.DefaultDataSet()) + .WithStatus(DataSetVersionStatus.Draft) + .WithVersionNumber(major: 2, minor: 0); + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var pathResolver = TestApp.Services.GetRequiredService(); + + var versionedFolder = pathResolver.DirectoryPath(dataSetVersion, dataSetVersion.SemVersion()); + Directory.CreateDirectory(versionedFolder); + File.Create(Path.Combine(versionedFolder, "versioned.txt")); + + var draftFolder = pathResolver.DirectoryPath(dataSetVersion); + Directory.CreateDirectory(draftFolder); + File.Create(Path.Combine(draftFolder, "draft.txt")); + + var migration = TestApp.Services.GetRequiredService(); + + var exception = Assert.Throws(migration.Apply); + + Assert.Equal("The following DataSetVersions have both a versioned " + + "and a draft folder: " + dataSetVersion.Id, exception.Message); + + // Assert that the versioned folder still exists. + Assert.True(Directory.Exists(versionedFolder)); + Assert.True(File.Exists(Path.Combine(versionedFolder, "versioned.txt"))); + + // Assert that the draft folder still exists. + Assert.True(Directory.Exists(draftFolder)); + Assert.True(File.Exists(Path.Combine(draftFolder, "draft.txt"))); + } + + [Theory] + [MemberData(nameof(PublicDataSetVersionStatusAndTypes))] + public async Task Success_PublicVersionsNotMigrated(Tuple statusAndType) + { + var (status, type) = statusAndType; + + DataSetVersion dataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithId(Guid.NewGuid()) + .WithDataSet(DataFixture + .DefaultDataSet() + .WithId(Guid.NewGuid())) + .WithStatus(status) + .WithVersionNumber( + major: type == DataSetVersionType.Major ? 2 : 1, + minor: type == DataSetVersionType.Major ? 0 : 1); + + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); + + var pathResolver = TestApp.Services.GetRequiredService(); + + var versionedFolder = pathResolver.DirectoryPath(dataSetVersion, dataSetVersion.SemVersion()); + Directory.CreateDirectory(versionedFolder); + File.Create(Path.Combine(versionedFolder, "test.txt")); + + var migration = TestApp.Services.GetRequiredService(); + + migration.Apply(); + + // Assert that the original folder has been left unaffected. + Assert.True(Directory.Exists(versionedFolder)); + Assert.True(File.Exists(Path.Combine(versionedFolder, "test.txt"))); + + // Assert that no draft folder has been created. + dataSetVersion.Status = DataSetVersionStatus.Draft; + var draftFolder = pathResolver.DirectoryPath(dataSetVersion); + Assert.False(Directory.Exists(draftFolder)); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Migrations/EES5660_MigrateDraftDataSetVersionFolderNames.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Migrations/EES5660_MigrateDraftDataSetVersionFolderNames.cs new file mode 100644 index 00000000000..92a5fe4845c --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Migrations/EES5660_MigrateDraftDataSetVersionFolderNames.cs @@ -0,0 +1,48 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Database; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Migrations; + +public class EES5660_MigrateDraftDataSetVersionFolderNames( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver pathResolver) : ICustomMigration +{ + public void Apply() + { + var draftDataSetVersions = publicDataDbContext + .DataSetVersions + .WherePrivateStatus() + .ToList(); + + var failureDataSetVersionIds = new List(); + + draftDataSetVersions.ForEach(dataSetVersion => + { + var versionedFolder = pathResolver.DirectoryPath(dataSetVersion, dataSetVersion.SemVersion()); + + if (Directory.Exists(versionedFolder)) + { + var newDraftFolder = pathResolver.DirectoryPath(dataSetVersion); + + if (Directory.Exists(newDraftFolder)) + { + failureDataSetVersionIds.Add(dataSetVersion.Id); + } + else + { + Directory.Move(versionedFolder, newDraftFolder); + } + } + }); + + if (failureDataSetVersionIds.Count > 0) + { + throw new Exception($"The following DataSetVersions have both a versioned " + + $"and a draft folder: {failureDataSetVersionIds.JoinToString(",")}"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs index 5863265cb28..d4dfff4d17a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs @@ -1,11 +1,17 @@ +using System.Diagnostics.CodeAnalysis; +using System.Net; +using System.Text.Json; +using System.Text.Json.Serialization; using AngleSharp.Io; using Dapper; using FluentValidation; using GovUk.Education.ExploreEducationStatistics.Common.Config; +using GovUk.Education.ExploreEducationStatistics.Common.Database; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.ModelBinding; using GovUk.Education.ExploreEducationStatistics.Common.Rules; using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Migrations; using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Options; using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Repository; using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Repository.Interfaces; @@ -26,10 +32,6 @@ using Microsoft.AspNetCore.Rewrite; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; -using System.Diagnostics.CodeAnalysis; -using System.Net; -using System.Text.Json; -using System.Text.Json.Serialization; using RequestTimeoutOptions = GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Options.RequestTimeoutOptions; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api; @@ -191,7 +193,8 @@ public void ConfigureServices(IServiceCollection services) // Services services.AddFluentValidation(); - services.AddValidatorsFromAssembly(typeof(DataSetGetQueryLocations.Validator).Assembly); // Adds *all* validators from Public.Data.Requests + services.AddValidatorsFromAssembly(typeof(DataSetGetQueryLocations.Validator) + .Assembly); // Adds *all* validators from Public.Data.Requests services.AddFluentValidationRulesToSwagger(); services.AddHttpClient((provider, httpClient) => @@ -219,6 +222,8 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); + + services.AddScoped(); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. @@ -289,5 +294,21 @@ private static void UpdateDatabase(IApplicationBuilder app) context.Database.SetCommandTimeout(300); context.Database.Migrate(); + + ApplyCustomMigrations(app); + } + + private static void ApplyCustomMigrations(IApplicationBuilder app) + { + using var serviceScope = app.ApplicationServices + .GetRequiredService() + .CreateScope(); + + var migrations = serviceScope.ServiceProvider.GetServices(); + + foreach (var migration in migrations) + { + migration.Apply(); + } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetGeneratorExtensions.cs index 91cb1679c09..cb09d7e8c09 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetGeneratorExtensions.cs @@ -13,6 +13,9 @@ public static Generator WithDefaults(this Generator generator) public static Generator WithPublicationId(this Generator generator, Guid publicationId) => generator.ForInstance(s => s.SetPublicationId(publicationId)); + public static Generator WithId(this Generator generator, Guid id) + => generator.ForInstance(s => s.SetId(id)); + public static Generator WithTitle(this Generator generator, string title) => generator.ForInstance(s => s.SetTitle(title)); @@ -39,7 +42,7 @@ public static Generator WithWithdrawn(this Generator generator public static Generator WithSupersedingDataSet(this Generator generator, DataSet? dataSet) => generator.ForInstance(s => s.SetSupersedingDataSet(dataSet)); - + public static Generator WithLatestDraftVersion( this Generator generator, DataSetVersion? dataSetVersion) @@ -78,6 +81,11 @@ public static InstanceSetters SetDefaults(this InstanceSetters .SetDefault(ds => ds.PublicationId) .Set(ds => ds.Status, DataSetStatus.Draft); + public static InstanceSetters SetId( + this InstanceSetters instanceSetter, + Guid id) + => instanceSetter.Set(ds => ds.Id, id); + public static InstanceSetters SetTitle( this InstanceSetters instanceSetter, string title) @@ -162,7 +170,7 @@ public static InstanceSetters SetLatestLiveVersion( ds.LatestLiveVersionId = dsv?.Id; } ); - + public static InstanceSetters SetLatestDraftVersion( this InstanceSetters instanceSetter, DataSetVersion? dataSetVersion) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetVersionGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetVersionGeneratorExtensions.cs index c681d890f4b..4ace459af53 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetVersionGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetVersionGeneratorExtensions.cs @@ -32,6 +32,11 @@ public static Generator DefaultDataSetVersion( public static Generator WithDefaults(this Generator generator) => generator.ForInstance(s => s.SetDefaults()); + public static Generator WithId( + this Generator generator, + Guid id) + => generator.ForInstance(s => s.SetId(id)); + public static Generator WithDataSet( this Generator generator, DataSet dataSet) @@ -212,6 +217,11 @@ public static InstanceSetters SetDefaults(this InstanceSetters dsv.TotalResults, f => f.Random.Long(min: 10000, max: 10_000_000)) .Set(dsv => dsv.Status, DataSetVersionStatus.Draft); + public static InstanceSetters SetId( + this InstanceSetters instanceSetter, + Guid id) + => instanceSetter.Set(dsv => dsv.Id, id); + public static InstanceSetters SetDataSet( this InstanceSetters instanceSetter, DataSet dataSet) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs index 44d01584a51..3216eea2872 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs @@ -12,4 +12,13 @@ public static IQueryable WherePublicStatus(this IQueryable> IsPublicStatus() => dataSetVersion => DataSetVersionStatusConstants.PublicStatuses.Contains(dataSetVersion.Status); + + public static bool IsPrivateStatus(this DataSetVersion dataSetVersion) + => IsPrivateStatus().Compile()(dataSetVersion); + + public static IQueryable WherePrivateStatus(this IQueryable queryable) + => queryable.Where(IsPrivateStatus()); + + private static Expression> IsPrivateStatus() + => dataSetVersion => DataSetVersionStatusConstants.PrivateStatuses.Contains(dataSetVersion.Status); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.csproj b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.csproj index 56b96f01507..b7a2eee43b5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.csproj +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.csproj @@ -1,44 +1,40 @@ - - net8.0 - enable - enable - GovUk.Education.ExploreEducationStatistics.Public.Data.Model - true - - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - - - - PreserveNewest - - + + net8.0 + enable + enable + GovUk.Education.ExploreEducationStatistics.Public.Data.Model + true + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + PreserveNewest + + diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs index c4fc07e53de..aeab7710c35 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs @@ -56,6 +56,11 @@ public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() var activitySequence = new MockSequence(); + var mockEntityFeature = new Mock(MockBehavior.Strict); + mockEntityFeature.SetupLockForActivity(ActivityNames.ImportMetadata); + mockOrchestrationContext.SetupGet(context => context.Entities) + .Returns(mockEntityFeature.Object); + mockOrchestrationContext .InSequence(activitySequence) .Setup(context => From b01b1f1792916655ca612d962422014178819ee1 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 18 Nov 2024 14:59:32 +0000 Subject: [PATCH 007/144] EES-5660 - tidy-up prior to raising PR --- .../Database/ICustomMigration.cs | 7 ++ ...rateDraftDataSetVersionFolderNamesTests.cs | 6 +- ...0_MigrateDraftDataSetVersionFolderNames.cs | 9 ++- .../Startup.cs | 1 + .../DataSetVersionStatus.cs | 17 ----- .../DataSetVersionAuthExtensions.cs | 18 ++++- ...ucationStatistics.Public.Data.Model.csproj | 76 ++++++++++--------- .../DataSetVersionPathResolverTests.cs | 7 +- .../Interfaces/IDataSetVersionPathResolver.cs | 4 +- 9 files changed, 81 insertions(+), 64 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Database/ICustomMigration.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Database/ICustomMigration.cs index a899c9b3e16..6497e05e27d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Database/ICustomMigration.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Database/ICustomMigration.cs @@ -1,5 +1,12 @@ namespace GovUk.Education.ExploreEducationStatistics.Common.Database; +/// +/// Custom migrations that run on deployment / startup. +/// These differ in use from EF database migrations in that they are not limited +/// to running against a single database / DbContext. These can be simple +/// standalone implementations or can use dependency injection if registered +/// with a DI container. +/// public interface ICustomMigration { void Apply(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs index 450d88f0f11..36e45765070 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs @@ -6,22 +6,24 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.TheoryData; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.Extensions.DependencyInjection; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Migrations; +// TODO EES-5660 - remove this migration after it has been run against each Public API-enabled environment. public class EES5660_MigrateDraftDataSetVersionFolderNamesTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { public static readonly TheoryData> - PrivateDataSetVersionStatusAndTypes = new(DataSetVersionStatusConstants + PrivateDataSetVersionStatusAndTypes = new(DataSetVersionAuthExtensions .PrivateStatuses .Cartesian(EnumUtil.GetEnums())); public static readonly TheoryData> - PublicDataSetVersionStatusAndTypes = new(DataSetVersionStatusConstants + PublicDataSetVersionStatusAndTypes = new(DataSetVersionAuthExtensions .PublicStatuses .Cartesian(EnumUtil.GetEnums())); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Migrations/EES5660_MigrateDraftDataSetVersionFolderNames.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Migrations/EES5660_MigrateDraftDataSetVersionFolderNames.cs index 92a5fe4845c..324c0075b65 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Migrations/EES5660_MigrateDraftDataSetVersionFolderNames.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Migrations/EES5660_MigrateDraftDataSetVersionFolderNames.cs @@ -7,6 +7,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Migrations; +// TODO EES-5660 - remove this migration after it has been run against each Public API-enabled environment. public class EES5660_MigrateDraftDataSetVersionFolderNames( PublicDataDbContext publicDataDbContext, IDataSetVersionPathResolver pathResolver) : ICustomMigration @@ -18,7 +19,7 @@ public void Apply() .WherePrivateStatus() .ToList(); - var failureDataSetVersionIds = new List(); + var failedDataSetVersionIds = new List(); draftDataSetVersions.ForEach(dataSetVersion => { @@ -30,7 +31,7 @@ public void Apply() if (Directory.Exists(newDraftFolder)) { - failureDataSetVersionIds.Add(dataSetVersion.Id); + failedDataSetVersionIds.Add(dataSetVersion.Id); } else { @@ -39,10 +40,10 @@ public void Apply() } }); - if (failureDataSetVersionIds.Count > 0) + if (failedDataSetVersionIds.Count > 0) { throw new Exception($"The following DataSetVersions have both a versioned " + - $"and a draft folder: {failureDataSetVersionIds.JoinToString(",")}"); + $"and a draft folder: {failedDataSetVersionIds.JoinToString(",")}"); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs index d4dfff4d17a..32a6574a0e6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs @@ -223,6 +223,7 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddScoped(); + // TODO EES-5660 - remove this migration after it has been run against each Public API-enabled environment. services.AddScoped(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs index 6124daf0eea..f62f4ac3578 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionStatus.cs @@ -1,5 +1,4 @@ using System.Text.Json.Serialization; -using GovUk.Education.ExploreEducationStatistics.Common.Utils; using Newtonsoft.Json.Converters; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model; @@ -17,19 +16,3 @@ public enum DataSetVersionStatus Withdrawn, Cancelled, } - -public class DataSetVersionStatusConstants -{ - public static readonly IReadOnlyList PublicStatuses = new List( - [ - DataSetVersionStatus.Published, - DataSetVersionStatus.Withdrawn, - DataSetVersionStatus.Deprecated - ] - ); - - public static readonly IReadOnlyList PrivateStatuses = EnumUtil - .GetEnums() - .Except(PublicStatuses) - .ToList(); -} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs index 3216eea2872..093ad188419 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Extensions/DataSetVersionAuthExtensions.cs @@ -1,9 +1,23 @@ using System.Linq.Expressions; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Extensions; public static class DataSetVersionAuthExtensions { + public static readonly IReadOnlyList PublicStatuses = new List( + [ + DataSetVersionStatus.Published, + DataSetVersionStatus.Withdrawn, + DataSetVersionStatus.Deprecated + ] + ); + + public static readonly IReadOnlyList PrivateStatuses = EnumUtil + .GetEnums() + .Except(PublicStatuses) + .ToList(); + public static bool IsPublicStatus(this DataSetVersion dataSetVersion) => IsPublicStatus().Compile()(dataSetVersion); @@ -11,7 +25,7 @@ public static IQueryable WherePublicStatus(this IQueryable queryable.Where(IsPublicStatus()); private static Expression> IsPublicStatus() - => dataSetVersion => DataSetVersionStatusConstants.PublicStatuses.Contains(dataSetVersion.Status); + => dataSetVersion => PublicStatuses.Contains(dataSetVersion.Status); public static bool IsPrivateStatus(this DataSetVersion dataSetVersion) => IsPrivateStatus().Compile()(dataSetVersion); @@ -20,5 +34,5 @@ public static IQueryable WherePrivateStatus(this IQueryable queryable.Where(IsPrivateStatus()); private static Expression> IsPrivateStatus() - => dataSetVersion => DataSetVersionStatusConstants.PrivateStatuses.Contains(dataSetVersion.Status); + => dataSetVersion => PrivateStatuses.Contains(dataSetVersion.Status); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.csproj b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.csproj index b7a2eee43b5..56b96f01507 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.csproj +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.csproj @@ -1,40 +1,44 @@ - - net8.0 - enable - enable - GovUk.Education.ExploreEducationStatistics.Public.Data.Model - true - - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - PreserveNewest - - + + net8.0 + enable + enable + GovUk.Education.ExploreEducationStatistics.Public.Data.Model + true + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + PreserveNewest + + diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs index e988bf36e26..14fcd86c36a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs @@ -3,6 +3,7 @@ using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; @@ -48,10 +49,10 @@ public class PathTests : DataSetVersionPathResolverTests public static readonly TheoryData GetEnvironmentNames = new(EnvironmentNames); public static readonly TheoryData> GetEnvironmentNamesAndPublicStatuses = - new(EnvironmentNames.Cartesian(DataSetVersionStatusConstants.PublicStatuses)); + new(EnvironmentNames.Cartesian(DataSetVersionAuthExtensions.PublicStatuses)); public static readonly TheoryData> GetEnvironmentNamesAndPrivateStatuses = - new(EnvironmentNames.Cartesian(DataSetVersionStatusConstants.PrivateStatuses)); + new(EnvironmentNames.Cartesian(DataSetVersionAuthExtensions.PrivateStatuses)); [Fact] public void DevelopmentEnv_ValidBasePath() @@ -178,6 +179,8 @@ public void ValidDirectoryPath_PrivateVersion(Tuple Path.Combine(BasePath(), DataSetFilenames.DataSetsDirectory); string DirectoryPath(DataSetVersion dataSetVersion); - + + // TODO EES-5660 - remove after draft DataSetVersion folders are migrated for each + // Public API-enabled environment. string DirectoryPath(DataSetVersion dataSetVersion, SemVersion versionNumber); string CsvDataPath(DataSetVersion dataSetVersion) From c5c4b05828fbd4d688b6e385aa31f8e59b8ef87c Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 19 Nov 2024 14:01:34 +0000 Subject: [PATCH 008/144] EES-5660 - making infrastructure changes to allow the Publisher to access the Public API fileshare --- infrastructure/templates/template.json | 28 ++++++++++++++++++- .../Startup.cs | 7 ++--- .../DataSetVersionPathResolver.cs | 2 +- .../appsettings.json | 3 ++ 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index 2262edd6e97..cbb3706dde4 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -1197,7 +1197,10 @@ "[concat('Microsoft.Web/sites/', variables('importerAppName'))]", "[concat('Microsoft.Web/sites/', variables('contentAppName'))]", "[concat('Microsoft.Web/sites/', variables('dataAppName'))]" - ] + ], + "publicDataFileshareMountPath": "/data/public-api-data", + "publicDataFileshareName": "[concat(parameters('subscription'), '-ees-papi-fs-data')]", + "publicDataStorageAccountName": "[concat(parameters('subscription'), 'eespapisa')]" }, "resources": [ { @@ -3283,9 +3286,32 @@ "App:NotifierStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-notifications')).secretUriWithVersion, ')')]", "App:PublicStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-public')).secretUriWithVersion, ')')]", "App:PublisherStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-publisher')).secretUriWithVersion, ')')]", + "DataFiles:BasePath": "[parameters('publicDataFileshareMountPath')]", "PublicDataDbExists": "[parameters('publicDataDbExists')]" } }, + { + "condition": "[parameters('publicDataDbExists')]", + "name": "[concat(variables('publisherAppName'), '/azurestorageaccounts')]", + "type": "Microsoft.Web/sites/config", + "location": "[resourceGroup().location]", + "apiVersion": "2019-08-01", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites', variables('publisherAppName'))]", + "[variables('publicDataFileshareMountPath')]", + "[variables('publicDataFileshareName')]", + "[variables('publicDataStorageAccountName')]" + ], + "properties": { + "[variables('publicDataFileshareName')]": { + "type": "AzureFiles", + "accountName": "[variables('publicDataStorageAccountName')]", + "shareName": "[variables('publicDataFileshareName')]", + "mountPath": "[variables('publicDataFileshareMountPath')]", + "protocol": "Smb" + } + } + }, { "type": "Microsoft.Storage/storageAccounts", "name": "[variables('publisherStorageAccountName')]", diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs index 32a6574a0e6..29eb5f30ded 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs @@ -224,7 +224,7 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); // TODO EES-5660 - remove this migration after it has been run against each Public API-enabled environment. - services.AddScoped(); + services.AddScoped(); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. @@ -307,9 +307,6 @@ private static void ApplyCustomMigrations(IApplicationBuilder app) var migrations = serviceScope.ServiceProvider.GetServices(); - foreach (var migration in migrations) - { - migration.Apply(); - } + migrations.ForEach(migration => migration.Apply()); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs index c8e16d37ce2..4f471d5ff42 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services/DataSetVersionPathResolver.cs @@ -39,7 +39,7 @@ public DataSetVersionPathResolver(IOptions options, IWebHostEn public string DirectoryPath(DataSetVersion dataSetVersion) { - var folderName = !dataSetVersion.IsPublicStatus() + var folderName = dataSetVersion.IsPrivateStatus() ? DraftVersionFolderName : $"v{dataSetVersion.SemVersion()}"; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/appsettings.json b/src/GovUk.Education.ExploreEducationStatistics.Publisher/appsettings.json index df8caea97e1..89cd0d9e249 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/appsettings.json +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/appsettings.json @@ -12,5 +12,8 @@ "PublicStorageConnectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://data-storage:10000/devstoreaccount1;TableEndpoint=http://data-storage:10002/devstoreaccount1;", "PublisherStorageConnectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://data-storage:10000/devstoreaccount1;QueueEndpoint=http://data-storage:10001/devstoreaccount1;TableEndpoint=http://data-storage:10002/devstoreaccount1;" }, + "DataFiles": { + "BasePath": "data/public-api-data" + }, "PublicDataDbExists": false } From fb0426b7907a3f08652e52d255eefca4369a5ce8 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 19 Nov 2024 17:53:30 +0000 Subject: [PATCH 009/144] EES-5660 - supressing the running of custom migrations during startup if running integration tests, as otherwise they fire on every individual integration test method run. --- .../Startup.cs | 5 ++- ...rateDraftDataSetVersionFolderNamesTests.cs | 36 ++++++++++--------- .../Startup.cs | 9 +++-- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs index cd6e7f0f16d..b04002a6bbd 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs @@ -783,7 +783,10 @@ private void UpdateDatabase(IApplicationBuilder app, IWebHostEnvironment env) context.Database.Migrate(); } - ApplyCustomMigrations(app); + if (!env.IsIntegrationTest()) + { + ApplyCustomMigrations(app); + } } if (env.IsDevelopment()) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs index 36e45765070..63c0806d5e6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs @@ -1,3 +1,4 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Database; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Utils; @@ -37,7 +38,8 @@ public async Task Success_NonMigratedDraft(Tuple(); - - migration.Apply(); + GetMigration().Apply(); // Assert that the original folder that used the draft's version in its name no longer exists. Assert.False(Directory.Exists(versionedFolder)); @@ -63,7 +63,7 @@ public async Task Success_NonMigratedDraft(Tuple statusAndType) @@ -89,9 +89,7 @@ public async Task Success_AlreadyMigratedDraft(Tuple(); - - migration.Apply(); + GetMigration().Apply(); // Assert that the existing draft folder is left untouched. Assert.True(Directory.Exists(draftFolder)); @@ -106,7 +104,7 @@ public async Task Failure_DraftFolderAndVersionedFolderExist() .WithDataSet(DataFixture.DefaultDataSet()) .WithStatus(DataSetVersionStatus.Draft) .WithVersionNumber(major: 2, minor: 0); - + await TestApp.AddTestData(context => context.DataSetVersions.Add(dataSetVersion)); var pathResolver = TestApp.Services.GetRequiredService(); @@ -119,17 +117,15 @@ public async Task Failure_DraftFolderAndVersionedFolderExist() Directory.CreateDirectory(draftFolder); File.Create(Path.Combine(draftFolder, "draft.txt")); - var migration = TestApp.Services.GetRequiredService(); + var exception = Assert.Throws(GetMigration().Apply); - var exception = Assert.Throws(migration.Apply); - Assert.Equal("The following DataSetVersions have both a versioned " + "and a draft folder: " + dataSetVersion.Id, exception.Message); // Assert that the versioned folder still exists. Assert.True(Directory.Exists(versionedFolder)); Assert.True(File.Exists(Path.Combine(versionedFolder, "versioned.txt"))); - + // Assert that the draft folder still exists. Assert.True(Directory.Exists(draftFolder)); Assert.True(File.Exists(Path.Combine(draftFolder, "draft.txt"))); @@ -160,9 +156,7 @@ public async Task Success_PublicVersionsNotMigrated(Tuple(); - - migration.Apply(); + GetMigration().Apply(); // Assert that the original folder has been left unaffected. Assert.True(Directory.Exists(versionedFolder)); @@ -173,4 +167,14 @@ public async Task Success_PublicVersionsNotMigrated(Tuple() + .Where(migration => migration.GetType() == typeof(EES5660_MigrateDraftDataSetVersionFolderNames)) + .Cast() + .Single(); + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs index 29eb5f30ded..b09040d725e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs @@ -230,7 +230,7 @@ public void ConfigureServices(IServiceCollection services) // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { - UpdateDatabase(app); + UpdateDatabase(app, env); if (_miniProfilerOptions.Enabled) { @@ -285,7 +285,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) app.UseHealthChecks("/health"); } - private static void UpdateDatabase(IApplicationBuilder app) + private static void UpdateDatabase(IApplicationBuilder app, IHostEnvironment env) { using var serviceScope = app.ApplicationServices .GetRequiredService() @@ -296,7 +296,10 @@ private static void UpdateDatabase(IApplicationBuilder app) context.Database.SetCommandTimeout(300); context.Database.Migrate(); - ApplyCustomMigrations(app); + if (!env.IsIntegrationTest()) + { + ApplyCustomMigrations(app); + } } private static void ApplyCustomMigrations(IApplicationBuilder app) From d65224b4a10e0ca432aaf5f91da47ba6178e52f5 Mon Sep 17 00:00:00 2001 From: dfe-sdt Date: Thu, 21 Nov 2024 09:44:36 +0000 Subject: [PATCH 010/144] chore(tests): update test snapshots --- .../tests/snapshots/data_catalogue_snapshot.json | 2 +- .../tests/snapshots/find_statistics_snapshot.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json index 6a7ec6c6863..37fe8803cbe 100644 --- a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json +++ b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json @@ -1,5 +1,5 @@ { - "num_datasets": "941 data sets", + "num_datasets": "947 data sets", "themes": [ { "publications": [ diff --git a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json index cebc24f63c0..76a5794fb4f 100644 --- a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json +++ b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json @@ -415,7 +415,7 @@ { "publication_summary": "Multiplication tables check attainment by pupil characteristics, school characteristics, region and local authority.", "publication_title": "Multiplication tables check attainment", - "published": "23 Nov 2023", + "published": "21 Nov 2024", "release_type": "Official statistics", "theme": "School and college outcomes and performance" }, @@ -534,7 +534,7 @@ { "publication_summary": "Pupil attendance and absence data including termly national statistics and fortnightly statistics in development derived from DfE\u2019s regular attendance data", "publication_title": "Pupil attendance in schools", - "published": "7 Nov 2024", + "published": "21 Nov 2024", "release_type": "Official statistics in development", "theme": "Pupils and schools" }, @@ -653,7 +653,7 @@ { "publication_summary": "Data on suspensions and permanent exclusions, including by reason, duration, by pupil characteristics and data on independent review panels.", "publication_title": "Suspensions and permanent exclusions in England", - "published": "18 Jul 2024", + "published": "21 Nov 2024", "release_type": "Accredited official statistics", "theme": "Pupils and schools" }, From 8846d993a34a8faeda0c1d065c7e11844cb15fa1 Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 21 Nov 2024 14:12:14 +0000 Subject: [PATCH 011/144] EES-5672 5673 fix ui tests --- tests/robot-tests/tests/admin/bau/release_content.robot | 2 ++ tests/robot-tests/tests/libs/admin/manage-content-common.robot | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/robot-tests/tests/admin/bau/release_content.robot b/tests/robot-tests/tests/admin/bau/release_content.robot index 1044ee51db4..ae6f5784bfe 100644 --- a/tests/robot-tests/tests/admin/bau/release_content.robot +++ b/tests/robot-tests/tests/admin/bau/release_content.robot @@ -214,6 +214,7 @@ Add content blocks to Test section one user adds content to autosaving accordion section text block Test section one 1 block one test text ... ${RELEASE_CONTENT_EDITABLE_ACCORDION} + user scrolls up 100 user adds text block to editable accordion section Test section one ${RELEASE_CONTENT_EDITABLE_ACCORDION} user adds content to autosaving accordion section text block Test section one 2 block two test text ... ${RELEASE_CONTENT_EDITABLE_ACCORDION} @@ -237,6 +238,7 @@ Validate two remaining content blocks ... ${RELEASE_CONTENT_EDITABLE_ACCORDION} Verify that validation prevents adding an image without alt text + user scrolls up 100 user adds image without alt text to accordion section text block Test section one 1 ... test-infographic.png ${RELEASE_CONTENT_EDITABLE_ACCORDION} diff --git a/tests/robot-tests/tests/libs/admin/manage-content-common.robot b/tests/robot-tests/tests/libs/admin/manage-content-common.robot index 1438ad91d92..f4873c5472c 100644 --- a/tests/robot-tests/tests/libs/admin/manage-content-common.robot +++ b/tests/robot-tests/tests/libs/admin/manage-content-common.robot @@ -467,7 +467,6 @@ user adds image to accordion section text block with retry ... xpath://button[span[.="Upload image from computer"]]/input[@type="file"] ... ${FILES_DIR}${filename} - user scrolls up 300 wait until keyword succeeds ${timeout} %{WAIT_SMALL} sec user clicks button ... Change image text alternative user enters text into element label:Text alternative ${alt_text} From 444bda517640ffb2fa5637da5acf93da91bb1aa4 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Tue, 12 Nov 2024 10:18:05 +0000 Subject: [PATCH 012/144] EES-5664 Update API docs to use correct host URL --- src/explore-education-statistics-api-docs/config/tech-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explore-education-statistics-api-docs/config/tech-docs.yml b/src/explore-education-statistics-api-docs/config/tech-docs.yml index bbd5d401bfb..a6a17235f33 100644 --- a/src/explore-education-statistics-api-docs/config/tech-docs.yml +++ b/src/explore-education-statistics-api-docs/config/tech-docs.yml @@ -1,5 +1,5 @@ # Host to use for canonical URL generation (without trailing slash) -host: https://docs.statistics.api.education.gov.uk +host: https://statistics.api.education.gov.uk/docs contact_email: explore.statistics@education.gov.uk # Header-related options From df6920ce75a31b4659c1be2b177edd63f9f20ec5 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Tue, 12 Nov 2024 10:22:44 +0000 Subject: [PATCH 013/144] EES-5664 Add `.env` to API docs for local development --- .../.gitignore | 2 ++ .../Gemfile | 2 ++ .../Gemfile.lock | 1 + .../README.md | 26 ++++++++++++++++++- .../config.rb | 3 +++ 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/explore-education-statistics-api-docs/.gitignore b/src/explore-education-statistics-api-docs/.gitignore index 313ccf02bd7..771c4aca9cc 100644 --- a/src/explore-education-statistics-api-docs/.gitignore +++ b/src/explore-education-statistics-api-docs/.gitignore @@ -14,3 +14,5 @@ .DS_Store Staticfile.auth + +.env diff --git a/src/explore-education-statistics-api-docs/Gemfile b/src/explore-education-statistics-api-docs/Gemfile index 1eee671b28c..ce9a9183ee3 100644 --- a/src/explore-education-statistics-api-docs/Gemfile +++ b/src/explore-education-statistics-api-docs/Gemfile @@ -20,3 +20,5 @@ gem 'chronic' gem 'http' gem 'rake' + +gem "dotenv", groups: [:development, :test] diff --git a/src/explore-education-statistics-api-docs/Gemfile.lock b/src/explore-education-statistics-api-docs/Gemfile.lock index 18d7ad6f4f5..b573eb93978 100644 --- a/src/explore-education-statistics-api-docs/Gemfile.lock +++ b/src/explore-education-statistics-api-docs/Gemfile.lock @@ -255,6 +255,7 @@ PLATFORMS DEPENDENCIES chronic + dotenv govuk_tech_docs html-proofer http diff --git a/src/explore-education-statistics-api-docs/README.md b/src/explore-education-statistics-api-docs/README.md index b8ce6004c97..6f4467a0084 100644 --- a/src/explore-education-statistics-api-docs/README.md +++ b/src/explore-education-statistics-api-docs/README.md @@ -40,10 +40,34 @@ Once the pre-requisites have been installed, follow these steps: This will start the Middleman development server on [https://localhost:4567](https://localhost:4567). -3. Optional. To automatically refresh the browser upon code changes, install the [LiveReload browser extension](https://chrome.google.com/webstore/detail/livereload/jnihajbhpnppcggbcgedagnkighmdlei?hl=en). +3. **Optional** - To automatically refresh the browser upon code changes, install the [LiveReload browser extension](https://chrome.google.com/webstore/detail/livereload/jnihajbhpnppcggbcgedagnkighmdlei?hl=en). For further guidance on how to develop this documentation, please visit the [Technical Documentation Template](https://tdt-documentation.london.cloudapps.digital/) website. +## Configuration + +The project is configured via the `config/tech_docs.yml` file, which should contain the base defaults. +Some parts of the project config can be changed on a per-environment basis using a `.env` file, +system environment variables or both. + +The environment variables permitted and the config options they affect can be found in `config.rb`. + +### Using `.env` file + +For convenience during local development, an `.env` file can be used to set environment variables. +Simply create a `.env` file in the project root and add any environment variables you want to set. + +Environment variables will only override default config options if they are present in `.env`. +Omit any variables you aren't interested in changing. + +### Using system environment variables + +Simply add environment variables to your task runner or CLI command e.g. + +```shell +TECH_DOCS_HOST=https://some-other-site bundle exec middleman +``` + ## Notifications for expired pages All pages are configured with expiry dates so that they can be re-reviewed. We have automated the diff --git a/src/explore-education-statistics-api-docs/config.rb b/src/explore-education-statistics-api-docs/config.rb index 40b304aed97..d034fa917c6 100644 --- a/src/explore-education-statistics-api-docs/config.rb +++ b/src/explore-education-statistics-api-docs/config.rb @@ -1,3 +1,4 @@ +require 'dotenv' require 'govuk_tech_docs' require 'lib/api_reference_pages_extension' require 'lib/helpers' @@ -7,6 +8,8 @@ # Check for broken links require 'html-proofer' +Dotenv.load('.env') + GovukTechDocs.configure(self, livereload: { js_host: "localhost", host: "127.0.0.1" }) # Override config from environment variables From 395d021192478f3fe0651659437189d5d92037eb Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Wed, 13 Nov 2024 23:06:19 +0000 Subject: [PATCH 014/144] EES-5664 Add configurable API URL option to public API docs --- azure-pipelines-main.yml | 2 ++ src/explore-education-statistics-api-docs/config.rb | 6 +++++- .../config/tech-docs.yml | 2 ++ .../lib/api_reference_pages_extension.rb | 4 +--- .../source/endpoints/index.html.md.erb | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/azure-pipelines-main.yml b/azure-pipelines-main.yml index d4e5321399e..d5fe21618fc 100644 --- a/azure-pipelines-main.yml +++ b/azure-pipelines-main.yml @@ -389,6 +389,8 @@ jobs: - task: Bash@3 displayName: Build + env: + TECH_DOCS_API_URL: https://dev.statistics.api.education.gov.uk inputs: workingDirectory: $(WorkingDirectory) targetType: inline diff --git a/src/explore-education-statistics-api-docs/config.rb b/src/explore-education-statistics-api-docs/config.rb index d034fa917c6..6549e0e669e 100644 --- a/src/explore-education-statistics-api-docs/config.rb +++ b/src/explore-education-statistics-api-docs/config.rb @@ -15,13 +15,17 @@ # Override config from environment variables if ENV.has_key?("TECH_DOCS_HOST") - config[:tech_docs][:host] = ENV["TECH_DOCS_HOST"] || config[:tech_docs][:host] + config[:tech_docs][:host] = ENV["TECH_DOCS_HOST"] end if ENV.has_key?("TECH_DOCS_PREVENT_INDEXING") config[:tech_docs][:prevent_indexing] = ENV["TECH_DOCS_PREVENT_INDEXING"] end +if ENV.has_key?("TECH_DOCS_API_URL") + config[:tech_docs][:api_url] = ENV["TECH_DOCS_API_URL"] +end + if ENV.has_key?("TECH_DOCS_API_DOCS_PATH") config[:tech_docs][:api_docs_path] = ENV["TECH_DOCS_API_DOCS_PATH"] end diff --git a/src/explore-education-statistics-api-docs/config/tech-docs.yml b/src/explore-education-statistics-api-docs/config/tech-docs.yml index a6a17235f33..6e5dba25a92 100644 --- a/src/explore-education-statistics-api-docs/config/tech-docs.yml +++ b/src/explore-education-statistics-api-docs/config/tech-docs.yml @@ -19,6 +19,8 @@ footer_links: # headings. max_toc_heading_level: 1 +# URL of the public API +api_url: https://localhost:5051 api_docs_path: https://dev.statistics.api.education.gov.uk/swagger/1.0/openapi.json # Multi-page options diff --git a/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb b/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb index 64449f821aa..1cc34213146 100644 --- a/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb +++ b/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb @@ -33,8 +33,6 @@ def manipulate_resource_list(resources) new_resources = [] - @base_url = document.servers[0]&.url || "" - document.paths.each do |uri, http_methods| get_operations(http_methods).each do |http_method, operation| new_resources << create_endpoint_page(uri, http_method, operation) @@ -77,7 +75,7 @@ def create_endpoint_page(uri, http_method, operation) # @param [String] uri # @return [String] def api_url(uri = "") - @base_url.chomp("/") + uri + @config[:api_url].chomp("/") + uri end private diff --git a/src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb b/src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb index bef4b101d9d..7b8ba0f1b69 100644 --- a/src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb @@ -9,7 +9,7 @@ weight: 100 This section lists all the endpoints available on the explore education statistics (EES) API. -The URL for the EES API is: <%= link_to api_url, api_url %> +The URL for the EES API is <%= link_to api_url, api_url %>. The documentation for each endpoint is composed of information about the request and response, with examples and schemas provided. From c545ea9897843309824c92dfdee99ec678c48187 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Sun, 17 Nov 2024 13:46:26 +0000 Subject: [PATCH 015/144] EES-5664 Update generated OpenAPI docs to use relative server URL --- .../Swagger/SwaggerConfig.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs index babd1bdec18..b44694bbe83 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs @@ -2,7 +2,6 @@ using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Options; using Microsoft.Extensions.Options; using Microsoft.OpenApi.Models; using Swashbuckle.AspNetCore.SwaggerGen; @@ -10,8 +9,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Swagger; public class SwaggerConfig( - IApiVersionDescriptionProvider provider, - IOptions appOptions) + IApiVersionDescriptionProvider provider) : IConfigureOptions { public void Configure(SwaggerGenOptions options) @@ -76,7 +74,7 @@ public void Configure(SwaggerGenOptions options) options.AddServer(new OpenApiServer { Description = "API server", - Url = appOptions.Value.Url + Url = "/" }); } From f63b640f07357d762c1e1e8f581c6bfbe213ea77 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Sun, 17 Nov 2024 13:50:45 +0000 Subject: [PATCH 016/144] EES-5664 Fix example for `DataSetLatestVersionViewModel.GeographicLevels` --- .../ViewModels/DataSetVersionViewModels.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs index a9c4b87a4bf..5102c49347f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs @@ -140,7 +140,7 @@ public record DataSetLatestVersionViewModel /// /// The geographic levels available in the data set. /// - /// ["NAT", "REG", "LA"] + /// ["National", "Regional", "Local authority"] [JsonConverter(typeof(ReadOnlyListJsonConverter>))] public required IReadOnlyList GeographicLevels { get; init; } From 2950242e2045d615505d6e97be0ab456b17883d7 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Sat, 16 Nov 2024 22:52:55 +0000 Subject: [PATCH 017/144] EES-5664 Rework public API docs to handle multiple versioned OpenAPI docs --- .../README.md | 8 + .../config.rb | 7 +- .../config/tech-docs.yml | 1 - .../lib/api_reference_helpers.rb | 15 +- .../lib/api_reference_pages_extension.rb | 112 +- .../source/endpoints/index.html.md.erb | 22 - .../index.html.md.erb | 8 +- .../index.html.md.erb | 18 +- .../how-to-get-csv-data/index.html.md.erb | 2 +- .../source/getting-started/index.html.md.erb | 4 +- .../quick-start/index.html.md.erb | 12 +- .../source/openapi-v1.json | 4057 +++++++++++++++++ .../overview/error-handling/index.html.md.erb | 2 +- .../openapi-documents/index.html.md.erb | 26 + .../openapi-specification/index.html.md.erb | 28 - .../overview/versioning/index.html.md.erb | 2 +- .../reference/endpoint.html.md.erb} | 7 +- .../reference/endpoint_index.html.md.erb | 20 + .../templates/reference/index.html.md.erb | 23 + .../reference/schema.html.md.erb} | 5 + .../reference/schema_index.html.md.erb} | 9 +- 21 files changed, 4263 insertions(+), 125 deletions(-) delete mode 100644 src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb create mode 100644 src/explore-education-statistics-api-docs/source/openapi-v1.json create mode 100644 src/explore-education-statistics-api-docs/source/overview/openapi-documents/index.html.md.erb delete mode 100644 src/explore-education-statistics-api-docs/source/overview/openapi-specification/index.html.md.erb rename src/explore-education-statistics-api-docs/source/{endpoints/template.html.md.erb => templates/reference/endpoint.html.md.erb} (98%) create mode 100644 src/explore-education-statistics-api-docs/source/templates/reference/endpoint_index.html.md.erb create mode 100644 src/explore-education-statistics-api-docs/source/templates/reference/index.html.md.erb rename src/explore-education-statistics-api-docs/source/{schemas/template.html.md.erb => templates/reference/schema.html.md.erb} (95%) rename src/explore-education-statistics-api-docs/source/{schemas/index.html.md.erb => templates/reference/schema_index.html.md.erb} (84%) diff --git a/src/explore-education-statistics-api-docs/README.md b/src/explore-education-statistics-api-docs/README.md index 6f4467a0084..a41b97b0a8f 100644 --- a/src/explore-education-statistics-api-docs/README.md +++ b/src/explore-education-statistics-api-docs/README.md @@ -68,6 +68,14 @@ Simply add environment variables to your task runner or CLI command e.g. TECH_DOCS_HOST=https://some-other-site bundle exec middleman ``` +## Generating API reference docs + +API reference documentation is generated at build time using any `openapi-v*.json` documents that +are found in the `source` directory. For example, to create the reference docs for a v2 API, add +a `openapi-v2.json`. + +Refer to `lib/api_reference_pages_extension.rb` for how we implement this. + ## Notifications for expired pages All pages are configured with expiry dates so that they can be re-reviewed. We have automated the diff --git a/src/explore-education-statistics-api-docs/config.rb b/src/explore-education-statistics-api-docs/config.rb index 6549e0e669e..a8c3f0fdd16 100644 --- a/src/explore-education-statistics-api-docs/config.rb +++ b/src/explore-education-statistics-api-docs/config.rb @@ -26,9 +26,10 @@ config[:tech_docs][:api_url] = ENV["TECH_DOCS_API_URL"] end -if ENV.has_key?("TECH_DOCS_API_DOCS_PATH") - config[:tech_docs][:api_docs_path] = ENV["TECH_DOCS_API_DOCS_PATH"] -end +ignore "**/template.html" +ignore "partials/*" +ignore "endpoints/*" +ignore "templates/*" helpers Helpers helpers ApiReferenceHelpers diff --git a/src/explore-education-statistics-api-docs/config/tech-docs.yml b/src/explore-education-statistics-api-docs/config/tech-docs.yml index 6e5dba25a92..4052c2a18d7 100644 --- a/src/explore-education-statistics-api-docs/config/tech-docs.yml +++ b/src/explore-education-statistics-api-docs/config/tech-docs.yml @@ -21,7 +21,6 @@ max_toc_heading_level: 1 # URL of the public API api_url: https://localhost:5051 -api_docs_path: https://dev.statistics.api.education.gov.uk/swagger/1.0/openapi.json # Multi-page options multipage_nav: true diff --git a/src/explore-education-statistics-api-docs/lib/api_reference_helpers.rb b/src/explore-education-statistics-api-docs/lib/api_reference_helpers.rb index 83b81a48451..b6fa73197de 100644 --- a/src/explore-education-statistics-api-docs/lib/api_reference_helpers.rb +++ b/src/explore-education-statistics-api-docs/lib/api_reference_helpers.rb @@ -1,4 +1,15 @@ module ApiReferenceHelpers + + # @return [ArrayString}>] + def get_openapi_links + FileList.glob("source/openapi-v*.json").map do |path| + url = path.split("source/")[1] + version = url.split("openapi-v")[1].split(".json")[0] + + { name: "v#{version}", url: url } + end + end + # @param [Openapi3Parser::Node::Schema] schema_data # @param [Array] references # @return [*] @@ -95,7 +106,7 @@ def render_schema_type(schema) return "object" end - href = ::Middleman::Util::url_for(@app, "/schemas/#{schema.name}/index.html", { + href = ::Middleman::Util::url_for(@app, "../../schemas/#{schema.name}/index.html", { current_resource: current_resource }) @@ -110,7 +121,7 @@ def render_schema_type(schema) "array (#{render_schema_type(items)})" else if schema.name - href = ::Middleman::Util::url_for(@app, "/schemas/#{schema.name}/index.html", { + href = ::Middleman::Util::url_for(@app, "../../schemas/#{schema.name}/index.html", { current_resource: current_resource }) diff --git a/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb b/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb index 1cc34213146..fec2807047f 100644 --- a/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb +++ b/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb @@ -6,65 +6,103 @@ def initialize(app, options_hash = {}, &block) @sitemap = app.sitemap @config = app.config[:tech_docs] - @endpoint_template = Middleman::Util.normalize_path("/endpoints/template.html") - @schema_template = Middleman::Util.normalize_path("/schemas/template.html") - - app.ignore @endpoint_template - app.ignore @schema_template + @base_template_path = "/templates/reference" end # @param [List] resources # @return [List] def manipulate_resource_list(resources) - api_docs_path = @config[:api_docs_path] + FileList.glob("source/openapi-v*.json").each do |path| + document = Openapi3Parser.load_file(path) + version = "v#{document.info.version}" - if api_docs_path.nil? - return resources - end + if path != "source/openapi-#{version}.json" + raise "OpenAPI document path '#{path}' does not match its version, expected 'source/openapi-#{version}.json'" + end - document = if uri?(api_docs_path) - Openapi3Parser.load_url(api_docs_path) - elsif File.exist?(api_docs_path) - # Load api file and set existence flag. - Openapi3Parser.load_file(api_docs_path) - else - raise "Unable to load `api_docs_path` from config/tech-docs.yml" - end + base_path = "/reference-#{version}" + + resources << Middleman::Sitemap::ProxyResource.new( + @sitemap, + Middleman::Util.normalize_path("#{base_path}/index.html"), + Middleman::Util.normalize_path("#{@base_template_path}/index.html") + ).tap do |p| + p.add_metadata locals: { + title: "API #{version} reference", + version: version, + }, page: { + title: "API #{version} reference", + } + end - new_resources = [] + resources << Middleman::Sitemap::ProxyResource.new( + @sitemap, + Middleman::Util.normalize_path("#{base_path}/endpoints/index.html"), + Middleman::Util.normalize_path("#{@base_template_path}/endpoint_index.html") + ).tap do |p| + p.add_metadata locals: { + title: "API #{version} endpoints", + version: version, + }, page: { + title: "Endpoints", + } + end - document.paths.each do |uri, http_methods| - get_operations(http_methods).each do |http_method, operation| - new_resources << create_endpoint_page(uri, http_method, operation) + resources << Middleman::Sitemap::ProxyResource.new( + @sitemap, + Middleman::Util.normalize_path("#{base_path}/schemas/index.html"), + Middleman::Util.normalize_path("#{@base_template_path}/schema_index.html") + ).tap do |p| + p.add_metadata locals: { + title: "API #{version} schemas", + version: version, + }, page: { + title: "Schemas" + } end - end - document.components.schemas.each do |_, schema| - new_resources << create_schema_page(schema) + document.paths.each do |uri, http_methods| + get_operations(http_methods).each do |http_method, operation| + resources << create_endpoint_page(uri, http_method, operation, version, base_path) + end + end + + document.components.schemas.each do |_, schema| + resources << create_schema_page(schema, version, base_path) + end end - resources + new_resources + resources end private + # @param [String] uri + # @return [String] + def api_url(uri = "") + @config[:api_url].chomp("/") + uri + end + # @param [String] uri # @param [String] http_method # @param [Openapi3Parser::Node::Operation] operation + # @param [String] version + # @param [String] base_path # @return [Middleman::Sitemap::ProxyResource] - def create_endpoint_page(uri, http_method, operation) + def create_endpoint_page(uri, http_method, operation, version, base_path) id = operation.operation_id Middleman::Sitemap::ProxyResource.new( @sitemap, - Middleman::Util.normalize_path("/endpoints/#{id}/index.html"), - @endpoint_template + Middleman::Util.normalize_path("#{base_path}/endpoints/#{id}/index.html"), + Middleman::Util.normalize_path("#{@base_template_path}/endpoint.html") ).tap do |p| p.add_metadata locals: { title: operation.summary, + version: version, url: api_url(uri), http_method: http_method.upcase, - description: operation.description, + endpoint_description: operation.description, parameters: operation.parameters, request_body: operation.request_body, responses: operation.responses @@ -72,26 +110,22 @@ def create_endpoint_page(uri, http_method, operation) end end - # @param [String] uri - # @return [String] - def api_url(uri = "") - @config[:api_url].chomp("/") + uri - end - - private # @param [Openapi3Parser::Node::Schema] schema + # @param [String] version + # @param [String] base_path # @return [Middleman::Sitemap::ProxyResource] - def create_schema_page(schema) + def create_schema_page(schema, version, base_path) name = schema.name Middleman::Sitemap::ProxyResource.new( @sitemap, - Middleman::Util.normalize_path("/schemas/#{name}/index.html"), - @schema_template + Middleman::Util.normalize_path("#{base_path}/schemas/#{name}/index.html"), + Middleman::Util.normalize_path("#{@base_template_path}/schema.html") ).tap do |p| p.add_metadata locals: { title: name, + version: version, schema: schema, } end diff --git a/src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb b/src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb deleted file mode 100644 index 7b8ba0f1b69..00000000000 --- a/src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: Endpoints -last_reviewed_on: 2024-09-16 -review_in: 24 months -weight: 100 ---- - -# API Endpoints - -This section lists all the endpoints available on the explore education statistics (EES) API. - -The URL for the EES API is <%= link_to api_url, api_url %>. - -The documentation for each endpoint is composed of information about the request and response, with -examples and schemas provided. - -For convenience, request examples in different languages/tools are provided. We currently support: - -- cURL -- JavaScript -- Python -- R diff --git a/src/explore-education-statistics-api-docs/source/getting-started/creating-advanced-data-set-queries/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/creating-advanced-data-set-queries/index.html.md.erb index 3106070fd30..67c50496354 100644 --- a/src/explore-education-statistics-api-docs/source/getting-started/creating-advanced-data-set-queries/index.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/getting-started/creating-advanced-data-set-queries/index.html.md.erb @@ -33,7 +33,7 @@ examples. ## The basic query syntax -Data set queries can be made using a POST request to the [Query a data set](/endpoints/QueryDataSetPost/index.html) +Data set queries can be made using a POST request to the [Query a data set](/reference-v1/endpoints/QueryDataSetPost/index.html) endpoint. At its most basic, such a request would look like the following: ``` @@ -46,7 +46,7 @@ POST <%= api_url "/api/v1/data-sets/{dataSetId}/query" %> The request body must contain `facets` and `indicators` properties. These filter the result data that can be in the response and must be populated using the IDs of facets from the data set metadata -(see the [Get a data set's metadata](/endpoints/GetDataSetMeta/index.html) endpoint). +(see the [Get a data set's metadata](/reference-v1/endpoints/GetDataSetMeta/index.html) endpoint). The `indicators` property controls what data values are shown in the results. This should simply contain a list of indicator IDs. For example, using the following indicators: @@ -120,8 +120,8 @@ The full list of comparators permitted is as follows: | `gt` | Greater than | No | `"gt": { "period": "2022/2023", "code": "AY" }` | Note that facet properties may only permit certain comparators to be used. Consult the -[schema documentation](/schemas/DataSetQueryCriteriaFacets/index.html) for each facet property to see if a -comparator is allowed. +[schema documentation](/reference-v1/schemas/DataSetQueryCriteriaFacets/index.html) for each facet property +to see if a comparator is allowed. Using all the above information, you could write a query that looks like the following: diff --git a/src/explore-education-statistics-api-docs/source/getting-started/debugging-data-set-queries/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/debugging-data-set-queries/index.html.md.erb index 8a7fb40ff29..28cd07a0010 100644 --- a/src/explore-education-statistics-api-docs/source/getting-started/debugging-data-set-queries/index.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/getting-started/debugging-data-set-queries/index.html.md.erb @@ -17,8 +17,8 @@ first as this guide will presume some prior knowledge. You should already be familiar with the basic usages of the following endpoints to proceed: -- [Query a data set](/endpoints/QueryDataSetPost/index.html) -- [Get a data set's metadata](/endpoints/GetDataSetMeta/index.html) +- [Query a data set](/reference-v1/endpoints/QueryDataSetPost/index.html) +- [Get a data set's metadata](/reference-v1/endpoints/GetDataSetMeta/index.html) ## Diagnosing error responses @@ -40,8 +40,8 @@ When there is an error response, the body will typically look like: } ``` -The response body is modelled by the [ProblemDetailsViewModel](/schemas/ProblemDetailsViewModel/index.html) schema, -which attempts to detail the reason(s) why the request failed. The following fields are always +The response body is modelled by the [ProblemDetailsViewModel](/reference-v1/schemas/ProblemDetailsViewModel/index.html) +schema, which attempts to detail the reason(s) why the request failed. The following fields are always included: | Property | Type | Description | @@ -84,7 +84,7 @@ Validation errors and error responses in general are covered in much more detail ## Validation errors for data set queries -The [Query a data set](/endpoints/QueryDataSetPost/index.html) endpoint will usually try to process a +The [Query a data set](/reference-v1/endpoints/QueryDataSetPost/index.html) endpoint will usually try to process a query as much as possible before a validation error response is sent (instead of failing early). Consequently, the response typically aggregates as many validation errors as possible. @@ -187,7 +187,7 @@ incompatible and will result in an error response like: #### Solution -Check that your query correctly follows the [DataSetQueryRequest](/schemas/DataSetQueryRequest/index.html) +Check that your query correctly follows the [DataSetQueryRequest](/reference-v1/schemas/DataSetQueryRequest/index.html) schema. Pay close attention to any usages of condition clauses. The `and` / `or` clauses accept **multiple** criteria or condition clauses in an array: @@ -315,7 +315,7 @@ missing (e.g. `invalid-filter-1`) in the `details` property. #### Solution Ensure that all facets in the data set query exist in the corresponding data set metadata. -You should check this by cross-referencing the missing facets with the [Get a data set's metadata](/endpoints/GetDataSetMeta/index.html) +You should check this by cross-referencing the missing facets with the [Get a data set's metadata](/reference-v1/endpoints/GetDataSetMeta/index.html) endpoint. Facets are not usually removed from existing data sets, so there may be a typo (or similar) in @@ -361,7 +361,7 @@ adjacent comment causing the JSON to be syntactically invalid. ## Debug mode -To assist in debugging unexpected results for a data set query, the [Query a data set](/endpoints/QueryDataSetPost/index.html) +To assist in debugging unexpected results for a data set query, the [Query a data set](/reference-v1/endpoints/QueryDataSetPost/index.html) endpoint also accepts a `debug` query parameter that enables debug mode. This can be set in the request's query string like so: @@ -399,7 +399,7 @@ The keys and values of `filters`, `locations` and `values` are changed to displa labels and facet IDs in the format `{facet ID} :: {label}`. Enabling debug mode is useful to avoid having to cross-reference the facets of each result with the -data set's metadata (using the [Get a data set's metadata](/endpoints/GetDataSetMeta/index.html) endpoint). +data set's metadata (using the [Get a data set's metadata](/reference-v1/endpoints/GetDataSetMeta/index.html) endpoint). However, it is important to note that debug mode **should not** be used outside of development / debugging purposes. When your query's issues have been resolved, you should disable debug mode before pushing diff --git a/src/explore-education-statistics-api-docs/source/getting-started/how-to-get-csv-data/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/how-to-get-csv-data/index.html.md.erb index 026836a6c7a..fc9ff30fb86 100644 --- a/src/explore-education-statistics-api-docs/source/getting-started/how-to-get-csv-data/index.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/getting-started/how-to-get-csv-data/index.html.md.erb @@ -28,7 +28,7 @@ Every data set is created from an underlying CSV file that contains **all** the it useful (or necessary) to work with the underlying CSV instead of interacting with the data set via the API. -The underlying CSV file can be downloaded via the [Download data set CSV](/endpoints/DownloadDataSetCsv/index.html) +The underlying CSV file can be downloaded via the [Download data set CSV](/reference-v1/endpoints/DownloadDataSetCsv/index.html) endpoint. To use this endpoint, you need to make a `GET` request: ``` diff --git a/src/explore-education-statistics-api-docs/source/getting-started/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/index.html.md.erb index 7b936ecd2c8..c5478e1b59e 100644 --- a/src/explore-education-statistics-api-docs/source/getting-started/index.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/getting-started/index.html.md.erb @@ -35,10 +35,10 @@ To assist in specific tasks once you've started using the EES API, there are add The [Overview section](/overview/index.html) provides high level documentation about the EES API. This details things such as message formats, error handling, versioning and the OpenAPI specification. -The [Endpoints section](/endpoints/index.html) provides reference documentation about the endpoints +The [Endpoints section](/reference-v1/endpoints/index.html) provides reference documentation about the endpoints available in the API. This details the requests that can be made and their responses. Code samples are also provided to illustrate how requests could be made. -The [Schemas section](/schemas/index.html) provides reference documentation about the structure of +The [Schemas section](/reference-v1/schemas/index.html) provides reference documentation about the structure of all the requests and responses (i.e. their schemas) across the API. Each schema provides in-depth detail about their properties, including their type and validation rules. diff --git a/src/explore-education-statistics-api-docs/source/getting-started/quick-start/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/quick-start/index.html.md.erb index 8bebd9eb796..7e3d179c5bc 100644 --- a/src/explore-education-statistics-api-docs/source/getting-started/quick-start/index.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/getting-started/quick-start/index.html.md.erb @@ -50,7 +50,7 @@ In the following sections, this guide will walk you through how to perform the a ### Step 1: Find a publication To find a publication that you may be interested in, you'll need to make a `GET` request to the -[List publications](/endpoints/ListPublications/index.html) endpoint: +[List publications](/reference-v1/endpoints/ListPublications/index.html) endpoint: ``` GET <%= api_url "/api/v1/publications" %> @@ -125,7 +125,7 @@ Once you find a publication you are interested in, proceed to the next step. Now that you have a publication that you are interested, you can use this to find data sets related to it. This can be done using -the [List a publication's data sets](/endpoints/ListPublicationDataSets/index.html) +the [List a publication's data sets](/reference-v1/endpoints/ListPublicationDataSets/index.html) endpoint: ``` @@ -207,7 +207,7 @@ Once you have chosen a data set, proceed to the next step. ### Step 3: Get the data set's metadata Now that you have a chosen a data set, you'll want to query it for some data. To create a query, -you'll need to use the [Get a data set's metadata](/endpoints/GetDataSetMeta/index.html) endpoint. +you'll need to use the [Get a data set's metadata](/reference-v1/endpoints/GetDataSetMeta/index.html) endpoint. This provides information about all the filterable facets and indicators available to a data set. **Facets** are specific features / characteristics of the data. These are used in a data set query @@ -340,14 +340,14 @@ In the above example, there are location options for: The `geographicLevels` property contains the different geographic levels that the data was collected at. Each geographic level is identified by its `code` and a full list of these can be found in the -[GeographicLevelCode schema](/schemas/GeographicLevelCode/index.html). +[GeographicLevelCode schema](/reference-v1/schemas/GeographicLevelCode/index.html). In the above example, there are geographic level options for 'National' (`NAT`) and 'Regional' (`REG`) geographic levels. The `timePeriods` property contains the time periods the data was collected at. The time period options are represented by a `code` that describes the time period's type and a `period` that describes the -date range. A full list of time period codes can be found in the [TimePeriodCode schema](/schemas/TimePeriodCode/index.html). +date range. A full list of time period codes can be found in the [TimePeriodCode schema](/reference-v1/schemas/TimePeriodCode/index.html). In the above example there is a single time period option for 'academic year 2022/23'. @@ -375,7 +375,7 @@ ready. ### Step 4: Create and run your data set query In this final step, you'll need to use the metadata from the previous step to create and run your -query against the [Query a data set](/endpoints/QueryDataSetPost/index.html) endpoint. +query against the [Query a data set](/reference-v1/endpoints/QueryDataSetPost/index.html) endpoint. To use this endpoint, a `POST` request needs to be sent to the endpoint URL with an appropriate request body. The most basic request would look like the following: diff --git a/src/explore-education-statistics-api-docs/source/openapi-v1.json b/src/explore-education-statistics-api-docs/source/openapi-v1.json new file mode 100644 index 00000000000..8c5aad8f217 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/openapi-v1.json @@ -0,0 +1,4057 @@ +{ + "openapi": "3.0.1", + "info": { + "title": "Explore education statistics - Public Data API", + "contact": { + "name": "Explore education statistics", + "url": "https://explore-education-statistics.service.gov.uk", + "email": "explore.statistics@education.gov.uk" + }, + "version": "1" + }, + "servers": [ + { + "url": "/", + "description": "API server" + } + ], + "paths": { + "/api/v1/data-sets/{dataSetId}": { + "get": { + "tags": [ + "DataSets" + ], + "summary": "Get a data set’s summary", + "description": "Gets a specific data set’s summary details.", + "operationId": "GetDataSet", + "parameters": [ + { + "name": "dataSetId", + "in": "path", + "description": "The ID of the data set.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "The requested data set summary.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataSetViewModel" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + } + }, + "/api/v1/data-sets/{dataSetId}/meta": { + "get": { + "tags": [ + "DataSets" + ], + "summary": "Get a data set’s metadata", + "description": "Get the metadata about a data set. Use this to create data set queries.", + "operationId": "GetDataSetMeta", + "parameters": [ + { + "name": "types", + "in": "query", + "description": "The types of meta to get for the requested data set version.\r\n\r\nCan be any combination of the following:\r\n\r\n- `Filters` - include all meta relating to *filters*\r\n- `Indicators` - include all meta relating to *indicators*\r\n- `Locations` - include all meta relating to *locations*\r\n- `TimePeriods` - include all meta relating to *time periods*", + "schema": { + "type": "array", + "items": { + "enum": [ + "Filters", + "Indicators", + "Locations", + "TimePeriods" + ], + "type": "string" + }, + "description": "The types of meta to get for the requested data set version.\r\n\r\nCan be any combination of the following:\r\n\r\n- `Filters` - include all meta relating to *filters*\r\n- `Indicators` - include all meta relating to *indicators*\r\n- `Locations` - include all meta relating to *locations*\r\n- `TimePeriods` - include all meta relating to *time periods*", + "nullable": true, + "example": [ + "Filters", + "Locations" + ] + }, + "example": [ + "Filters", + "Locations" + ] + }, + { + "name": "dataSetVersion", + "in": "query", + "description": "The version of the data set to use e.g. 2.0, 1.1, etc.", + "schema": { + "type": "string" + } + }, + { + "name": "dataSetId", + "in": "path", + "description": "The ID of the data set.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "The requested data set version metadata.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataSetMetaViewModel" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + } + }, + "/api/v1/data-sets/{dataSetId}/query": { + "get": { + "tags": [ + "DataSets" + ], + "summary": "Query a data set (GET)", + "description": "Query a data set using a `GET` request, returning the filtered results.\r\n \r\nNote that there is a `POST` variant of this endpoint which provides a more complete set\r\nof querying functionality. The `GET` variant is only recommended for initial exploratory\r\ntesting or simple queries that do not need advanced functionality.\r\n \r\nUnlike the `POST` variant, this endpoint does not allow condition clauses (`and`, `or`, `not`)\r\nand consequently cannot express more complex queries. Use the `POST` variant instead for these\r\ntypes of queries.\r\n \r\n## Indicators\r\n \r\nThe `indicators` query parameter is required and **at least one** indicator must be specified.\r\n \r\nEach indicator should be a string containing the indicator ID e.g. `4xbOu`, `8g1RI`.\r\n \r\n## Filters\r\n \r\nThe `filters` query parameter is used to filter by other filter options (not locations,\r\ngeographic levels or time periods).\r\n \r\nEach filter should be a string containing the required filter option ID e.g. `z4FQE`, `DcQeg`.\r\n \r\n## Geographic levels\r\n \r\nThe `geographicLevels` query parameter is used to filter results by geographic level.\r\n \r\nThe geographic levels are specified as codes, and can be one of the following:\r\n \r\n- `EDA` - English devolved area\r\n- `INST` - Institution\r\n- `LA` - Local authority\r\n- `LAD` - Local authority district\r\n- `LEP` - Local enterprise partnership\r\n- `LSIP` - Local skills improvement plan area\r\n- `MCA` - Mayoral combined authority\r\n- `MAT` - MAT\r\n- `NAT` - National\r\n- `OA` - Opportunity area\r\n- `PA` - Planning area\r\n- `PCON` - Parliamentary constituency\r\n- `PROV` - Provider\r\n- `REG` - Regional\r\n- `RSC` - RSC region\r\n- `SCH` - School\r\n- `SPON` - Sponsor\r\n- `WARD` - Ward\r\n \r\n## Locations\r\n \r\nThe `locations` query parameter is used to filter results by location.\r\n \r\nThe locations should be strings formatted like `{level}|{property}|{value}` where:\r\n \r\n- `{level}` is the location's level code (e.g. `NAT`, `REG`, `LA`)\r\n- `{property}` is the name of the identifying property to match on (e.g. `id, `code`, `urn`)\r\n- `{value}` is the value for the property to match\r\n \r\nAn ID or a code can be used to identify a location, with the following differences:\r\n \r\n- IDs only match a **single location**\r\n- Codes may match **multiple locations**\r\n \r\nWhilst codes are generally unique to a single location, they can be used for multiple locations.\r\nThis may match more results than you expect so it's recommended to use IDs where possible.\r\n \r\n### Examples\r\n \r\n- `LA|code|E08000019` matches any local authority with code `E08000019`\r\n- `REG|id|6bQgZ` matches any region with ID `6bQgZ`\r\n- `SCH|urn|140821` matches any school with URN `140821`\r\n \r\n## Time periods\r\n \r\nThe `timePeriods` query parameter is used to filter results by time period.\r\n \r\nThe time periods should be strings formatted like `{period}|{code}` where:\r\n \r\n- `period` is the time period or range (e.g. `2020` or `2020/2021`)\r\n- `code` is the code identifying the time period type (e.g. `AY`, `CY`, `M1`, `W20`)\r\n \r\nThe `period` should be a single year like `2020`, or a range like `2020/2021`.\r\nCurrently, only years (or year ranges) are supported.\r\n \r\nSome time period types span two years e.g. financial year part 2 (`P2`), or may fall in a\r\nlatter year e.g. academic year summer term (`T3`). For these types, a singular year `period`\r\nlike `2020` is considered as `2020/2021`.\r\n \r\nFor example, a `period` value of `2020` is applicable to the following time periods:\r\n \r\n- 2020 calendar year\r\n- 2020/2021 academic year\r\n- 2020/2021 financial year part 2 (October to March)\r\n- 2020/2021 academic year's summer term\r\n \r\nIf you wish to be more explicit, you may use a range for the `period` e.g. `2020/2021`.\r\nHowever, a range cannot be used with time period types which only span a single year,\r\nfor example, `2020/21` cannot be used with `CY`, `M` or `W` codes.\r\n \r\n### Examples\r\n \r\n- `2020|AY` is the 2020/21 academic year\r\n- `2021|FY` is the 2021/22 financial year\r\n- `2020|T3` is the 2020/21 academic year's summer term\r\n- `2020|P2` is the 2020/21 financial year part 2 (October to March)\r\n- `2020|CY` is the 2020 calendar year\r\n- `2020|W32` is 2020 week 32\r\n- `2020/2021|AY` is the 2020/21 academic year\r\n- `2021/2022|FY` is the 2021/22 financial year\r\n \r\n## Sorts\r\n \r\nThe `sorts` query parameter is used to sort the results.\r\n \r\nSorts are applied in the order they are provided and should be strings\r\nformatted like `{field}|{direction}` where:\r\n \r\n- `field` is the name of the field to sort e.g. `timePeriod`\r\n- `direction` is the direction to sort in e.g. ascending (`Asc`) or descending (`Desc`)\r\n \r\nThe `field` can be one of the following:\r\n \r\n- `timePeriod` to sort by time period\r\n- `geographicLevel` to sort by the geographic level of the data\r\n- `location|{level}` to sort by locations in a geographic level where `{level}` is the level code (e.g. `REG`, `LA`)\r\n- `filter|{id}` to sort by the options in a filter where `{id}` is the filter ID (e.g. `3RxWP`)\r\n- `indicator|{id}` to sort by the values in a indicator where `{id}` is the indicator ID (e.g. `6VfPgZ`)\r\n \r\n### Examples\r\n \r\n- `timePeriod|Desc` sorts by time period in descending order\r\n- `geographicLevel|Asc` sorts by geographic level in ascending order\r\n- `location|REG|Asc` sorts by regions in ascending order\r\n- `filter|3RxWP|Desc` sorts by options in filter `3RxWP` in descending order\r\n- `indicator|7a1dk|Asc` sorts by values in indicator `7a1dk` in ascending order", + "operationId": "QueryDataSetGet", + "parameters": [ + { + "name": "dataSetId", + "in": "path", + "description": "The ID of the data set.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "name": "dataSetVersion", + "in": "query", + "description": "The version of the data set to use e.g. 2.0, 1.1, etc.", + "schema": { + "type": "string" + } + }, + { + "name": "filters.in", + "in": "query", + "description": "Filter the results to have a filter option matching at least one of these IDs.", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "n0WqP", + "3dCWP" + ] + }, + "example": [ + "n0WqP", + "3dCWP" + ] + }, + { + "name": "filters.notIn", + "in": "query", + "description": "Filter the results to not have a filter option matching any of these IDs.", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "RMxeh", + "wUzft" + ] + }, + "example": [ + "RMxeh", + "wUzft" + ] + }, + { + "name": "filters.eq", + "in": "query", + "description": "Filter the results to have a filter option matching this ID.", + "schema": { + "type": "string", + "example": "pVAkV" + }, + "example": "pVAkV" + }, + { + "name": "filters.notEq", + "in": "query", + "description": "Filter the results to not have a filter option matching this ID.", + "schema": { + "type": "string", + "example": "wUzft" + }, + "example": "wUzft" + }, + { + "name": "geographicLevels.in", + "in": "query", + "description": "Filter the results to be in one of these geographic levels.", + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GeographicLevelCode" + }, + "example": [ + "NAT", + "LA" + ] + }, + "example": [ + "NAT", + "LA" + ] + }, + { + "name": "geographicLevels.notIn", + "in": "query", + "description": "Filter the results to not be in one of these geographic levels.", + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GeographicLevelCode" + }, + "example": [ + "REG", + "LAD" + ] + }, + "example": [ + "REG", + "LAD" + ] + }, + { + "name": "geographicLevels.eq", + "in": "query", + "description": "Filter the results to be in this geographic level.", + "schema": { + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "example": "NAT" + }, + "example": "NAT" + }, + { + "name": "geographicLevels.notEq", + "in": "query", + "description": "Filter the results to not be in this geographic level.", + "schema": { + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "example": "PROV" + }, + "example": "PROV" + }, + { + "name": "locations.eq", + "in": "query", + "description": "Filter the results to be in this location.", + "schema": { + "type": "string", + "example": "NAT|id|3dCWP" + }, + "example": "NAT|id|3dCWP" + }, + { + "name": "locations.notEq", + "in": "query", + "description": "Filter the results to not be in this location.", + "schema": { + "type": "string", + "example": "REG|code|E12000003" + }, + "example": "REG|code|E12000003" + }, + { + "name": "locations.in", + "in": "query", + "description": "Filter the results to be in one of these locations.", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "LA|code|E08000003", + "LA|oldCode|373" + ] + }, + "example": [ + "LA|code|E08000003", + "LA|oldCode|373" + ] + }, + { + "name": "locations.notIn", + "in": "query", + "description": "Filter the results not to be in one of these locations.", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "SCH|urn|123456", + "SCH|laEstab|1234567" + ] + }, + "example": [ + "SCH|urn|123456", + "SCH|laEstab|1234567" + ] + }, + { + "name": "timePeriods.eq", + "in": "query", + "description": "Filter the results to be in this time period.", + "schema": { + "type": "string", + "example": "2024|CY" + }, + "example": "2024|CY" + }, + { + "name": "timePeriods.notEq", + "in": "query", + "description": "Filter the results to not be in this time period.", + "schema": { + "type": "string", + "example": "2024/2025|AY" + }, + "example": "2024/2025|AY" + }, + { + "name": "timePeriods.in", + "in": "query", + "description": "Filter the results to be in one of these time periods.", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "2022|CY", + "2023|CY", + "2024|CY" + ] + }, + "example": [ + "2022|CY", + "2023|CY", + "2024|CY" + ] + }, + { + "name": "timePeriods.notIn", + "in": "query", + "description": "Filter the results to not be in one of these time periods.", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "2020|M1", + "2020|M2", + "2020|M3" + ] + }, + "example": [ + "2020|M1", + "2020|M2", + "2020|M3" + ] + }, + { + "name": "timePeriods.gt", + "in": "query", + "description": "Filter the results to be in a time period that is\r\nchronologically greater than the one specified.", + "schema": { + "type": "string", + "example": "2017/2018|AY" + }, + "example": "2017/2018|AY" + }, + { + "name": "timePeriods.gte", + "in": "query", + "description": "Filter the results to be in a time period that is\r\nchronologically greater than or equal to the one specified.", + "schema": { + "type": "string", + "example": "2017|CY" + }, + "example": "2017|CY" + }, + { + "name": "timePeriods.lt", + "in": "query", + "description": "Filter the results to be in a time period that is\r\nchronologically less than the one specified.", + "schema": { + "type": "string", + "example": "2023/2024|AY" + }, + "example": "2023/2024|AY" + }, + { + "name": "timePeriods.lte", + "in": "query", + "description": "Filter the results to be in a time period that is\r\nchronologically less than or equal to the one specified.", + "schema": { + "type": "string", + "example": "2023|CY" + }, + "example": "2023|CY" + }, + { + "name": "indicators", + "in": "query", + "description": "The IDs of indicators to return values for.", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The IDs of indicators to return values for." + } + }, + { + "name": "sorts", + "in": "query", + "description": "The sorts to apply to the results. Sorts at the start of the\r\nlist will be applied first.\r\n \r\nBy default, results are sorted by time period in descending order.", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The sorts to apply to the results. Sorts at the start of the\r\nlist will be applied first.\r\n \r\nBy default, results are sorted by time period in descending order.", + "nullable": true + } + }, + { + "name": "debug", + "in": "query", + "description": "Enable debug mode. Results will be formatted with human-readable\r\nlabels to assist in identification.\r\n \r\nThis **should not** be enabled in a production environment.", + "schema": { + "type": "boolean", + "description": "Enable debug mode. Results will be formatted with human-readable\r\nlabels to assist in identification.\r\n \r\nThis **should not** be enabled in a production environment." + } + }, + { + "name": "page", + "in": "query", + "description": "The page of results to fetch.", + "schema": { + "minimum": 1, + "type": "integer", + "description": "The page of results to fetch.", + "format": "int32", + "default": 1 + } + }, + { + "name": "pageSize", + "in": "query", + "description": "The maximum number of results per page.", + "schema": { + "maximum": 10000, + "minimum": 1, + "type": "integer", + "description": "The maximum number of results per page.", + "format": "int32", + "default": 1000 + } + } + ], + "responses": { + "200": { + "description": "The paginated list of query results.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataSetQueryPaginatedResultsViewModel" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationProblemViewModel" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + }, + "post": { + "tags": [ + "DataSets" + ], + "summary": "Query a data set (POST)", + "description": "Query a data set using a `POST` request, returning the filtered results.\r\n \r\nNote that for simpler queries or exploratory testing, there is also GET variant of this endpoint\r\nonly handles a smaller subset of querying functionality. However, for most use-cases,\r\nthis endpoint is recommended as it provides the complete set of functionality.\r\n \r\nUnlike the `GET` endpoint, the `POST` endpoint allows condition criteria (`and`, `or`, `not`)\r\nand consequently can express more complex queries.", + "operationId": "QueryDataSetPost", + "parameters": [ + { + "name": "dataSetId", + "in": "path", + "description": "The ID of the data set.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "name": "dataSetVersion", + "in": "query", + "description": "The version of the data set to use e.g. 2.0, 1.1, etc.", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryRequest" + } + ], + "description": "A data set query to match results against." + } + } + } + }, + "responses": { + "200": { + "description": "The paginated list of query results.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataSetQueryPaginatedResultsViewModel" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationProblemViewModel" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + } + }, + "/api/v1/data-sets/{dataSetId}/csv": { + "get": { + "tags": [ + "DataSets" + ], + "summary": "Download a data set as CSV", + "description": "The CSV response will render its metadata in a human-readable format (instead of\r\nmachine-readable IDs). The CSV is not subject to the same backward compatibility\r\nguarantees as the data set's JSON representation in other endpoints.", + "operationId": "DownloadDataSetCsv", + "parameters": [ + { + "name": "dataSetId", + "in": "path", + "description": "The ID of the data set.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "name": "dataSetVersion", + "in": "query", + "description": "The version of the data set to use e.g. 2.0, 1.1, etc.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The data set CSV file.", + "content": { + "text/csv": { + "schema": { + "type": "string" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + } + }, + "/api/v1/data-sets/{dataSetId}/versions": { + "get": { + "tags": [ + "DataSetVersions" + ], + "summary": "List a data set’s versions", + "description": "List a data set’s versions. Only provides summary information of each version.", + "operationId": "ListDataSetVersions", + "parameters": [ + { + "name": "page", + "in": "query", + "description": "The page of results to fetch.", + "schema": { + "minimum": 1, + "type": "integer", + "description": "The page of results to fetch.", + "format": "int32", + "default": 1 + } + }, + { + "name": "pageSize", + "in": "query", + "description": "The maximum number of results per page.", + "schema": { + "maximum": 20, + "minimum": 1, + "type": "integer", + "description": "The maximum number of results per page.", + "format": "int32", + "default": 10 + } + }, + { + "name": "dataSetId", + "in": "path", + "description": "The ID of the data set.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "The paginated list of data set versions.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataSetVersionPaginatedListViewModel" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationProblemViewModel" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + } + }, + "/api/v1/data-sets/{dataSetId}/versions/{dataSetVersion}": { + "get": { + "tags": [ + "DataSetVersions" + ], + "summary": "Get a data set version", + "description": "Get a data set version's summary details.", + "operationId": "GetDataSetVersion", + "parameters": [ + { + "name": "dataSetId", + "in": "path", + "description": "The ID of the data set.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "name": "dataSetVersion", + "in": "path", + "description": "The data set version e.g. 1.0, 1.1, 2.0, etc.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The requested data set version.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataSetVersionViewModel" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + } + }, + "/api/v1/data-sets/{dataSetId}/versions/{dataSetVersion}/changes": { + "get": { + "tags": [ + "DataSetVersions" + ], + "summary": "Get a data set version's changes", + "description": "Lists the changes made by a data set version relative to the version prior to it.", + "operationId": "GetDataSetVersionChanges", + "parameters": [ + { + "name": "dataSetId", + "in": "path", + "description": "The ID of the data set.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "name": "dataSetVersion", + "in": "path", + "description": "The data set version e.g. 1.0, 1.1, 2.0, etc.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The changes for the data set version.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataSetVersionChangesViewModel" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + } + }, + "/api/v1/publications": { + "get": { + "tags": [ + "Publications" + ], + "summary": "List publications", + "description": "Lists details about publications with data available for querying.", + "operationId": "ListPublications", + "parameters": [ + { + "name": "search", + "in": "query", + "description": "A search term to find matching publications.", + "schema": { + "minLength": 3, + "type": "string", + "description": "A search term to find matching publications.", + "nullable": true, + "example": "Pupil absence" + }, + "example": "Pupil absence" + }, + { + "name": "page", + "in": "query", + "description": "The page of results to fetch.", + "schema": { + "minimum": 1, + "type": "integer", + "description": "The page of results to fetch.", + "format": "int32", + "default": 1 + } + }, + { + "name": "pageSize", + "in": "query", + "description": "The maximum number of results per page.", + "schema": { + "maximum": 40, + "minimum": 1, + "type": "integer", + "description": "The maximum number of results per page.", + "format": "int32", + "default": 20 + } + } + ], + "responses": { + "200": { + "description": "The paginated list of publications.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PublicationPaginatedListViewModel" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationProblemViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + } + }, + "/api/v1/publications/{publicationId}": { + "get": { + "tags": [ + "Publications" + ], + "summary": "Get a publication’s details", + "description": "Get a specific publication's summary details.", + "operationId": "GetPublication", + "parameters": [ + { + "name": "publicationId", + "in": "path", + "description": "The ID of the publication.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "The requested publication summary.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PublicationSummaryViewModel" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetailsViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + } + }, + "/api/v1/publications/{publicationId}/data-sets": { + "get": { + "tags": [ + "Publications" + ], + "summary": "List a publication’s data sets", + "description": "Lists summary details of all the data sets related to a publication.", + "operationId": "ListPublicationDataSets", + "parameters": [ + { + "name": "page", + "in": "query", + "description": "The page of results to fetch.", + "schema": { + "minimum": 1, + "type": "integer", + "description": "The page of results to fetch.", + "format": "int32", + "default": 1 + } + }, + { + "name": "pageSize", + "in": "query", + "description": "The maximum number of results per page.", + "schema": { + "maximum": 20, + "minimum": 1, + "type": "integer", + "description": "The maximum number of results per page.", + "format": "int32", + "default": 10 + } + }, + { + "name": "publicationId", + "in": "path", + "description": "The ID of the publication.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "The paginated list of data sets.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataSetPaginatedListViewModel" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationProblemViewModel" + } + } + } + }, + "504": { + "description": "Timeout" + } + } + } + } + }, + "components": { + "schemas": { + "ChangeSetViewModel": { + "type": "object", + "properties": { + "filters": { + "type": "array", + "items": { + "$ref": "#/components/schemas/FilterChangeViewModel" + }, + "description": "A list of any filter changes made to the data set.", + "nullable": true + }, + "filterOptions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/FilterOptionChangesViewModel" + }, + "description": "A list of any filter option changes made to the data set.", + "nullable": true + }, + "geographicLevels": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GeographicLevelChangeViewModel" + }, + "description": "A list of any geographic level changes made to the data set.", + "nullable": true + }, + "indicators": { + "type": "array", + "items": { + "$ref": "#/components/schemas/IndicatorChangeViewModel" + }, + "description": "A list of any indicator changes made to the data set.", + "nullable": true + }, + "locationGroups": { + "type": "array", + "items": { + "$ref": "#/components/schemas/LocationGroupChangeViewModel" + }, + "description": "A list of any location group changes made to the data set.", + "nullable": true + }, + "locationOptions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/LocationOptionChangesViewModel" + }, + "description": "A list of any location option changes made to the data set.", + "nullable": true + }, + "timePeriods": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TimePeriodOptionChangeViewModel" + }, + "description": "A list of any time period changes made to the data set.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "A set of changes grouped by their type (major or minor)." + }, + "DataSetGetQueryFilters": { + "type": "object", + "properties": { + "eq": { + "type": "string", + "description": "Filter the results to have a filter option matching this ID.", + "nullable": true, + "example": "pVAkV" + }, + "notEq": { + "type": "string", + "description": "Filter the results to not have a filter option matching this ID.", + "nullable": true, + "example": "wUzft" + }, + "in": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Filter the results to have a filter option matching at least one of these IDs.", + "nullable": true, + "example": [ + "n0WqP", + "3dCWP" + ] + }, + "notIn": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Filter the results to not have a filter option matching any of these IDs.", + "nullable": true, + "example": [ + "RMxeh", + "wUzft" + ] + } + }, + "additionalProperties": false, + "description": "The filter option criteria to filter results by in a data set GET query." + }, + "DataSetGetQueryGeographicLevels": { + "type": "object", + "properties": { + "eq": { + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "Filter the results to be in this geographic level.", + "nullable": true, + "example": "NAT" + }, + "notEq": { + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "Filter the results to not be in this geographic level.", + "nullable": true, + "example": "PROV" + }, + "in": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GeographicLevelCode" + }, + "description": "Filter the results to be in one of these geographic levels.", + "nullable": true, + "example": [ + "NAT", + "LA" + ] + }, + "notIn": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GeographicLevelCode" + }, + "description": "Filter the results to not be in one of these geographic levels.", + "nullable": true, + "example": [ + "REG", + "LAD" + ] + } + }, + "additionalProperties": false, + "description": "The geographic levels criteria to filter results by in a data set GET query." + }, + "DataSetGetQueryLocations": { + "type": "object", + "properties": { + "eq": { + "type": "string", + "description": "Filter the results to be in this location.", + "nullable": true, + "example": "NAT|id|3dCWP" + }, + "notEq": { + "type": "string", + "description": "Filter the results to not be in this location.", + "nullable": true, + "example": "REG|code|E12000003" + }, + "in": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Filter the results to be in one of these locations.", + "nullable": true, + "example": [ + "LA|code|E08000003", + "LA|oldCode|373" + ] + }, + "notIn": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Filter the results not to be in one of these locations.", + "nullable": true, + "example": [ + "SCH|urn|123456", + "SCH|laEstab|1234567" + ] + } + }, + "additionalProperties": false, + "description": "The location option criteria to filter results by in a data set GET query.\r\n \r\nThe results can be matched by either the location option's ID or a code.\r\nNote the following differences:\r\n \r\n- IDs only match a **single location**\r\n- Codes may match **multiple locations**\r\n \r\nWhilst codes are generally unique to a single location, they can be\r\nused for multiple locations. This may match more results than you\r\nexpect so it's recommended to use IDs where possible." + }, + "DataSetGetQueryRequest": { + "required": [ + "indicators" + ], + "type": "object", + "properties": { + "filters": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetGetQueryFilters" + } + ], + "description": "Query criteria relating to filter options.", + "nullable": true + }, + "geographicLevels": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetGetQueryGeographicLevels" + } + ], + "description": "Query criteria relating to geographic levels.", + "nullable": true + }, + "locations": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetGetQueryLocations" + } + ], + "description": "Query criteria relating to location options.", + "nullable": true + }, + "timePeriods": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetGetQueryTimePeriods" + } + ], + "description": "Query criteria relating to time periods.", + "nullable": true + }, + "indicators": { + "minItems": 1, + "type": "array", + "items": { + "maxLength": 40, + "minLength": 1, + "type": "string" + }, + "description": "The IDs of indicators to return values for." + }, + "sorts": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The sorts to apply to the results. Sorts at the start of the\r\nlist will be applied first.\r\n \r\nBy default, results are sorted by time period in descending order.", + "nullable": true + }, + "debug": { + "type": "boolean", + "description": "Enable debug mode. Results will be formatted with human-readable\r\nlabels to assist in identification.\r\n \r\nThis **should not** be enabled in a production environment." + }, + "page": { + "minimum": 1, + "type": "integer", + "description": "The page of results to fetch.", + "format": "int32", + "default": 1 + }, + "pageSize": { + "maximum": 10000, + "minimum": 1, + "type": "integer", + "description": "The maximum number of results per page.", + "format": "int32", + "default": 1000 + } + }, + "additionalProperties": false, + "description": "A data set GET query request." + }, + "DataSetGetQueryTimePeriods": { + "type": "object", + "properties": { + "eq": { + "type": "string", + "description": "Filter the results to be in this time period.", + "nullable": true, + "example": "2024|CY" + }, + "notEq": { + "type": "string", + "description": "Filter the results to not be in this time period.", + "nullable": true, + "example": "2024/2025|AY" + }, + "in": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Filter the results to be in one of these time periods.", + "nullable": true, + "example": [ + "2022|CY", + "2023|CY", + "2024|CY" + ] + }, + "notIn": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Filter the results to not be in one of these time periods.", + "nullable": true, + "example": [ + "2020|M1", + "2020|M2", + "2020|M3" + ] + }, + "gt": { + "type": "string", + "description": "Filter the results to be in a time period that is\r\nchronologically greater than the one specified.", + "nullable": true, + "example": "2017/2018|AY" + }, + "gte": { + "type": "string", + "description": "Filter the results to be in a time period that is\r\nchronologically greater than or equal to the one specified.", + "nullable": true, + "example": "2017|CY" + }, + "lt": { + "type": "string", + "description": "Filter the results to be in a time period that is\r\nchronologically less than the one specified.", + "nullable": true, + "example": "2023/2024|AY" + }, + "lte": { + "type": "string", + "description": "Filter the results to be in a time period that is\r\nchronologically less than or equal to the one specified.", + "nullable": true, + "example": "2023|CY" + } + }, + "additionalProperties": false, + "description": "The time period criteria to filter results by in a data set GET query." + }, + "DataSetLatestVersionViewModel": { + "required": [ + "file", + "filters", + "geographicLevels", + "indicators", + "published", + "timePeriods", + "totalResults", + "version" + ], + "type": "object", + "properties": { + "version": { + "type": "string", + "description": "The version number. Follows semantic versioning e.g. 2.0 (major), 1.1 (minor).", + "example": "2.0" + }, + "published": { + "type": "string", + "description": "When the version was published.", + "format": "date-time", + "example": "2024-03-01T09:30:00+00:00" + }, + "totalResults": { + "type": "integer", + "description": "The total number of results available to query in the data set.", + "format": "int64", + "example": 1000000 + }, + "file": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetVersionFileViewModel" + } + ], + "description": "The file that this data set version is based on." + }, + "timePeriods": { + "allOf": [ + { + "$ref": "#/components/schemas/TimePeriodRangeViewModel" + } + ], + "description": "The time period range covered by the data set." + }, + "geographicLevels": { + "type": "array", + "items": { + "enum": [ + "English devolved area", + "Local authority", + "Local authority district", + "Local enterprise partnership", + "Local skills improvement plan area", + "Institution", + "Mayoral combined authority", + "MAT", + "National", + "Opportunity area", + "Parliamentary constituency", + "Provider", + "Regional", + "RSC region", + "School", + "Sponsor", + "Ward", + "Planning area" + ], + "type": "string" + }, + "description": "The geographic levels available in the data set.", + "example": [ + "National", + "Regional", + "Local authority" + ] + }, + "filters": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The filters available in the data set.", + "example": [ + "Characteristic", + "School type" + ] + }, + "indicators": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The indicators available in the data set." + } + }, + "additionalProperties": false, + "description": "Provides high-level information about the latest version of a data set." + }, + "DataSetListRequest": { + "type": "object", + "properties": { + "page": { + "minimum": 1, + "type": "integer", + "description": "The page of results to fetch.", + "format": "int32", + "default": 1 + }, + "pageSize": { + "maximum": 20, + "minimum": 1, + "type": "integer", + "description": "The maximum number of results per page.", + "format": "int32", + "default": 10 + } + }, + "additionalProperties": false + }, + "DataSetMetaRequest": { + "type": "object", + "properties": { + "types": { + "type": "array", + "items": { + "enum": [ + "Filters", + "Indicators", + "Locations", + "TimePeriods" + ], + "type": "string" + }, + "description": "The types of meta to get for the requested data set version.\r\n\r\nCan be any combination of the following:\r\n\r\n- `Filters` - include all meta relating to *filters*\r\n- `Indicators` - include all meta relating to *indicators*\r\n- `Locations` - include all meta relating to *locations*\r\n- `TimePeriods` - include all meta relating to *time periods*", + "nullable": true, + "example": [ + "Filters", + "Locations" + ] + } + }, + "additionalProperties": false + }, + "DataSetMetaViewModel": { + "required": [ + "filters", + "geographicLevels", + "indicators", + "locations", + "timePeriods" + ], + "type": "object", + "properties": { + "filters": { + "type": "array", + "items": { + "$ref": "#/components/schemas/FilterOptionsViewModel" + }, + "description": "All the filters associated with the data set." + }, + "indicators": { + "type": "array", + "items": { + "$ref": "#/components/schemas/IndicatorViewModel" + }, + "description": "All the indicators associated with the data set." + }, + "geographicLevels": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GeographicLevelViewModel" + }, + "description": "All the geographic levels associated with the data set." + }, + "locations": { + "type": "array", + "items": { + "$ref": "#/components/schemas/LocationGroupOptionsViewModel" + }, + "description": "All the locations associated with the data set, grouped by geographic level." + }, + "timePeriods": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TimePeriodOptionViewModel" + }, + "description": "All the time periods associated with the data set." + } + }, + "additionalProperties": false, + "description": "All the metadata associated with a data set." + }, + "DataSetPaginatedListViewModel": { + "required": [ + "paging", + "results" + ], + "type": "object", + "properties": { + "paging": { + "allOf": [ + { + "$ref": "#/components/schemas/PagingViewModel" + } + ], + "description": "Provides metadata for use in pagination." + }, + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DataSetViewModel" + }, + "description": "The list of results for this page." + } + }, + "additionalProperties": false, + "description": "A paginated list of data sets." + }, + "DataSetQueryCriteriaAnd": { + "required": [ + "and" + ], + "type": "object", + "properties": { + "and": { + "minItems": 1, + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/DataSetQueryCriteriaAnd" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaOr" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaNot" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaFacets" + } + ] + }, + "description": "The sub-criteria which all must resolve to true.", + "example": [ + { + "filters": { + "eq": "pVAkV" + } + }, + { + "locations": { + "eq": { + "level": "LA", + "code": "E08000019" + } + } + } + ] + } + }, + "additionalProperties": false, + "description": "A condition criteria where one or more sub-criteria must all resolve\r\nto true for the overall query to match any results.\r\n \r\nThis is equivalent to the `AND` operator in SQL." + }, + "DataSetQueryCriteriaFacets": { + "type": "object", + "properties": { + "filters": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryCriteriaFilters" + } + ], + "description": "Query criteria relating to filter options.", + "nullable": true + }, + "geographicLevels": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryCriteriaGeographicLevels" + } + ], + "description": "Query criteria relating to geographic levels.", + "nullable": true + }, + "locations": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryCriteriaLocations" + } + ], + "description": "Query criteria relating to location options.", + "nullable": true + }, + "timePeriods": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryCriteriaTimePeriods" + } + ], + "description": "Query criteria relating to time periods.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "A set of criteria specifying which facets the query should match results with.\r\n \r\nAll parts of the criteria must resolve to true to match a result." + }, + "DataSetQueryCriteriaFilters": { + "type": "object", + "properties": { + "eq": { + "type": "string", + "description": "Filter the results to have a filter option matching this ID.", + "nullable": true, + "example": "pVAkV" + }, + "notEq": { + "type": "string", + "description": "Filter the results to not have a filter option matching this ID.", + "nullable": true, + "example": "wUzft" + }, + "in": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Filter the results to have a filter option matching at least one of these IDs.", + "nullable": true, + "example": [ + "q1g3J", + "ufp2K" + ] + }, + "notIn": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Filter the results to not have a filter option matching any of these IDs.", + "nullable": true, + "example": [ + "ksrK9", + "s1J8a" + ] + } + }, + "additionalProperties": false, + "description": "The filter option criteria to filter results by in a data set query." + }, + "DataSetQueryCriteriaGeographicLevels": { + "type": "object", + "properties": { + "eq": { + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "Filter the results to be in this geographic level.", + "nullable": true, + "example": "NAT" + }, + "notEq": { + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "Filter the results to not be in this geographic level.", + "nullable": true, + "example": "PROV" + }, + "in": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GeographicLevelCode" + }, + "description": "Filter the results to be in one of these geographic levels.", + "nullable": true, + "example": [ + "REG", + "LA" + ] + }, + "notIn": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GeographicLevelCode" + }, + "description": "Filter the results to not be in one of these geographic levels.", + "nullable": true, + "example": [ + "LAD", + "SCH" + ] + } + }, + "additionalProperties": false, + "description": "The geographic levels criteria to filter results by in a data set query." + }, + "DataSetQueryCriteriaLocations": { + "type": "object", + "properties": { + "eq": { + "oneOf": [ + { + "$ref": "#/components/schemas/DataSetQueryLocationId" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationLocalAuthorityCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationLocalAuthorityOldCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationProviderUkprn" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationSchoolLaEstab" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationSchoolUrn" + } + ], + "description": "Filter the results to be in this location.", + "nullable": true + }, + "notEq": { + "oneOf": [ + { + "$ref": "#/components/schemas/DataSetQueryLocationId" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationLocalAuthorityCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationLocalAuthorityOldCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationProviderUkprn" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationSchoolLaEstab" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationSchoolUrn" + } + ], + "description": "Filter the results to not be in this location.", + "nullable": true + }, + "in": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/DataSetQueryLocationId" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationLocalAuthorityCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationLocalAuthorityOldCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationProviderUkprn" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationSchoolLaEstab" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationSchoolUrn" + } + ] + }, + "description": "Filter the results to be in one of these locations.", + "nullable": true + }, + "notIn": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/DataSetQueryLocationId" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationLocalAuthorityCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationLocalAuthorityOldCode" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationProviderUkprn" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationSchoolLaEstab" + }, + { + "$ref": "#/components/schemas/DataSetQueryLocationSchoolUrn" + } + ] + }, + "description": "Filter the results to not be in one of these locations.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "The location option criteria to filter results by in a data set query.\r\n \r\nThe results can be matched by either the location option's ID or a code.\r\nNote the following differences:\r\n \r\n- IDs only match a **single location**\r\n- Codes may match **multiple locations**\r\n \r\nWhilst codes are generally unique to a single location, they can be\r\nused for multiple locations. This may match more results than you\r\nexpect so it's recommended to use IDs where possible." + }, + "DataSetQueryCriteriaNot": { + "required": [ + "not" + ], + "type": "object", + "properties": { + "not": { + "oneOf": [ + { + "$ref": "#/components/schemas/DataSetQueryCriteriaAnd" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaOr" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaNot" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaFacets" + } + ], + "description": "The sub-criteria which must resolve to false.", + "example": { + "filters": { + "eq": "pVAkV" + } + } + } + }, + "additionalProperties": false, + "description": "A condition criteria where its sub-criteria must resolve\r\nto *false* for the overall query to match any results.\r\n \r\nThis is equivalent to the `NOT` operator in SQL." + }, + "DataSetQueryCriteriaOr": { + "required": [ + "or" + ], + "type": "object", + "properties": { + "or": { + "minItems": 1, + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/DataSetQueryCriteriaAnd" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaOr" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaNot" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaFacets" + } + ] + }, + "description": "The sub-criteria where one must resolve to true.", + "example": [ + { + "filters": { + "eq": "pVAkV" + } + }, + { + "locations": { + "eq": { + "level": "LA", + "code": "E08000019" + } + } + } + ] + } + }, + "additionalProperties": false, + "description": "A condition criteria where at least one sub-criteria must resolve\r\nto true for the overall query to match any results.\r\n \r\nThis is equivalent to the `OR` operator in SQL." + }, + "DataSetQueryCriteriaTimePeriods": { + "type": "object", + "properties": { + "eq": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryTimePeriod" + } + ], + "description": "Filter the results to be in this time period.", + "nullable": true + }, + "notEq": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryTimePeriod" + } + ], + "description": "Filter the results to not be in this time period.", + "nullable": true + }, + "in": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DataSetQueryTimePeriod" + }, + "description": "Filter the results to be in one of these time periods.", + "nullable": true + }, + "notIn": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DataSetQueryTimePeriod" + }, + "description": "Filter the results to not be in one of these time periods.", + "nullable": true + }, + "gt": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryTimePeriod" + } + ], + "description": "Filter the results to be in a time period that is\r\nchronologically greater than the one specified.", + "nullable": true + }, + "gte": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryTimePeriod" + } + ], + "description": "Filter the results to be in a time period that is\r\nchronologically greater than or equal to the one specified.", + "nullable": true + }, + "lt": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryTimePeriod" + } + ], + "description": "Filter the results to be in a time period that is\r\nchronologically less than the one specified.", + "nullable": true + }, + "lte": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetQueryTimePeriod" + } + ], + "description": "Filter the results to be in a time period that is\r\nchronologically less than or equal to the one specified.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "The time period criteria to filter results by in a data set query." + }, + "DataSetQueryLocationCode": { + "required": [ + "code", + "level" + ], + "type": "object", + "properties": { + "code": { + "maxLength": 30, + "minLength": 1, + "type": "string", + "description": "The code of the location. This may be an ONS code, or some\r\nother code that identifies the location.", + "example": "E12000003" + }, + "level": { + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "The geographic level of the location.\r\n \r\nThis should be a valid geographic level code e.g. `NAT`, `REG`, `LA`.", + "example": "NAT" + } + }, + "additionalProperties": false, + "description": "A location (identified by a code) to filter results by.\r\n \r\nNote that location codes may correspond to multiple locations in the same geographic level." + }, + "DataSetQueryLocationId": { + "required": [ + "id", + "level" + ], + "type": "object", + "properties": { + "id": { + "maxLength": 10, + "minLength": 1, + "type": "string", + "description": "The ID of the location. If specified, this will be used\r\ninstead of any codes or other types of identifier.", + "example": "2tYX" + }, + "level": { + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "The geographic level of the location.\r\n \r\nThis should be a valid geographic level code e.g. `NAT`, `REG`, `LA`.", + "example": "NAT" + } + }, + "additionalProperties": false, + "description": "A location (identified by an ID) to filter results by.\r\n \r\nNote that location IDs are guaranteed to be unique to a single location\r\nunlike location codes (which may correspond to multiple locations)." + }, + "DataSetQueryLocationLocalAuthorityCode": { + "required": [ + "code", + "level" + ], + "type": "object", + "properties": { + "code": { + "maxLength": 30, + "minLength": 1, + "type": "string", + "description": "The ONS code of the local authority. This is typically 9 characters\r\nin the standard ONS format for local authorities e.g. `E08000019`,\r\nbut may be a combination of multiple codes e.g. `E08000019 / E08000020`.", + "example": "E08000019" + }, + "level": { + "minLength": 1, + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "The geographic level of the local authority. Must be set to `LA`.", + "example": "LA" + } + }, + "additionalProperties": false, + "description": "A local authority (identified by its ONS code) to filter results by." + }, + "DataSetQueryLocationLocalAuthorityOldCode": { + "required": [ + "oldCode" + ], + "type": "object", + "properties": { + "oldCode": { + "maxLength": 20, + "minLength": 1, + "type": "string", + "description": "The old code (previously the LEA code) of the local authority.\r\nThis is typically a 3-digit number e.g. `318`, but may be a\r\ncombination of multiple codes e.g. `318 / 319`.", + "example": "373" + }, + "level": { + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "The geographic level of the local authority. Must be set to `LA`.", + "example": "LA" + } + }, + "additionalProperties": false, + "description": "A local authority (identified by its old code) to filter results by." + }, + "DataSetQueryLocationProviderUkprn": { + "required": [ + "level", + "ukprn" + ], + "type": "object", + "properties": { + "ukprn": { + "maxLength": 20, + "minLength": 1, + "type": "string", + "description": "The UKPRN (UK provider reference number) of the provider.\r\nThis is typically an 8-digit number.", + "example": "123454678" + }, + "level": { + "minLength": 1, + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "The geographic level of the provider. Must be set to `PROV`.", + "example": "PROV" + } + }, + "additionalProperties": false, + "description": "A provider (identified by its UKPRN) to filter results by." + }, + "DataSetQueryLocationSchoolLaEstab": { + "required": [ + "laEstab", + "level" + ], + "type": "object", + "properties": { + "laEstab": { + "maxLength": 20, + "minLength": 1, + "type": "string", + "description": "The LAESTAB (local authority establishment number) of the school.\r\nThis is typically a 7-digit number.", + "example": "1234567" + }, + "level": { + "minLength": 1, + "type": "string", + "description": "The geographic level of the location. Must be set to `SCH`.", + "example": "SCH" + } + }, + "additionalProperties": false, + "description": "A school (identified by its LAESTAB) to filter results by." + }, + "DataSetQueryLocationSchoolUrn": { + "required": [ + "level", + "urn" + ], + "type": "object", + "properties": { + "urn": { + "maxLength": 20, + "minLength": 1, + "type": "string", + "description": "The URN (unique reference number) of the school.\r\nThis is typically a 6-digit number.", + "example": "123456" + }, + "level": { + "minLength": 1, + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "The geographic level of the school. Must be set to `SCH`.", + "example": "SCH" + } + }, + "additionalProperties": false, + "description": "A school (identified by its URN) to filter results by." + }, + "DataSetQueryPaginatedResultsViewModel": { + "required": [ + "paging", + "results", + "warnings" + ], + "type": "object", + "properties": { + "paging": { + "allOf": [ + { + "$ref": "#/components/schemas/PagingViewModel" + } + ], + "description": "Provides metadata for use in pagination." + }, + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DataSetQueryResultViewModel" + }, + "description": "The list of results for this page." + }, + "warnings": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WarningViewModel" + }, + "description": "A list of warnings, highlighting any potential issues with the request." + } + }, + "additionalProperties": false, + "description": "A paginated list of results from a data set query." + }, + "DataSetQueryRequest": { + "required": [ + "indicators" + ], + "type": "object", + "properties": { + "criteria": { + "oneOf": [ + { + "$ref": "#/components/schemas/DataSetQueryCriteriaAnd" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaOr" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaNot" + }, + { + "$ref": "#/components/schemas/DataSetQueryCriteriaFacets" + } + ], + "description": "The criteria to match.", + "nullable": true + }, + "indicators": { + "minItems": 1, + "type": "array", + "items": { + "maxLength": 40, + "minLength": 1, + "type": "string" + }, + "description": "The IDs of indicators in the data set to return values for.", + "example": [ + "C2ySJ", + "q4X3J" + ] + }, + "sorts": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DataSetQuerySort" + }, + "description": "The sorts to sort the results by. Sorts at the start of the\r\nlist will be applied first.\r\n \r\nBy default, results are sorted by time period in descending order.", + "nullable": true + }, + "debug": { + "type": "boolean", + "description": "Enable debug mode. Results will be formatted with human-readable\r\nlabels to assist in identification.\r\n \r\nThis **should not** be enabled in a production environment." + }, + "page": { + "minimum": 1, + "type": "integer", + "description": "The page of results to fetch.", + "format": "int32", + "default": 1 + }, + "pageSize": { + "maximum": 10000, + "minimum": 1, + "type": "integer", + "description": "The maximum number of results per page.", + "format": "int32", + "default": 1000 + } + }, + "additionalProperties": false, + "description": "A data set query to match results against." + }, + "DataSetQueryResultViewModel": { + "required": [ + "filters", + "geographicLevel", + "locations", + "timePeriod", + "values" + ], + "type": "object", + "properties": { + "timePeriod": { + "allOf": [ + { + "$ref": "#/components/schemas/TimePeriodViewModel" + } + ], + "description": "The time period relevant to this result's data." + }, + "geographicLevel": { + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "The geographic level relevant to this result's data." + }, + "locations": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "The locations relevant to this result's data.\r\n \r\nThis is a dictionary where the key is the location's geographic\r\nlevel and the value is the location's ID.", + "example": { + "NAT": "04bTr", + "REG": "4veOu", + "LA": "owqlK" + } + }, + "filters": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "The filters associated with this result.\r\n \r\nThis is a dictionary where the key is the filter ID and\r\nthe value is the specific filter option ID.", + "example": { + "ups2K": "n0WqP", + "j51wV": "AnZsi", + "hAkBQ": "dvB4z" + } + }, + "values": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "The data values for the result's indicators.\r\n \r\nThis is a dictionary where the key is the indicator ID\r\nand the value is the data value.", + "example": { + "wLcft": "23593018", + "4S8Ou": "50.342", + "9kVFg": "25369172" + } + } + }, + "additionalProperties": false, + "description": "A result from a data set query containing its values and any facets relevant to it." + }, + "DataSetQuerySort": { + "required": [ + "direction", + "field" + ], + "type": "object", + "properties": { + "field": { + "maxLength": 40, + "minLength": 1, + "type": "string", + "description": "The name of the field to sort by. This can be one of the following:\r\n \r\n- `timePeriod` to sort by time period\r\n- `geographicLevel` to sort by the geographic level of the data\r\n- `location|{level}` to sort by locations in a geographic level where `{level}` is the level code (e.g. `REG`, `LA`)\r\n- `filter|{id}` to sort by the options in a filter where `{id}` is the filter ID (e.g. `3RxWP`)\r\n- `indicator|{id}` to sort by the values in a indicator where `{id}` is the indicator ID (e.g. `6VfPgZ`)", + "example": "timePeriod" + }, + "direction": { + "enum": [ + "Asc", + "Desc" + ], + "type": "string", + "description": "The direction to sort by. This can be one of the following:\r\n \r\n- `Asc` - sort by ascending order\r\n- `Desc` - sort by descending order", + "example": "Asc" + } + }, + "additionalProperties": false, + "description": "Specifies how the results from a data set query should be sorted.\r\n \r\nThis consists of a field in the data set and a direction to sort by." + }, + "DataSetQueryTimePeriod": { + "required": [ + "code", + "period" + ], + "type": "object", + "properties": { + "period": { + "type": "string", + "description": "The time period to match results by.\r\n \r\nThis should be a single year like `2020` or a range like `2020/2021`.", + "example": "2020/2021" + }, + "code": { + "type": "string", + "allOf": [ + { + "$ref": "#/components/schemas/TimePeriodCode" + } + ], + "description": "The code identifying the time period type to match results by.\r\n \r\nThis should be a valid time period code e.g. `AY`, `CY`, `M1`, `W20`.", + "example": "AY" + } + }, + "additionalProperties": false, + "description": "A time period to filter results by.\r\n \r\n- `period` is the time period or range (e.g. `2020` or `2020/2021`)\r\n- `code` is the code identifying the time period type (e.g. `AY`, `CY`, `M1`, `W20`)\r\n \r\nThe `period` should be a single year like `2020`, or a range like `2020/2021`.\r\nCurrently, only years (or year ranges) are supported.\r\n \r\nSome time period types span two years e.g. financial year part 2 (`P2`), or may fall in a\r\nlatter year e.g. academic year summer term (`T3`). For these types, a singular year `period`\r\nlike `2020` is considered as `2020/2021`.\r\n \r\nFor example, a `period` value of `2020` is applicable to the following time periods:\r\n \r\n- 2020 calendar year\r\n- 2020/2021 academic year\r\n- 2020/2021 financial year part 2 (October to March)\r\n- 2020/2021 academic year's summer term\r\n \r\nIf you wish to be more explicit, you may use a range for the `period` e.g. `2020/2021`.\r\nHowever, a range cannot be used with time period types which only span a single year,\r\nfor example, `2020/21` cannot be used with `CY`, `M` or `W` codes.\r\n \r\nSome examples:\r\n \r\n- `2020|AY` is the 2020/21 academic year\r\n- `2021|FY` is the 2021/22 financial year\r\n- `2020|T3` is the 2020/21 academic year's summer term\r\n- `2020|P2` is the 2020/21 financial year part 2 (October to March)\r\n- `2020|CY` is the 2020 calendar year\r\n- `2020|W32` is 2020 week 32\r\n- `2020/2021|AY` is the 2020/21 academic year\r\n- `2021/2022|FY` is the 2021/22 financial year" + }, + "DataSetStatus": { + "enum": [ + "Published", + "Deprecated", + "Withdrawn" + ], + "type": "string" + }, + "DataSetVersionChangesViewModel": { + "required": [ + "majorChanges", + "minorChanges" + ], + "type": "object", + "properties": { + "majorChanges": { + "allOf": [ + { + "$ref": "#/components/schemas/ChangeSetViewModel" + } + ], + "description": "Any major changes that were made to the data set." + }, + "minorChanges": { + "allOf": [ + { + "$ref": "#/components/schemas/ChangeSetViewModel" + } + ], + "description": "Any minor changes that were made to the data set." + } + }, + "additionalProperties": false, + "description": "A set of changes made to a data set version." + }, + "DataSetVersionFileViewModel": { + "required": [ + "id" + ], + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The ID of the file.", + "format": "uuid", + "example": "e0754872-3206-4918-aad4-029eaaae191f" + } + }, + "additionalProperties": false, + "description": "Details about the file a data set version is based on." + }, + "DataSetVersionListRequest": { + "type": "object", + "properties": { + "page": { + "minimum": 1, + "type": "integer", + "description": "The page of results to fetch.", + "format": "int32", + "default": 1 + }, + "pageSize": { + "maximum": 20, + "minimum": 1, + "type": "integer", + "description": "The maximum number of results per page.", + "format": "int32", + "default": 10 + } + }, + "additionalProperties": false + }, + "DataSetVersionPaginatedListViewModel": { + "required": [ + "paging", + "results" + ], + "type": "object", + "properties": { + "paging": { + "allOf": [ + { + "$ref": "#/components/schemas/PagingViewModel" + } + ], + "description": "Provides metadata for use in pagination." + }, + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DataSetVersionViewModel" + }, + "description": "The list of results for this page." + } + }, + "additionalProperties": false, + "description": "A paginated list of data set versions." + }, + "DataSetVersionReleaseViewModel": { + "required": [ + "slug", + "title" + ], + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "The title of the release.", + "example": "Spring term 2023/24" + }, + "slug": { + "type": "string", + "description": "The slug of the release.", + "example": "2023-24-spring-term" + } + }, + "additionalProperties": false, + "description": "Details about the statistical release a data set version was published with." + }, + "DataSetVersionStatus": { + "enum": [ + "Published", + "Deprecated", + "Withdrawn" + ], + "type": "string" + }, + "DataSetVersionType": { + "enum": [ + "Major", + "Minor" + ], + "type": "string" + }, + "DataSetVersionViewModel": { + "required": [ + "file", + "filters", + "geographicLevels", + "indicators", + "notes", + "release", + "status", + "timePeriods", + "totalResults", + "type", + "version" + ], + "type": "object", + "properties": { + "version": { + "type": "string", + "description": "The version number. Follows semantic versioning e.g. 2.0 (major), 1.1 (minor).", + "example": "1.0" + }, + "type": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetVersionType" + } + ], + "description": "The version type. Can be one of the following:\r\n \r\n- `Major` - backwards incompatible changes are being introduced\r\n- `Minor` - backwards compatible changes are being introduced\r\n \r\nMajor versions typically indicate that some action may be required\r\nto ensure code that consumes the data set continues to work.\r\n \r\nMinor versions should not cause issues in the functionality of existing code.", + "example": "Major" + }, + "status": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetVersionStatus" + } + ], + "description": "The version’s status. Can be one of the following:\r\n \r\n- `Published` - the version is published and can be used\r\n- `Deprecated` - the version is being deprecated and will not be usable in the future\r\n- `Withdrawn` - the version has been withdrawn and can no longer be used", + "example": "Published" + }, + "published": { + "type": "string", + "description": "When the version was published.", + "format": "date-time", + "nullable": true, + "example": "2024-03-01T09:30:00+00:00" + }, + "withdrawn": { + "type": "string", + "description": "When the version was withdrawn.", + "format": "date-time", + "nullable": true, + "example": "2024-06-01T12:00:00+00:00" + }, + "notes": { + "type": "string", + "description": "Any notes about this version and its changes.", + "example": "Some notes about the version." + }, + "totalResults": { + "type": "integer", + "description": "The total number of results available to query in the data set.", + "format": "int64", + "example": 1000000 + }, + "file": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetVersionFileViewModel" + } + ], + "description": "The file that this data set version is based on." + }, + "release": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetVersionReleaseViewModel" + } + ], + "description": "The statistical release that this version was published with." + }, + "timePeriods": { + "allOf": [ + { + "$ref": "#/components/schemas/TimePeriodRangeViewModel" + } + ], + "description": "The time period range covered by the data set." + }, + "geographicLevels": { + "type": "array", + "items": { + "enum": [ + "English devolved area", + "Local authority", + "Local authority district", + "Local enterprise partnership", + "Local skills improvement plan area", + "Institution", + "Mayoral combined authority", + "MAT", + "National", + "Opportunity area", + "Parliamentary constituency", + "Provider", + "Regional", + "RSC region", + "School", + "Sponsor", + "Ward", + "Planning area" + ], + "type": "string" + }, + "description": "The geographic levels available in the data set.", + "example": [ + "National", + "Regional", + "Local authority" + ] + }, + "filters": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The filters available in the data set.", + "example": [ + "Characteristic", + "School type" + ] + }, + "indicators": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The indicators available in the data set.", + "example": [ + "Authorised absence rate", + "Overall absence rate" + ] + } + }, + "additionalProperties": false, + "description": "Provides high-level information about a data set version." + }, + "DataSetViewModel": { + "required": [ + "id", + "latestVersion", + "status", + "summary", + "title" + ], + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The ID of the data set.", + "format": "uuid", + "example": "7588c2d6-9e8a-4d84-8f19-6b8d52a01fbd" + }, + "title": { + "type": "string", + "description": "The title of the data set.", + "example": "Absence rates by geographic level" + }, + "summary": { + "type": "string", + "description": "A summary of the data set’s contents.", + "example": "Absence information for all enrolments in schools." + }, + "status": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetStatus" + } + ], + "description": "The status of the data set. Can be one of the following:\r\n \r\n- `Published` - the data set has been published and will receive updates\r\n- `Deprecated` - the data set is being discontinued and will no longer receive updates\r\n- `Withdrawn` - the data set has been withdrawn and can no longer be used", + "example": "Published" + }, + "supersedingDataSetId": { + "type": "string", + "description": "The ID of the data set that supersedes this data set (if it has been deprecated).", + "format": "uuid", + "nullable": true, + "example": "2118a6df-4934-4a1f-ad2e-4589d2b9ccaf" + }, + "latestVersion": { + "allOf": [ + { + "$ref": "#/components/schemas/DataSetLatestVersionViewModel" + } + ], + "description": "The latest published data set version." + } + }, + "additionalProperties": false, + "description": "Provides high-level information about a data set." + }, + "ErrorViewModel": { + "required": [ + "code", + "message" + ], + "type": "object", + "properties": { + "message": { + "type": "string", + "description": "The error message.", + "example": "Must be 50 characters or fewer." + }, + "path": { + "type": "string", + "description": "The path to the property on the request that the error relates to.\r\nMay be omitted or empty if no specific property of the\r\nrequest relates to the error (it is a 'global' error).", + "nullable": true, + "example": "person.name" + }, + "code": { + "type": "string", + "description": "The error's machine-readable code. Can be used for further\r\nprocessing of the error before presenting to users.\r\nMay be omitted if there is none.", + "example": "MaximumLength" + }, + "detail": { + "description": "Additional detail about the error that can be used to provide\r\nmore context to users. May be omitted if there is none.", + "nullable": true, + "example": { + "maxLength": 50 + } + } + }, + "additionalProperties": false, + "description": "Describes an error that occurred with the request. This will typically\r\nneed to be rectified before the request can be fully processed." + }, + "FilterChangeViewModel": { + "type": "object", + "properties": { + "currentState": { + "allOf": [ + { + "$ref": "#/components/schemas/FilterViewModel" + } + ], + "description": "The current state after the change was made.\r\nIf the change is an addition, this will be null.", + "nullable": true + }, + "previousState": { + "allOf": [ + { + "$ref": "#/components/schemas/FilterViewModel" + } + ], + "description": "The previous state before the change was made.\r\nIf the change is a deletion, this will be null.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "A change to a filter in a data set." + }, + "FilterOptionChangeViewModel": { + "type": "object", + "properties": { + "currentState": { + "allOf": [ + { + "$ref": "#/components/schemas/FilterOptionViewModel" + } + ], + "description": "The current state after the change was made.\r\nIf the change is an addition, this will be null.", + "nullable": true + }, + "previousState": { + "allOf": [ + { + "$ref": "#/components/schemas/FilterOptionViewModel" + } + ], + "description": "The previous state before the change was made.\r\nIf the change is a deletion, this will be null.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "A change to a filter option in a data set." + }, + "FilterOptionChangesViewModel": { + "required": [ + "filter", + "options" + ], + "type": "object", + "properties": { + "filter": { + "allOf": [ + { + "$ref": "#/components/schemas/FilterViewModel" + } + ], + "description": "The filter the option changes belong to." + }, + "options": { + "type": "array", + "items": { + "$ref": "#/components/schemas/FilterOptionChangeViewModel" + }, + "description": "The list of filter option changes." + } + }, + "additionalProperties": false, + "description": "A set of filter option changes and details of the filter they belong to." + }, + "FilterOptionViewModel": { + "required": [ + "id", + "label" + ], + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The ID of the filter option.", + "example": "q1g3J" + }, + "label": { + "type": "string", + "description": "The human-readable label describing the filter option.", + "example": "State-funded primary" + }, + "isAggregate": { + "type": "boolean", + "description": "Whether the filter option is an aggregate (i.e. ‘all’ or a ‘total’) of the other filter options.", + "nullable": true, + "example": false + } + }, + "additionalProperties": false, + "description": "A filterable option that can be used to filter a data set." + }, + "FilterOptionsViewModel": { + "required": [ + "column", + "id", + "label", + "options" + ], + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The ID of the filter.", + "example": "BRlj4" + }, + "column": { + "type": "string", + "description": "The name of the filter CSV column.", + "example": "school_type" + }, + "label": { + "type": "string", + "description": "The human-readable label describing the filter.", + "example": "School type" + }, + "hint": { + "type": "string", + "description": "A hint to assist in describing the filter.", + "example": "Additional detail about the filter." + }, + "options": { + "type": "array", + "items": { + "$ref": "#/components/schemas/FilterOptionViewModel" + }, + "description": "The filter options belonging to this filter." + } + }, + "additionalProperties": false, + "description": "The options available for a filterable characteristic about the data set\r\n(excluding geography or time)." + }, + "FilterViewModel": { + "required": [ + "column", + "id", + "label" + ], + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The ID of the filter.", + "example": "BRlj4" + }, + "column": { + "type": "string", + "description": "The name of the filter CSV column.", + "example": "school_type" + }, + "label": { + "type": "string", + "description": "The human-readable label describing the filter.", + "example": "School type" + }, + "hint": { + "type": "string", + "description": "A hint to assist in describing the filter.", + "example": "Additional detail about the filter." + } + }, + "additionalProperties": false, + "description": "A filterable characteristic (excluding geography or time) of a data set." + }, + "GeographicLevelChangeViewModel": { + "type": "object", + "properties": { + "currentState": { + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelViewModel" + } + ], + "description": "The current state after the change was made.\r\nIf the change is an addition, this will be null.", + "nullable": true + }, + "previousState": { + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelViewModel" + } + ], + "description": "The previous state before the change was made.\r\nIf the change is a deletion, this will be null.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "A change to a geographic level option in a data set." + }, + "GeographicLevelCode": { + "enum": [ + "EDA", + "INST", + "LA", + "LAD", + "LEP", + "LSIP", + "MAT", + "MCA", + "NAT", + "OA", + "PA", + "PCON", + "PROV", + "REG", + "RSC", + "SCH", + "SPON", + "WARD" + ], + "type": "string", + "description": "The code for a geographic level that locations are grouped by.\n\nThe allowed values are:\n\n- `EDA` - English devolved area\n- `INST` - Institution\n- `LA` - Local authority\n- `LAD` - Local authority district\n- `LEP` - Local enterprise partnership\n- `LSIP` - Local skills improvement plan area\n- `MCA` - Mayoral combined authority\n- `MAT` - MAT\n- `NAT` - National\n- `OA` - Opportunity area\n- `PA` - Planning area\n- `PCON` - Parliamentary constituency\n- `PROV` - Provider\n- `REG` - Regional\n- `RSC` - RSC region\n- `SCH` - School\n- `SPON` - Sponsor\n- `WARD` - Ward", + "example": "NAT" + }, + "GeographicLevelViewModel": { + "required": [ + "code" + ], + "type": "object", + "properties": { + "code": { + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelCode" + } + ], + "description": "The code for the geographic level.", + "example": "NAT" + }, + "label": { + "type": "string", + "description": "The human-readable label for the geographic level.", + "readOnly": true, + "example": "National" + } + }, + "additionalProperties": false, + "description": "A geographic level (e.g. national, regional) covered by a data set." + }, + "IndicatorChangeViewModel": { + "type": "object", + "properties": { + "currentState": { + "allOf": [ + { + "$ref": "#/components/schemas/IndicatorViewModel" + } + ], + "description": "The current state after the change was made.\r\nIf the change is an addition, this will be null.", + "nullable": true + }, + "previousState": { + "allOf": [ + { + "$ref": "#/components/schemas/IndicatorViewModel" + } + ], + "description": "The previous state before the change was made.\r\nIf the change is a deletion, this will be null.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "A change to an indicator in a data set." + }, + "IndicatorUnit": { + "enum": [ + "", + "%", + "£", + "£m", + "pp", + "string" + ], + "type": "string", + "description": "The recommended unit to format an indicator with.\n\nThe allowed values are:\n\n- `\"\"` - No units with numeric formatting where possible e.g. `123,500,600`\n- `£` - Pound sterling with numeric formatting e.g. `£1,234,500`\n- `£m` - Pound sterling in millions with numeric formatting e.g. `£1,234m`\n- `%` - Percentage with numeric formatting e.g. `1,234.56%`\n- `pp` - Percentage points with numeric formatting e.g. `1,234pp`\n- `string` - String with no units and no numeric formatting e.g. `123500600`", + "format": "int32", + "example": "%" + }, + "IndicatorViewModel": { + "required": [ + "column", + "id", + "label", + "unit" + ], + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The ID of the indicator.", + "example": "enW68" + }, + "column": { + "type": "string", + "description": "The name of the indicator CSV column.", + "example": "sess_authorised" + }, + "label": { + "type": "string", + "description": "The human-readable label of the indicator.", + "example": "Percentage of authorised sessions" + }, + "unit": { + "allOf": [ + { + "$ref": "#/components/schemas/IndicatorUnit" + } + ], + "description": "The type of unit that should be used when formatting the indicator.", + "nullable": true, + "example": "%" + }, + "decimalPlaces": { + "type": "integer", + "description": "The recommended number of decimal places to use when formatting the indicator.", + "format": "int32", + "nullable": true, + "example": 2 + } + }, + "additionalProperties": false, + "description": "A type of data point measured by a data set." + }, + "LocationCodedOptionViewModel": { + "allOf": [ + { + "$ref": "#/components/schemas/LocationOptionViewModel" + }, + { + "required": [ + "code" + ], + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "The code of the location.", + "example": "E12000003" + } + }, + "additionalProperties": false + } + ], + "description": "A location option that can be identified by code and used to filter a data set." + }, + "LocationGroupChangeViewModel": { + "type": "object", + "properties": { + "currentState": { + "allOf": [ + { + "$ref": "#/components/schemas/LocationGroupViewModel" + } + ], + "description": "The current state after the change was made.\r\nIf the change is an addition, this will be null.", + "nullable": true + }, + "previousState": { + "allOf": [ + { + "$ref": "#/components/schemas/LocationGroupViewModel" + } + ], + "description": "The previous state before the change was made.\r\nIf the change is a deletion, this will be null.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "A change to a location group in a data set." + }, + "LocationGroupOptionsViewModel": { + "required": [ + "level", + "options" + ], + "type": "object", + "properties": { + "level": { + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelViewModel" + } + ], + "description": "The geographic level of the locations in this group." + }, + "options": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/LocationCodedOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationLocalAuthorityOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationProviderOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationRscRegionOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationSchoolOptionViewModel" + } + ], + "description": "A location option that can be used to filter a data set." + }, + "description": "The locations belonging to this level." + } + }, + "additionalProperties": false, + "description": "The options available for a location group in the data set." + }, + "LocationGroupViewModel": { + "required": [ + "level" + ], + "type": "object", + "properties": { + "level": { + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelViewModel" + } + ], + "description": "The geographic level of the locations in this group." + } + }, + "additionalProperties": false, + "description": "A group of locations in a data set based on their geographic level." + }, + "LocationLocalAuthorityOptionViewModel": { + "allOf": [ + { + "$ref": "#/components/schemas/LocationOptionViewModel" + }, + { + "required": [ + "code", + "oldCode" + ], + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "The ONS code of the local authority.", + "example": "E08000019" + }, + "oldCode": { + "type": "string", + "description": "The old code (previously the LEA code) of the local authority.", + "example": "373" + } + }, + "additionalProperties": false + } + ], + "description": "A location option for a local authority that can be used to filter a data set." + }, + "LocationOptionChangeViewModel": { + "type": "object", + "properties": { + "currentState": { + "oneOf": [ + { + "$ref": "#/components/schemas/LocationCodedOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationLocalAuthorityOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationProviderOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationRscRegionOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationSchoolOptionViewModel" + } + ], + "description": "The current state after the change was made.\r\nIf the change is an addition, this will be null.", + "nullable": true + }, + "previousState": { + "oneOf": [ + { + "$ref": "#/components/schemas/LocationCodedOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationLocalAuthorityOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationProviderOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationRscRegionOptionViewModel" + }, + { + "$ref": "#/components/schemas/LocationSchoolOptionViewModel" + } + ], + "description": "The previous state before the change was made.\r\nIf the change is a deletion, this will be null.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "A change to a location option in a data set." + }, + "LocationOptionChangesViewModel": { + "required": [ + "level", + "options" + ], + "type": "object", + "properties": { + "level": { + "allOf": [ + { + "$ref": "#/components/schemas/GeographicLevelViewModel" + } + ], + "description": "The geographic level the changes belong to." + }, + "options": { + "type": "array", + "items": { + "$ref": "#/components/schemas/LocationOptionChangeViewModel" + }, + "description": "The list of location option changes." + } + }, + "additionalProperties": false, + "description": "A set of location option changes and details of the geographic level they belong to." + }, + "LocationOptionViewModel": { + "required": [ + "id", + "label" + ], + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The ID of the location.", + "example": "bOmZ4" + }, + "label": { + "type": "string", + "description": "The human-readable label of the location.", + "example": "Sheffield" + } + }, + "additionalProperties": false, + "description": "A location option that can be used to filter a data set." + }, + "LocationProviderOptionViewModel": { + "allOf": [ + { + "$ref": "#/components/schemas/LocationOptionViewModel" + }, + { + "required": [ + "ukprn" + ], + "type": "object", + "properties": { + "ukprn": { + "type": "string", + "description": "The UKPRN (UK provider reference number) of the provider.", + "example": "12345678" + } + }, + "additionalProperties": false + } + ], + "description": "A location option for a provider that can be used to filter a data set." + }, + "LocationRscRegionOptionViewModel": { + "allOf": [ + { + "$ref": "#/components/schemas/LocationOptionViewModel" + }, + { + "type": "object", + "additionalProperties": false + } + ], + "description": "A location option for an RSC region that can be used to filter a data set." + }, + "LocationSchoolOptionViewModel": { + "allOf": [ + { + "$ref": "#/components/schemas/LocationOptionViewModel" + }, + { + "required": [ + "laEstab", + "urn" + ], + "type": "object", + "properties": { + "urn": { + "type": "string", + "description": "The URN (unique reference number) of the school.", + "example": "123456" + }, + "laEstab": { + "type": "string", + "description": "The LAESTAB (local authority establishment number) of the school.", + "example": "1234567" + } + }, + "additionalProperties": false + } + ], + "description": "A location option for a school that can be used to filter a data set." + }, + "PagingViewModel": { + "required": [ + "page", + "pageSize", + "totalPages", + "totalResults" + ], + "type": "object", + "properties": { + "page": { + "type": "integer", + "description": "The current page number.", + "format": "int32", + "example": 1 + }, + "pageSize": { + "type": "integer", + "description": "The maximum number of results per page.", + "format": "int32", + "example": 20 + }, + "totalResults": { + "type": "integer", + "description": "The total number of results across all pages.", + "format": "int32", + "example": 60 + }, + "totalPages": { + "type": "integer", + "description": "The total number of pages.", + "format": "int32", + "readOnly": true, + "example": 3 + } + }, + "additionalProperties": false + }, + "ProblemDetailsViewModel": { + "required": [ + "status", + "title", + "type" + ], + "type": "object", + "properties": { + "type": { + "minLength": 1, + "type": "string", + "description": "A URI reference (RFC3986) that identifies the problem type. It should\r\nprovide human-readable documentation for the problem type.", + "example": "https://tools.ietf.org/html/rfc9110#section-15.5.1" + }, + "title": { + "minLength": 1, + "type": "string", + "description": "A short, human-readable summary of the problem type.", + "example": "One or more validation errors occurred." + }, + "status": { + "type": "integer", + "description": "The HTTP status code generated by the origin server for the problem.", + "format": "int32", + "example": 400 + }, + "detail": { + "type": "string", + "description": "A human-readable explanation specific to the problem.", + "nullable": true, + "example": "More detail about the problem." + }, + "instance": { + "type": "string", + "description": "A URI reference that identifies the specific occurrence of the problem.\r\nIt may or may not yield further information about the problem.", + "nullable": true, + "example": "https://example.com" + } + }, + "additionalProperties": { }, + "description": "Describes a problem that occurred whilst processing the request.\r\nThe format is based on RFC7807." + }, + "PublicationListRequest": { + "type": "object", + "properties": { + "search": { + "minLength": 3, + "type": "string", + "description": "A search term to find matching publications.", + "nullable": true, + "example": "Pupil absence" + }, + "page": { + "minimum": 1, + "type": "integer", + "description": "The page of results to fetch.", + "format": "int32", + "default": 1 + }, + "pageSize": { + "maximum": 40, + "minimum": 1, + "type": "integer", + "description": "The maximum number of results per page.", + "format": "int32", + "default": 20 + } + }, + "additionalProperties": false + }, + "PublicationPaginatedListViewModel": { + "required": [ + "paging", + "results" + ], + "type": "object", + "properties": { + "paging": { + "allOf": [ + { + "$ref": "#/components/schemas/PagingViewModel" + } + ], + "description": "Provides metadata for use in pagination." + }, + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PublicationSummaryViewModel" + }, + "description": "The list of results for this page." + } + }, + "additionalProperties": false, + "description": "A paginated list of publication summaries." + }, + "PublicationSummaryViewModel": { + "required": [ + "id", + "lastPublished", + "slug", + "summary", + "title" + ], + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The ID of the publication.", + "format": "uuid", + "example": "d851c09e-7f5a-4750-9191-ed67ba5e8f8b" + }, + "title": { + "type": "string", + "description": "The title of the publication.", + "example": "Pupil absence in schools in England" + }, + "slug": { + "type": "string", + "description": "The URL slug of the publication.", + "example": "pupil-absence-in-schools-in-england" + }, + "summary": { + "type": "string", + "description": "The summary of the publication.", + "example": "Summary of the publication." + }, + "lastPublished": { + "type": "string", + "description": "When the publication was last published.", + "format": "date-time", + "example": "2024-06-01T09:30:00+00:00" + } + }, + "additionalProperties": false, + "description": "Provides summary information about a publication." + }, + "TimePeriodCode": { + "enum": [ + "AY", + "AYQ1", + "AYQ2", + "AYQ3", + "AYQ4", + "CY", + "CYQ1", + "CYQ2", + "CYQ3", + "CYQ4", + "FY", + "FYQ1", + "FYQ2", + "FYQ3", + "FYQ4", + "M1", + "M2", + "M3", + "M4", + "M5", + "M6", + "M7", + "M8", + "M9", + "M10", + "M11", + "M12", + "P1", + "P2", + "RY", + "T1", + "T1T2", + "T2", + "T3", + "TY", + "TYQ1", + "TYQ2", + "TYQ3", + "TYQ4", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "W9", + "W10", + "W11", + "W12", + "W13", + "W14", + "W15", + "W16", + "W17", + "W18", + "W19", + "W20", + "W21", + "W22", + "W23", + "W24", + "W25", + "W26", + "W27", + "W28", + "W29", + "W30", + "W31", + "W32", + "W33", + "W34", + "W35", + "W36", + "W37", + "W38", + "W39", + "W40", + "W41", + "W42", + "W43", + "W44", + "W45", + "W46", + "W47", + "W48", + "W49", + "W50", + "W51", + "W52" + ], + "type": "string", + "description": "The code identifying the time period's type.\n\nThe allowed values are:\n\n- `AY` - academic year\n- `AYQ1 - AYQ4` - academic year quarter 1 to 4\n- `T1` - academic year's autumn term\n- `T2` - academic year's spring term\n- `T3` - academic year's summer term\n- `T1T2` - academic year's autumn and spring term\n- `CY` - calendar year\n- `CYQ1 - CYQ4` - calendar year quarter 1 to 4\n- `RY` - reporting year\n- `P1` - financial year part 1 (April to September)\n- `P2` - financial year part 2 (October to March)\n- `FY` - financial year\n- `FYQ1 - FYQ4` - financial year quarter 1 to 4\n- `TY` - tax year\n- `TYQ1 - FYQ4` - tax year quarter 1 to 4\n- `W1 - W52` - week 1 to 52\n- `M1 - M12` - month 1 to 12", + "example": "CY" + }, + "TimePeriodOptionChangeViewModel": { + "type": "object", + "properties": { + "currentState": { + "allOf": [ + { + "$ref": "#/components/schemas/TimePeriodOptionViewModel" + } + ], + "description": "The current state after the change was made.\r\nIf the change is an addition, this will be null.", + "nullable": true + }, + "previousState": { + "allOf": [ + { + "$ref": "#/components/schemas/TimePeriodOptionViewModel" + } + ], + "description": "The previous state before the change was made.\r\nIf the change is a deletion, this will be null.", + "nullable": true + } + }, + "additionalProperties": false, + "description": "A change to a time period option in a data set." + }, + "TimePeriodOptionViewModel": { + "required": [ + "code", + "label", + "period" + ], + "type": "object", + "properties": { + "code": { + "allOf": [ + { + "$ref": "#/components/schemas/TimePeriodCode" + } + ], + "description": "The code identifying the time period's type.", + "example": "AYQ1" + }, + "period": { + "type": "string", + "description": "The period covered by the data e.g. '2020' or '2020/2021'.", + "example": "2020/2021" + }, + "label": { + "type": "string", + "description": "The time period in human-readable format.", + "example": "2020/21 Q1" + } + }, + "additionalProperties": false, + "description": "A time period option that can be used to filter a data set." + }, + "TimePeriodRangeViewModel": { + "required": [ + "end", + "start" + ], + "type": "object", + "properties": { + "start": { + "type": "string", + "description": "The starting time period in human-readable format.", + "example": "2024 January" + }, + "end": { + "type": "string", + "description": "The ending time period in human-readable format.", + "example": "2024 December" + } + }, + "additionalProperties": false, + "description": "Describes a time period range in human-readable format." + }, + "TimePeriodViewModel": { + "required": [ + "code", + "period" + ], + "type": "object", + "properties": { + "code": { + "allOf": [ + { + "$ref": "#/components/schemas/TimePeriodCode" + } + ], + "description": "The code identifying the time period's type.", + "example": "AYQ1" + }, + "period": { + "type": "string", + "description": "The period covered by the data e.g. '2020' or '2020/2021'.", + "example": "2020/2021" + } + }, + "additionalProperties": false, + "description": "A time period relating to some data." + }, + "ValidationProblemViewModel": { + "required": [ + "title", + "type" + ], + "type": "object", + "properties": { + "type": { + "minLength": 1, + "type": "string", + "description": "A URI reference (RFC3986) that identifies the problem type. It should\r\nprovide human-readable documentation for the problem type.", + "example": "https://tools.ietf.org/html/rfc9110#section-15.5.1" + }, + "title": { + "minLength": 1, + "type": "string", + "description": "A short, human-readable summary of the problem type.", + "example": "One or more validation errors occurred." + }, + "detail": { + "type": "string", + "description": "A human-readable explanation specific to the problem.", + "nullable": true, + "example": "More detail about the problem." + }, + "instance": { + "type": "string", + "description": "A URI reference that identifies the specific occurrence of the problem.\r\nIt may or may not yield further information about the problem.", + "nullable": true, + "example": "https://example.com" + }, + "status": { + "type": "integer", + "description": "The HTTP status code generated by the origin server for the problem.", + "format": "int32", + "example": 400 + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorViewModel" + }, + "description": "The errors relating to the validation problem." + } + }, + "additionalProperties": { }, + "description": "A validation problem that has occurred with the API request.\r\nThis type of response will be composed of one or more errors\r\nrelating to the request and its properties." + }, + "WarningViewModel": { + "required": [ + "code", + "message" + ], + "type": "object", + "properties": { + "message": { + "type": "string", + "description": "The warning message.", + "example": "There are no results." + }, + "path": { + "type": "string", + "description": "The path to the property on the request that the warning relates to.\r\nMay be omitted or be empty if no specific property of the\r\nrequest relates to the warning (it is a 'global' warning).", + "nullable": true, + "example": "results" + }, + "code": { + "type": "string", + "description": "The warning's machine-readable code. Can be used for further\r\nprocessing of the warning before presenting to users.", + "example": "NoResults" + }, + "detail": { + "description": "Additional detail about the warning that can be used to provide\r\nmore context to users. May be omitted if there is none.", + "nullable": true, + "example": { + "total": 0 + } + } + }, + "additionalProperties": false, + "description": "A warning that points to a potential issue. This is not a critical error,\r\nbut may require attention to get the desired response." + } + } + } +} diff --git a/src/explore-education-statistics-api-docs/source/overview/error-handling/index.html.md.erb b/src/explore-education-statistics-api-docs/source/overview/error-handling/index.html.md.erb index 316324ac50b..d7ee1e0dedf 100644 --- a/src/explore-education-statistics-api-docs/source/overview/error-handling/index.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/overview/error-handling/index.html.md.erb @@ -39,7 +39,7 @@ If an error occurs, the EES API will respond with a body that looks like the fol } ``` -The response body is modelled by the [ProblemDetailsViewModel](/schemas/ProblemDetailsViewModel/index.html) +The response body is modelled by the [ProblemDetailsViewModel](/reference-v1/schemas/ProblemDetailsViewModel/index.html) schema, which attempts to detail the reason(s) why the request failed. The following fields are always included: diff --git a/src/explore-education-statistics-api-docs/source/overview/openapi-documents/index.html.md.erb b/src/explore-education-statistics-api-docs/source/overview/openapi-documents/index.html.md.erb new file mode 100644 index 00000000000..e6ee47102da --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/overview/openapi-documents/index.html.md.erb @@ -0,0 +1,26 @@ +--- +title: OpenAPI documents +last_reviewed_on: 2024-09-18 +review_in: 12 months +weight: 3 +--- + +# OpenAPI documents + +The explore education statistics (EES) API is fully compliant with the [OpenAPI 3.0 specification](https://swagger.io/specification/v3/). + +The OpenAPI specification provides a standard language for documenting REST APIs and their capabilities. +Using a document that conforms to this specification, users can understand how to use an API and navigate +through it with minimal effort. + +OpenAPI documents for the EES API can be downloaded below: + +<% get_openapi_links.each do |link| %> +- <%= link_to "#{link[:name]} document", link[:url] %> +<% end %> + +With an OpenAPI document, you can do things like: + +- explore it with a third-party HTTP client tool such as Postman or Insomnia +- generate an API client for writing integrations +- run automated tests against it to validate your integrations diff --git a/src/explore-education-statistics-api-docs/source/overview/openapi-specification/index.html.md.erb b/src/explore-education-statistics-api-docs/source/overview/openapi-specification/index.html.md.erb deleted file mode 100644 index 5529019b3e7..00000000000 --- a/src/explore-education-statistics-api-docs/source/overview/openapi-specification/index.html.md.erb +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: OpenAPI specification -last_reviewed_on: 2024-09-18 -review_in: 12 months -weight: 3 ---- - -# OpenAPI specification - -The explore education statistics API is fully compliant with the OpenAPI 3.0 specification. - -## About OpenAPI - -The [OpenAPI specification](https://swagger.io/specification/) provides a standard language for -documenting REST APIs and their capabilities. Using a document that conforms to this specification, -users should be able to understand how to use an API and navigate through it with minimal effort. - -This service's API reference documentation is generated using the OpenAPI specification. - -## Getting our OpenAPI specification - -You can download the service's OpenAPI specification in <%= link_to "JSON format", config[:tech_docs][:api_path] %>. - -Once you have our specification, you can: - -- explore it with a third-party tool such as Postman or Insomnia -- generate an API client for writing integrations -- run automated tests against it to validate your integrations diff --git a/src/explore-education-statistics-api-docs/source/overview/versioning/index.html.md.erb b/src/explore-education-statistics-api-docs/source/overview/versioning/index.html.md.erb index 7afdc27c1e9..085ad78f189 100644 --- a/src/explore-education-statistics-api-docs/source/overview/versioning/index.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/overview/versioning/index.html.md.erb @@ -55,4 +55,4 @@ changes introduced by the version and consider if they affect your consuming cod You can find lists of changes via: - the data set's changelog on its details page on the EES website (see [Data catalogue](https://explore-education-statistics.service.gov.uk/data-catalogue)) -- the [Get a data set version's changes](/endpoints/GetDataSetVersionChanges/index.html) endpoint +- the [Get a data set version's changes](/reference-v1/endpoints/GetDataSetVersionChanges/index.html) endpoint diff --git a/src/explore-education-statistics-api-docs/source/endpoints/template.html.md.erb b/src/explore-education-statistics-api-docs/source/templates/reference/endpoint.html.md.erb similarity index 98% rename from src/explore-education-statistics-api-docs/source/endpoints/template.html.md.erb rename to src/explore-education-statistics-api-docs/source/templates/reference/endpoint.html.md.erb index ddba5950b3f..4d4fd4cc6b3 100644 --- a/src/explore-education-statistics-api-docs/source/endpoints/template.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/templates/reference/endpoint.html.md.erb @@ -1,6 +1,11 @@ +--- +last_reviewed_on: 2024-11-15 +review_in: 24 months +--- + # <%= title %> -<%= description %> +<%= endpoint_description %> The URL for this endpoint is: diff --git a/src/explore-education-statistics-api-docs/source/templates/reference/endpoint_index.html.md.erb b/src/explore-education-statistics-api-docs/source/templates/reference/endpoint_index.html.md.erb new file mode 100644 index 00000000000..1d2dc283aed --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/templates/reference/endpoint_index.html.md.erb @@ -0,0 +1,20 @@ +--- +last_reviewed_on: 2024-11-15 +review_in: 24 months +--- + +# <%= title %> + +This section lists all the endpoints available on the explore education statistics (EES) <%= version %> API. + +The URL for the <%= version %> API is <%= link_to "#{api_url}/#{version}", "#{api_url}/#{version}" %>. + +The documentation for each endpoint contains information about the respective request(s) and response(s), +with examples and schemas provided. + +For convenience, request examples in different languages/tools are provided. We currently support: + +- cURL +- JavaScript +- Python +- R diff --git a/src/explore-education-statistics-api-docs/source/templates/reference/index.html.md.erb b/src/explore-education-statistics-api-docs/source/templates/reference/index.html.md.erb new file mode 100644 index 00000000000..a398e166b2d --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/templates/reference/index.html.md.erb @@ -0,0 +1,23 @@ +--- +last_reviewed_on: 2024-09-18 +review_in: 24 months +--- + +# <%= title %> + +This section contains the reference documentation for the explore education statistics (EES) +<%= version %> API. + +The URL for the <%= version %> API is <%= link_to "#{api_url}/#{version}", "#{api_url}/#{version}" %>. + +The reference documentation has been generated from an <%= link_to "OpenAPI document", "/openapi-#{version}.json" %> +for the EES API. You can download this for your own use with HTTP client tools like Postman. + +## Documentation structure + +The [Endpoints](endpoints/index.html) section lists all the endpoints supported by the EES API. +Information is provided about each endpoint's functionality and its respective request(s) and +response(s). + +The [Schemas](schemas/index.html) section lists all the schemas used by requests and responses across +the EES API. Each schema contains details about its purpose, properties, types and validation rules. diff --git a/src/explore-education-statistics-api-docs/source/schemas/template.html.md.erb b/src/explore-education-statistics-api-docs/source/templates/reference/schema.html.md.erb similarity index 95% rename from src/explore-education-statistics-api-docs/source/schemas/template.html.md.erb rename to src/explore-education-statistics-api-docs/source/templates/reference/schema.html.md.erb index 3deb4f5b190..f82c6cb7a53 100644 --- a/src/explore-education-statistics-api-docs/source/schemas/template.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/templates/reference/schema.html.md.erb @@ -1,3 +1,8 @@ +--- +last_reviewed_on: 2024-11-15 +review_in: 24 months +--- + # <%= title %> <% if schema.description %> diff --git a/src/explore-education-statistics-api-docs/source/schemas/index.html.md.erb b/src/explore-education-statistics-api-docs/source/templates/reference/schema_index.html.md.erb similarity index 84% rename from src/explore-education-statistics-api-docs/source/schemas/index.html.md.erb rename to src/explore-education-statistics-api-docs/source/templates/reference/schema_index.html.md.erb index 1c7690e7544..4613e129a59 100644 --- a/src/explore-education-statistics-api-docs/source/schemas/index.html.md.erb +++ b/src/explore-education-statistics-api-docs/source/templates/reference/schema_index.html.md.erb @@ -1,10 +1,9 @@ --- -title: Schemas -last_reviewed_on: 2024-09-18 -review_in: 12 months +last_reviewed_on: 2024-11-15 +review_in: 24 months --- -# API Schemas +# <%= title %> This section lists all the schemas available on the explore education statistics API. @@ -13,7 +12,7 @@ typically composed of a set of properties and some information about each of the - the property name - the property type e.g. string, number, boolean, array, object, etc -- if it is required or optional +- is it required or optional - a description of the property - any validation rules that apply From 99101cda96986296f16a9f73974034a207d64fc0 Mon Sep 17 00:00:00 2001 From: Nusrath Moh Date: Mon, 25 Nov 2024 13:07:13 +0000 Subject: [PATCH 018/144] EES-5535 Remove Test data file with no observations --- .../grouped-filters-and-indicators.csv | 101 ++++++++++++++++++ .../grouped-filters-and-indicators.meta.csv | 7 ++ .../institution_and_provider.csv | 11 -- .../institution_and_provider.meta.csv | 2 - .../public_api/public_api_restricted.robot | 4 +- .../small-files/institution_and_provider.csv | 11 -- .../institution_and_provider.meta.csv | 2 - 7 files changed, 110 insertions(+), 28 deletions(-) create mode 100644 tests/robot-tests/tests/files/public-api-data-files/grouped-filters-and-indicators.csv create mode 100644 tests/robot-tests/tests/files/public-api-data-files/grouped-filters-and-indicators.meta.csv delete mode 100644 tests/robot-tests/tests/files/public-api-data-files/institution_and_provider.csv delete mode 100644 tests/robot-tests/tests/files/public-api-data-files/institution_and_provider.meta.csv delete mode 100644 tests/test-data/files/small-files/institution_and_provider.csv delete mode 100644 tests/test-data/files/small-files/institution_and_provider.meta.csv diff --git a/tests/robot-tests/tests/files/public-api-data-files/grouped-filters-and-indicators.csv b/tests/robot-tests/tests/files/public-api-data-files/grouped-filters-and-indicators.csv new file mode 100644 index 00000000000..3f433b1ea3b --- /dev/null +++ b/tests/robot-tests/tests/files/public-api-data-files/grouped-filters-and-indicators.csv @@ -0,0 +1,101 @@ +time_period,time_identifier,geographic_level,country_code,country_name,provider_ukprn,provider_name,filter_1_group,filter_1,filter_2_group,filter_2,indicator_1,indicator_2,indicator_3,indicator_4 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-1,Filter 2 group 1,F2G1-1,519987,851738,508866,721563 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-2,Filter 2 group 1,F2G1-1,890260,684500,730929,900942 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-1,Filter 2 group 1,F2G1-1,898281,194658,626890,179607 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-2,Filter 2 group 1,F2G1-1,442204,520880,752327,357358 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-1,Filter 2 group 1,F2G1-2,845035,980359,396950,787580 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-2,Filter 2 group 1,F2G1-2,21092,24298,965843,657526 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-1,Filter 2 group 1,F2G1-2,268499,34940,827630,73584 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-2,Filter 2 group 1,F2G1-2,498805,484674,210648,958899 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-1,Filter 2 group 2,F2G2-1,116897,301509,988763,562842 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-2,Filter 2 group 2,F2G2-1,583667,901339,853121,826380 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-1,Filter 2 group 2,F2G2-1,207260,423338,682544,360786 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-2,Filter 2 group 2,F2G2-1,640172,911376,148064,234958 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-1,Filter 2 group 2,F2G2-2,501494,747088,979616,135628 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-2,Filter 2 group 2,F2G2-2,363825,32131,260954,603809 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-1,Filter 2 group 2,F2G2-2,967308,110093,788718,162605 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-2,Filter 2 group 2,F2G2-2,974348,546568,267548,382114 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Total,Total,Filter 2 group 1,F2G1-1,1982083,2207028,3618158,3826163 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Total,Total,Filter 2 group 1,F2G1-2,3240042,8827119,1644168,3189486 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Total,Total,Filter 2 group 2,F2G2-1,5469024,7980560,3848042,2378061 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Total,Total,Filter 2 group 2,F2G2-2,3932207,4963282,8725026,1680956 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-1,Total,Total,5545322,8638316,2318343,4750265 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-2,Total,Total,4754866,6148310,3105203,5826699 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-1,Total,Total,9723794,9280871,2144881,9817514 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-2,Total,Total,2941718,8547200,1541122,8742396 +2022,Calendar year,Provider,E92000001,England,1,Provider 1,Total,Total,Total,Total,19264353,79307567,43651917,93261352 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-1,Filter 2 group 1,F2G1-1,83794,966035,682390,863661 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-2,Filter 2 group 1,F2G1-1,938914,146961,46666,280967 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-1,Filter 2 group 1,F2G1-1,866122,37150,56798,96061 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-2,Filter 2 group 1,F2G1-1,793284,549160,633867,264223 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-1,Filter 2 group 1,F2G1-2,139909,306329,425033,900882 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-2,Filter 2 group 1,F2G1-2,679377,985554,594334,849442 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-1,Filter 2 group 1,F2G1-2,995781,785727,451027,656006 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-2,Filter 2 group 1,F2G1-2,266941,212332,967279,637448 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-1,Filter 2 group 2,F2G2-1,498220,427783,602431,250688 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-2,Filter 2 group 2,F2G2-1,265187,215196,183561,439819 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-1,Filter 2 group 2,F2G2-1,829030,4198,112319,855808 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-2,Filter 2 group 2,F2G2-1,927889,982443,924178,401248 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-1,Filter 2 group 2,F2G2-2,338532,607964,578208,272318 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-2,Filter 2 group 2,F2G2-2,502979,427059,777613,323772 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-1,Filter 2 group 2,F2G2-2,195032,520449,570641,75940 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-2,Filter 2 group 2,F2G2-2,731269,792087,138370,891238 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Total,Total,Filter 2 group 1,F2G1-1,6896452,8060420,5135948,7794604 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Total,Total,Filter 2 group 1,F2G1-2,7760898,8671334,5200061,5662784 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Total,Total,Filter 2 group 2,F2G2-1,5544223,5056471,2649363,8489236 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Total,Total,Filter 2 group 2,F2G2-2,2129360,8087774,4547218,8905245 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-1,Total,Total,2239729,3691997,1467644,1696872 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-2,Total,Total,4213007,3876222,7044831,9069167 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-1,Total,Total,6612665,1394364,2810933,1830711 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-2,Total,Total,9611462,7699216,4677741,6485417 +2022,Calendar year,Provider,E92000001,England,2,Provider 2,Total,Total,Total,Total,20006389,96030253,25187338,52603644 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-1,Filter 2 group 1,F2G1-1,424456,523756,532778,784121 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-2,Filter 2 group 1,F2G1-1,731288,321298,504769,74678 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-1,Filter 2 group 1,F2G1-1,691613,488448,700915,548068 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-2,Filter 2 group 1,F2G1-1,47473,767840,871517,974949 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-1,Filter 2 group 1,F2G1-2,65866,805593,98663,505646 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-2,Filter 2 group 1,F2G1-2,661656,451353,586477,825657 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-1,Filter 2 group 1,F2G1-2,117853,454350,585981,147043 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-2,Filter 2 group 1,F2G1-2,692128,332137,545458,368991 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-1,Filter 2 group 2,F2G2-1,409108,210780,644056,648560 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-2,Filter 2 group 2,F2G2-1,73227,534108,126294,752982 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-1,Filter 2 group 2,F2G2-1,886706,549623,980458,311184 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-2,Filter 2 group 2,F2G2-1,757235,360995,589706,120728 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-1,Filter 2 group 2,F2G2-2,750413,946653,227681,59962 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-2,Filter 2 group 2,F2G2-2,633998,595601,617520,786904 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-1,Filter 2 group 2,F2G2-2,900284,705619,903759,883138 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-2,Filter 2 group 2,F2G2-2,873285,123902,368887,854288 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Total,Total,Filter 2 group 1,F2G1-1,3800361,5926696,1129011,3012649 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Total,Total,Filter 2 group 1,F2G1-2,3848960,6244513,2869219,4722341 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Total,Total,Filter 2 group 2,F2G2-1,5394621,8106152,4792742,7240877 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Total,Total,Filter 2 group 2,F2G2-2,1837813,9873198,8641270,7726271 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-1,Total,Total,2553275,9669468,2561438,5861471 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 1,F1G1-2,Total,Total,7552538,9721299,1890102,2670993 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-1,Total,Total,8683582,8783737,7873126,7292120 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Filter 1 group 2,F1G2-2,Total,Total,7392787,1615391,8184700,3587650 +2021,Calendar year,Provider,E92000001,England,1,Provider 1,Total,Total,Total,Total,13203655,18384170,52220504,51794885 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-1,Filter 2 group 1,F2G1-1,507350,229265,628790,252563 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-2,Filter 2 group 1,F2G1-1,506521,502138,363244,955008 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-1,Filter 2 group 1,F2G1-1,651914,850564,209111,407476 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-2,Filter 2 group 1,F2G1-1,83425,181396,660786,785317 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-1,Filter 2 group 1,F2G1-2,444556,650615,592598,388728 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-2,Filter 2 group 1,F2G1-2,430134,957886,432065,213302 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-1,Filter 2 group 1,F2G1-2,479509,853964,334090,133803 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-2,Filter 2 group 1,F2G1-2,525528,766621,115447,722768 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-1,Filter 2 group 2,F2G2-1,266667,44970,204982,298586 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-2,Filter 2 group 2,F2G2-1,212195,192795,692080,788605 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-1,Filter 2 group 2,F2G2-1,649235,767364,378570,152624 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-2,Filter 2 group 2,F2G2-1,60089,413454,455145,744538 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-1,Filter 2 group 2,F2G2-2,44178,607208,147312,856832 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-2,Filter 2 group 2,F2G2-2,785203,473856,562676,599506 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-1,Filter 2 group 2,F2G2-2,548157,924384,16384,40223 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-2,Filter 2 group 2,F2G2-2,164898,929131,183449,390387 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Total,Total,Filter 2 group 1,F2G1-1,7050152,8169706,5494845,3858265 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Total,Total,Filter 2 group 1,F2G1-2,6552976,2443206,8667092,7802975 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Total,Total,Filter 2 group 2,F2G2-1,2668348,7113547,2009895,4985845 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Total,Total,Filter 2 group 2,F2G2-2,2303707,1941501,8969219,2991816 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-1,Total,Total,1411290,9026393,4632174,7903124 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 1,F1G1-2,Total,Total,4462203,3335909,2858860,7178183 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-1,Total,Total,5637258,7413917,1214481,9280212 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Filter 1 group 2,F1G2-2,Total,Total,8815716,5825935,5124154,6824399 +2021,Calendar year,Provider,E92000001,England,2,Provider 2,Total,Total,Total,Total,79115205,40336058,40244077,54043187 diff --git a/tests/robot-tests/tests/files/public-api-data-files/grouped-filters-and-indicators.meta.csv b/tests/robot-tests/tests/files/public-api-data-files/grouped-filters-and-indicators.meta.csv new file mode 100644 index 00000000000..ed9a84a2b82 --- /dev/null +++ b/tests/robot-tests/tests/files/public-api-data-files/grouped-filters-and-indicators.meta.csv @@ -0,0 +1,7 @@ +col_name,col_type,label,indicator_grouping,indicator_unit,indicator_dp,filter_hint,filter_grouping_column +filter_1,Filter,Filter 1,,,,,filter_1_group +filter_2,Filter,Filter 2,,,,,filter_2_group +indicator_1,Indicator,Indicator 1,Indicator group 1,,,, +indicator_2,Indicator,Indicator 2,Indicator group 1,,,, +indicator_3,Indicator,Indicator 3,Indicator group 2,,,, +indicator_4,Indicator,Indicator 4,Indicator group 2,,,, diff --git a/tests/robot-tests/tests/files/public-api-data-files/institution_and_provider.csv b/tests/robot-tests/tests/files/public-api-data-files/institution_and_provider.csv deleted file mode 100644 index bf7e7eca6f1..00000000000 --- a/tests/robot-tests/tests/files/public-api-data-files/institution_and_provider.csv +++ /dev/null @@ -1,11 +0,0 @@ -time_period,time_identifier,geographic_level,country_code,country_name,institution_name,institution_id,provider_ukprn,provider_name,admission_numbers -2011,Calendar year,Institution,E92000001,England,One,1,,,712196 -2011,Calendar year,Institution,E92000001,England,Two,2,,,709548 -2011,Calendar year,Institution,E92000001,England,Three,3,,,709143 -2011,Calendar year,Institution,E92000001,England,Four,4,,,607928 -2222,Calendar year,Institution,E92000001,England,Five,5,,,600903 -2011,Calendar year,Provider,E92000001,England,,,6,Six,712196 -2011,Calendar year,Provider,E92000001,England,,,7,Seven,709548 -2011,Calendar year,Provider,E92000001,England,,,8,Eight,709143 -2011,Calendar year,Provider,E92000001,England,,,9,Nine,607928 -2222,Calendar year,Provider,E92000001,England,,,10,Ten,600903 diff --git a/tests/robot-tests/tests/files/public-api-data-files/institution_and_provider.meta.csv b/tests/robot-tests/tests/files/public-api-data-files/institution_and_provider.meta.csv deleted file mode 100644 index 322518824c7..00000000000 --- a/tests/robot-tests/tests/files/public-api-data-files/institution_and_provider.meta.csv +++ /dev/null @@ -1,2 +0,0 @@ -col_name,col_type,label,indicator_grouping,indicator_unit,indicator_dp,filter_hint,filter_grouping_column -admission_numbers,Indicator,Admission Numbers,,,,, diff --git a/tests/robot-tests/tests/public_api/public_api_restricted.robot b/tests/robot-tests/tests/public_api/public_api_restricted.robot index 7b3a711d77f..958c13f06b4 100644 --- a/tests/robot-tests/tests/public_api/public_api_restricted.robot +++ b/tests/robot-tests/tests/public_api/public_api_restricted.robot @@ -297,8 +297,8 @@ Create a third draft release via API user creates release from publication page ${PUBLICATION_NAME} Academic year 3020 Upload subject to the third release - user uploads subject and waits until complete ${SUBJECT_NAME_5} institution_and_provider.csv - ... institution_and_provider.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_5} grouped-filters-and-indicators.csv + ... grouped-filters-and-indicators.meta.csv ${PUBLIC_API_FILES_DIR} Add data guidance to the third release user clicks link Data and files diff --git a/tests/test-data/files/small-files/institution_and_provider.csv b/tests/test-data/files/small-files/institution_and_provider.csv deleted file mode 100644 index bf7e7eca6f1..00000000000 --- a/tests/test-data/files/small-files/institution_and_provider.csv +++ /dev/null @@ -1,11 +0,0 @@ -time_period,time_identifier,geographic_level,country_code,country_name,institution_name,institution_id,provider_ukprn,provider_name,admission_numbers -2011,Calendar year,Institution,E92000001,England,One,1,,,712196 -2011,Calendar year,Institution,E92000001,England,Two,2,,,709548 -2011,Calendar year,Institution,E92000001,England,Three,3,,,709143 -2011,Calendar year,Institution,E92000001,England,Four,4,,,607928 -2222,Calendar year,Institution,E92000001,England,Five,5,,,600903 -2011,Calendar year,Provider,E92000001,England,,,6,Six,712196 -2011,Calendar year,Provider,E92000001,England,,,7,Seven,709548 -2011,Calendar year,Provider,E92000001,England,,,8,Eight,709143 -2011,Calendar year,Provider,E92000001,England,,,9,Nine,607928 -2222,Calendar year,Provider,E92000001,England,,,10,Ten,600903 diff --git a/tests/test-data/files/small-files/institution_and_provider.meta.csv b/tests/test-data/files/small-files/institution_and_provider.meta.csv deleted file mode 100644 index 322518824c7..00000000000 --- a/tests/test-data/files/small-files/institution_and_provider.meta.csv +++ /dev/null @@ -1,2 +0,0 @@ -col_name,col_type,label,indicator_grouping,indicator_unit,indicator_dp,filter_hint,filter_grouping_column -admission_numbers,Indicator,Admission Numbers,,,,, From 33b12517f796d717dd35d771dc1a92734e3ae810 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Mon, 25 Nov 2024 15:56:59 +0000 Subject: [PATCH 019/144] EES-5682: Replace privacy policy with external URL. --- .../pages/bau/__tests__/BauUsersPage.test.tsx | 1 - .../components/EditableKeyStatTextForm.tsx | 1 - .../next-sitemap.config.js | 1 - .../src/components/PageFooter.tsx | 2 +- .../src/pages/privacy-notice.tsx | 216 ------------------ .../tests/general_public/miscellaneous.robot | 12 - 6 files changed, 1 insertion(+), 232 deletions(-) delete mode 100644 src/explore-education-statistics-frontend/src/pages/privacy-notice.tsx diff --git a/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx b/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx index 17694f85497..8e97984fe56 100644 --- a/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx @@ -6,7 +6,6 @@ import { MemoryRouter } from 'react-router'; import { TestConfigContextProvider } from '@admin/contexts/ConfigContext'; import render from '@common-test/render'; import { screen } from '@testing-library/react'; -import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import userEvent from '@testing-library/user-event'; import { waitFor } from '@testing-library/dom'; import BauUsersPage from '../BauUsersPage'; diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatTextForm.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatTextForm.tsx index f2842d8d889..33c9270acd5 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatTextForm.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatTextForm.tsx @@ -7,7 +7,6 @@ import FormFieldTextArea from '@common/components/form/FormFieldTextArea'; import styles from '@common/modules/find-statistics/components/KeyStat.module.scss'; import { KeyStatisticText } from '@common/services/publicationService'; import React from 'react'; -import classNames from 'classnames'; import Yup from '@common/validation/yup'; export interface KeyStatTextFormValues { diff --git a/src/explore-education-statistics-frontend/next-sitemap.config.js b/src/explore-education-statistics-frontend/next-sitemap.config.js index 2497341321d..fa2cdfae625 100644 --- a/src/explore-education-statistics-frontend/next-sitemap.config.js +++ b/src/explore-education-statistics-frontend/next-sitemap.config.js @@ -38,7 +38,6 @@ module.exports = { '/contact-us', '/cookies', '/cookies/details', - '/privacy-notice', '/help-support', ]; if (lowerPriorityPages.includes(path)) { diff --git a/src/explore-education-statistics-frontend/src/components/PageFooter.tsx b/src/explore-education-statistics-frontend/src/components/PageFooter.tsx index 86fb2eee0cd..0484893871c 100644 --- a/src/explore-education-statistics-frontend/src/components/PageFooter.tsx +++ b/src/explore-education-statistics-frontend/src/components/PageFooter.tsx @@ -29,7 +29,7 @@ const PageFooter = ({ wide }: Props) => (
  • Privacy notice diff --git a/src/explore-education-statistics-frontend/src/pages/privacy-notice.tsx b/src/explore-education-statistics-frontend/src/pages/privacy-notice.tsx deleted file mode 100644 index 046925dc5cb..00000000000 --- a/src/explore-education-statistics-frontend/src/pages/privacy-notice.tsx +++ /dev/null @@ -1,216 +0,0 @@ -import Page from '@frontend/components/Page'; -import React from 'react'; - -function PrivacyNoticePage() { - return ( - -
    -
    -
    -

    Who we are

    -

    - The Explore education statistics service is operated by the - Department for Education (DfE). -

    -

    - For the purpose of data protection legislation, DfE is the data - controller for the personal data processed as part of the explore - education statistics service. -

    -
    -
    -

    How we'll use your information

    -

    - We receive your personal data when you sign up and subscribe to - receive notifications about the Explore education statistics - service. -

    -

    - We’ll process your data in order to email you updates to let you - know when the service has been updated and keep you informed about - any changes to the statistics and data it contains. -

    -

    - Your use of our service means you agree to our use of your - personal data as set out in this privacy notice. -

    -
    -
    -

    - The nature of your personal data we'll be using -

    -

    - In terms of your personal data, we’ll be using your email address - to send you notifications about the service. -

    -
    -
    -

    - Why our use of your personal data is lawful -

    -

    - In order for our use of your personal data to be lawful, we need - to meet 1 (or more) conditions in the data protection legislation. -

    -

    - For the purpose of this service, the relevant condition we're - relying on is Article 6(1)(e) of the General Data Protection - Regulation (GDPR). -

    -
    -
    -

    - Who we’ll make your personal data available to -

    -

    - We sometimes need to make personal data available to other - organisations. -

    -

    - These might include contracted partners (who we have employed to - process your personal data on our behalf) and/or other - organisations (with whom we need to share your personal data for - specific purposes). -

    -

    - Where we need to share your personal data with others, we make - sure this data sharing complies with data protection legislation. -

    -

    - For the purpose of this service we need to share your personal - data with{' '} - - GOV.UK Notify - {' '} - (a service provided by the Government Digital Service which is - part of the Cabinet Office) so we can send you email notifications - about this service. -

    -
    -
    -

    Where your data is stored

    -

    - We store your data on secure servers in the{' '} - - European Economic Area (EEA) - - . By submitting your personal data, you agree to this. -

    -
    -
    -

    - How long we'll keep your personal data -

    -

    - We’ll only keep your personal data for as long as we need it for - the purpose of sending you notifications, after which your - personal data will be securely destroyed. -

    -

    - Under Data Protection legislation, and in compliance with the - relevant data processing conditions, personal data can be kept for - longer periods of time when processed purely for archiving in the - public interest, statistical purposes and historical or scientific - research. For further information refer to GDPR Article 6(1)(e). -

    -
    -
    -

    Your data protection rights

    -

    Under certain circumstances, you have the right to:

    -
      -
    • ask us for access to information about you that we hold
      • -
    • - have your personal data rectified - if it's inaccurate or - incomplete -
      • -
    • - request the deletion or removal of personal data where there's - no compelling reason for its continued processing -
      • -
    • - restrict our processing of your personal data (ie permitting its - storage but no further processing) -
      • -
    • - object to direct marketing (including profiling) and processing - for the purposes of scientific/historical research and - statistics -
      • -
    • - not to be subject to decisions based purely on automated - processing where it produces a legal or similarly significant - effect on you -
      • -
    -

    - If you need to contact us regarding any of the above use the - details on DfE's{' '} - - Contact the Department for Education (DfE) - {' '} - page. -

    -

    - Further information about your data protection rights can be found - on the{' '} - - Information Commissioner’s Office (ICO) - {' '} - website. -

    -
    -
    -

    - Withdrawal of consent and the right to lodge a complaint -

    -

    - Where we’re processing your personal data with your consent - you - have the right to withdraw that consent. -

    -

    - If you change your mind, or you are unhappy with our use of your - personal data, let us know by contacting the explore education - statistics service at{' '} - - explore.statistics@education.gov.uk - - . -

    -

    - You have the right to raise any concerns via the{' '} - - Information Commissioner’s Office (ICO) - {' '} - website. -

    -
    -
    -

    Last updated

    -

    - We may need to update this privacy notice periodically so we - recommend you revisit from time to time. This version was last - updated on 14/04/2020. -

    -
    -
    -

    Contact information

    -

    - If you have any questions about how your personal information will - be used contact the Explore education statistics service at - - explore.statistics@education.gov.uk - {' '} - or contact DfE's Data Protection Officer (DPO) via our{' '} - - Contact the Department for Education (DfE) - {' '} - page. -

    -
    -
    -
    -     - ); -} - -export default PrivacyNoticePage; diff --git a/tests/robot-tests/tests/general_public/miscellaneous.robot b/tests/robot-tests/tests/general_public/miscellaneous.robot index 144a07924a3..4df3159f03a 100644 --- a/tests/robot-tests/tests/general_public/miscellaneous.robot +++ b/tests/robot-tests/tests/general_public/miscellaneous.robot @@ -94,18 +94,6 @@ Validate Cookies Details page cookie names should be on page -Validate Privacy notice page - user clicks link Privacy notice - user waits until h1 is visible Privacy notice - user waits until page contains - ... The Explore education statistics service is operated by the Department for Education - - user checks url contains %{PUBLIC_URL}/privacy-notice - - user checks breadcrumb count should be 2 - user checks nth breadcrumb contains 1 Home - user checks nth breadcrumb contains 2 Privacy notice - Validate Contact page user clicks link Contact us user waits until page contains Contact Explore education statistics From d538755a07181d7e702d565a878e6b4bb880ced7 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 26 Nov 2024 02:31:15 +0000 Subject: [PATCH 020/144] EES-5541 - Boundary level per dataset form changes (#5308) * EES-5402 minor bugfix * EES-5541 add dataset boundary level configuration to form * EES-5541 data groupings tab form init massively simplified, ammended data set boundary form interaction and select * EES-5541 - wired up forms to reducer state * EES-5541 simplified form layout and updated reducer * EES-5541 added onChange to Form and simplified boundary form implementation * EES-5541 added feature flag, fixed build errors * EES-5541 Various fixes across boundary level and data groupings - Hoists initialization of map `dataSetConfigs` into `chartBuilderReducer` - Refactors form types for boundary level and data grouping tabs - Fixes boundary level tab not showing invalid form errors for other tabs - Refactors out new `getMapDataSetCategoryConfigs` function from `getDataSetCategoryConfigs`. * EES-5541 fixes stale boundaryLevel form state bug when removing datasets * EES-5541 removed defaultBoundaryLevel, which lead to unset configs becoming set * EES-5541 updated getDataSetCategoryConfigs tests, no longer used props removed etc. * EES-5541 added 2 simple tests for getMapDataSetCategoryConfigs * EES-5541 generateFeaturesAndDataGroups: fixed types * EES-5541 fix up test:ChartBoundaryLevelsConfiguration * EES-5541 mostly fixed tests: ChartDataGroupingsConfiguration * EES-5541 chartBuilderReducer tests fix * EES-5541 ChartDataGroupingsConfiguration tests fix * EES-5541 various typo fixes and test improvements --------- Co-authored-by: Nicholas Tsim --- .../ChartBoundaryLevelsConfiguration.tsx | 145 ++--- .../chart/ChartBoundaryLevelsForm.tsx | 193 ++++++ .../components/chart/ChartBuilder.tsx | 68 +-- .../chart/ChartDataGroupingsConfiguration.tsx | 99 +--- .../ChartBoundaryLevelsConfiguration.test.tsx | 193 +++++- .../chart/__tests__/ChartBuilder.test.tsx | 212 +++++++ .../ChartDataGroupingsConfiguration.test.tsx | 331 +++-------- .../contexts/ChartBuilderFormsContext.tsx | 2 +- .../__tests__/chartBuilderReducer.test.ts | 556 +++++++++++++++++- .../chart/reducers/chartBuilderReducer.ts | 450 +++++++++----- .../components/chart/types/mapConfig.ts | 17 + .../src/components/form/Form.tsx | 11 +- .../components/form/__tests__/Form.test.tsx | 32 + .../modules/charts/components/MapBlock.tsx | 30 +- .../modules/charts/components/MapGeoJSON.tsx | 5 +- .../generateFeaturesAndDataGroups.test.ts | 34 +- .../utils/generateFeaturesAndDataGroups.ts | 4 +- .../src/modules/charts/types/chart.ts | 1 + .../getDataSetCategoryConfigs.test.ts | 198 ------- .../getMapDataSetCategoryConfigs.test.ts | 221 +++++++ .../charts/util/getDataSetCategoryConfigs.ts | 55 +- .../util/getMapDataSetCategoryConfigs.ts | 77 +++ 22 files changed, 2000 insertions(+), 934 deletions(-) create mode 100644 src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBoundaryLevelsForm.tsx create mode 100644 src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/types/mapConfig.ts create mode 100644 src/explore-education-statistics-common/src/modules/charts/util/__tests__/getMapDataSetCategoryConfigs.test.ts create mode 100644 src/explore-education-statistics-common/src/modules/charts/util/getMapDataSetCategoryConfigs.ts diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBoundaryLevelsConfiguration.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBoundaryLevelsConfiguration.tsx index 9e06cbb6e50..9a5d5a353c3 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBoundaryLevelsConfiguration.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBoundaryLevelsConfiguration.tsx @@ -1,118 +1,87 @@ -import ChartBuilderSaveActions from '@admin/pages/release/datablocks/components/chart/ChartBuilderSaveActions'; -import { useChartBuilderFormsContext } from '@admin/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext'; +import ChartBoundaryLevelsForm, { + ChartBoundaryLevelsFormValues, +} from '@admin/pages/release/datablocks/components/chart/ChartBoundaryLevelsForm'; import { ChartOptions } from '@admin/pages/release/datablocks/components/chart/reducers/chartBuilderReducer'; -import Effect from '@common/components/Effect'; -import FormProvider from '@common/components/form/FormProvider'; -import Form from '@common/components/form/Form'; -import FormFieldSelect from '@common/components/form/FormFieldSelect'; +import { MapBoundaryLevelConfig } from '@admin/pages/release/datablocks/components/chart/types/mapConfig'; +import { MapConfig } from '@common/modules/charts/types/chart'; import { FullTableMeta } from '@common/modules/table-tool/types/fullTable'; import parseNumber from '@common/utils/number/parseNumber'; -import Yup from '@common/validation/yup'; -import merge from 'lodash/merge'; -import React, { ReactNode, useCallback } from 'react'; - -const formId = 'chartBoundaryLevelsConfigurationForm'; - -interface FormValues { - boundaryLevel?: number; -} +import React, { ReactNode, useCallback, useMemo } from 'react'; interface Props { buttons?: ReactNode; + // TODO: EES-5402 - Remove when all boundary level changes are done + hasDataSetBoundaryLevels?: boolean; + map: MapConfig; meta: FullTableMeta; options: ChartOptions; - onChange: (values: ChartOptions) => void; - onSubmit: (chartOptions: ChartOptions) => void; + onChange: (values: MapBoundaryLevelConfig) => void; + onSubmit: (values: MapBoundaryLevelConfig) => void; } export default function ChartBoundaryLevelsConfiguration({ buttons, + // TODO: EES-5402 - Remove when all boundary level changes are done + hasDataSetBoundaryLevels = false, + map, meta, options, onChange, onSubmit, }: Props) { - const { updateForm, submitForms } = useChartBuilderFormsContext(); + const initialValues = useMemo(() => { + return { + boundaryLevel: options.boundaryLevel?.toString(), + dataSetConfigs: + map?.dataSetConfigs.map(dataSetConfig => { + return { + boundaryLevel: dataSetConfig.boundaryLevel?.toString(), + }; + }) ?? [], + }; + }, [options.boundaryLevel, map?.dataSetConfigs]); const normalizeValues = useCallback( - (values: FormValues): ChartOptions => { - // Use `merge` as we want to avoid potential undefined - // values from overwriting existing values - return merge({}, options, values, { - boundaryLevel: values.boundaryLevel - ? parseNumber(values.boundaryLevel) - : undefined, - }); + ( + values: Partial, + ): MapBoundaryLevelConfig => { + return { + boundaryLevel: parseNumber(values.boundaryLevel), + dataSetConfigs: + values.dataSetConfigs?.map(({ boundaryLevel }, index) => { + return { + boundaryLevel: parseNumber(boundaryLevel), + dataSet: map.dataSetConfigs[index].dataSet, + }; + }) ?? [], + }; }, - [options], + [map.dataSetConfigs], + ); + + const handleSubmit = useCallback( + (values: ChartBoundaryLevelsFormValues) => { + onSubmit(normalizeValues(values)); + }, + [onSubmit, normalizeValues], ); const handleChange = useCallback( - (values: FormValues) => { + (values: Partial) => { onChange(normalizeValues(values)); }, - [normalizeValues, onChange], + [onChange, normalizeValues], ); return ( - ({ - boundaryLevel: Yup.number() - .transform(value => (Number.isNaN(value) ? undefined : value)) - .nullable() - .oneOf(meta.boundaryLevels.map(level => level.id)) - .required('Choose a boundary level'), - })} - > - {({ formState, watch }) => { - const values = watch(); - return ( -
    { - onSubmit(normalizeValues(values)); - await submitForms(); - }} - > - - - - label="Boundary level" - hint="Select a version of geographical data to use" - name="boundaryLevel" - order={[]} - options={[ - { - label: 'Please select', - value: '', - }, - ...meta.boundaryLevels.map(({ id, label }) => ({ - value: id, - label, - })), - ]} - /> - - - {buttons} - - - ); - }} -
    + ); } diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBoundaryLevelsForm.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBoundaryLevelsForm.tsx new file mode 100644 index 00000000000..208d7d8f781 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBoundaryLevelsForm.tsx @@ -0,0 +1,193 @@ +import ChartBuilderSaveActions from '@admin/pages/release/datablocks/components/chart/ChartBuilderSaveActions'; +import { useChartBuilderFormsContext } from '@admin/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext'; +import generateDataSetLabel from '@admin/pages/release/datablocks/components/chart/utils/generateDataSetLabel'; +import Effect from '@common/components/Effect'; +import Form from '@common/components/form/Form'; +import FormFieldSelect from '@common/components/form/FormFieldSelect'; +import FormProvider from '@common/components/form/FormProvider'; +import { SelectOption } from '@common/components/form/FormSelect'; +import { MapDataSetConfig } from '@common/modules/charts/types/chart'; +import expandDataSet from '@common/modules/charts/util/expandDataSet'; +import generateDataSetKey from '@common/modules/charts/util/generateDataSetKey'; +import { FullTableMeta } from '@common/modules/table-tool/types/fullTable'; +import Yup from '@common/validation/yup'; +import React, { ReactNode, useCallback, useMemo } from 'react'; +import { ObjectSchema } from 'yup'; + +const formId = 'chartBoundaryLevelsConfigurationForm'; + +export interface ChartBoundaryLevelsFormValues { + boundaryLevel?: string; + dataSetConfigs: { + boundaryLevel?: string; + }[]; +} + +interface Props { + buttons?: ReactNode; + initialValues: ChartBoundaryLevelsFormValues; + dataSetConfigs: MapDataSetConfig[]; + // TODO: EES-5402 - Remove when all boundary level changes are done + hasDataSetBoundaryLevels?: boolean; + meta: FullTableMeta; + onChange: (values: Partial) => void; + onSubmit: (values: ChartBoundaryLevelsFormValues) => void; +} + +export default function ChartBoundaryLevelsForm({ + buttons, + dataSetConfigs, + // TODO: EES-5402 - Remove when all boundary level changes are done + hasDataSetBoundaryLevels, + initialValues, + meta, + onChange, + onSubmit, +}: Props) { + const { updateForm, submitForms } = useChartBuilderFormsContext(); + + const boundaryLevelOptions = useMemo[]>(() => { + return meta.boundaryLevels.map(level => { + return { + label: level.label, + value: level.id.toString(), + }; + }); + }, [meta.boundaryLevels]); + + const dataSetRows = useMemo(() => { + return dataSetConfigs.map(dataSetConfig => { + const expandedDataSet = expandDataSet(dataSetConfig.dataSet, meta); + const label = generateDataSetLabel(expandedDataSet); + const key = generateDataSetKey(dataSetConfig.dataSet); + + return { + key, + label, + }; + }); + }, [dataSetConfigs, meta]); + + const validationSchema = useMemo< + ObjectSchema + >(() => { + return Yup.object({ + boundaryLevel: Yup.string().required('Choose a boundary level'), + dataSetConfigs: Yup.array() + .of( + Yup.object({ + boundaryLevel: Yup.string().optional(), + }), + ) + .required(), + }); + }, []); + + const handleSubmit = useCallback( + async (values: ChartBoundaryLevelsFormValues) => { + onSubmit(values); + await submitForms(); + }, + [onSubmit, submitForms], + ); + + return ( + + {({ formState, reset }) => { + return ( + + id={formId} + onChange={onChange} + onSubmit={handleSubmit} + > + { + reset(); + }} + /> + + + label={ + hasDataSetBoundaryLevels + ? 'Default boundary level' + : 'Boundary level' + } + hint={`Select a version of geographical data to use${ + hasDataSetBoundaryLevels + ? " across any data sets that don't have a specific one set" + : '' + }`} + name="boundaryLevel" + order={[]} + options={[ + { + label: 'Please select', + value: '', + }, + ...boundaryLevelOptions, + ]} + /> + {hasDataSetBoundaryLevels && dataSetRows.length > 1 && ( + <> +

    Set boundary levels per data set

    + + + + + + + + + {dataSetRows.map(({ key, label }, index) => { + return ( + + + + + ); + })} + +
    Data setBoundary level
    {label} + +
    + + )} + + + {buttons} + + + ); + }} +     + ); +} diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBuilder.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBuilder.tsx index e2e26b008b4..aa8f3a0fdc3 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBuilder.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartBuilder.tsx @@ -1,17 +1,15 @@ import useGetChartFile from '@admin/hooks/useGetChartFile'; import ChartAxisConfiguration from '@admin/pages/release/datablocks/components/chart/ChartAxisConfiguration'; +import ChartBoundaryLevelsConfiguration from '@admin/pages/release/datablocks/components/chart/ChartBoundaryLevelsConfiguration'; import ChartBuilderPreview from '@admin/pages/release/datablocks/components/chart/ChartBuilderPreview'; import ChartConfiguration from '@admin/pages/release/datablocks/components/chart/ChartConfiguration'; +import ChartDataGroupingsConfiguration from '@admin/pages/release/datablocks/components/chart/ChartDataGroupingsConfiguration'; import ChartDataSetsConfiguration from '@admin/pages/release/datablocks/components/chart/ChartDataSetsConfiguration'; import ChartDefinitionSelector from '@admin/pages/release/datablocks/components/chart/ChartDefinitionSelector'; import ChartLegendConfiguration from '@admin/pages/release/datablocks/components/chart/ChartLegendConfiguration'; -import ChartBoundaryLevelsConfiguration from '@admin/pages/release/datablocks/components/chart/ChartBoundaryLevelsConfiguration'; -import ChartDataGroupingsConfiguration from '@admin/pages/release/datablocks/components/chart/ChartDataGroupingsConfiguration'; import { ChartBuilderFormsContextProvider } from '@admin/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext'; -import { - ChartOptions, - useChartBuilderReducer, -} from '@admin/pages/release/datablocks/components/chart/reducers/chartBuilderReducer'; +import { useChartBuilderReducer } from '@admin/pages/release/datablocks/components/chart/reducers/chartBuilderReducer'; +import { MapBoundaryLevelConfig } from '@admin/pages/release/datablocks/components/chart/types/mapConfig'; import Button from '@common/components/Button'; import ModalConfirm from '@common/components/ModalConfirm'; import Tabs from '@common/components/Tabs'; @@ -109,7 +107,7 @@ interface Props { onTableQueryUpdate: TableQueryUpdateHandler; } -const ChartBuilder = ({ +export default function ChartBuilder({ data, meta, releaseId, @@ -118,7 +116,7 @@ const ChartBuilder = ({ onChartSave, onChartDelete, onTableQueryUpdate, -}: Props) => { +}: Props) { const containerRef = useRef(null); const [showDeleteModal, toggleDeleteModal] = useToggle(false); @@ -126,10 +124,12 @@ const ChartBuilder = ({ const [isDataLoading, setDataLoading] = useState(false); const [isDeleting, setDeleting] = useState(false); - const { state: chartBuilderState, actions } = useChartBuilderReducer( - initialChart, + const { state: chartBuilderState, actions } = useChartBuilderReducer({ + chart: initialChart, + data, + meta, tableTitle, - ); + }); const { axes, map, definition, options, legend } = chartBuilderState; @@ -264,8 +264,8 @@ const ChartBuilder = ({ 200, ); - const [handleMapConfigurationChange] = useDebouncedCallback( - actions.updateChartMapConfiguration, + const [handleMapDataGroupingsChange] = useDebouncedCallback( + actions.updateMapDataGroupings, 200, ); @@ -274,14 +274,14 @@ const ChartBuilder = ({ 200, ); - const handleBoundaryLevelChange = useCallback( - async (values: ChartOptions) => { - actions.updateChartOptions(values); + const handleDefaultBoundaryLevelChange = useCallback( + async (config: MapBoundaryLevelConfig) => { + actions.updateMapBoundaryLevels(config); setDataLoading(true); await onTableQueryUpdate({ - boundaryLevel: parseNumber(values.boundaryLevel), + boundaryLevel: parseNumber(config.boundaryLevel), }); setDataLoading(false); @@ -358,8 +358,8 @@ const ChartBuilder = ({ {forms.boundaryLevels && definition?.type === 'map' && - options && - meta.boundaryLevels.length && ( + map && + options && ( )} - {axes.major && - forms.dataGroupings && + {forms.dataGroupings && definition?.type === 'map' && - options && - legend && ( + map && + options && ( { - actions.updateChartMapConfiguration(values); + onChange={handleMapDataGroupingsChange} + onSubmit={config => { + actions.updateMapDataGroupings(config); + if (options.dataClassification) { actions.updateChartOptions({ ...options, @@ -406,7 +403,6 @@ const ChartBuilder = ({ /> )} - {forms.legend && axes.major && legend && ( ); -}; - -export default ChartBuilder; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartDataGroupingsConfiguration.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartDataGroupingsConfiguration.tsx index 2c35dfa26c3..07bd55fcc57 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartDataGroupingsConfiguration.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartDataGroupingsConfiguration.tsx @@ -1,94 +1,46 @@ import ChartBuilderSaveActions from '@admin/pages/release/datablocks/components/chart/ChartBuilderSaveActions'; -import { useChartBuilderFormsContext } from '@admin/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext'; import ChartDataGroupingForm from '@admin/pages/release/datablocks/components/chart/ChartDataGroupingForm'; +import { useChartBuilderFormsContext } from '@admin/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext'; +import { MapDataGroupingConfig } from '@admin/pages/release/datablocks/components/chart/types/mapConfig'; import generateDataSetLabel from '@admin/pages/release/datablocks/components/chart/utils/generateDataSetLabel'; -import { ChartOptions } from '@admin/pages/release/datablocks/components/chart/reducers/chartBuilderReducer'; import ButtonText from '@common/components/ButtonText'; import Effect from '@common/components/Effect'; import Modal from '@common/components/Modal'; import { - AxisConfiguration, + dataGroupingTypes, MapConfig, MapDataSetConfig, - dataGroupingTypes, } from '@common/modules/charts/types/chart'; -import { LegendConfiguration } from '@common/modules/charts/types/legend'; -import createDataSetCategories from '@common/modules/charts/util/createDataSetCategories'; import expandDataSet from '@common/modules/charts/util/expandDataSet'; import generateDataSetKey from '@common/modules/charts/util/generateDataSetKey'; -import getDataSetCategoryConfigs from '@common/modules/charts/util/getDataSetCategoryConfigs'; import { FullTableMeta } from '@common/modules/table-tool/types/fullTable'; -import { TableDataResult } from '@common/services/tableBuilderService'; import isEqual from 'lodash/isEqual'; -import React, { ReactNode, useMemo, useState } from 'react'; +import React, { ReactNode, useState } from 'react'; const formId = 'chartDataGroupingsConfigurationForm'; -interface FormValues { - dataSetConfigs: MapDataSetConfig[]; -} - interface Props { - axisMajor: AxisConfiguration; buttons?: ReactNode; - data: TableDataResult[]; - legend: LegendConfiguration; map?: MapConfig; meta: FullTableMeta; - options: ChartOptions; - onChange: (dataSetConfigs: MapDataSetConfig[]) => void; - onSubmit: (dataSetConfigs: MapDataSetConfig[]) => void; + onChange: (values: MapDataGroupingConfig) => void; + onSubmit: (values: MapDataGroupingConfig) => void; } -const ChartDataGroupingsConfiguration = ({ - axisMajor, +export default function ChartDataGroupingsConfiguration({ buttons, - data, - legend, map, meta, - options, onChange, onSubmit, -}: Props) => { +}: Props) { const [editDataSetConfig, setEditDataSetConfig] = useState<{ dataSetConfig: MapDataSetConfig; unit: string; }>(); const { forms, updateForm, submitForms } = useChartBuilderFormsContext(); - const initialValues = useMemo(() => { - const dataSetCategories = createDataSetCategories({ - axisConfiguration: { - ...axisMajor, - groupBy: 'locations', - }, - data, - meta, - }); - - const dataSetCategoryConfigs = getDataSetCategoryConfigs({ - dataSetCategories, - legendItems: legend.items, - meta, - deprecatedDataClassification: options.dataClassification, - deprecatedDataGroups: options.dataGroups, - }); - - return { - dataSetConfigs: dataSetCategoryConfigs.map( - ({ rawDataSet, dataGrouping }) => ({ - dataSet: rawDataSet, - dataGrouping: - map?.dataSetConfigs.find(config => - isEqual(config.dataSet, rawDataSet), - )?.dataGrouping ?? dataGrouping, - }), - ), - }; - }, [axisMajor, data, meta, legend.items, map, options]); - - if (!initialValues.dataSetConfigs?.length) { + if (!map?.dataSetConfigs.length) { return

    No data groupings to edit.

    ; } @@ -112,7 +64,7 @@ const ChartDataGroupingsConfiguration = ({ - {initialValues.dataSetConfigs.map(dataSetConfig => { + {map.dataSetConfigs.map(dataSetConfig => { const expandedDataSet = expandDataSet(dataSetConfig.dataSet, meta); const label = generateDataSetLabel(expandedDataSet); const key = generateDataSetKey(dataSetConfig.dataSet); @@ -146,21 +98,26 @@ const ChartDataGroupingsConfiguration = ({ {editDataSetConfig && ( - + setEditDataSetConfig(undefined)} + > setEditDataSetConfig(undefined)} onSubmit={values => { - const updated = initialValues.dataSetConfigs.map(config => { - if (isEqual(config.dataSet, values.dataSet)) { - return values; - } - return config; + onChange({ + dataSetConfigs: map.dataSetConfigs.map(config => { + return isEqual(config.dataSet, values.dataSet) + ? values + : config; + }), }); - onChange(updated); + setEditDataSetConfig(undefined); }} /> @@ -177,7 +134,11 @@ const ChartDataGroupingsConfiguration = ({ ? forms.dataGroupings.submitCount + 1 : 1, }); - onSubmit(initialValues.dataSetConfigs); + + onSubmit({ + dataSetConfigs: map.dataSetConfigs, + }); + await submitForms(); }} > @@ -185,6 +146,4 @@ const ChartDataGroupingsConfiguration = ({ ); -}; - -export default ChartDataGroupingsConfiguration; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBoundaryLevelsConfiguration.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBoundaryLevelsConfiguration.test.tsx index b811a38c0f4..9387194638e 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBoundaryLevelsConfiguration.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBoundaryLevelsConfiguration.test.tsx @@ -5,11 +5,15 @@ import { ChartBuilderFormsContextProvider, } from '@admin/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext'; import { ChartOptions } from '@admin/pages/release/datablocks/components/chart/reducers/chartBuilderReducer'; -import { FullTableMeta } from '@common/modules/table-tool/types/fullTable'; import baseRender from '@common-test/render'; +import { MapConfig } from '@common/modules/charts/types/chart'; +import { DataSet } from '@common/modules/charts/types/dataSet'; +import { defaultDataGrouping } from '@common/modules/charts/util/getMapDataSetCategoryConfigs'; +import { FullTableMeta } from '@common/modules/table-tool/types/fullTable'; import { screen, waitFor, within } from '@testing-library/react'; import noop from 'lodash/noop'; -import React, { ReactElement } from 'react'; +import { ReactElement } from 'react'; +import { MapBoundaryLevelConfig } from '../types/mapConfig'; describe('ChartBoundaryLevelsConfiguration', () => { const testDefaultChartOptions: ChartOptions = { @@ -35,7 +39,6 @@ describe('ChartBoundaryLevelsConfiguration', () => { }, ], }; - const testFormState: ChartBuilderForms = { options: { isValid: true, @@ -50,6 +53,30 @@ describe('ChartBoundaryLevelsConfiguration', () => { title: 'Boundary levels configuration', }, }; + const testDataSets: DataSet[] = [ + { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + ]; + const testDefaultMap: MapConfig = { + dataSetConfigs: [ + { + dataGrouping: defaultDataGrouping, + dataSet: testDataSets[0], + }, + { + dataGrouping: defaultDataGrouping, + dataSet: testDataSets[1], + }, + ], + }; function render(element: ReactElement) { return baseRender( @@ -63,46 +90,133 @@ describe('ChartBoundaryLevelsConfiguration', () => { ); } - test('renders correctly without initial values', () => { + test('renders without data sets table', () => { render( , ); + expect(screen.getByLabelText('Default boundary level')).toBeInTheDocument(); + expect( + screen.queryByText('Set boundary levels per data set'), + ).not.toBeInTheDocument(); + expect(screen.queryByRole('table')).not.toBeInTheDocument(); + }); - expect(screen.getByLabelText('Boundary level')).not.toHaveValue(); - const boundaryLevels = within( - screen.getByLabelText('Boundary level'), + test('renders data sets without initial boundary levels', () => { + render( + , + ); + + expect(screen.getByLabelText('Default boundary level')).not.toHaveValue(); + const defaultBoundaryLevels = within( + screen.getByLabelText('Default boundary level'), ).getAllByRole('option'); - expect(boundaryLevels).toHaveLength(4); - expect(boundaryLevels[0]).toHaveTextContent('Please select'); - expect(boundaryLevels[0]).toHaveValue(''); - expect(boundaryLevels[1]).toHaveTextContent('Boundary level 1'); - expect(boundaryLevels[1]).toHaveValue('1'); - expect(boundaryLevels[2]).toHaveTextContent('Boundary level 2'); - expect(boundaryLevels[2]).toHaveValue('2'); - expect(boundaryLevels[3]).toHaveTextContent('Boundary level 3'); - expect(boundaryLevels[3]).toHaveValue('3'); + expect(defaultBoundaryLevels).toHaveLength(4); + + expect(defaultBoundaryLevels[0]).toHaveTextContent('Please select'); + expect(defaultBoundaryLevels[0]).toHaveValue(''); + expect(defaultBoundaryLevels[1]).toHaveTextContent('Boundary level 1'); + expect(defaultBoundaryLevels[1]).toHaveValue('1'); + expect(defaultBoundaryLevels[2]).toHaveTextContent('Boundary level 2'); + expect(defaultBoundaryLevels[2]).toHaveValue('2'); + expect(defaultBoundaryLevels[3]).toHaveTextContent('Boundary level 3'); + expect(defaultBoundaryLevels[3]).toHaveValue('3'); + + expect( + screen.getByText('Set boundary levels per data set'), + ).toBeInTheDocument(); + expect(screen.queryByRole('table')).toBeInTheDocument(); + + const rows = screen.getAllByRole('row'); + expect(rows).toHaveLength(3); + + const row1Cells = within(rows[1]).getAllByRole('cell'); + expect(row1Cells[0]).toHaveTextContent( + 'Number of authorised absence sessions (Ethnicity Major Chinese, State-funded primary, All locations, 2014/15)', + ); + expect(within(row1Cells[1]).getByRole('combobox')).toHaveValue(''); + + const row1BoundaryLevels = within(row1Cells[1]).getAllByRole('option'); + expect(row1BoundaryLevels[0]).toHaveTextContent('Use default'); + expect(row1BoundaryLevels[0]).toHaveValue(''); + expect(row1BoundaryLevels[1]).toHaveTextContent('Boundary level 1'); + expect(row1BoundaryLevels[1]).toHaveValue('1'); + expect(row1BoundaryLevels[2]).toHaveTextContent('Boundary level 2'); + expect(row1BoundaryLevels[2]).toHaveValue('2'); + expect(row1BoundaryLevels[3]).toHaveTextContent('Boundary level 3'); + expect(row1BoundaryLevels[3]).toHaveValue('3'); + + const row2Cells = within(rows[2]).getAllByRole('cell'); + expect(row2Cells[0]).toHaveTextContent( + 'Number of authorised absence sessions (Ethnicity Major Chinese, State-funded primary, All locations, 2015/16)', + ); + expect(within(row2Cells[1]).getByRole('combobox')).toHaveValue(''); + + const row2BoundaryLevels = within(row2Cells[1]).getAllByRole('option'); + expect(row2BoundaryLevels[0]).toHaveTextContent('Use default'); + expect(row2BoundaryLevels[0]).toHaveValue(''); + expect(row2BoundaryLevels[1]).toHaveTextContent('Boundary level 1'); + expect(row2BoundaryLevels[1]).toHaveValue('1'); + expect(row2BoundaryLevels[2]).toHaveTextContent('Boundary level 2'); + expect(row2BoundaryLevels[2]).toHaveValue('2'); + expect(row2BoundaryLevels[3]).toHaveTextContent('Boundary level 3'); + expect(row2BoundaryLevels[3]).toHaveValue('3'); }); - test('renders correctly with initial values', () => { + test('renders data sets with initial boundary levels', () => { render( , ); - expect(screen.getByLabelText('Boundary level')).toHaveValue('2'); + expect(screen.getByLabelText('Default boundary level')).toHaveValue( + String(testMeta.boundaryLevels[2].id), + ); + const rows = screen.getAllByRole('row'); + expect(rows).toHaveLength(3); + + expect(within(rows[1]).getByRole('combobox')).toHaveValue( + String(testMeta.boundaryLevels[0].id), + ); + expect(within(rows[2]).getByRole('combobox')).toHaveValue( + String(testMeta.boundaryLevels[1].id), + ); }); test('calls `onChange` handler when form values change', async () => { @@ -110,28 +224,47 @@ describe('ChartBoundaryLevelsConfiguration', () => { const { user } = render( , ); - await user.selectOptions(screen.getByLabelText('Boundary level'), ['2']); + await user.selectOptions(screen.getByLabelText('Default boundary level'), [ + '2', + ]); + + expect(handleChange).toHaveBeenCalledWith<[MapBoundaryLevelConfig]>({ + boundaryLevel: 2, + dataSetConfigs: testDataSets.map(dataSet => ({ + dataSet, + })), + }); + + const rows = screen.getAllByRole('row'); + await user.selectOptions(within(rows[1]).getByRole('combobox'), ['2']); - expect(handleChange).toHaveBeenCalledWith<[ChartOptions]>({ - ...testDefaultChartOptions, + expect(handleChange).toHaveBeenCalledWith<[MapBoundaryLevelConfig]>({ boundaryLevel: 2, + dataSetConfigs: [ + { dataSet: testDataSets[0], boundaryLevel: 2 }, + { dataSet: testDataSets[1] }, + ], }); }); test('submitting fails with validation errors if no boundary level set', async () => { const { user } = render( , ); @@ -156,14 +289,18 @@ describe('ChartBoundaryLevelsConfiguration', () => { const { user } = render( , ); - await user.selectOptions(screen.getByLabelText('Boundary level'), ['2']); + await user.selectOptions(screen.getByLabelText('Default boundary level'), [ + '2', + ]); expect(handleSubmit).not.toHaveBeenCalled(); @@ -172,9 +309,9 @@ describe('ChartBoundaryLevelsConfiguration', () => { ); await waitFor(() => { - expect(handleSubmit).toHaveBeenCalledWith<[ChartOptions]>({ - ...testDefaultChartOptions, + expect(handleSubmit).toHaveBeenCalledWith<[MapBoundaryLevelConfig]>({ boundaryLevel: 2, + dataSetConfigs: testDataSets.map(dataSet => ({ dataSet })), }); }); }); @@ -184,6 +321,7 @@ describe('ChartBoundaryLevelsConfiguration', () => { const { user } = render( { }} onChange={noop} onSubmit={handleSubmit} + hasDataSetBoundaryLevels />, ); @@ -201,9 +340,9 @@ describe('ChartBoundaryLevelsConfiguration', () => { ); await waitFor(() => { - expect(handleSubmit).toHaveBeenCalledWith<[ChartOptions]>({ - ...testDefaultChartOptions, + expect(handleSubmit).toHaveBeenCalledWith<[MapBoundaryLevelConfig]>({ boundaryLevel: 3, + dataSetConfigs: testDataSets.map(dataSet => ({ dataSet })), }); }); }); diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBuilder.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBuilder.test.tsx index 91a2211648e..4a282044a9c 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBuilder.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBuilder.test.tsx @@ -447,4 +447,216 @@ describe('ChartBuilder', () => { expect(handleUpdate).toHaveBeenCalledWith({ boundaryLevel: 1 }); }); + + describe('data gorupings tab', () => { + const testInitialChart: Chart = { + type: 'map', + boundaryLevel: 2, + map: { + dataSetConfigs: [ + { + dataGrouping: { + customGroups: [], + numberOfGroups: 5, + type: 'EqualIntervals', + }, + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + }, + { + dataGrouping: { + customGroups: [], + numberOfGroups: 5, + type: 'EqualIntervals', + }, + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + }, + ], + }, + title: 'Data block title', + subtitle: '', + alt: 'd', + height: 600, + includeNonNumericData: false, + axes: { + major: { + type: 'major', + groupBy: 'locations', + groupByFilter: '', + groupByFilterGroups: false, + sortBy: 'name', + sortAsc: false, + dataSets: [ + { + order: 0, + indicator: 'overall-absence-sessions', + filters: ['state-funded-primary'], + timePeriod: '2014_AY', + }, + + { + order: 1, + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + { + order: 2, + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + ], + referenceLines: [], + visible: true, + unit: '', + showGrid: false, + label: { + text: '', + rotated: false, + }, + min: 0, + size: 50, + tickConfig: 'default', + tickSpacing: 1, + }, + }, + legend: { + items: [ + { + colour: '#12436D', + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + inlinePosition: undefined, + label: + 'Number of authorised absence sessions (Ethnicity Major Chinese, State-funded primary, 2014/15)', + lineStyle: undefined, + symbol: undefined, + }, + { + colour: '#F46A25', + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + inlinePosition: undefined, + label: + 'Number of authorised absence sessions (Ethnicity Major Chinese, State-funded primary, 2015/16)', + lineStyle: undefined, + symbol: undefined, + }, + ], + }, + }; + + test('save chart with updates data groupings', async () => { + const handleSubmit = jest.fn(); + + const { user } = render( + + + , + ); + + expect( + screen.getByRole('button', { name: 'Chart preview' }), + ).toBeInTheDocument(); + + expect( + screen.getByRole('tab', { name: 'Data groupings' }), + ).toBeInTheDocument(); + + await user.click(screen.getByRole('tab', { name: 'Data groupings' })); + + expect(await screen.findByRole('heading', { name: 'Data groupings' })); + + await user.click(screen.getAllByRole('button', { name: 'Edit' })[0]); + + expect(await screen.findByRole('dialog')).toBeInTheDocument(); + expect(screen.getByText('Edit groupings')).toBeInTheDocument(); + + await user.click(screen.getByLabelText('Quantiles')); + await user.click(screen.getByRole('button', { name: 'Done' })); + + await user.click( + screen.getByRole('button', { name: 'Save chart options' }), + ); + + await waitFor(() => { + expect(handleSubmit).toHaveBeenCalledTimes(1); + expect(handleSubmit).toHaveBeenCalledWith( + { + ...testInitialChart, + map: { + dataSetConfigs: [ + { + dataGrouping: { + customGroups: [], + numberOfGroups: 5, + type: 'Quantiles', + }, + dataSet: { + filters: [ + 'ethnicity-major-chinese', + 'state-funded-primary', + ], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + }, + { + dataGrouping: { + customGroups: [], + numberOfGroups: 5, + type: 'EqualIntervals', + }, + dataSet: { + filters: [ + 'ethnicity-major-chinese', + 'state-funded-primary', + ], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + }, + ], + }, + }, + undefined, + ); + }); + }); + }); }); diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartDataGroupingsConfiguration.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartDataGroupingsConfiguration.test.tsx index 63703d835e4..bcc581bba00 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartDataGroupingsConfiguration.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartDataGroupingsConfiguration.test.tsx @@ -4,9 +4,9 @@ import { ChartBuilderForms, ChartBuilderFormsContextProvider, } from '@admin/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext'; -import { ChartOptions } from '@admin/pages/release/datablocks/components/chart/reducers/chartBuilderReducer'; import baseRender from '@common-test/render'; -import { screen, within } from '@testing-library/react'; +import { defaultDataGrouping } from '@common/modules/charts/util/getMapDataSetCategoryConfigs'; +import { screen, waitFor, within } from '@testing-library/react'; import noop from 'lodash/noop'; import React, { ReactElement } from 'react'; @@ -34,12 +34,6 @@ describe('ChartDataGroupingsConfiguration', () => { }, }; - const testDefaultChartOptions: ChartOptions = { - alt: '', - height: 600, - titleType: 'default', - }; - function render( element: ReactElement, initialForms: ChartBuilderForms = testFormState, @@ -54,22 +48,9 @@ describe('ChartDataGroupingsConfiguration', () => { test('renders correctly with no data groupings', () => { render( , ); @@ -80,25 +61,18 @@ describe('ChartDataGroupingsConfiguration', () => { render( , @@ -121,30 +95,26 @@ describe('ChartDataGroupingsConfiguration', () => { render( , @@ -176,20 +146,6 @@ describe('ChartDataGroupingsConfiguration', () => { render( { }, ], }} - legend={{ - position: 'top', - items: [], - }} - options={testDefaultChartOptions} onSubmit={noop} onChange={noop} />, @@ -233,30 +184,6 @@ describe('ChartDataGroupingsConfiguration', () => { render( { }, ], }} - legend={{ - position: 'top', - items: [], - }} - options={testDefaultChartOptions} onSubmit={noop} onChange={noop} />, @@ -337,29 +259,10 @@ describe('ChartDataGroupingsConfiguration', () => { ).toBeInTheDocument(); }); - test('renders correctly when data sets are removed', () => { - render( + test('shows a validation error if another form is invalid', async () => { + const { user } = render( { }, dataGrouping: { customGroups: [], - numberOfGroups: 6, + numberOfGroups: 5, type: 'EqualIntervals', }, }, @@ -399,174 +302,96 @@ describe('ChartDataGroupingsConfiguration', () => { }, ], }} - legend={{ - position: 'top', - items: [], - }} - options={testDefaultChartOptions} onSubmit={noop} onChange={noop} />, + { + ...testFormState, + options: { + ...testFormState.options, + isValid: false, + }, + }, ); - const rows = screen.getAllByRole('row'); - expect(rows).toHaveLength(3); - - const row1Cells = within(rows[1]).getAllByRole('cell'); - expect(row1Cells[0]).toHaveTextContent( - 'Number of authorised absence sessions (Ethnicity Major Chinese, State-funded primary, All locations, 2015/16)', + await user.click( + screen.getByRole('button', { name: 'Save chart options' }), ); - expect(row1Cells[1]).toHaveTextContent('2 quantiles'); - expect( - within(row1Cells[2]).getByRole('button', { name: 'Edit' }), - ).toBeInTheDocument(); - const row2Cells = within(rows[2]).getAllByRole('cell'); - expect(row2Cells[0]).toHaveTextContent( - 'Number of authorised absence sessions (Ethnicity Major Chinese, State-funded secondary, All locations, 2015/16)', - ); - expect(row2Cells[1]).toHaveTextContent('Custom'); - expect( - within(row2Cells[2]).getByRole('button', { name: 'Edit' }), - ).toBeInTheDocument(); + expect(await screen.findByText('Cannot save chart')).toBeInTheDocument(); + expect(screen.getByText('Options tab is invalid')).toBeInTheDocument(); }); - test('renders correctly when data sets are added', () => { - render( + test('making a data grouping change and submitting the form', async () => { + const handleChange = jest.fn(); + const handleSubmit = jest.fn(); + + const { user } = render( , ); - const rows = screen.getAllByRole('row'); - expect(rows).toHaveLength(4); - - const row1Cells = within(rows[1]).getAllByRole('cell'); - expect(row1Cells[0]).toHaveTextContent( - 'Number of authorised absence sessions (Ethnicity Major Chinese, State-funded primary, All locations, 2014/15)', - ); - expect(row1Cells[1]).toHaveTextContent('5 equal intervals'); - expect( - within(row1Cells[2]).getByRole('button', { name: 'Edit' }), - ).toBeInTheDocument(); + await user.click(screen.getAllByRole('button', { name: 'Edit' })[0]); - const row2Cells = within(rows[2]).getAllByRole('cell'); - expect(row2Cells[0]).toHaveTextContent( - 'Number of authorised absence sessions (Ethnicity Major Chinese, State-funded primary, All locations, 2015/16)', - ); - expect(row2Cells[1]).toHaveTextContent('2 quantiles'); - expect( - within(row2Cells[2]).getByRole('button', { name: 'Edit' }), - ).toBeInTheDocument(); + expect(await screen.findByRole('dialog')).toBeInTheDocument(); + expect(screen.getByText('Edit groupings')).toBeInTheDocument(); - const row3Cells = within(rows[3]).getAllByRole('cell'); - expect(row3Cells[0]).toHaveTextContent( - 'Number of authorised absence sessions (Ethnicity Major Chinese, State-funded secondary, All locations, 2015/16)', - ); - expect(row3Cells[1]).toHaveTextContent('Custom'); - expect( - within(row3Cells[2]).getByRole('button', { name: 'Edit' }), - ).toBeInTheDocument(); - }); + await user.click(screen.getByLabelText('Quantiles')); + await user.click(screen.getByRole('button', { name: 'Done' })); - test('shows a validation error if another form is invalid', async () => { - const { user } = render( - { + expect(handleChange).toHaveBeenCalledTimes(1); + expect(handleChange).toHaveBeenCalledWith({ + dataSetConfigs: [ + { + dataGrouping: { ...defaultDataGrouping, type: 'Quantiles' }, + dataSet: { filters: ['ethnicity-major-chinese', 'state-funded-primary'], indicator: 'authorised-absence-sessions', timePeriod: '2014_AY', }, - ], - groupBy: 'locations', - referenceLines: [], - type: 'major', - visible: true, - }} - legend={{ - position: 'top', - items: [], - }} - options={testDefaultChartOptions} - onSubmit={noop} - onChange={noop} - />, - { - ...testFormState, - options: { - ...testFormState.options, - isValid: false, - }, - }, - ); + }, + { + dataGrouping: defaultDataGrouping, + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + }, + ], + }); + }); await user.click( screen.getByRole('button', { name: 'Save chart options' }), ); - expect(await screen.findByText('Cannot save chart')).toBeInTheDocument(); - expect(screen.getByText('Options tab is invalid')).toBeInTheDocument(); + await waitFor(() => { + expect(handleSubmit).toHaveBeenCalledTimes(1); + }); }); }); diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext.tsx index b676d44579a..1d717551356 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/contexts/ChartBuilderFormsContext.tsx @@ -50,7 +50,7 @@ export interface ChartBuilderFormsContextValue { hasSubmitted: boolean; isSubmitting: boolean; isValid: boolean; - submitForms: () => void; + submitForms: () => Promise; updateForm: (nextState: UpdateFormState) => void; } diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/reducers/__tests__/chartBuilderReducer.test.ts b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/reducers/__tests__/chartBuilderReducer.test.ts index 7122336a13e..6c33e4fd5a3 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/reducers/__tests__/chartBuilderReducer.test.ts +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/reducers/__tests__/chartBuilderReducer.test.ts @@ -6,16 +6,19 @@ import { useChartBuilderReducer, } from '@admin/pages/release/datablocks/components/chart/reducers/chartBuilderReducer'; import { lineChartBlockDefinition } from '@common/modules/charts/components/LineChartBlock'; +import { mapBlockDefinition } from '@common/modules/charts/components/MapBlock'; import { AxisConfiguration, AxisType, ChartDefinition, + DataGroupingConfig, } from '@common/modules/charts/types/chart'; import { DataSet } from '@common/modules/charts/types/dataSet'; import { LegendConfiguration } from '@common/modules/charts/types/legend'; import { Chart } from '@common/services/types/blocks'; import { renderHook } from '@testing-library/react'; import { produce } from 'immer'; +import { testFullTable } from '../../__tests__/__data__/testTableData'; describe('chartBuilderReducer', () => { const testChartDefinition: ChartDefinition = { @@ -100,13 +103,23 @@ describe('chartBuilderReducer', () => { payload: testChartDefinition, }; - const nextState = produce(chartBuilderReducer)(initialState, action); + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, action); expect(nextState.definition).toEqual(testChartDefinition); }); test('sets `options` with defaults from the definition', () => { - const nextState = produce(chartBuilderReducer)(initialState, { + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, { type: 'UPDATE_CHART_DEFINITION', payload: testChartDefinition, }); @@ -126,7 +139,12 @@ describe('chartBuilderReducer', () => { payload: testChartDefinition, }; - const nextState = produce(chartBuilderReducer)(initialState, action); + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, action); expect(nextState.legend).toEqual({ position: 'top', @@ -164,10 +182,12 @@ describe('chartBuilderReducer', () => { payload: testChartDefinition, }; - const nextState = produce(chartBuilderReducer)( - initialStateWithOptions, - action, - ); + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialStateWithOptions, action); expect(nextState.options).toEqual({ // Height is set to the definition default @@ -198,7 +218,12 @@ describe('chartBuilderReducer', () => { payload: testChartDefinition, }; - const nextState = produce(chartBuilderReducer)(initialState, action); + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, action); expect(nextState.axes.major).toEqual({ dataSets: [], @@ -257,10 +282,12 @@ describe('chartBuilderReducer', () => { payload: testChartDefinition, }; - const nextState = produce(chartBuilderReducer)( - initialStateWithAxes, - action, - ); + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialStateWithAxes, action); expect(nextState.axes.major).toMatchObject>({ dataSets: [], @@ -310,7 +337,12 @@ describe('chartBuilderReducer', () => { }, }; - const nextState = produce(chartBuilderReducer)(initialState, action); + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, action); expect(nextState.axes.major).toEqual({ dataSets: [], @@ -349,7 +381,12 @@ describe('chartBuilderReducer', () => { }, }; - produce(chartBuilderReducer)(initialState, action); + produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, action); }).toThrow("Could not find chart axis definition for type 'not valid'"); }); }); @@ -380,7 +417,12 @@ describe('chartBuilderReducer', () => { }, }; - const nextState = produce(chartBuilderReducer)(initialState, action); + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, action); expect(nextState.options).toEqual({ height: 500, @@ -417,10 +459,12 @@ describe('chartBuilderReducer', () => { }, }; - const nextState = produce(chartBuilderReducer)( - initialStateWithExistingOptions, - action, - ); + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialStateWithExistingOptions, action); expect(nextState.options).toEqual({ height: 500, @@ -458,10 +502,12 @@ describe('chartBuilderReducer', () => { }, }; - const nextState = produce(chartBuilderReducer)( - initialStateWithExistingOptions, - action, - ); + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialStateWithExistingOptions, action); expect(nextState.options).toEqual({ height: 300, @@ -503,7 +549,12 @@ describe('chartBuilderReducer', () => { ], }; - const nextState = produce(chartBuilderReducer)(initialState, action); + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, action); expect(nextState.axes.major?.dataSets).toEqual([ { @@ -534,7 +585,12 @@ describe('chartBuilderReducer', () => { }, }; - const nextState = produce(chartBuilderReducer)(initialState, { + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, { type: 'RESET', }); @@ -545,10 +601,105 @@ describe('chartBuilderReducer', () => { }); }); + describe('UPDATE_MAP_BOUNDARY_LEVELS', () => { + const initialState: ChartBuilderState = { + axes: {}, + options: { + height: 300, + subtitle: '', + title: '', + titleType: 'default', + alt: '', + }, + map: { + dataSetConfigs: [ + { + dataSet: { filters: [] }, + dataGrouping: { type: 'EqualIntervals', customGroups: [] }, + }, + ], + }, + }; + + test('sets the boundary level to the payload boundaryLevel', () => { + const action: ChartBuilderActions = { + type: 'UPDATE_MAP_BOUNDARY_LEVELS', + payload: { + boundaryLevel: 2, + dataSetConfigs: [{ dataSet: { filters: [] }, boundaryLevel: 1 }], + }, + }; + + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, action); + + expect(nextState.options?.boundaryLevel).toEqual(2); + expect(nextState.map?.dataSetConfigs[0]?.boundaryLevel).toEqual(1); + }); + }); + + describe('UPDATE_MAP_DATA_GROUPINGS', () => { + const initialState: ChartBuilderState = { + axes: {}, + options: { + height: 300, + subtitle: '', + title: '', + titleType: 'default', + alt: '', + }, + map: { + dataSetConfigs: [ + { + dataSet: { filters: [] }, + dataGrouping: { type: 'EqualIntervals', customGroups: [] }, + }, + ], + }, + }; + test('sets the dataGrouping to the payload dataGrouping', () => { + const testDataGrouping: DataGroupingConfig = { + type: 'Custom', + customGroups: [{ min: 0, max: 999 }], + numberOfGroups: 1, + }; + const action: ChartBuilderActions = { + type: 'UPDATE_MAP_DATA_GROUPINGS', + payload: { + dataSetConfigs: [ + { + dataSet: { filters: [] }, + dataGrouping: testDataGrouping, + }, + ], + }, + }; + + const nextState = produce( + chartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + }), + )(initialState, action); + + expect(nextState.map?.dataSetConfigs[0]?.dataGrouping).toEqual( + testDataGrouping, + ); + }); + }); + describe('useChartBuilderReducer', () => { test('has correct state when no initial configuration', () => { const { result } = renderHook(() => - useChartBuilderReducer(undefined, 'Table title'), + useChartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + tableTitle: 'Table title', + }), ); expect(result.current.state).toEqual({ @@ -612,7 +763,12 @@ describe('chartBuilderReducer', () => { }; const { result } = renderHook(() => - useChartBuilderReducer(initialConfiguration, 'Table title'), + useChartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + tableTitle: 'Table title', + chart: initialConfiguration, + }), ); expect(result.current.state).toEqual({ @@ -725,7 +881,12 @@ describe('chartBuilderReducer', () => { }; const { result } = renderHook(() => - useChartBuilderReducer(initialConfiguration, 'Table title'), + useChartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + tableTitle: 'Table title', + chart: initialConfiguration, + }), ); expect(result.current.state).toEqual({ @@ -823,7 +984,12 @@ describe('chartBuilderReducer', () => { }; const { result } = renderHook(() => - useChartBuilderReducer(initialConfiguration), + useChartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + + chart: initialConfiguration, + }), ); expect(result.current.state.axes.minor).toEqual({ @@ -897,7 +1063,12 @@ describe('chartBuilderReducer', () => { }; const { result } = renderHook(() => - useChartBuilderReducer(initialConfiguration, 'Table title'), + useChartBuilderReducer({ + data: [], + meta: testFullTable.subjectMeta, + tableTitle: 'Table title', + chart: initialConfiguration, + }), ); expect(result.current.state).toEqual({ @@ -966,5 +1137,330 @@ describe('chartBuilderReducer', () => { }, }); }); + + test('has correct state with initial configuration for a map', () => { + const initialConfiguration: Chart = { + legend: { + position: 'top', + items: [], + }, + axes: { + major: { + type: 'major', + dataSets: [ + { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + ], + referenceLines: [], + }, + }, + type: 'map', + height: 300, + title: 'Chart title', + alt: '', + boundaryLevel: 1, + }; + + const { result } = renderHook(() => + useChartBuilderReducer({ + data: testFullTable.results, + meta: testFullTable.subjectMeta, + tableTitle: 'Table title', + chart: initialConfiguration, + }), + ); + + expect(result.current.state).toEqual({ + axes: { + major: { + type: 'major', + groupBy: 'locations', + dataSets: [ + { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + ], + referenceLines: [], + visible: true, + label: { + text: '', + }, + }, + }, + definition: mapBlockDefinition, + map: { + dataSetConfigs: [ + { + dataGrouping: { + customGroups: [], + numberOfGroups: 5, + type: 'EqualIntervals', + }, + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + }, + { + dataGrouping: { + customGroups: [], + numberOfGroups: 5, + type: 'EqualIntervals', + }, + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + }, + ], + }, + options: { + alt: '', + height: 300, + subtitle: '', + title: 'Chart title', + titleType: 'alternative', + boundaryLevel: 1, + }, + legend: { + position: 'top', + items: [], + }, + }); + }); + + test('setting boundary levels does not change data groupings', () => { + const initialConfiguration: Chart = { + legend: { + position: 'top', + items: [], + }, + axes: { + major: { + type: 'major', + dataSets: [ + { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + ], + referenceLines: [], + }, + }, + type: 'map', + height: 300, + title: 'Chart title', + alt: '', + boundaryLevel: 1, + map: { + dataSetConfigs: [ + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + dataGrouping: { type: 'EqualIntervals', customGroups: [] }, + boundaryLevel: 2, + }, + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + dataGrouping: { type: 'Custom', customGroups: [] }, + boundaryLevel: 3, + }, + ], + }, + }; + + const { result, rerender } = renderHook(() => + useChartBuilderReducer({ + data: testFullTable.results, + meta: testFullTable.subjectMeta, + tableTitle: 'Table title', + chart: initialConfiguration, + }), + ); + + result.current.actions.updateMapBoundaryLevels({ + boundaryLevel: 10, + dataSetConfigs: [ + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + boundaryLevel: 20, + }, + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + boundaryLevel: 30, + }, + ], + }); + rerender(); + + expect(result.current.state.map).toEqual({ + dataSetConfigs: [ + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + dataGrouping: { type: 'EqualIntervals', customGroups: [] }, + boundaryLevel: 20, + }, + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + dataGrouping: { type: 'Custom', customGroups: [] }, + boundaryLevel: 30, + }, + ], + }); + }); + + test('setting data groupings does not change boundary levels', () => { + const initialConfiguration: Chart = { + legend: { + position: 'top', + items: [], + }, + axes: { + major: { + type: 'major', + dataSets: [ + { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + ], + referenceLines: [], + }, + }, + type: 'map', + height: 300, + title: 'Chart title', + alt: '', + boundaryLevel: 1, + map: { + dataSetConfigs: [ + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + dataGrouping: { type: 'EqualIntervals', customGroups: [] }, + boundaryLevel: 2, + }, + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + dataGrouping: { type: 'Custom', customGroups: [] }, + boundaryLevel: 3, + }, + ], + }, + }; + + const { result, rerender } = renderHook(() => + useChartBuilderReducer({ + data: testFullTable.results, + meta: testFullTable.subjectMeta, + tableTitle: 'Table title', + chart: initialConfiguration, + }), + ); + + result.current.actions.updateMapDataGroupings({ + dataSetConfigs: [ + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + dataGrouping: { type: 'Quantiles', customGroups: [] }, + }, + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + dataGrouping: { type: 'Quantiles', customGroups: [] }, + }, + ], + }); + rerender(); + + expect(result.current.state.map).toEqual({ + dataSetConfigs: [ + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2015_AY', + }, + dataGrouping: { type: 'Quantiles', customGroups: [] }, + boundaryLevel: 2, + }, + { + dataSet: { + filters: ['ethnicity-major-chinese', 'state-funded-primary'], + indicator: 'authorised-absence-sessions', + timePeriod: '2014_AY', + }, + dataGrouping: { type: 'Quantiles', customGroups: [] }, + boundaryLevel: 3, + }, + ], + }); + }); }); }); diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/reducers/chartBuilderReducer.ts b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/reducers/chartBuilderReducer.ts index 165174a8696..0d1641f4ae2 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/reducers/chartBuilderReducer.ts +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/reducers/chartBuilderReducer.ts @@ -1,3 +1,7 @@ +import { + MapBoundaryLevelConfig, + MapDataGroupingConfig, +} from '@admin/pages/release/datablocks/components/chart/types/mapConfig'; import { useLoggedImmerReducer } from '@common/hooks/useLoggedReducer'; import { AxesConfiguration, @@ -12,11 +16,21 @@ import { } from '@common/modules/charts/types/chart'; import { DataSet } from '@common/modules/charts/types/dataSet'; import { LegendConfiguration } from '@common/modules/charts/types/legend'; +import createDataSetCategories from '@common/modules/charts/util/createDataSetCategories'; +import getMapDataSetCategoryConfigs from '@common/modules/charts/util/getMapDataSetCategoryConfigs'; +import { FullTableMeta } from '@common/modules/table-tool/types/fullTable'; +import { TableDataResult } from '@common/services/tableBuilderService'; import { Chart } from '@common/services/types/blocks'; import deepMerge from 'deepmerge'; import mapValues from 'lodash/mapValues'; -import { useCallback, useMemo } from 'react'; -import { Reducer } from 'use-immer'; +import { Reducer, useCallback, useMemo } from 'react'; + +export interface ChartBuilderReducerOptions { + chart?: Chart; + data: TableDataResult[]; + meta: FullTableMeta; + tableTitle?: string; +} export interface ChartOptions extends ChartDefinitionOptions { file?: File; @@ -50,8 +64,12 @@ export type ChartBuilderActions = payload: LegendConfiguration; } | { - type: 'UPDATE_CHART_MAP_CONFIGURATION'; - payload: MapDataSetConfig[]; + type: 'UPDATE_MAP_BOUNDARY_LEVELS'; + payload: MapBoundaryLevelConfig; + } + | { + type: 'UPDATE_MAP_DATA_GROUPINGS'; + payload: MapDataGroupingConfig; } | { type: 'UPDATE_CHART_AXIS'; @@ -106,171 +124,313 @@ const updateAxis = ( }, ], { - arrayMerge: (target, source) => source, + arrayMerge: (_, source) => source, }, ); }; -const getInitialState = ( - initialChart?: Chart, - tableTitle?: string, -): ChartBuilderState => { - if (!initialChart) { +const getInitialState = ({ + chart, + data, + meta, + tableTitle, +}: ChartBuilderReducerOptions): ChartBuilderState => { + if (!chart) { return { titleType: 'default', axes: {}, }; } - const { type, axes, legend, map, ...options } = initialChart; + const { + type, + axes: initialAxes, + legend: initialLegend, + map, + ...initialOptions + } = chart; const definition = chartDefinitions.find( chartDefinition => chartDefinition.type === type, ); + if (!definition) { + throw new Error(`Could not find chart definition for type: ${type}`); + } + + const options: ChartOptions = { + ...defaultOptions, + ...(definition?.options.defaults ?? {}), + ...initialOptions, + titleType: chart.title === tableTitle ? 'default' : 'alternative', + }; + + const axes: AxesConfiguration = mapValues( + definition?.axes ?? {}, + (axisDefinition: ChartDefinitionAxis, axisType: AxisType) => + updateAxis( + axisDefinition, + (initialAxes?.[axisType] ?? {}) as AxisConfiguration, + ), + ); + + const legend: LegendConfiguration = { + ...defaultLegend, + ...(initialLegend ?? {}), + }; + return { definition, - options: { - ...defaultOptions, - ...(definition?.options.defaults ?? {}), - ...options, - titleType: initialChart.title === tableTitle ? 'default' : 'alternative', - }, - legend: { - ...defaultLegend, - ...(legend ?? {}), - }, - axes: mapValues( - definition?.axes ?? {}, - (axisDefinition: ChartDefinitionAxis, axisType: AxisType) => - updateAxis( - axisDefinition, - (axes?.[axisType] ?? {}) as AxisConfiguration, - ), - ), - map, + options, + legend, + axes, + map: getInitialMapState({ + axes, + data, + definition, + legend, + map, + meta, + options, + }), }; }; -export const chartBuilderReducer: Reducer< - ChartBuilderState, - ChartBuilderActions -> = (draft, action) => { - switch (action.type) { - case 'UPDATE_CHART_DEFINITION': { - draft.definition = action.payload; - - draft.options = { - ...defaultOptions, - ...(action.payload.options.defaults ?? {}), - ...draft.options, - // Set height/width to definition defaults - // as this seems to surprise users the least. - height: - action.payload.options.defaults?.height ?? - draft.options?.height ?? - defaultOptions.height, - width: action.payload.options.defaults?.width ?? draft.options?.width, - }; - - if (action.payload.capabilities.hasLegend) { - draft.legend = { - ...defaultLegend, - ...(action.payload.legend.defaults ?? {}), - ...(draft.legend ?? {}), - }; - } else { - draft.legend = undefined; - } +function getInitialMapState({ + axes, + data, + definition, + legend, + map, + meta, + options, +}: { + axes: AxesConfiguration; + data: TableDataResult[]; + definition: ChartDefinition; + legend?: LegendConfiguration; + map?: MapConfig; + meta: FullTableMeta; + options: ChartOptions; +}): MapConfig | undefined { + if (definition.type !== 'map' || !axes.major) { + return undefined; + } - draft.axes = mapValues( - action.payload.axes, - (axisDefinition: ChartDefinitionAxis, type: AxisType) => { - return updateAxis(axisDefinition, draft.axes[type]); - }, - ); + return { + dataSetConfigs: getMapDataSetConfigs({ + axisMajor: axes.major, + data, + legend, + map, + meta, + options, + }), + }; +} - break; - } - case 'UPDATE_CHART_AXIS': { - const axisDefinition = draft?.definition?.axes?.[action.payload.type]; +function getMapDataSetConfigs({ + axisMajor, + data, + legend, + map, + meta, + options, +}: { + axisMajor: AxisConfiguration; + data: TableDataResult[]; + legend?: LegendConfiguration; + map?: MapConfig; + meta: FullTableMeta; + options?: ChartOptions; +}): MapDataSetConfig[] { + const dataSetCategories = createDataSetCategories({ + axisConfiguration: { + ...axisMajor, + groupBy: 'locations', + }, + data, + meta, + }); + + const dataSetCategoryConfigs = getMapDataSetCategoryConfigs({ + dataSetCategories, + dataSetConfigs: map?.dataSetConfigs, + legendItems: legend?.items ?? [], + meta, + deprecatedDataClassification: options?.dataClassification, + deprecatedDataGroups: options?.dataGroups, + }); + + return dataSetCategoryConfigs.map(config => { + return { + boundaryLevel: config.boundaryLevel, + dataSet: config.rawDataSet, + dataGrouping: config.dataGrouping, + }; + }); +} - if (!axisDefinition) { - throw new Error( - `Could not find chart axis definition for type '${action.payload.type}'`, - ); - } +export function chartBuilderReducer( + reducerOptions: ChartBuilderReducerOptions, +): Reducer { + const { data, meta } = reducerOptions; + + return (draft, action) => { + switch (action.type) { + case 'UPDATE_CHART_DEFINITION': { + draft.definition = action.payload; + + draft.options = { + ...defaultOptions, + ...(action.payload.options.defaults ?? {}), + ...draft.options, + // Set height/width to definition defaults + // as this seems to surprise users the least. + height: + action.payload.options.defaults?.height ?? + draft.options?.height ?? + defaultOptions.height, + width: action.payload.options.defaults?.width ?? draft.options?.width, + }; - if (!draft.axes[action.payload.type]) { - throw new Error( - `Could not find axis configuration for type '${action.payload.type}'`, + if (action.payload.capabilities.hasLegend) { + draft.legend = { + ...defaultLegend, + ...(action.payload.legend.defaults ?? {}), + ...(draft.legend ?? {}), + }; + } else { + draft.legend = undefined; + } + + draft.axes = mapValues( + action.payload.axes, + (axisDefinition: ChartDefinitionAxis, type: AxisType) => { + return updateAxis(axisDefinition, draft.axes[type]); + }, ); - } - draft.axes[action.payload.type] = updateAxis( - axisDefinition, - draft.axes[action.payload.type] as AxisConfiguration, - action.payload, - ); + if (draft.definition.type === 'map' && !draft.map) { + draft.map = getInitialMapState({ + axes: draft.axes, + data, + definition: action.payload, + legend: draft.legend, + map: draft.map, + meta, + options: draft.options, + }); + } + + break; + } + case 'UPDATE_CHART_AXIS': { + const axisDefinition = draft?.definition?.axes?.[action.payload.type]; + + if (!axisDefinition) { + throw new Error( + `Could not find chart axis definition for type '${action.payload.type}'`, + ); + } + + if (!draft.axes[action.payload.type]) { + throw new Error( + `Could not find axis configuration for type '${action.payload.type}'`, + ); + } + + draft.axes[action.payload.type] = updateAxis( + axisDefinition, + draft.axes[action.payload.type] as AxisConfiguration, + action.payload, + ); - break; - } - case 'UPDATE_CHART_LEGEND': { - draft.legend = { - ...defaultLegend, - ...(draft?.definition?.legend.defaults ?? {}), - ...draft.legend, - ...action.payload, - }; - - break; - } - case 'UPDATE_CHART_MAP_CONFIGURATION': { - draft.map = { - ...draft.map, - dataSetConfigs: action.payload, - }; + break; + } + case 'UPDATE_CHART_LEGEND': { + draft.legend = { + ...defaultLegend, + ...(draft?.definition?.legend.defaults ?? {}), + ...draft.legend, + ...action.payload, + }; - break; - } - case 'UPDATE_CHART_OPTIONS': { - draft.options = { - ...defaultOptions, - ...(draft?.definition?.options.defaults ?? {}), - ...draft.options, - ...action.payload, - }; - - break; - } - case 'UPDATE_DATA_SETS': { - if (draft.axes.major) { - draft.axes.major.dataSets = action.payload; + break; } + case 'UPDATE_MAP_BOUNDARY_LEVELS': { + if (!draft.map) { + throw new Error('Map config has not been initialised'); + } + + if (draft.options) { + draft.options.boundaryLevel = action.payload.boundaryLevel; + } + + draft.map.dataSetConfigs.forEach((dataSetConfig, index) => { + // eslint-disable-next-line no-param-reassign + dataSetConfig.boundaryLevel = + action.payload.dataSetConfigs[index].boundaryLevel; + }); + + break; + } + case 'UPDATE_MAP_DATA_GROUPINGS': { + if (!draft.map) { + throw new Error('Map config has not been initialised'); + } + + draft.map.dataSetConfigs.forEach((dataSetConfig, index) => { + // eslint-disable-next-line no-param-reassign + dataSetConfig.dataGrouping = + action.payload.dataSetConfigs[index].dataGrouping; + }); + + break; + } + case 'UPDATE_CHART_OPTIONS': { + draft.options = { + ...defaultOptions, + ...(draft?.definition?.options.defaults ?? {}), + ...draft.options, + ...action.payload, + }; - break; + break; + } + case 'UPDATE_DATA_SETS': { + if (draft.axes.major) { + draft.axes.major.dataSets = action.payload; + } + + if (draft.map && draft.axes.major) { + draft.map.dataSetConfigs = getMapDataSetConfigs({ + axisMajor: draft.axes.major, + data, + legend: draft.legend, + map: draft.map, + meta, + options: draft.options, + }); + } + + break; + } + case 'RESET': + return getInitialState(reducerOptions); + default: + break; } - case 'RESET': - return getInitialState(); - default: - break; - } - return draft; -}; + return draft; + }; +} -export function useChartBuilderReducer( - initialChart?: Chart, - tableTitle?: string, -) { +export function useChartBuilderReducer(options: ChartBuilderReducerOptions) { const [state, dispatch] = useLoggedImmerReducer< ChartBuilderState, ChartBuilderActions - >( - 'Chart builder', - chartBuilderReducer, - getInitialState(initialChart, tableTitle), - ); + >('Chart builder', chartBuilderReducer(options), getInitialState(options)); const updateDataSets = useCallback( (dataSets: DataSet[]) => { @@ -312,10 +472,20 @@ export function useChartBuilderReducer( [dispatch], ); - const updateChartMapConfiguration = useCallback( - (dataSetConfigs: MapDataSetConfig[]) => { + const updateMapBoundaryLevels = useCallback( + (payload: MapBoundaryLevelConfig) => { dispatch({ - type: 'UPDATE_CHART_MAP_CONFIGURATION', + type: 'UPDATE_MAP_BOUNDARY_LEVELS', + payload, + }); + }, + [dispatch], + ); + + const updateMapDataGroupings = useCallback( + (dataSetConfigs: MapDataGroupingConfig) => { + dispatch({ + type: 'UPDATE_MAP_DATA_GROUPINGS', payload: dataSetConfigs, }); }, @@ -343,18 +513,20 @@ export function useChartBuilderReducer( updateDataSets, updateChartDefinition, updateChartLegend, - updateChartMapConfiguration, + updateMapBoundaryLevels, + updateMapDataGroupings, updateChartOptions, updateChartAxis, resetState, }), [ updateDataSets, - updateChartAxis, updateChartDefinition, updateChartLegend, - updateChartMapConfiguration, + updateMapBoundaryLevels, + updateMapDataGroupings, updateChartOptions, + updateChartAxis, resetState, ], ); diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/types/mapConfig.ts b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/types/mapConfig.ts new file mode 100644 index 00000000000..c78fecef992 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/types/mapConfig.ts @@ -0,0 +1,17 @@ +import { DataGroupingConfig } from '@common/modules/charts/types/chart'; +import { DataSet } from '@common/modules/charts/types/dataSet'; + +export interface MapBoundaryLevelConfig { + boundaryLevel?: number; + dataSetConfigs: { + boundaryLevel?: number; + dataSet: DataSet; + }[]; +} + +export interface MapDataGroupingConfig { + dataSetConfigs: { + dataGrouping: DataGroupingConfig; + dataSet: DataSet; + }[]; +} diff --git a/src/explore-education-statistics-common/src/components/form/Form.tsx b/src/explore-education-statistics-common/src/components/form/Form.tsx index 83124b5294e..07aa9424a8e 100644 --- a/src/explore-education-statistics-common/src/components/form/Form.tsx +++ b/src/explore-education-statistics-common/src/components/form/Form.tsx @@ -5,6 +5,7 @@ import { FormIdContextProvider } from '@common/components/form/contexts/FormIdCo import createErrorHelper from '@common/components/form/validation/createErrorHelper'; import useMountedRef from '@common/hooks/useMountedRef'; import useToggle from '@common/hooks/useToggle'; +import { isEqual } from 'lodash'; import camelCase from 'lodash/camelCase'; import React, { FormEvent, @@ -24,6 +25,7 @@ interface Props { submitId?: string; showErrorSummary?: boolean; visuallyHiddenErrorSummary?: boolean; + onChange?: (values: Partial) => Promise | void; onSubmit: (values: TFormValues) => Promise | void; } @@ -52,6 +54,7 @@ export default function Form({ submitId = `${id}-submit`, showErrorSummary = true, visuallyHiddenErrorSummary = false, + onChange, onSubmit, }: Props) { const isMounted = useMountedRef(); @@ -67,10 +70,16 @@ export default function Form({ handleSubmit: submit, } = useFormContext(); - const values = useWatch(); + const values = useWatch(); const previousValues = useRef(values); const previousSubmitCount = useRef(submitCount); + useEffect(() => { + if (!isEqual(previousValues.current, values)) { + onChange?.(values); + } + }, [previousValues, values, onChange]); + const { getAllErrors } = createErrorHelper({ errors, initialTouched, diff --git a/src/explore-education-statistics-common/src/components/form/__tests__/Form.test.tsx b/src/explore-education-statistics-common/src/components/form/__tests__/Form.test.tsx index c5a1911e3fa..47cc5a47a67 100644 --- a/src/explore-education-statistics-common/src/components/form/__tests__/Form.test.tsx +++ b/src/explore-education-statistics-common/src/components/form/__tests__/Form.test.tsx @@ -7,6 +7,7 @@ import Yup from '@common/validation/yup'; import render from '@common-test/render'; import { screen, waitFor } from '@testing-library/react'; import React from 'react'; +import FormFieldTextInput from '../FormFieldTextInput'; describe('Form', () => { test('renders error summary from form errors when form is submitted', async () => { @@ -141,6 +142,37 @@ describe('Form', () => { }); }); + test('calls `onChange` handler as form values are updated', async () => { + const handleChange = jest.fn(); + + const { user } = render( + +
    + The form + + + +
    , + ); + + const firstName = 'first name'; + + await user.type(screen.getByLabelText('First name'), firstName); + + await waitFor(() => { + expect(handleChange).toHaveBeenCalledTimes(firstName.length); + }); + + expect(handleChange).toHaveBeenCalledWith({ firstName }); + }); + test('calls `onSubmit` handler when form is submitted successfully', async () => { const handleSubmit = jest.fn(); diff --git a/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.tsx b/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.tsx index c35985fa1fe..4157248d7da 100644 --- a/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.tsx +++ b/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.tsx @@ -1,14 +1,14 @@ import { SelectOption } from '@common/components/form/FormSelect'; import styles from '@common/modules/charts/components/MapBlock.module.scss'; -import createMapDataSetCategories, { - MapDataSetCategory, -} from '@common/modules/charts/components/utils/createMapDataSetCategories'; -import { LegendDataGroup } from '@common/modules/charts/components/utils/generateLegendDataGroups'; -import MapGeoJSON from '@common/modules/charts/components/MapGeoJSON'; import MapControls from '@common/modules/charts/components/MapControls'; +import MapGeoJSON from '@common/modules/charts/components/MapGeoJSON'; import MapLegend from '@common/modules/charts/components/MapLegend'; import MapSelectedItem from '@common/modules/charts/components/MapSelectedItem'; +import createMapDataSetCategories, { + MapDataSetCategory, +} from '@common/modules/charts/components/utils/createMapDataSetCategories'; import generateFeaturesAndDataGroups from '@common/modules/charts/components/utils/generateFeaturesAndDataGroups'; +import { LegendDataGroup } from '@common/modules/charts/components/utils/generateLegendDataGroups'; import { AxisConfiguration, ChartDefinition, @@ -18,16 +18,16 @@ import { } from '@common/modules/charts/types/chart'; import { DataSetCategory } from '@common/modules/charts/types/dataSet'; import { LegendConfiguration } from '@common/modules/charts/types/legend'; -import getDataSetCategoryConfigs, { - DataSetCategoryConfig, -} from '@common/modules/charts/util/getDataSetCategoryConfigs'; +import getMapDataSetCategoryConfigs, { + MapDataSetCategoryConfig, +} from '@common/modules/charts/util/getMapDataSetCategoryConfigs'; import { GeoJsonFeatureProperties } from '@common/services/tableBuilderService'; import { Dictionary } from '@common/types'; +import naturalOrderBy from '@common/utils/array/naturalOrderBy'; import classNames from 'classnames'; import { Feature, FeatureCollection, Geometry } from 'geojson'; import { Layer, Path, Polyline } from 'leaflet'; import keyBy from 'lodash/keyBy'; -import orderBy from 'lodash/orderBy'; import React, { useEffect, useMemo, useState } from 'react'; import { MapContainer } from 'react-leaflet'; @@ -136,14 +136,14 @@ export default function MapBlock({ [axisMajor, data, meta], ); - const dataSetCategoryConfigs = useMemo>( + const dataSetCategoryConfigs = useMemo>( () => keyBy( - getDataSetCategoryConfigs({ + getMapDataSetCategoryConfigs({ dataSetCategories, + dataSetConfigs: map?.dataSetConfigs, legendItems: legend.items, meta, - dataSetConfigs: map?.dataSetConfigs, deprecatedDataGroups, deprecatedDataClassification, }), @@ -151,16 +151,16 @@ export default function MapBlock({ ), [ dataSetCategories, - map?.dataSetConfigs, - legend, + legend.items, meta, + map, deprecatedDataGroups, deprecatedDataClassification, ], ); const dataSetOptions = useMemo(() => { - return orderBy( + return naturalOrderBy( Object.values(dataSetCategoryConfigs).map(dataSet => ({ label: dataSet.config.label, value: dataSet.dataKey, diff --git a/src/explore-education-statistics-common/src/modules/charts/components/MapGeoJSON.tsx b/src/explore-education-statistics-common/src/modules/charts/components/MapGeoJSON.tsx index 4fa35519306..accaced31be 100644 --- a/src/explore-education-statistics-common/src/modules/charts/components/MapGeoJSON.tsx +++ b/src/explore-education-statistics-common/src/modules/charts/components/MapGeoJSON.tsx @@ -1,13 +1,12 @@ import useCallbackRef from '@common/hooks/useCallbackRef'; import useIntersectionObserver from '@common/hooks/useIntersectionObserver'; import styles from '@common/modules/charts/components/MapBlock.module.scss'; -import { DataSetCategoryConfig } from '@common/modules/charts/util/getDataSetCategoryConfigs'; import { MapFeature, MapFeatureCollection, MapFeatureProperties, } from '@common/modules/charts/components/MapBlock'; - +import { MapDataSetCategoryConfig } from '@common/modules/charts/util/getMapDataSetCategoryConfigs'; import { Dictionary } from '@common/types'; import formatPretty from '@common/utils/number/formatPretty'; import { FeatureCollection } from 'geojson'; @@ -16,7 +15,7 @@ import React, { useEffect, useRef, useState } from 'react'; import { GeoJSON, useMap } from 'react-leaflet'; interface Props { - dataSetCategoryConfigs: Dictionary; + dataSetCategoryConfigs: Dictionary; features?: MapFeatureCollection; selectedFeature?: MapFeature; selectedDataSetKey: string; diff --git a/src/explore-education-statistics-common/src/modules/charts/components/utils/__tests__/generateFeaturesAndDataGroups.test.ts b/src/explore-education-statistics-common/src/modules/charts/components/utils/__tests__/generateFeaturesAndDataGroups.test.ts index cb443701583..6230d23a748 100644 --- a/src/explore-education-statistics-common/src/modules/charts/components/utils/__tests__/generateFeaturesAndDataGroups.test.ts +++ b/src/explore-education-statistics-common/src/modules/charts/components/utils/__tests__/generateFeaturesAndDataGroups.test.ts @@ -1,24 +1,24 @@ -import generateFeaturesAndDataGroups from '@common/modules/charts/components/utils/generateFeaturesAndDataGroups'; -import generateDataSetKey from '@common/modules/charts/util/generateDataSetKey'; -import { MapDataSetCategory } from '@common/modules/charts/components/utils/createMapDataSetCategories'; -import { DataSetCategoryConfig } from '@common/modules/charts/util/getDataSetCategoryConfigs'; -import expandDataSet from '@common/modules/charts/util/expandDataSet'; -import { MapFeatureCollection } from '@common/modules/charts/components/MapBlock'; -import { LegendDataGroup } from '@common/modules/charts/components/utils/generateLegendDataGroups'; import { - testLocation1, - testLocation2, - testLocation3, - testTimePeriod1, testDataGrouping, testDataSet1, + testDataSet2, + testDataSet3, testGeoJsonFeature1, testGeoJsonFeature2, testGeoJsonFeature3, - testDataSet2, - testDataSet3, + testLocation1, + testLocation2, + testLocation3, testSubjectMeta, + testTimePeriod1, } from '@common/modules/charts/components/__tests__/__data__/testMapBlockData'; +import { MapFeatureCollection } from '@common/modules/charts/components/MapBlock'; +import { MapDataSetCategory } from '@common/modules/charts/components/utils/createMapDataSetCategories'; +import generateFeaturesAndDataGroups from '@common/modules/charts/components/utils/generateFeaturesAndDataGroups'; +import { LegendDataGroup } from '@common/modules/charts/components/utils/generateLegendDataGroups'; +import expandDataSet from '@common/modules/charts/util/expandDataSet'; +import generateDataSetKey from '@common/modules/charts/util/generateDataSetKey'; +import { MapDataSetCategoryConfig } from '@common/modules/charts/util/getMapDataSetCategoryConfigs'; import { Indicator } from '@common/modules/table-tool/types/filters'; describe('generateFeaturesAndDataGroups', () => { @@ -55,7 +55,7 @@ describe('generateFeaturesAndDataGroups', () => { geoJson: testGeoJsonFeature3, }, ]; - const testSelectedDataSetConfig: DataSetCategoryConfig = { + const testSelectedDataSetConfig: MapDataSetCategoryConfig = { config: { label: 'Indicator 1 (Time period 1)', colour: '#12436D', @@ -203,7 +203,7 @@ describe('generateFeaturesAndDataGroups', () => { ], }; - const testSelectedDataSetConfig: DataSetCategoryConfig = { + const testSelectedDataSetConfig: MapDataSetCategoryConfig = { config: { label: 'Indicator 1 (Time period 1)', colour: '#12436D', @@ -312,7 +312,7 @@ describe('generateFeaturesAndDataGroups', () => { ], }; - const testSelectedDataSetConfig: DataSetCategoryConfig = { + const testSelectedDataSetConfig: MapDataSetCategoryConfig = { config: { label: 'Indicator 1 (Time period 1)', colour: '#12436D', @@ -421,7 +421,7 @@ describe('generateFeaturesAndDataGroups', () => { ], }; - const testSelectedDataSetConfig: DataSetCategoryConfig = { + const testSelectedDataSetConfig: MapDataSetCategoryConfig = { config: { label: 'Indicator 1 (Time period 1)', colour: '#4763a5', diff --git a/src/explore-education-statistics-common/src/modules/charts/components/utils/generateFeaturesAndDataGroups.ts b/src/explore-education-statistics-common/src/modules/charts/components/utils/generateFeaturesAndDataGroups.ts index 3f9a3bb4e44..18480d9111a 100644 --- a/src/explore-education-statistics-common/src/modules/charts/components/utils/generateFeaturesAndDataGroups.ts +++ b/src/explore-education-statistics-common/src/modules/charts/components/utils/generateFeaturesAndDataGroups.ts @@ -2,7 +2,7 @@ import { MapDataSetCategory } from '@common/modules/charts/components/utils/crea import generateLegendDataGroups, { LegendDataGroup, } from '@common/modules/charts/components/utils/generateLegendDataGroups'; -import { DataSetCategoryConfig } from '@common/modules/charts/util/getDataSetCategoryConfigs'; +import { MapDataSetCategoryConfig } from '@common/modules/charts/util/getMapDataSetCategoryConfigs'; import generateHslColour from '@common/utils/colour/generateHslColour'; import { MapFeature, @@ -14,7 +14,7 @@ export default function generateFeaturesAndDataGroups({ selectedDataSetConfig, }: { dataSetCategories: MapDataSetCategory[]; - selectedDataSetConfig: DataSetCategoryConfig; + selectedDataSetConfig: MapDataSetCategoryConfig; }): { features: MapFeatureCollection; dataGroups: LegendDataGroup[]; diff --git a/src/explore-education-statistics-common/src/modules/charts/types/chart.ts b/src/explore-education-statistics-common/src/modules/charts/types/chart.ts index 85c31b779ea..3e23bf25703 100644 --- a/src/explore-education-statistics-common/src/modules/charts/types/chart.ts +++ b/src/explore-education-statistics-common/src/modules/charts/types/chart.ts @@ -102,6 +102,7 @@ export interface DataGroupingConfig { export interface MapDataSetConfig { dataSet: DataSet; dataGrouping: DataGroupingConfig; + boundaryLevel?: number; } export interface MapConfig { diff --git a/src/explore-education-statistics-common/src/modules/charts/util/__tests__/getDataSetCategoryConfigs.test.ts b/src/explore-education-statistics-common/src/modules/charts/util/__tests__/getDataSetCategoryConfigs.test.ts index 7d7a9e20f57..c5bf18a6968 100644 --- a/src/explore-education-statistics-common/src/modules/charts/util/__tests__/getDataSetCategoryConfigs.test.ts +++ b/src/explore-education-statistics-common/src/modules/charts/util/__tests__/getDataSetCategoryConfigs.test.ts @@ -1,7 +1,3 @@ -import { - DataGroupingConfig, - MapDataSetConfig, -} from '@common/modules/charts/types/chart'; import { DataSet, DataSetCategory } from '@common/modules/charts/types/dataSet'; import getDataSetCategoryConfigs, { DataSetCategoryConfig, @@ -125,12 +121,6 @@ describe('getDataSetCategoryConfigs', () => { }, }; - const testDataGrouping: DataGroupingConfig = { - customGroups: [], - numberOfGroups: 5, - type: 'EqualIntervals', - }; - test('returns configs correctly when grouped by time period', () => { const testDataSetCategories: DataSetCategory[] = [ { @@ -168,7 +158,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 1, Location 1)', colour: '#12436D', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet1, testTimePeriod1), dataSet: expandDataSet(testDataSet1, testSubjectMeta), rawDataSet: JSON.parse( @@ -180,7 +169,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 2, Location 1)', colour: '#F46A25', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet2, testTimePeriod1), dataSet: expandDataSet(testDataSet2, testSubjectMeta), rawDataSet: JSON.parse( @@ -192,7 +180,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 1, Location 1)', colour: '#801650', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet3, testTimePeriod1), dataSet: expandDataSet(testDataSet3, testSubjectMeta), rawDataSet: JSON.parse( @@ -204,7 +191,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 2, Location 1)', colour: '#28A197', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet4, testTimePeriod1), dataSet: expandDataSet(testDataSet4, testSubjectMeta), rawDataSet: JSON.parse( @@ -253,7 +239,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 1, 2020/21)', colour: '#12436D', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet1, testLocation1), dataSet: expandDataSet(testDataSet1, testSubjectMeta), rawDataSet: JSON.parse(generateDataSetKey(testDataSet1, testLocation1)), @@ -263,7 +248,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 2, 2020/21)', colour: '#F46A25', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet2, testLocation1), dataSet: expandDataSet(testDataSet2, testSubjectMeta), rawDataSet: JSON.parse(generateDataSetKey(testDataSet2, testLocation1)), @@ -273,7 +257,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 1, 2020/21)', colour: '#801650', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet3, testLocation1), dataSet: expandDataSet(testDataSet3, testSubjectMeta), rawDataSet: JSON.parse(generateDataSetKey(testDataSet3, testLocation1)), @@ -283,7 +266,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 2, 2020/21)', colour: '#28A197', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet4, testLocation1), dataSet: expandDataSet(testDataSet4, testSubjectMeta), rawDataSet: JSON.parse(generateDataSetKey(testDataSet4, testLocation1)), @@ -330,7 +312,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Filter group 1 item 1, Location 1, 2020/21', colour: '#12436D', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet1, testIndicator1), dataSet: expandDataSet(testDataSet1, testSubjectMeta), rawDataSet: JSON.parse( @@ -342,7 +323,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Filter group 1 item 2, Location 1, 2020/21', colour: '#F46A25', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet2, testIndicator1), dataSet: expandDataSet(testDataSet2, testSubjectMeta), rawDataSet: JSON.parse( @@ -354,7 +334,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Filter group 2 item 1, Location 1, 2020/21', colour: '#801650', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet3, testIndicator1), dataSet: expandDataSet(testDataSet3, testSubjectMeta), rawDataSet: JSON.parse( @@ -366,7 +345,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Filter group 2 item 2, Location 1, 2020/21', colour: '#28A197', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet4, testIndicator1), dataSet: expandDataSet(testDataSet4, testSubjectMeta), rawDataSet: JSON.parse( @@ -429,7 +407,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Location 1, 2020/21)', colour: '#12436D', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet1, testFilterGroup1Item1), dataSet: expandDataSet(testDataSet1, testSubjectMeta), rawDataSet: JSON.parse( @@ -494,7 +471,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 1, Location 1, 2020/21)', colour: '#12436D', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet1), dataSet: expandDataSet(testDataSet1, testSubjectMeta), rawDataSet: testDataSet1, @@ -504,7 +480,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 2, Location 1, 2020/21)', colour: '#F46A25', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet2), dataSet: expandDataSet(testDataSet2, testSubjectMeta), rawDataSet: testDataSet2, @@ -514,7 +489,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 1, Location 1, 2020/21)', colour: '#801650', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet3), dataSet: expandDataSet(testDataSet3, testSubjectMeta), rawDataSet: testDataSet3, @@ -524,7 +498,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 2, Location 1, 2020/21)', colour: '#28A197', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet4), dataSet: expandDataSet(testDataSet4, testSubjectMeta), rawDataSet: testDataSet4, @@ -594,7 +567,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Label 1', colour: '#28A197', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet1, testTimePeriod1), dataSet: expandDataSet(testDataSet1, testSubjectMeta), rawDataSet: JSON.parse( @@ -606,7 +578,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Label 2', colour: '#6BACE6', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet2, testTimePeriod1), dataSet: expandDataSet(testDataSet2, testSubjectMeta), rawDataSet: JSON.parse( @@ -618,7 +589,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Label 3', colour: '#28A197', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet3, testTimePeriod1), dataSet: expandDataSet(testDataSet3, testSubjectMeta), rawDataSet: JSON.parse( @@ -630,7 +600,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Label 4', colour: '#6BACE6', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet4, testTimePeriod1), dataSet: expandDataSet(testDataSet4, testSubjectMeta), rawDataSet: JSON.parse( @@ -703,7 +672,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Label 1', colour: '#28A197', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet1, testTimePeriod1), dataSet: expandDataSet(testDataSet1, testSubjectMeta), rawDataSet: JSON.parse( @@ -715,7 +683,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Label 2', colour: '#6BACE6', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet2, testTimePeriod1), dataSet: expandDataSet(testDataSet2, testSubjectMeta), rawDataSet: JSON.parse( @@ -727,7 +694,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Label 3', colour: '#28A197', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet3, testTimePeriod1), dataSet: expandDataSet(testDataSet3, testSubjectMeta), rawDataSet: JSON.parse( @@ -739,7 +705,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Label 4', colour: '#6BACE6', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet4, testTimePeriod1), dataSet: expandDataSet(testDataSet4, testSubjectMeta), rawDataSet: JSON.parse( @@ -788,7 +753,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 1, Location 1)', colour: '#12436D', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet1, testTimePeriod1), dataSet: expandDataSet(testDataSet1, testSubjectMeta), rawDataSet: JSON.parse( @@ -800,7 +764,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 2, Location 1)', colour: '#F46A25', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet2, testTimePeriod1), dataSet: expandDataSet(testDataSet2, testSubjectMeta), rawDataSet: JSON.parse( @@ -812,7 +775,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 1, Location 1)', colour: '#801650', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet3, testTimePeriod1), dataSet: expandDataSet(testDataSet3, testSubjectMeta), rawDataSet: JSON.parse( @@ -824,7 +786,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 2, Location 1)', colour: '#28A197', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet4, testTimePeriod1), dataSet: expandDataSet(testDataSet4, testSubjectMeta), rawDataSet: JSON.parse( @@ -861,140 +822,8 @@ describe('getDataSetCategoryConfigs', () => { }, ]; - const testDeprecatedDataGrouping: DataGroupingConfig = { - customGroups: [], - numberOfGroups: 7, - type: 'Quantiles', - }; - - const result = getDataSetCategoryConfigs({ - dataSetCategories: testDataSetCategories, - deprecatedDataClassification: 'Quantiles', - deprecatedDataGroups: 7, - legendItems: [], - meta: testSubjectMeta, - }); - - const expected: DataSetCategoryConfig[] = [ - { - config: { - label: 'Indicator 1 (Filter group 1 item 1, Location 1)', - colour: '#12436D', - }, - dataGrouping: testDeprecatedDataGrouping, - dataKey: generateDataSetKey(testDataSet1, testTimePeriod1), - dataSet: expandDataSet(testDataSet1, testSubjectMeta), - rawDataSet: JSON.parse( - generateDataSetKey(testDataSet1, testTimePeriod1), - ), - }, - { - config: { - label: 'Indicator 1 (Filter group 1 item 2, Location 1)', - colour: '#F46A25', - }, - dataGrouping: testDeprecatedDataGrouping, - dataKey: generateDataSetKey(testDataSet2, testTimePeriod1), - dataSet: expandDataSet(testDataSet2, testSubjectMeta), - rawDataSet: JSON.parse( - generateDataSetKey(testDataSet2, testTimePeriod1), - ), - }, - { - config: { - label: 'Indicator 1 (Filter group 2 item 1, Location 1)', - colour: '#801650', - }, - dataGrouping: testDeprecatedDataGrouping, - dataKey: generateDataSetKey(testDataSet3, testTimePeriod1), - dataSet: expandDataSet(testDataSet3, testSubjectMeta), - rawDataSet: JSON.parse( - generateDataSetKey(testDataSet3, testTimePeriod1), - ), - }, - { - config: { - label: 'Indicator 1 (Filter group 2 item 2, Location 1)', - colour: '#28A197', - }, - dataGrouping: testDeprecatedDataGrouping, - dataKey: generateDataSetKey(testDataSet4, testTimePeriod1), - dataSet: expandDataSet(testDataSet4, testSubjectMeta), - rawDataSet: JSON.parse( - generateDataSetKey(testDataSet4, testTimePeriod1), - ), - }, - ]; - - expect(result).toEqual(expected); - }); - - test('returns the data set data grouping if present', () => { - const testDataSetCategories: DataSetCategory[] = [ - { - dataSets: { - [generateDataSetKey(testDataSet1, testTimePeriod1)]: { - dataSet: testDataSet1, - value: 30, - }, - [generateDataSetKey(testDataSet2, testTimePeriod1)]: { - dataSet: testDataSet2, - value: 70, - }, - [generateDataSetKey(testDataSet3, testTimePeriod1)]: { - dataSet: testDataSet3, - value: 60, - }, - [generateDataSetKey(testDataSet4, testTimePeriod1)]: { - dataSet: testDataSet4, - value: 40, - }, - }, - filter: testTimePeriod1, - }, - ]; - - const testDataSetConfigs: MapDataSetConfig[] = [ - { - dataSet: omit(testDataSet1, 'timePeriod'), - dataGrouping: { - customGroups: [], - numberOfGroups: 2, - type: 'EqualIntervals', - }, - }, - { - dataSet: omit(testDataSet2, 'timePeriod'), - dataGrouping: { - customGroups: [], - numberOfGroups: 9, - type: 'Quantiles', - }, - }, - { - dataSet: omit(testDataSet3, 'timePeriod'), - dataGrouping: { - customGroups: [{ min: 0, max: 10 }], - numberOfGroups: 9, - type: 'Custom', - }, - }, - { - dataSet: omit(testDataSet4, 'timePeriod'), - dataGrouping: { - customGroups: [ - { min: 0, max: 10 }, - { min: 11, max: 20 }, - ], - numberOfGroups: 9, - type: 'Custom', - }, - }, - ]; - const result = getDataSetCategoryConfigs({ dataSetCategories: testDataSetCategories, - dataSetConfigs: testDataSetConfigs, legendItems: [], meta: testSubjectMeta, }); @@ -1005,11 +834,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 1, Location 1)', colour: '#12436D', }, - dataGrouping: { - customGroups: [], - numberOfGroups: 2, - type: 'EqualIntervals', - }, dataKey: generateDataSetKey(testDataSet1, testTimePeriod1), dataSet: expandDataSet(testDataSet1, testSubjectMeta), rawDataSet: JSON.parse( @@ -1021,11 +845,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 2, Location 1)', colour: '#F46A25', }, - dataGrouping: { - customGroups: [], - numberOfGroups: 9, - type: 'Quantiles', - }, dataKey: generateDataSetKey(testDataSet2, testTimePeriod1), dataSet: expandDataSet(testDataSet2, testSubjectMeta), rawDataSet: JSON.parse( @@ -1037,11 +856,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 1, Location 1)', colour: '#801650', }, - dataGrouping: { - customGroups: [{ min: 0, max: 10 }], - numberOfGroups: 9, - type: 'Custom', - }, dataKey: generateDataSetKey(testDataSet3, testTimePeriod1), dataSet: expandDataSet(testDataSet3, testSubjectMeta), rawDataSet: JSON.parse( @@ -1053,14 +867,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 2, Location 1)', colour: '#28A197', }, - dataGrouping: { - customGroups: [ - { min: 0, max: 10 }, - { min: 11, max: 20 }, - ], - numberOfGroups: 9, - type: 'Custom', - }, dataKey: generateDataSetKey(testDataSet4, testTimePeriod1), dataSet: expandDataSet(testDataSet4, testSubjectMeta), rawDataSet: JSON.parse( @@ -1109,7 +915,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 1, Location 1)', colour: '#12436D', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet1, testTimePeriod1), dataSet: expandDataSet(testDataSet1, testSubjectMeta), rawDataSet: JSON.parse( @@ -1121,7 +926,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 1 item 2, Location 1)', colour: '#F46A25', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet2, testTimePeriod1), dataSet: expandDataSet(testDataSet2, testSubjectMeta), rawDataSet: JSON.parse( @@ -1133,7 +937,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 1, Location 1)', colour: '#801650', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet3, testTimePeriod1), dataSet: expandDataSet(testDataSet3, testSubjectMeta), rawDataSet: JSON.parse( @@ -1145,7 +948,6 @@ describe('getDataSetCategoryConfigs', () => { label: 'Indicator 1 (Filter group 2 item 2, Location 1)', colour: '#28A197', }, - dataGrouping: testDataGrouping, dataKey: generateDataSetKey(testDataSet4, testTimePeriod1), dataSet: expandDataSet(testDataSet4, testSubjectMeta), rawDataSet: JSON.parse( diff --git a/src/explore-education-statistics-common/src/modules/charts/util/__tests__/getMapDataSetCategoryConfigs.test.ts b/src/explore-education-statistics-common/src/modules/charts/util/__tests__/getMapDataSetCategoryConfigs.test.ts new file mode 100644 index 00000000000..76cbabf8811 --- /dev/null +++ b/src/explore-education-statistics-common/src/modules/charts/util/__tests__/getMapDataSetCategoryConfigs.test.ts @@ -0,0 +1,221 @@ +import { + DataGroupingConfig, + MapDataSetConfig, +} from '@common/modules/charts/types/chart'; +import { DataSet, DataSetCategory } from '@common/modules/charts/types/dataSet'; +import { + CategoryFilter, + Indicator, + LocationFilter, + TimePeriodFilter, +} from '@common/modules/table-tool/types/filters'; +import generateDataSetKey from '@common/modules/charts/util/generateDataSetKey'; +import { FullTableMeta } from '@common/modules/table-tool/types/fullTable'; +import expandDataSet from '@common/modules/charts/util/expandDataSet'; +import getMapDataSetCategoryConfigs, { + defaultDataGrouping, +} from '../getMapDataSetCategoryConfigs'; + +describe('getMapDataSetCategoryConfigs', () => { + const testFilterGroupItem = new CategoryFilter({ + category: 'Filter1', + group: 'Filter group 1', + label: 'Filter group 1 item 1', + value: 'filter-group-1-item-1', + }); + const testFilterGroupItem2 = new CategoryFilter({ + category: 'Filter2', + group: 'Filter group 2', + label: 'Filter group 2 item 2', + value: 'filter-group-2-item-2', + }); + const testIndicator = new Indicator({ + label: 'Indicator 1', + name: 'indicator-1-name', + unit: '', + value: 'indicator-1', + }); + const testLocation1 = new LocationFilter({ + id: 'location-1-id', + label: 'Location 1', + level: 'country', + value: 'location-1', + }); + const testTimePeriod = new TimePeriodFilter({ + year: 2020, + code: 'AY', + label: '2020/21', + order: 0, + }); + + const testSubjectMeta: FullTableMeta = { + boundaryLevels: [], + filters: { + Filter1: { + name: 'filter-1', + options: [testFilterGroupItem], + order: 0, + }, + Filter2: { + name: 'filter-2', + options: [testFilterGroupItem2], + order: 0, + }, + }, + footnotes: [], + geoJsonAvailable: false, + indicators: [testIndicator], + locations: [testLocation1], + publicationName: 'Publication 1', + subjectName: 'Subject 1', + timePeriodRange: [testTimePeriod], + }; + + const testDataSets: DataSet[] = [ + { + filters: [testFilterGroupItem.id], + indicator: testIndicator.id, + timePeriod: testTimePeriod.id, + location: { + level: testLocation1.level, + value: testLocation1.value, + }, + }, + { + filters: [testFilterGroupItem2.id], + indicator: testIndicator.id, + timePeriod: testTimePeriod.id, + location: { + level: testLocation1.level, + value: testLocation1.value, + }, + }, + ]; + + const testDataGrouping: DataGroupingConfig = { + customGroups: [{ min: 0, max: 999 }], + numberOfGroups: 6, + type: 'Quantiles', + }; + + test('boundaryLevel and dataGrouping are set to defaults when no dataSetConfigs provided', () => { + const testDataSetCategories: DataSetCategory[] = [ + { + dataSets: { + [generateDataSetKey(testDataSets[0])]: { + dataSet: testDataSets[0], + value: 30, + }, + [generateDataSetKey(testDataSets[1])]: { + dataSet: testDataSets[1], + value: 20, + }, + }, + filter: testTimePeriod, + }, + ]; + + const result = getMapDataSetCategoryConfigs({ + dataSetConfigs: [], + dataSetCategories: testDataSetCategories, + legendItems: [], + meta: testSubjectMeta, + }); + + expect(result).toEqual([ + { + config: { + label: 'Indicator 1 (Filter group 1 item 1, Location 1)', + colour: '#12436D', + }, + dataGrouping: defaultDataGrouping, + dataKey: generateDataSetKey(testDataSets[0]), + dataSet: expandDataSet(testDataSets[0], testSubjectMeta), + rawDataSet: testDataSets[0], + boundaryLevel: undefined, + }, + { + config: { + label: 'Indicator 1 (Filter group 2 item 2, Location 1)', + colour: '#F46A25', + }, + dataGrouping: defaultDataGrouping, + dataKey: generateDataSetKey(testDataSets[1]), + dataSet: expandDataSet(testDataSets[1], testSubjectMeta), + rawDataSet: testDataSets[1], + boundaryLevel: undefined, + }, + ]); + }); + + test('boundaryLevel and dataGrouping are set when dataSetConfigs are provided', () => { + const testDataSetCategories: DataSetCategory[] = [ + { + dataSets: { + [generateDataSetKey(testDataSets[0])]: { + dataSet: testDataSets[0], + value: 30, + }, + [generateDataSetKey(testDataSets[1])]: { + dataSet: testDataSets[1], + value: 90, + }, + }, + filter: testTimePeriod, + }, + ]; + const customDataGrouping: MapDataSetConfig['dataGrouping'] = { + customGroups: [ + { min: 0, max: 59 }, + { min: 60, max: 119 }, + ], + type: 'Custom', + numberOfGroups: 2, + }; + + const result = getMapDataSetCategoryConfigs({ + dataSetConfigs: [ + { + dataSet: testDataSets[0], + dataGrouping: testDataGrouping, + boundaryLevel: 15, + }, + { + dataSet: testDataSets[1], + dataGrouping: customDataGrouping, + boundaryLevel: 6, + }, + ], + dataSetCategories: testDataSetCategories, + legendItems: [ + { colour: '#aa0000', dataSet: testDataSets[1], label: 'custom-label' }, + ], + meta: testSubjectMeta, + }); + + expect(result).toEqual([ + { + config: { + label: 'Indicator 1 (Filter group 1 item 1, Location 1)', + colour: '#12436D', + }, + dataGrouping: testDataGrouping, + dataKey: generateDataSetKey(testDataSets[0]), + dataSet: expandDataSet(testDataSets[0], testSubjectMeta), + rawDataSet: testDataSets[0], + boundaryLevel: 15, + }, + { + config: { + label: 'custom-label', + colour: '#aa0000', + }, + dataGrouping: customDataGrouping, + dataKey: generateDataSetKey(testDataSets[1]), + dataSet: expandDataSet(testDataSets[1], testSubjectMeta), + rawDataSet: testDataSets[1], + boundaryLevel: 6, + }, + ]); + }); +}); diff --git a/src/explore-education-statistics-common/src/modules/charts/util/getDataSetCategoryConfigs.ts b/src/explore-education-statistics-common/src/modules/charts/util/getDataSetCategoryConfigs.ts index 47b04b01341..d046d9724db 100644 --- a/src/explore-education-statistics-common/src/modules/charts/util/getDataSetCategoryConfigs.ts +++ b/src/explore-education-statistics-common/src/modules/charts/util/getDataSetCategoryConfigs.ts @@ -8,11 +8,6 @@ import { LegendItem, LegendItemConfiguration, } from '@common/modules/charts/types/legend'; -import { - DataGroupingConfig, - DataGroupingType, - MapDataSetConfig, -} from '@common/modules/charts/types/chart'; import { colours } from '@common/modules/charts/util/chartUtils'; import expandDataSet from '@common/modules/charts/util/expandDataSet'; import generateDataSetKey from '@common/modules/charts/util/generateDataSetKey'; @@ -24,39 +19,19 @@ import keyBy from 'lodash/keyBy'; import omit from 'lodash/omit'; import uniqBy from 'lodash/uniqBy'; -const defaultDataGrouping: DataGroupingConfig = { - customGroups: [], - numberOfGroups: 5, - type: 'EqualIntervals', -}; - export interface DataSetCategoryConfig { config: LegendItemConfiguration; dataKey: string; dataSet: ExpandedDataSet; rawDataSet: DataSet; - dataGrouping: DataGroupingConfig; } /** * Get the data set configurations that are used to * style/modify how they look in the chart. */ -interface Options { +export interface GetDataSetCategoryConfigsOptions { dataSetCategories: DataSetCategory[]; - dataSetConfigs?: MapDataSetConfig[]; - /** - * Data classification and data groups are now in `dataSetConfigs` - * as they are per data set instead of for all data sets (EES-3858). - * The deprecated versions are to retain backwards compatibility - * with maps created before this change. - * @deprecated - */ - deprecatedDataClassification?: DataGroupingType; - /** - * @deprecated - */ - deprecatedDataGroups?: number; groupByFilterGroups?: boolean; legendItems: LegendItem[]; meta: FullTableMeta; @@ -64,40 +39,22 @@ interface Options { export default function getDataSetCategoryConfigs({ dataSetCategories, - dataSetConfigs, - deprecatedDataClassification, - deprecatedDataGroups, groupByFilterGroups = false, legendItems, meta, -}: Options): DataSetCategoryConfig[] { +}: GetDataSetCategoryConfigsOptions): DataSetCategoryConfig[] { const legendItemsByDataSet = keyBy(legendItems, item => generateDataSetKey(item.dataSet), ); - const dataSetConfigsByDataSet = keyBy(dataSetConfigs, item => - generateDataSetKey(item.dataSet), - ); - - const deprecatedGrouping: DataGroupingConfig | undefined = - !dataSetConfigs?.length && deprecatedDataClassification - ? { - customGroups: [], - numberOfGroups: deprecatedDataGroups ?? 5, - type: deprecatedDataClassification, - } - : undefined; - const dataSets = dataSetCategories.reduce( (acc, category) => { Object.entries(category.dataSets).forEach( ([dataSetKey, dataSetValue]) => { acc.push( toLegendConfig({ - dataSetConfigsByDataSet, dataSetKey, dataSetValue, - deprecatedGrouping, filter: groupByFilterGroups ? undefined : category.filter, index: acc.length, legendItemsByDataSet, @@ -117,9 +74,7 @@ export default function getDataSetCategoryConfigs({ function toLegendConfig({ dataSetValue, - dataSetConfigsByDataSet, dataSetKey, - deprecatedGrouping, filter, index, legendItemsByDataSet, @@ -129,9 +84,7 @@ function toLegendConfig({ dataSet: DataSetConfiguration; value: number; }; - dataSetConfigsByDataSet: Dictionary; dataSetKey: string; - deprecatedGrouping?: DataGroupingConfig; filter?: Filter; index: number; legendItemsByDataSet: Dictionary; @@ -161,14 +114,10 @@ function toLegendConfig({ const config = legendItemConfig ?? dataSetValue.dataSet.config ?? getDefaultConfig(); - const dataGrouping = - dataSetConfigsByDataSet[dataSetKey]?.dataGrouping ?? defaultDataGrouping; - return { config, dataSet, rawDataSet, dataKey: dataSetKey, - dataGrouping: deprecatedGrouping ?? dataGrouping, }; } diff --git a/src/explore-education-statistics-common/src/modules/charts/util/getMapDataSetCategoryConfigs.ts b/src/explore-education-statistics-common/src/modules/charts/util/getMapDataSetCategoryConfigs.ts new file mode 100644 index 00000000000..05d1649ed7b --- /dev/null +++ b/src/explore-education-statistics-common/src/modules/charts/util/getMapDataSetCategoryConfigs.ts @@ -0,0 +1,77 @@ +import { + DataGroupingConfig, + DataGroupingType, + MapDataSetConfig, +} from '@common/modules/charts/types/chart'; +import generateDataSetKey from '@common/modules/charts/util/generateDataSetKey'; +import getDataSetCategoryConfigs, { + DataSetCategoryConfig, + GetDataSetCategoryConfigsOptions, +} from '@common/modules/charts/util/getDataSetCategoryConfigs'; +import keyBy from 'lodash/keyBy'; + +export const defaultDataGrouping: DataGroupingConfig = { + customGroups: [], + numberOfGroups: 5, + type: 'EqualIntervals', +}; + +export interface MapDataSetCategoryConfig extends DataSetCategoryConfig { + boundaryLevel?: number; + dataGrouping: DataGroupingConfig; +} + +interface GetMapDataSetCategoryConfigsOptions + extends GetDataSetCategoryConfigsOptions { + dataSetConfigs?: MapDataSetConfig[]; + /** + * Data classification and data groups are now in `dataSetConfigs` + * as they are per data set instead of for all data sets (EES-3858). + * The deprecated versions are to retain backwards compatibility + * with maps created before this change. + * @deprecated + */ + deprecatedDataClassification?: DataGroupingType; + /** + * @deprecated + */ + deprecatedDataGroups?: number; +} + +export default function getMapDataSetCategoryConfigs({ + dataSetConfigs = [], + deprecatedDataClassification, + deprecatedDataGroups, + ...options +}: GetMapDataSetCategoryConfigsOptions): MapDataSetCategoryConfig[] { + const dataSetConfigsByDataSet = keyBy(dataSetConfigs, item => + generateDataSetKey(item.dataSet), + ); + + const deprecatedGrouping: DataGroupingConfig | undefined = + !dataSetConfigs?.length && deprecatedDataClassification + ? { + customGroups: [], + numberOfGroups: + deprecatedDataGroups ?? defaultDataGrouping.numberOfGroups, + type: deprecatedDataClassification, + } + : undefined; + + const dataSetCategoryConfigs = getDataSetCategoryConfigs(options); + + return dataSetCategoryConfigs.map(dataSetCategoryConfig => { + const dataSetConfig = + dataSetConfigsByDataSet[dataSetCategoryConfig.dataKey]; + + const { dataGrouping, boundaryLevel } = dataSetConfig ?? { + dataGrouping: defaultDataGrouping, + }; + + return { + ...dataSetCategoryConfig, + boundaryLevel, + dataGrouping: deprecatedGrouping ?? dataGrouping, + }; + }); +} From 622b1c09c4c830af2a26e05d58d669a90c0f2e26 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Tue, 19 Nov 2024 16:59:17 +0000 Subject: [PATCH 021/144] Delete non-cancelled release versions first and cancelled (soft deleted) ones last --- .../Services/ThemeServiceTests.cs | 2 +- .../Services/ThemeService.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ThemeServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ThemeServiceTests.cs index c383373f782..9eed4ab4f9b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ThemeServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ThemeServiceTests.cs @@ -594,8 +594,8 @@ public async Task DeleteTheme_ReleaseVersionsDeletedByVersionOrder() { // Expect the ReleaseVersions from the more recent Release to be deleted first. release1Version3.Id, - release1Version2Cancelled.Id, release1Version2.Id, + release1Version2Cancelled.Id, release1Version1.Id, release2Version2.Id, release2Version1.Id, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ThemeService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ThemeService.cs index 945953b2ef4..6fe005fb36a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ThemeService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ThemeService.cs @@ -352,7 +352,7 @@ public int Compare(ReleaseVersionAndDataSetVersions? version1, ReleaseVersionAnd // Delete non-cancelled ReleaseVersions first. if (releaseVersion1.SoftDeleted != releaseVersion2.SoftDeleted) { - return releaseVersion1.SoftDeleted ? 1 : 0; + return releaseVersion1.SoftDeleted ? 1 : -1; } return -releaseVersion1.Created.CompareTo(releaseVersion2.Created); From 826c142d49459c26332e363e9934548209886f19 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Tue, 19 Nov 2024 16:59:59 +0000 Subject: [PATCH 022/144] EES-5625 Add new Release attributes and copy values from ReleaseVersion --- .../ManageContentPageServiceTests.cs | 323 +-- .../Methodologies/MethodologyServiceTests.cs | 66 +- .../ReleaseAmendmentServicePermissionTests.cs | 25 +- .../Services/ReleaseAmendmentServiceTests.cs | 30 +- .../Services/ReleaseServiceTests.cs | 329 +-- .../Services/ReleaseVersionRepositoryTests.cs | 10 +- .../Services/ThemeServiceTests.cs | 146 +- ...yReleaseVersionFieldsToRelease.Designer.cs | 2233 +++++++++++++++++ ...S5625_CopyReleaseVersionFieldsToRelease.cs | 158 ++ ...5625_CopyReleaseVersionFieldsToRelease.sql | 17 + .../ContentDbContextModelSnapshot.cs | 34 +- .../Requests/ReleaseRequests.cs | 4 +- .../Services/ReleaseService.cs | 52 +- .../Controllers/ReleaseFileControllerTests.cs | 27 +- .../MethodologyVersionGeneratorExtensions.cs | 29 +- .../PublicationGeneratorExtensions.cs | 145 +- .../Fixtures/ReleaseGeneratorExtensions.cs | 176 +- .../ReleaseSeriesItemGeneratorExtensions.cs | 59 + .../ReleaseVersionGeneratorExtensions.cs | 52 +- .../Database/ContentDbContext.cs | 12 +- .../Publication.cs | 2 + .../Release.cs | 43 +- .../ReleaseServiceTests.cs | 6 +- .../TableBuilderControllerTests.cs | 10 +- .../TableBuilderService.cs | 2 +- ...teNextDataSetVersionImportFunctionTests.cs | 3 +- ...NextDataSetVersionMappingsFunctionTests.cs | 10 +- 27 files changed, 3340 insertions(+), 663 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.Designer.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.sql create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseSeriesItemGeneratorExtensions.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs index f4c7c07a371..7d7b01ece26 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs @@ -1,4 +1,9 @@ #nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; using AutoMapper; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Admin.Services.ManageContent; @@ -6,103 +11,57 @@ using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using Microsoft.AspNetCore.Mvc; using Moq; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.DbUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Model.FileType; using static GovUk.Education.ExploreEducationStatistics.Common.Model.TimeIdentifier; using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; using HtmlBlockViewModel = GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.HtmlBlockViewModel; -using IReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces.IReleaseVersionRepository; -using ReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.ReleaseVersionRepository; +using IReleaseVersionRepository = + GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces.IReleaseVersionRepository; +using ReleaseVersionRepository = + GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.ReleaseVersionRepository; namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.ManageContent { public class ManageContentPageServiceTests { + private readonly DataFixture _dataFixture = new(); + [Fact] public async Task GetManageContentPageViewModel() { - var releaseId = Guid.NewGuid(); - var releaseVersionId = Guid.NewGuid(); - - var publication = new Publication - { - LatestPublishedReleaseVersionId = releaseVersionId, - Contact = new Contact - { - ContactName = "Name", - ContactTelNo = "01234 567890", - TeamEmail = "test@test.com", - TeamName = "Team Name" - }, - ReleaseSeries = new List + Publication publication = _dataFixture.DefaultPublication() + .WithTheme(_dataFixture.DefaultTheme()) + .WithReleases(_dataFixture.DefaultRelease(publishedVersions: 1, draftVersion: true) + .Generate(1)) + .WithLegacyLinks(_dataFixture.DefaultLegacyReleaseSeriesItem() + .Generate(2)); + + var publishedReleaseVersion = publication.ReleaseVersions.Single(rv => rv.Published is not null); + var otherReleaseVersion = publication.ReleaseVersions.Single(rv => rv.Published is null); + + publishedReleaseVersion.RelatedInformation.Add( + new Link { - new() - { - Id = Guid.NewGuid(), - ReleaseId = releaseId, - }, - new() - { - Id = Guid.NewGuid(), - LegacyLinkDescription = "Legacy 2018/19", - LegacyLinkUrl = "https://legacy-2018-19" - }, - new() - { - Id = Guid.NewGuid(), - LegacyLinkDescription = "Legacy 2017/18", - LegacyLinkUrl = "https://legacy-2017-18" - }, - }, - Slug = "test-publication", - Title = "Publication", - Theme = new Theme - { - Title = "Theme" - } - }; - - var releaseVersion = new ReleaseVersion - { - Id = releaseVersionId, - NextReleaseDate = new PartialDate { Day = "9", Month = "9", Year = "2040" }, - PreReleaseAccessList = "Test access list", - Publication = publication, - Release = new Release { Id = releaseId }, - PublishScheduled = DateTime.Parse("2020-09-08T23:00:00.00Z", styles: DateTimeStyles.AdjustToUniversal), - Published = DateTime.UtcNow, - ReleaseName = "2020", - RelatedInformation = new List - { - new() - { - Description = "Related 1", - Url = "https://related-1" - } - }, - Slug = "2020-21", - TimePeriodCoverage = AcademicYear, - Type = ReleaseType.OfficialStatistics, - Updates = new List(), - }; + Description = "Related 1", + Url = "https://related-1" + }); var unattachedDataBlockParent = new DataBlockParent { LatestPublishedVersion = new DataBlockVersion { - ReleaseVersionId = releaseVersion.Id, + ReleaseVersionId = publishedReleaseVersion.Id, Id = Guid.NewGuid() } }; @@ -111,7 +70,7 @@ public async Task GetManageContentPageViewModel() { LatestPublishedVersion = new DataBlockVersion { - ReleaseVersionId = releaseVersion.Id, + ReleaseVersionId = publishedReleaseVersion.Id, Id = Guid.NewGuid(), } }; @@ -122,7 +81,7 @@ public async Task GetManageContentPageViewModel() { LatestPublishedVersion = new DataBlockVersion { - ReleaseVersionId = releaseVersion.Id, + ReleaseVersionId = publishedReleaseVersion.Id, Id = inContentDataBlockVersionId, ContentBlock = new DataBlock { @@ -132,115 +91,91 @@ public async Task GetManageContentPageViewModel() } }; - releaseVersion.KeyStatistics = new List - { - new KeyStatisticText {Order = 1}, + publishedReleaseVersion.KeyStatistics = + [ + new KeyStatisticText + { + Order = 1 + }, new KeyStatisticDataBlock { Order = 0, DataBlockId = keyStatDataBlockParent.LatestPublishedVersion!.Id } - }; - - var otherReleaseVersion = new ReleaseVersion - { - NextReleaseDate = new PartialDate { Day = "8", Month = "8", Year = "2040" }, - Publication = publication, - PublishScheduled = DateTime.Parse("2020-08-07T23:00:00.00Z", styles: DateTimeStyles.AdjustToUniversal), - Published = null, - ReleaseName = "2019", - Slug = "2019-20", - TimePeriodCoverage = AcademicYear, - Type = ReleaseType.OfficialStatistics, - }; + ]; - var unattachedDataBlocks = new List - { + List unattachedDataBlocks = + [ new() { Id = unattachedDataBlockParent.LatestPublishedVersion!.Id } - }; - - var ancillaryFileId = Guid.NewGuid(); - var dataFileId = Guid.NewGuid(); - var files = new List - { - new() - { - Id = ancillaryFileId, - FileName = "ancillary.pdf", - Name = "Ancillary File", - Size = "10 Kb", - Type = Ancillary - }, - new() - { - Id = dataFileId, - FileName = "data.csv", - Name = "Subject File", - Size = "20 Kb", - Type = FileType.Data - } - }; - - var methodologies = AsList( - new MethodologyVersion - { - Id = Guid.NewGuid(), - AlternativeTitle = "Methodology 1 title", - Status = MethodologyApprovalStatus.Approved, - }, - new MethodologyVersion - { - Id = Guid.NewGuid(), - AlternativeTitle = "Methodology 2 title", - Status = MethodologyApprovalStatus.Draft, - } - ); + ]; + + var files = _dataFixture + .DefaultReleaseFile() + .WithReleaseVersion(publishedReleaseVersion) + .WithFiles([ + _dataFixture.DefaultFile(Ancillary), + _dataFixture.DefaultFile(FileType.Data) + ]) + .Generate(2) + .Select(rf => rf.ToFileInfo()) + .ToList(); + + var methodology = _dataFixture + .DefaultMethodology() + .WithMethodologyVersions(_dataFixture + .DefaultMethodologyVersion() + .ForIndex(0, mv => mv.SetApprovalStatus(MethodologyApprovalStatus.Approved)) + .ForIndex(1, mv => mv.SetAlternativeTitle("Alternative title")) + .Generate(2)) + .FinishWith(m => m.LatestPublishedVersion = m.Versions[0]) + .WithOwningPublication(publication) + .Generate(); var genericContentSection = new ContentSection { Heading = "Test section 1", Type = ContentSectionType.Generic, - Content = new List - { + Content = + [ new HtmlBlock { Order = 0, Body = "Test block 1" }, inContentDataBlockParent.LatestPublishedVersion!.ContentBlock - }, - ReleaseVersion = releaseVersion + ], + ReleaseVersion = publishedReleaseVersion }; var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - contentDbContext.ReleaseVersions.AddRange(releaseVersion, otherReleaseVersion); + contentDbContext.Publications.Add(publication); contentDbContext.DataBlockParents.AddRange( unattachedDataBlockParent, keyStatDataBlockParent, inContentDataBlockParent); contentDbContext.ContentSections.AddRange( new ContentSection { - ReleaseVersion = releaseVersion, + ReleaseVersion = publishedReleaseVersion, Type = ContentSectionType.Headlines }, new ContentSection { - ReleaseVersion = releaseVersion, + ReleaseVersion = publishedReleaseVersion, Type = ContentSectionType.KeyStatisticsSecondary }, new ContentSection { - ReleaseVersion = releaseVersion, + ReleaseVersion = publishedReleaseVersion, Type = ContentSectionType.ReleaseSummary }, new ContentSection { - ReleaseVersion = releaseVersion, + ReleaseVersion = publishedReleaseVersion, Type = ContentSectionType.RelatedDashboards }, genericContentSection); @@ -253,15 +188,15 @@ public async Task GetManageContentPageViewModel() var releaseFileService = new Mock(MockBehavior.Strict); dataBlockService.Setup(mock => - mock.GetUnattachedDataBlocks(releaseVersion.Id)) + mock.GetUnattachedDataBlocks(publishedReleaseVersion.Id)) .ReturnsAsync(unattachedDataBlocks); methodologyVersionRepository.Setup(mock => mock.GetLatestVersionByPublication(publication.Id)) - .ReturnsAsync(methodologies); + .ReturnsAsync(methodology.Versions); releaseFileService.Setup(mock => - mock.ListAll(releaseVersion.Id, Ancillary, FileType.Data)) + mock.ListAll(publishedReleaseVersion.Id, Ancillary, FileType.Data)) .ReturnsAsync(files); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) @@ -271,76 +206,68 @@ public async Task GetManageContentPageViewModel() methodologyVersionRepository: methodologyVersionRepository.Object, releaseFileService: releaseFileService.Object); - var result = await service.GetManageContentPageViewModel(releaseVersion.Id); + var result = await service.GetManageContentPageViewModel(publishedReleaseVersion.Id); - var viewModel = result.AssertRight(); + MockUtils.VerifyAllMocks(dataBlockService, methodologyVersionRepository, releaseFileService); - dataBlockService.Verify(mock => - mock.GetUnattachedDataBlocks(releaseVersion.Id), Times.Once); - - releaseFileService.Verify(mock => - mock.ListAll(releaseVersion.Id, Ancillary, FileType.Data), Times.Once); + var viewModel = result.AssertRight(); Assert.Equal(unattachedDataBlocks, viewModel.UnattachedDataBlocks); var contentRelease = viewModel.Release; Assert.NotNull(contentRelease); - Assert.Equal(releaseVersion.Id, contentRelease.Id); + Assert.Equal(publishedReleaseVersion.Id, contentRelease.Id); Assert.Equal("Academic year", contentRelease.CoverageTitle); Assert.True(contentRelease.HasDataGuidance); Assert.True(contentRelease.HasPreReleaseAccessList); Assert.Equal(2, contentRelease.KeyStatistics.Count); - Assert.Equal(releaseVersion.KeyStatistics[1].Id, contentRelease.KeyStatistics[0].Id); + Assert.Equal(publishedReleaseVersion.KeyStatistics[1].Id, contentRelease.KeyStatistics[0].Id); Assert.Equal(0, contentRelease.KeyStatistics[0].Order); - var originalKeyStatDataBlock = (releaseVersion.KeyStatistics[1] as KeyStatisticDataBlock)!; - var keyStatDataBlockViewModel = Assert.IsType(contentRelease.KeyStatistics[0]); + var originalKeyStatDataBlock = (publishedReleaseVersion.KeyStatistics[1] as KeyStatisticDataBlock)!; + var keyStatDataBlockViewModel = + Assert.IsType(contentRelease.KeyStatistics[0]); Assert.Equal(originalKeyStatDataBlock.DataBlockId, keyStatDataBlockViewModel.DataBlockId); Assert.Equal( keyStatDataBlockParent.LatestPublishedVersion!.DataBlockParentId, keyStatDataBlockViewModel.DataBlockParentId); - Assert.Equal(releaseVersion.KeyStatistics[0].Id, contentRelease.KeyStatistics[1].Id); + Assert.Equal(publishedReleaseVersion.KeyStatistics[0].Id, contentRelease.KeyStatistics[1].Id); Assert.Equal(1, contentRelease.KeyStatistics[1].Order); Assert.IsType(contentRelease.KeyStatistics[1]); - Assert.Equal(releaseVersion.KeyStatisticsSecondarySection.Id, + Assert.Equal(publishedReleaseVersion.KeyStatisticsSecondarySection.Id, contentRelease.KeyStatisticsSecondarySection.Id); - Assert.Equal(releaseVersion.HeadlinesSection.Id, contentRelease.HeadlinesSection.Id); - Assert.Equal(releaseVersion.RelatedDashboardsSection.Id, - contentRelease.RelatedDashboardsSection.Id); + Assert.Equal(publishedReleaseVersion.HeadlinesSection.Id, contentRelease.HeadlinesSection.Id); + Assert.Equal(publishedReleaseVersion.RelatedDashboardsSection.Id, contentRelease.RelatedDashboardsSection.Id); Assert.True(contentRelease.LatestRelease); - Assert.Equal("9", contentRelease.NextReleaseDate.Day); - Assert.Equal("9", contentRelease.NextReleaseDate.Month); - Assert.Equal("2040", contentRelease.NextReleaseDate.Year); - Assert.Equal("2020", contentRelease.ReleaseName); - Assert.NotNull(contentRelease.Published); - Assert.InRange(DateTime.UtcNow.Subtract(contentRelease.Published!.Value).Milliseconds, - 0, 1500); + Assert.Equal(publishedReleaseVersion.NextReleaseDate, contentRelease.NextReleaseDate); + Assert.Equal(publishedReleaseVersion.ReleaseName, contentRelease.ReleaseName); + Assert.Equal(publishedReleaseVersion.PublishScheduled, contentRelease.PublishScheduled); + Assert.Equal(publishedReleaseVersion.Published, contentRelease.Published); Assert.Equal(publication.Id, contentRelease.PublicationId); - Assert.Equal(DateTime.Parse("2020-09-09T00:00:00.00"), contentRelease.PublishScheduled); - Assert.Equal("2020-21", contentRelease.Slug); - Assert.Equal(releaseVersion.SummarySection.Id, contentRelease.SummarySection.Id); - Assert.Equal("Academic year 2020/21", contentRelease.Title); - Assert.Equal(ReleaseType.OfficialStatistics, contentRelease.Type); - Assert.Equal("2020/21", contentRelease.YearTitle); + Assert.Equal(publishedReleaseVersion.Slug, contentRelease.Slug); + Assert.Equal(publishedReleaseVersion.SummarySection.Id, contentRelease.SummarySection.Id); + Assert.Equal(publishedReleaseVersion.Title, contentRelease.Title); + Assert.Equal(publishedReleaseVersion.Type, contentRelease.Type); + Assert.Equal(publishedReleaseVersion.YearTitle, contentRelease.YearTitle); Assert.Empty(contentRelease.Updates); var contentDownloadFiles = contentRelease.DownloadFiles.ToList(); Assert.Equal(2, contentDownloadFiles.Count); Assert.Equal(files[0].Id, contentDownloadFiles[0].Id); - Assert.Equal("pdf", contentDownloadFiles[0].Extension); - Assert.Equal("ancillary.pdf", contentDownloadFiles[0].FileName); - Assert.Equal("Ancillary File", contentDownloadFiles[0].Name); - Assert.Equal("10 Kb", contentDownloadFiles[0].Size); - Assert.Equal(Ancillary, contentDownloadFiles[0].Type); + Assert.Equal(files[0].Extension, contentDownloadFiles[0].Extension); + Assert.Equal(files[0].FileName, contentDownloadFiles[0].FileName); + Assert.Equal(files[0].Name, contentDownloadFiles[0].Name); + Assert.Equal(files[0].Size, contentDownloadFiles[0].Size); + Assert.Equal(files[0].Type, contentDownloadFiles[0].Type); Assert.Equal(files[1].Id, contentDownloadFiles[1].Id); - Assert.Equal("csv", contentDownloadFiles[1].Extension); - Assert.Equal("data.csv", contentDownloadFiles[1].FileName); - Assert.Equal("Subject File", contentDownloadFiles[1].Name); - Assert.Equal("20 Kb", contentDownloadFiles[1].Size); - Assert.Equal(FileType.Data, contentDownloadFiles[1].Type); + Assert.Equal(files[1].Extension, contentDownloadFiles[1].Extension); + Assert.Equal(files[1].FileName, contentDownloadFiles[1].FileName); + Assert.Equal(files[1].Name, contentDownloadFiles[1].Name); + Assert.Equal(files[1].Size, contentDownloadFiles[1].Size); + Assert.Equal(files[1].Type, contentDownloadFiles[1].Type); var contentRelatedInformation = contentRelease.RelatedInformation; Assert.Single(contentRelatedInformation); @@ -372,44 +299,46 @@ public async Task GetManageContentPageViewModel() Assert.Equal(publication.Contact.TeamEmail, contentPublication.Contact.TeamEmail); Assert.Equal(publication.Contact.ContactTelNo, contentPublication.Contact.ContactTelNo); Assert.Null(contentPublication.ExternalMethodology); - Assert.Equal("test-publication", contentPublication.Slug); - Assert.Equal("Publication", contentPublication.Title); + Assert.Equal(publication.Slug, contentPublication.Slug); + Assert.Equal(publication.Title, contentPublication.Title); var contentPublicationReleaseSeries = contentPublication.ReleaseSeries; Assert.Equal(3, contentPublicationReleaseSeries.Count); Assert.False(contentPublicationReleaseSeries[0].IsLegacyLink); - Assert.Equal(releaseId, contentPublicationReleaseSeries[0].ReleaseId); - Assert.Equal(releaseVersion.Slug, contentPublicationReleaseSeries[0].ReleaseSlug); - Assert.Equal(releaseVersion.Title, contentPublicationReleaseSeries[0].Description); + Assert.Equal(publishedReleaseVersion.ReleaseId, contentPublicationReleaseSeries[0].ReleaseId); + Assert.Equal(publishedReleaseVersion.Slug, contentPublicationReleaseSeries[0].ReleaseSlug); + Assert.Equal(publishedReleaseVersion.Title, contentPublicationReleaseSeries[0].Description); Assert.Null(contentPublicationReleaseSeries[0].LegacyLinkUrl); Assert.True(contentPublicationReleaseSeries[1].IsLegacyLink); Assert.Null(contentPublicationReleaseSeries[1].ReleaseId); Assert.Null(contentPublicationReleaseSeries[1].ReleaseSlug); - Assert.Equal("Legacy 2018/19", contentPublicationReleaseSeries[1].Description); - Assert.Equal("https://legacy-2018-19", contentPublicationReleaseSeries[1].LegacyLinkUrl); + Assert.Equal(publication.ReleaseSeries[1].LegacyLinkDescription, + contentPublicationReleaseSeries[1].Description); + Assert.Equal(publication.ReleaseSeries[1].LegacyLinkUrl, + contentPublicationReleaseSeries[1].LegacyLinkUrl); Assert.True(contentPublicationReleaseSeries[2].IsLegacyLink); Assert.Null(contentPublicationReleaseSeries[2].ReleaseId); Assert.Null(contentPublicationReleaseSeries[2].ReleaseSlug); - Assert.Equal("Legacy 2017/18", contentPublicationReleaseSeries[2].Description); - Assert.Equal("https://legacy-2017-18", contentPublicationReleaseSeries[2].LegacyLinkUrl); + Assert.Equal(publication.ReleaseSeries[2].LegacyLinkDescription, + contentPublicationReleaseSeries[2].Description); + Assert.Equal(publication.ReleaseSeries[2].LegacyLinkUrl, + contentPublicationReleaseSeries[2].LegacyLinkUrl); var contentPublicationReleases = contentPublication.Releases; Assert.Single(contentPublicationReleases); Assert.Equal(otherReleaseVersion.Id, contentPublicationReleases[0].Id); - Assert.Equal("2019-20", contentPublicationReleases[0].Slug); - Assert.Equal("Academic year 2019/20", contentPublicationReleases[0].Title); + Assert.Equal(otherReleaseVersion.Slug, contentPublicationReleases[0].Slug); + Assert.Equal(otherReleaseVersion.Title, contentPublicationReleases[0].Title); Assert.Equal(2, contentPublication.Methodologies.Count); - Assert.Equal(methodologies[0].Id, contentPublication.Methodologies[0].Id); - Assert.Equal("Methodology 1 title", contentPublication.Methodologies[0].Title); - Assert.Equal(methodologies[1].Id, contentPublication.Methodologies[1].Id); - Assert.Equal("Methodology 2 title", contentPublication.Methodologies[1].Title); + Assert.Equal(methodology.Versions[0].Id, contentPublication.Methodologies[0].Id); + Assert.Equal(methodology.Versions[0].Title, contentPublication.Methodologies[0].Title); + Assert.Equal(methodology.Versions[1].Id, contentPublication.Methodologies[1].Id); + Assert.Equal(methodology.Versions[1].Title, contentPublication.Methodologies[1].Title); } - - MockUtils.VerifyAllMocks(dataBlockService, methodologyVersionRepository, releaseFileService); } [Fact] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs index bd7c36fcf17..20e83050f45 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs @@ -1399,6 +1399,7 @@ public async Task ListLatestMethodologyVersions() Assert.Null(viewModels[3].PreviousVersionId); } } + [Fact] public async Task ListLatestMethodologyVersions_IsPrerelease() { @@ -1924,13 +1925,13 @@ public async Task UpdateMethodology_UpdatingAmendmentWithAlternativeSlug() }), Versions = new List { - new () + new() { Id = latestPublishedVersionId, Status = Approved, Version = 0, }, - new () + new() { Id = latestVersionId, Status = Draft, @@ -2039,13 +2040,13 @@ public async Task UpdateMethodology_UpdatingAmendmentUnsetsAlternativeTitleAndSl }), Versions = new List { - new () + new() { Id = latestPublishedVersionId, Status = Approved, Version = 0, }, - new () + new() { Id = latestVersionId, AlternativeTitle = "Alternative Methodology Title", @@ -2590,7 +2591,6 @@ public async Task DeleteMethodologyVersion() { MethodologyId = methodologyId, PublicationId = Guid.NewGuid(), - }) } }; @@ -3221,13 +3221,13 @@ public class ListUsersMethodologyVersionsForApprovalForReleaseRoles [Fact] public async Task UserIsApproverOnOwningPublicationRelease_Included() { - var releaseVersion = _fixture.DefaultReleaseVersion().Generate(); - - var publication = _fixture + Publication publication = _fixture .DefaultPublication() .WithContact(MockContact) - .WithReleaseVersions(ListOf(releaseVersion)) - .Generate(); + .WithReleases(_fixture.DefaultRelease(publishedVersions: 1) + .Generate(1)); + + var releaseVersion = publication.Releases.Single().Versions.Single(); var methodology = _fixture .DefaultMethodology() @@ -3271,18 +3271,14 @@ public async Task UserIsApproverOnOwningPublicationRelease_Included() [Fact] public async Task UserIsApproverOnOwningPublicationOldRelease_Included() { - var releaseVersions = _fixture - .DefaultReleaseVersion() - .WithApprovalStatuses(ListOf( - ReleaseApprovalStatus.Approved, - ReleaseApprovalStatus.Draft)) - .GenerateList(); - - var publication = _fixture + Publication publication = _fixture .DefaultPublication() .WithContact(MockContact) - .WithReleaseVersions(releaseVersions) - .Generate(); + .WithReleases(_fixture.DefaultRelease(publishedVersions: 1, draftVersion: true) + .Generate(1)); + + var publishedReleaseVersion = publication.Releases.Single().Versions + .Single(rv => rv is { Published: not null, Version: 0 }); var methodology = _fixture .DefaultMethodology() @@ -3296,7 +3292,7 @@ public async Task UserIsApproverOnOwningPublicationOldRelease_Included() var releaseRoleForUserOnOldRelease = _fixture .DefaultUserReleaseRole() .WithUser(User) - .WithReleaseVersion(releaseVersions[0]) + .WithReleaseVersion(publishedReleaseVersion) .WithRole(ReleaseRole.Approver) .Generate(); @@ -3505,13 +3501,13 @@ public async Task DifferentUserIsApproverOnOwningPublicationRelease_NotIncluded( [Fact] public async Task UserIsPublicationAndReleaseApprover_NoDuplication() { - var releaseVersion = _fixture.DefaultReleaseVersion().Generate(); - - var publication = _fixture + Publication publication = _fixture .DefaultPublication() .WithContact(MockContact) - .WithReleaseVersions(ListOf(releaseVersion)) - .Generate(); + .WithReleases(_fixture.DefaultRelease(publishedVersions: 1) + .Generate(1)); + + var publishedReleaseVersion = publication.Releases.Single().Versions.Single(); var methodology = _fixture .DefaultMethodology() @@ -3532,7 +3528,7 @@ public async Task UserIsPublicationAndReleaseApprover_NoDuplication() var releaseRoleForUser = _fixture .DefaultUserReleaseRole() .WithUser(User) - .WithReleaseVersion(releaseVersion) + .WithReleaseVersion(publishedReleaseVersion) .WithRole(ReleaseRole.Approver) .Generate(); @@ -3692,8 +3688,6 @@ await service.PublicationTitleOrSlugChanged(publicationId, } } - - [Fact] public async Task PublicationTitleOrSlugChanged_DoesNotAffectUnrelatedMethodologies() { @@ -3937,7 +3931,8 @@ public async Task PublicationTitleOrSlugChanged_MethodologyIsLive() } [Fact] - public async Task PublicationTitleOrSlugChanged_CurrentInheritedPubSlugChangesWithUnpublishedAmendmentWithAlternativeSlug() + public async Task + PublicationTitleOrSlugChanged_CurrentInheritedPubSlugChangesWithUnpublishedAmendmentWithAlternativeSlug() { var publicationId = Guid.NewGuid(); var latestPublishedVersionId = Guid.NewGuid(); @@ -3959,12 +3954,12 @@ public async Task PublicationTitleOrSlugChanged_CurrentInheritedPubSlugChangesWi LatestPublishedVersionId = latestPublishedVersionId, Versions = new List { - new () + new() { Id = latestPublishedVersionId, Version = 0, }, - new () + new() { Id = latestVersionId, Version = 1, @@ -4032,7 +4027,8 @@ await service.PublicationTitleOrSlugChanged(publicationId, "current-slug", } [Fact] - public async Task PublicationTitleOrSlugChanged_UnpublishedMethodologyAmendmentDoesNotNeedRedirectIfRedirectAlreadyExists() + public async Task + PublicationTitleOrSlugChanged_UnpublishedMethodologyAmendmentDoesNotNeedRedirectIfRedirectAlreadyExists() { var publicationId = Guid.NewGuid(); var latestPublishedVersionId = Guid.NewGuid(); @@ -4054,12 +4050,12 @@ public async Task PublicationTitleOrSlugChanged_UnpublishedMethodologyAmendmentD LatestPublishedVersionId = latestPublishedVersionId, Versions = new List { - new () + new() { Id = latestPublishedVersionId, Version = 0, }, - new () + new() { Id = latestVersionId, Version = 1, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServicePermissionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServicePermissionTests.cs index 234a57cd21d..fdd1e9f332e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServicePermissionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServicePermissionTests.cs @@ -1,11 +1,13 @@ #nullable enable -using System; +using System.Linq; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Security; using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using Moq; @@ -17,27 +19,30 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services { public class ReleaseAmendmentServicePermissionTests { - private readonly ReleaseVersion _releaseVersion = new() - { - Id = Guid.NewGuid(), - Publication = new Publication(), - Release = new Release() - }; + private readonly DataFixture _fixture = new(); [Fact] public async Task CreateReleaseAmendment() { + Publication publication = _fixture + .DefaultPublication() + .WithReleases(_fixture + .DefaultRelease(publishedVersions: 1) + .Generate(1)); + + var releaseVersion = publication.Releases.Single().Versions.Single(); + await PolicyCheckBuilder() - .SetupResourceCheckToFail(_releaseVersion, CanMakeAmendmentOfSpecificRelease) + .SetupResourceCheckToFail(releaseVersion, CanMakeAmendmentOfSpecificRelease) .AssertForbidden( userService => { using var contentDbContext = InMemoryApplicationDbContext(); - contentDbContext.ReleaseVersions.Add(_releaseVersion); + contentDbContext.Publications.Add(publication); contentDbContext.SaveChanges(); var service = BuildService(userService.Object, contentDbContext); - return service.CreateReleaseAmendment(_releaseVersion.Id); + return service.CreateReleaseAmendment(releaseVersion.Id); } ); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServiceTests.cs index 28351eeee65..81eb5bff414 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseAmendmentServiceTests.cs @@ -53,10 +53,10 @@ public async Task CreateReleaseAmendment() ReleaseVersion originalReleaseVersion = _fixture .DefaultReleaseVersion() - .WithPublication(_fixture - .DefaultPublication()) .WithRelease(_fixture - .DefaultRelease()) + .DefaultRelease() + .WithPublication(_fixture + .DefaultPublication())) .WithCreated( created: DateTime.UtcNow.AddDays(-2), createdById: originalCreatedBy.Id) @@ -771,10 +771,10 @@ Content 1 ReleaseVersion originalReleaseVersion = _fixture .DefaultReleaseVersion() - .WithPublication(_fixture - .DefaultPublication()) .WithRelease(_fixture - .DefaultRelease()) + .DefaultRelease() + .WithPublication(_fixture + .DefaultPublication())) .WithCreated(createdById: _userId) .WithContent(_fixture .DefaultContentSection() @@ -843,10 +843,10 @@ public async Task NullHtmlBlockBody() { ReleaseVersion originalReleaseVersion = _fixture .DefaultReleaseVersion() - .WithPublication(_fixture - .DefaultPublication()) .WithRelease(_fixture - .DefaultRelease()) + .DefaultRelease() + .WithPublication(_fixture + .DefaultPublication())) .WithCreated(createdById: _userId) .WithContent(_fixture .DefaultContentSection() @@ -893,10 +893,10 @@ public async Task CreatesRelatedDashboardsSectionIfNotOnOriginal() { ReleaseVersion originalReleaseVersion = _fixture .DefaultReleaseVersion() - .WithPublication(_fixture - .DefaultPublication()) .WithRelease(_fixture - .DefaultRelease()) + .DefaultRelease() + .WithPublication(_fixture + .DefaultPublication())) .WithCreated(createdById: _userId); var contentDbContextId = Guid.NewGuid().ToString(); @@ -937,10 +937,10 @@ public async Task CopyFootnotes() { ReleaseVersion originalReleaseVersion = _fixture .DefaultReleaseVersion() - .WithPublication(_fixture - .DefaultPublication()) .WithRelease(_fixture - .DefaultRelease()) + .DefaultRelease() + .WithPublication(_fixture + .DefaultPublication())) .WithCreated(createdById: _userId) .WithContent(_fixture .DefaultContentSection() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServiceTests.cs index 9216d9ca3de..66b836a0788 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseServiceTests.cs @@ -14,6 +14,7 @@ using GovUk.Education.ExploreEducationStatistics.Admin.Validators.ErrorDetails; using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; using GovUk.Education.ExploreEducationStatistics.Common.Cache; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; @@ -47,6 +48,7 @@ using static Moq.MockBehavior; using IReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IReleaseVersionRepository; +using Release = GovUk.Education.ExploreEducationStatistics.Content.Model.Release; using ReleaseVersion = GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion; using StatsReleaseVersion = GovUk.Education.ExploreEducationStatistics.Data.Model.ReleaseVersion; using Unit = GovUk.Education.ExploreEducationStatistics.Common.Model.Unit; @@ -78,16 +80,18 @@ public async Task ReleaseTypeExperimentalStatistics_ReturnsValidationActionResul [Fact] public async Task NoTemplate() { - var publication = new Publication { Title = "Publication" }; + Publication publication = _dataFixture.DefaultPublication(); var contextId = Guid.NewGuid().ToString(); await using (var context = InMemoryApplicationDbContext(contextId)) { - await context.AddAsync(publication); + context.Publications.Add(publication); await context.SaveChangesAsync(); } + Guid? newReleaseVersionId; + await using (var context = InMemoryApplicationDbContext(contextId)) { var releaseService = BuildReleaseService(context); @@ -101,6 +105,7 @@ public async Task NoTemplate() Type = ReleaseType.OfficialStatistics } )).AssertRight(); + newReleaseVersionId = result.Id; Assert.Equal("Academic year 2018/19", result.Title); Assert.Equal("2018/19", result.YearTitle); @@ -120,32 +125,37 @@ public async Task NoTemplate() await using (var context = InMemoryApplicationDbContext(contextId)) { - var actual = await context - .ReleaseVersions - .SingleAsync(rv => rv.PublicationId == publication.Id); - - Assert.Equal(2018, actual.Year); - Assert.Equal(TimeIdentifier.AcademicYear, actual.TimePeriodCoverage); - Assert.Equal(ReleaseType.OfficialStatistics, actual.Type); - Assert.Equal(ReleaseApprovalStatus.Draft, actual.ApprovalStatus); - Assert.Equal(0, actual.Version); - Assert.NotEqual(Guid.Empty, actual.ReleaseId); - - Assert.Null(actual.PreviousVersionId); - Assert.Null(actual.PublishScheduled); - Assert.Null(actual.Published); - Assert.Null(actual.NextReleaseDate); - Assert.Null(actual.NotifiedOn); - Assert.False(actual.NotifySubscribers); - Assert.False(actual.UpdatePublishedDate); + var actualReleaseVersion = await context.ReleaseVersions + .Include(rv => rv.Release) + .SingleAsync(rv => rv.Id == newReleaseVersionId); + + var actualRelease = actualReleaseVersion.Release; + + Assert.Equal(publication.Id, actualRelease.PublicationId); + Assert.Equal(2018, actualRelease.Year); + Assert.Equal(TimeIdentifier.AcademicYear, actualRelease.TimePeriodCoverage); + Assert.Equal("2018-19", actualRelease.Slug); + + Assert.Equal(2018, actualReleaseVersion.Year); + Assert.Equal(TimeIdentifier.AcademicYear, actualReleaseVersion.TimePeriodCoverage); + Assert.Equal("2018-19", actualReleaseVersion.Slug); + Assert.Equal(ReleaseType.OfficialStatistics, actualReleaseVersion.Type); + Assert.Equal(ReleaseApprovalStatus.Draft, actualReleaseVersion.ApprovalStatus); + Assert.Equal(0, actualReleaseVersion.Version); + + Assert.Null(actualReleaseVersion.PreviousVersionId); + Assert.Null(actualReleaseVersion.PublishScheduled); + Assert.Null(actualReleaseVersion.Published); + Assert.Null(actualReleaseVersion.NextReleaseDate); + Assert.Null(actualReleaseVersion.NotifiedOn); + Assert.False(actualReleaseVersion.NotifySubscribers); + Assert.False(actualReleaseVersion.UpdatePublishedDate); } } [Fact] public async Task WithTemplate() { - var templateReleaseId = Guid.NewGuid(); - var dataBlock1 = new DataBlock { Id = Guid.NewGuid(), @@ -153,31 +163,27 @@ public async Task WithTemplate() Order = 2, Comments = [ - new() + new Comment { Id = Guid.NewGuid(), Content = "Comment 1 Text" }, - new() + new Comment { Id = Guid.NewGuid(), Content = "Comment 2 Text" } - ], - ReleaseVersionId = templateReleaseId + ] }; var dataBlock2 = new DataBlock { Id = Guid.NewGuid(), - Name = "Data Block 2", - ReleaseVersionId = templateReleaseId + Name = "Data Block 2" }; - var templateRelease = new ReleaseVersion + var templateReleaseVersion = new ReleaseVersion { - Id = templateReleaseId, - ReleaseName = "2018", Content = ListOf(new ContentSection { Id = Guid.NewGuid(), @@ -194,37 +200,34 @@ public async Task WithTemplate() Order = 1, Comments = [ - new() + new Comment { Id = Guid.NewGuid(), Content = "Comment 1 Text" }, - new() + new Comment { Id = Guid.NewGuid(), Content = "Comment 2 Text" } ] }, - dataBlock1 + dataBlock1, + dataBlock2 ] - }), - Version = 0 + }) }; + Publication publication = _dataFixture.DefaultPublication() + .WithReleases(_dataFixture.DefaultRelease() + .WithVersions([templateReleaseVersion]) + .Generate(1)); + var contextId = Guid.NewGuid().ToString(); await using (var context = InMemoryApplicationDbContext(contextId)) { - await context.AddAsync( - new Publication - { - Id = new Guid("403d3c5d-a8cd-4d54-a029-0c74c86c55b2"), - Title = "Publication", - ReleaseVersions = ListOf(templateRelease) - } - ); - await context.ContentBlocks.AddRangeAsync(dataBlock1, dataBlock2); + context.Publications.Add(publication); await context.SaveChangesAsync(); } @@ -237,8 +240,8 @@ await context.AddAsync( var result = await releaseService.CreateRelease( new ReleaseCreateRequest { - PublicationId = new Guid("403d3c5d-a8cd-4d54-a029-0c74c86c55b2"), - TemplateReleaseId = templateReleaseId, + PublicationId = publication.Id, + TemplateReleaseId = templateReleaseVersion.Id, Year = 2018, TimePeriodCoverage = TimeIdentifier.AcademicYear, Type = ReleaseType.OfficialStatistics @@ -849,27 +852,18 @@ public class UpdateReleaseVersionTests : ReleaseServiceTests [Fact] public async Task Success() { - var releaseVersion = new ReleaseVersion - { - Type = ReleaseType.AdHocStatistics, - Publication = new Publication(), - ReleaseName = "2030", - PublishScheduled = DateTime.UtcNow, - NextReleaseDate = new PartialDate - { - Day = "15", - Month = "6", - Year = "2039" - }, - PreReleaseAccessList = "Old access list", - Version = 0 - }; + Publication publication = _dataFixture.DefaultPublication() + .WithReleases(_dataFixture.DefaultRelease(publishedVersions: 1) + .Generate(1)); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); var contextId = Guid.NewGuid().ToString(); await using (var context = InMemoryApplicationDbContext(contextId)) { - context.ReleaseVersions.AddRange(releaseVersion); + context.Publications.Add(publication); await context.SaveChangesAsync(); } @@ -877,11 +871,15 @@ public async Task Success() dataSetVersionService.Setup(service => service.UpdateVersionsForReleaseVersion( releaseVersion.Id, - "2035-march", - "March 2035", + release.Slug, + release.Title, It.IsAny())) .Returns(Task.CompletedTask); + var updatedType = EnumUtil.GetEnums() + .Except([releaseVersion.Type, ReleaseType.ExperimentalStatistics]) + .First(); + await using (var context = InMemoryApplicationDbContext(contextId)) { var releaseService = BuildReleaseService( @@ -893,10 +891,10 @@ public async Task Success() releaseVersion.Id, new ReleaseUpdateRequest { - Type = ReleaseType.OfficialStatistics, - Year = 2035, - TimePeriodCoverage = TimeIdentifier.March, - PreReleaseAccessList = "New access list", + Type = updatedType, + Year = release.Year, + TimePeriodCoverage = release.TimePeriodCoverage, + PreReleaseAccessList = "New access list" } ); @@ -906,60 +904,55 @@ public async Task Success() Assert.Equal(releaseVersion.Publication.Id, viewModel.PublicationId); Assert.Equal(releaseVersion.NextReleaseDate, viewModel.NextReleaseDate); - Assert.Equal(ReleaseType.OfficialStatistics, viewModel.Type); - Assert.Equal(2035, viewModel.Year); - Assert.Equal("2035", viewModel.YearTitle); - Assert.Equal(TimeIdentifier.March, viewModel.TimePeriodCoverage); + Assert.Equal(updatedType, viewModel.Type); + Assert.Equal(release.Year, viewModel.Year); + Assert.Equal(release.YearTitle, viewModel.YearTitle); + Assert.Equal(release.TimePeriodCoverage, viewModel.TimePeriodCoverage); Assert.Equal("New access list", viewModel.PreReleaseAccessList); + Assert.Equal(release.Title, viewModel.Title); } await using (var context = InMemoryApplicationDbContext(contextId)) { - var saved = await context.ReleaseVersions + var actualReleaseVersion = await context.ReleaseVersions + .Include(rv => rv.Release) .Include(rv => rv.ReleaseStatuses) - .FirstAsync(rv => rv.Id == releaseVersion.Id); + .SingleAsync(rv => rv.Id == releaseVersion.Id); + + var actualRelease = actualReleaseVersion.Release; + + Assert.Equal(publication.Id, actualRelease.PublicationId); + Assert.Equal(release.Year, actualRelease.Year); + Assert.Equal(release.TimePeriodCoverage, actualRelease.TimePeriodCoverage); + Assert.Equal(release.Slug, actualRelease.Slug); - Assert.Equal(releaseVersion.Publication.Id, saved.PublicationId); - Assert.Equal(releaseVersion.NextReleaseDate, saved.NextReleaseDate); - Assert.Equal(ReleaseType.OfficialStatistics, saved.Type); - Assert.Equal("2035-march", saved.Slug); - Assert.Equal("2035", saved.ReleaseName); - Assert.Equal(TimeIdentifier.March, saved.TimePeriodCoverage); - Assert.Equal("New access list", saved.PreReleaseAccessList); + Assert.Equal(release.Year, actualReleaseVersion.Year); + Assert.Equal(release.TimePeriodCoverage, actualReleaseVersion.TimePeriodCoverage); + Assert.Equal(release.Slug, actualReleaseVersion.Slug); + Assert.Equal(releaseVersion.NextReleaseDate, actualReleaseVersion.NextReleaseDate); + Assert.Equal(updatedType, actualReleaseVersion.Type); + Assert.Equal("New access list", actualReleaseVersion.PreReleaseAccessList); - Assert.Empty(saved.ReleaseStatuses); + Assert.Empty(actualReleaseVersion.ReleaseStatuses); } } [Fact] public async Task FailsNonUniqueSlug() { - var publication = new Publication(); + Publication publication = _dataFixture.DefaultPublication(); - var releaseVersion = new ReleaseVersion - { - Type = ReleaseType.AdHocStatistics, - Publication = publication, - ReleaseName = "2030", - Slug = "2030", - PublishScheduled = DateTime.UtcNow, - Version = 0 - }; + var (release, otherRelease) = _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true) + .WithPublication(publication) + .Generate(2) + .ToTuple2(); - var otherRelease = new ReleaseVersion - { - Type = ReleaseType.AdHocStatistics, - Publication = publication, - ReleaseName = "2035", - Slug = "2035", - PublishScheduled = DateTime.UtcNow, - Version = 0 - }; + var releaseVersion = release.Versions.Single(); var contextId = Guid.NewGuid().ToString(); await using (var context = InMemoryApplicationDbContext(contextId)) { - context.ReleaseVersions.AddRange(releaseVersion, otherRelease); + context.Publications.Add(publication); await context.SaveChangesAsync(); } @@ -972,10 +965,10 @@ public async Task FailsNonUniqueSlug() releaseVersion.Id, new ReleaseUpdateRequest { - Type = ReleaseType.AdHocStatistics, - Year = 2035, - TimePeriodCoverage = TimeIdentifier.CalendarYear, - PreReleaseAccessList = "Test" + Year = otherRelease.Year, + TimePeriodCoverage = otherRelease.TimePeriodCoverage, + Type = releaseVersion.Type, + PreReleaseAccessList = releaseVersion.PreReleaseAccessList } ); @@ -2439,7 +2432,7 @@ public class ListUsersReleasesForApprovalTests : ReleaseServiceTests private readonly DataFixture _fixture = new(); [Fact] - public async Task UserHasApproverRoleOnRelease() + public async Task UserHasApproverRoleOnReleaseVersion() { var contextId = Guid.NewGuid().ToString(); @@ -2447,52 +2440,63 @@ public async Task UserHasApproverRoleOnRelease() var publications = _fixture .DefaultPublication() - .WithReleaseVersions(_ => _fixture - .DefaultReleaseVersion() - .WithApprovalStatuses(ListOf( - ReleaseApprovalStatus.Draft, - ReleaseApprovalStatus.HigherLevelReview, - ReleaseApprovalStatus.Approved)) - .GenerateList()) + .WithReleases(_ => ListOf( + _fixture.DefaultRelease() + .WithVersions(_ => _fixture + .DefaultReleaseVersion() + .WithApprovalStatus(ReleaseApprovalStatus.Draft) + .Generate(1)), + _fixture.DefaultRelease() + .WithVersions(_ => _fixture + .DefaultReleaseVersion() + .WithApprovalStatus(ReleaseApprovalStatus.HigherLevelReview) + .Generate(1)), + _fixture.DefaultRelease() + .WithVersions(_ => _fixture + .DefaultReleaseVersion() + .WithApprovalStatus(ReleaseApprovalStatus.Approved) + .Generate(1)) + )) .GenerateList(4); var contributorReleaseRolesForUser = _fixture .DefaultUserReleaseRole() .WithUser(User) .WithRole(ReleaseRole.Contributor) - .WithReleaseVersions(publications[0].ReleaseVersions) + .WithReleaseVersions(publications[0].Releases.SelectMany(r => r.Versions)) .GenerateList(); var approverReleaseRolesForUser = _fixture .DefaultUserReleaseRole() .WithUser(User) .WithRole(ReleaseRole.Approver) - .WithReleaseVersions(publications[1].ReleaseVersions) + .WithReleaseVersions(publications[1].Releases.SelectMany(r => r.Versions)) .GenerateList(); var prereleaseReleaseRolesForUser = _fixture .DefaultUserReleaseRole() .WithUser(User) .WithRole(ReleaseRole.PrereleaseViewer) - .WithReleaseVersions(publications[2].ReleaseVersions) + .WithReleaseVersions(publications[2].Releases.SelectMany(r => r.Versions)) .GenerateList(); var approverReleaseRolesForOtherUser = _fixture .DefaultUserReleaseRole() .WithUser(otherUser) .WithRole(ReleaseRole.Approver) - .WithReleaseVersions(publications.SelectMany(publication => publication.ReleaseVersions)) + .WithReleaseVersions(publications.SelectMany(publication => + publication.Releases.SelectMany(r => r.Versions))) .GenerateList(); - var higherReviewReleaseWithApproverRoleForUser = publications[1].ReleaseVersions[1]; + var higherReviewReleaseVersionWithApproverRoleForUser = publications[1].Releases[1].Versions.Single(); await using (var context = InMemoryApplicationDbContext(contextId)) { - await context.Publications.AddRangeAsync(publications); - await context.UserReleaseRoles.AddRangeAsync(contributorReleaseRolesForUser); - await context.UserReleaseRoles.AddRangeAsync(approverReleaseRolesForUser); - await context.UserReleaseRoles.AddRangeAsync(prereleaseReleaseRolesForUser); - await context.UserReleaseRoles.AddRangeAsync(approverReleaseRolesForOtherUser); + context.Publications.AddRange(publications); + context.UserReleaseRoles.AddRange(contributorReleaseRolesForUser); + context.UserReleaseRoles.AddRange(approverReleaseRolesForUser); + context.UserReleaseRoles.AddRange(prereleaseReleaseRolesForUser); + context.UserReleaseRoles.AddRange(approverReleaseRolesForOtherUser); await context.SaveChangesAsync(); } @@ -2505,14 +2509,14 @@ public async Task UserHasApproverRoleOnRelease() var viewModels = result.AssertRight(); // Assert that the only Release returned for this user is the Release where they have a direct - // Approver role on and it is in Higher Review. + // Approver role on, and it is in Higher Review. var viewModel = Assert.Single(viewModels); - Assert.Equal(higherReviewReleaseWithApproverRoleForUser.Id, viewModel.Id); + Assert.Equal(higherReviewReleaseVersionWithApproverRoleForUser.Id, viewModel.Id); // Assert that we have a fully populated ReleaseSummaryViewModel, including details from the owning // Publication. Assert.Equal( - higherReviewReleaseWithApproverRoleForUser.Publication.Title, + higherReviewReleaseVersionWithApproverRoleForUser.Publication.Title, viewModel.Publication!.Title); } } @@ -2526,14 +2530,28 @@ public async Task UserHasApproverRoleOnPublications() var publications = _fixture .DefaultPublication() - .WithReleaseVersions(_ => _fixture - .DefaultReleaseVersion() - .WithApprovalStatuses(ListOf( - ReleaseApprovalStatus.Draft, - ReleaseApprovalStatus.HigherLevelReview, - ReleaseApprovalStatus.Approved, - ReleaseApprovalStatus.HigherLevelReview)) - .GenerateList()) + .WithReleases(_ => ListOf( + _fixture.DefaultRelease() + .WithVersions(_ => _fixture + .DefaultReleaseVersion() + .WithApprovalStatus(ReleaseApprovalStatus.Draft) + .Generate(1)), + _fixture.DefaultRelease() + .WithVersions(_ => _fixture + .DefaultReleaseVersion() + .WithApprovalStatus(ReleaseApprovalStatus.HigherLevelReview) + .Generate(1)), + _fixture.DefaultRelease() + .WithVersions(_ => _fixture + .DefaultReleaseVersion() + .WithApprovalStatus(ReleaseApprovalStatus.Approved) + .Generate(1)), + _fixture.DefaultRelease() + .WithVersions(_ => _fixture + .DefaultReleaseVersion() + .WithApprovalStatus(ReleaseApprovalStatus.HigherLevelReview) + .Generate(1)) + )) .GenerateList(3); var ownerPublicationRoleForUser = _fixture @@ -2564,17 +2582,15 @@ public async Task UserHasApproverRoleOnPublications() .WithPublications(publications) .GenerateList(); - var release1WithApproverRoleForUser = publications[1].ReleaseVersions[1]; - var release2WithApproverRoleForUser = publications[1].ReleaseVersions[3]; + var releaseVersion1WithApproverRoleForUser = publications[1].Releases[1].Versions.Single(); + var releaseVersion2WithApproverRoleForUser = publications[1].Releases[3].Versions.Single(); await using (var context = InMemoryApplicationDbContext(contextId)) { - await context.Publications.AddRangeAsync(publications); - await context.UserPublicationRoles.AddRangeAsync( - ownerPublicationRoleForUser, - approverPublicationRoleForUser); - await context.UserPublicationRoles.AddRangeAsync(ownerPublicationRolesForOtherUser); - await context.UserPublicationRoles.AddRangeAsync(approverPublicationRolesForOtherUser); + context.Publications.AddRange(publications); + context.UserPublicationRoles.AddRange(ownerPublicationRoleForUser, approverPublicationRoleForUser); + context.UserPublicationRoles.AddRange(ownerPublicationRolesForOtherUser); + context.UserPublicationRoles.AddRange(approverPublicationRolesForOtherUser); await context.SaveChangesAsync(); } @@ -2589,8 +2605,8 @@ await context.UserPublicationRoles.AddRangeAsync( // Assert that the only Releases returned for this user are the Releases where they have Approver // role on the overarching Publication and the Releases are in Higher Review. Assert.Equal(2, viewModels.Count); - Assert.Equal(release1WithApproverRoleForUser.Id, viewModels[0].Id); - Assert.Equal(release2WithApproverRoleForUser.Id, viewModels[1].Id); + Assert.Equal(releaseVersion1WithApproverRoleForUser.Id, viewModels[0].Id); + Assert.Equal(releaseVersion2WithApproverRoleForUser.Id, viewModels[1].Id); } } @@ -2599,19 +2615,20 @@ public async Task UserIsPublicationAndReleaseApprover_NoDuplication() { var contextId = Guid.NewGuid().ToString(); - var publication = _fixture + Publication publication = _fixture .DefaultPublication() - .WithReleaseVersions(_ => _fixture - .DefaultReleaseVersion() - .WithApprovalStatus(ReleaseApprovalStatus.HigherLevelReview) - .Generate(1)) - .Generate(); + .WithReleases(_ => _fixture.DefaultRelease() + .WithVersions(_ => _fixture + .DefaultReleaseVersion() + .WithApprovalStatus(ReleaseApprovalStatus.HigherLevelReview) + .Generate(1)) + .Generate(1)); var approverReleaseRolesForUser = _fixture .DefaultUserReleaseRole() .WithUser(User) .WithRole(ReleaseRole.Approver) - .WithReleaseVersions(publication.ReleaseVersions) + .WithReleaseVersions(publication.Releases.Single().Versions) .GenerateList(); var approverPublicationRoleForUser = _fixture @@ -2623,9 +2640,9 @@ public async Task UserIsPublicationAndReleaseApprover_NoDuplication() await using (var context = InMemoryApplicationDbContext(contextId)) { - await context.Publications.AddRangeAsync(publication); - await context.UserReleaseRoles.AddRangeAsync(approverReleaseRolesForUser); - await context.UserPublicationRoles.AddRangeAsync(approverPublicationRoleForUser); + context.Publications.AddRange(publication); + context.UserReleaseRoles.AddRange(approverReleaseRolesForUser); + context.UserPublicationRoles.AddRange(approverPublicationRoleForUser); await context.SaveChangesAsync(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseVersionRepositoryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseVersionRepositoryTests.cs index 0fdc02c6bde..c93d5901040 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseVersionRepositoryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseVersionRepositoryTests.cs @@ -290,8 +290,9 @@ public async Task StatsReleaseDoesNotExist_CreatesStatsReleaseAndReleaseSubject( { ReleaseVersion releaseVersion = _fixture .DefaultReleaseVersion() - .WithPublication(_fixture - .DefaultPublication()); + .WithRelease(_fixture.DefaultRelease() + .WithPublication(_fixture + .DefaultPublication())); var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) @@ -328,8 +329,9 @@ public async Task StatsReleaseExists_CreatesReleaseSubject() { ReleaseVersion releaseVersion = _fixture .DefaultReleaseVersion() - .WithPublication(_fixture - .DefaultPublication()); + .WithRelease(_fixture.DefaultRelease() + .WithPublication(_fixture + .DefaultPublication())); Data.Model.ReleaseVersion existingStatsReleaseVersion = _fixture .DefaultStatsReleaseVersion() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ThemeServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ThemeServiceTests.cs index 9eed4ab4f9b..656de0e9fff 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ThemeServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ThemeServiceTests.cs @@ -18,6 +18,7 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using Microsoft.EntityFrameworkCore; @@ -29,6 +30,7 @@ using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; using static Moq.MockBehavior; +using Release = GovUk.Education.ExploreEducationStatistics.Content.Model.Release; using Theme = GovUk.Education.ExploreEducationStatistics.Content.Model.Theme; namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services @@ -312,21 +314,17 @@ public async Task DeleteTheme() Title = "UI test theme to delete" }; - var releaseVersion = _fixture + ReleaseVersion releaseVersion = _fixture .DefaultReleaseVersion() .WithId(releaseVersionId) - .WithRelease(_fixture.DefaultRelease()) - .WithPublication( - _fixture + .WithRelease(_fixture.DefaultRelease() + .WithPublication(_fixture .DefaultPublication() - .WithTheme(theme) - .Generate()) - .Generate(); + .WithTheme(theme))); - var methodology = _fixture + Methodology methodology = _fixture .DefaultMethodology() - .WithOwningPublication(releaseVersion.Publication) - .Generate(); + .WithOwningPublication(releaseVersion.Publication); var contextId = Guid.NewGuid().ToString(); @@ -390,45 +388,38 @@ public async Task DeleteTheme_ReleaseVersionsDeletedByDataSetVersionOrder() Title = "UI test theme to delete" }; - var publication = _fixture + Publication publication = _fixture .DefaultPublication() - .WithTheme(theme) - .Generate(); + .WithTheme(theme); - var releaseVersion1 = _fixture + ReleaseVersion releaseVersion1 = _fixture .DefaultReleaseVersion() - .WithRelease(_fixture.DefaultRelease()) - .WithPublication(publication) - .Generate(); + .WithRelease(_fixture.DefaultRelease() + .WithPublication(publication)); - var releaseVersion1ReleaseFile = _fixture + ReleaseFile releaseVersion1ReleaseFile = _fixture .DefaultReleaseFile() - .WithReleaseVersion(releaseVersion1) - .Generate(); + .WithReleaseVersion(releaseVersion1); - var releaseVersion2 = _fixture + ReleaseVersion releaseVersion2 = _fixture .DefaultReleaseVersion() - .WithRelease(_fixture.DefaultRelease()) - .WithPublication(publication) - .Generate(); + .WithRelease(_fixture.DefaultRelease() + .WithPublication(publication)); - var releaseVersion2ReleaseFile = _fixture + ReleaseFile releaseVersion2ReleaseFile = _fixture .DefaultReleaseFile() - .WithReleaseVersion(releaseVersion2) - .Generate(); + .WithReleaseVersion(releaseVersion2); - var releaseVersion3 = _fixture + ReleaseVersion releaseVersion3 = _fixture .DefaultReleaseVersion() - .WithRelease(_fixture.DefaultRelease()) - .WithPublication(publication) - .Generate(); + .WithRelease(_fixture.DefaultRelease() + .WithPublication(publication)); - var releaseVersion3ReleaseFile = _fixture + ReleaseFile releaseVersion3ReleaseFile = _fixture .DefaultReleaseFile() - .WithReleaseVersion(releaseVersion3) - .Generate(); + .WithReleaseVersion(releaseVersion3); - var dataSet = _fixture.DefaultDataSet().Generate(); + DataSet dataSet = _fixture.DefaultDataSet(); var dataSetVersions = _fixture .DefaultDataSetVersion() @@ -507,63 +498,50 @@ public async Task DeleteTheme_ReleaseVersionsDeletedByVersionOrder() Title = "UI test theme to delete" }; - var publication = _fixture + Publication publication = _fixture .DefaultPublication() - .WithTheme(theme) - .Generate(); + .WithTheme(theme); - var release1 = _fixture + Release release1 = _fixture .DefaultRelease() - .WithCreated(DateTime.UtcNow.AddDays(-1)) - .Generate(); + .WithPublication(publication) + .WithCreated(DateTime.UtcNow.AddDays(-1)); - var release2 = _fixture + Release release2 = _fixture .DefaultRelease() - .WithCreated(DateTime.UtcNow.AddDays(-2)) - .Generate(); + .WithPublication(publication) + .WithCreated(DateTime.UtcNow.AddDays(-2)); - var release1Version1 = _fixture + ReleaseVersion release1Version1 = _fixture .DefaultReleaseVersion() .WithVersion(0) - .WithRelease(release1) - .WithPublication(publication) - .Generate(); + .WithRelease(release1); - var release1Version2 = _fixture + ReleaseVersion release1Version2 = _fixture .DefaultReleaseVersion() .WithVersion(1) - .WithRelease(release1) - .WithPublication(publication) - .Generate(); + .WithRelease(release1); - var release1Version2Cancelled = _fixture + ReleaseVersion release1Version2Cancelled = _fixture .DefaultReleaseVersion() .WithVersion(1) .WithSoftDeleted() - .WithRelease(release1) - .WithPublication(publication) - .Generate(); + .WithRelease(release1); - var release1Version3 = _fixture + ReleaseVersion release1Version3 = _fixture .DefaultReleaseVersion() .WithVersion(2) - .WithRelease(release1) - .WithPublication(publication) - .Generate(); + .WithRelease(release1); - var release2Version1 = _fixture + ReleaseVersion release2Version1 = _fixture .DefaultReleaseVersion() .WithVersion(0) - .WithRelease(release2) - .WithPublication(publication) - .Generate(); + .WithRelease(release2); - var release2Version2 = _fixture + ReleaseVersion release2Version2 = _fixture .DefaultReleaseVersion() .WithVersion(1) - .WithRelease(release2) - .WithPublication(publication) - .Generate(); + .WithRelease(release2); var contextId = Guid.NewGuid().ToString(); @@ -787,21 +765,17 @@ public async Task DeleteTheme_OtherThemesUnaffected() Title = "UI test theme to delete" }; - var releaseVersion = _fixture + ReleaseVersion releaseVersion = _fixture .DefaultReleaseVersion() .WithId(releaseVersionId) - .WithRelease(_fixture.DefaultRelease()) - .WithPublication( - _fixture + .WithRelease(_fixture.DefaultRelease() + .WithPublication(_fixture .DefaultPublication() - .WithTheme(theme) - .Generate()) - .Generate(); + .WithTheme(theme))); - var methodology = _fixture + Methodology methodology = _fixture .DefaultMethodology() - .WithOwningPublication(releaseVersion.Publication) - .Generate(); + .WithOwningPublication(releaseVersion.Publication); var otherReleaseVersionId = Guid.NewGuid(); @@ -811,21 +785,17 @@ public async Task DeleteTheme_OtherThemesUnaffected() Title = "UI test theme to retain" }; - var otherReleaseVersion = _fixture + ReleaseVersion otherReleaseVersion = _fixture .DefaultReleaseVersion() .WithId(otherReleaseVersionId) - .WithRelease(_fixture.DefaultRelease()) - .WithPublication( - _fixture + .WithRelease(_fixture.DefaultRelease() + .WithPublication(_fixture .DefaultPublication() - .WithTheme(otherTheme) - .Generate()) - .Generate(); + .WithTheme(otherTheme))); - var otherMethodology = _fixture + Methodology otherMethodology = _fixture .DefaultMethodology() - .WithOwningPublication(otherReleaseVersion.Publication) - .Generate(); + .WithOwningPublication(otherReleaseVersion.Publication); var contextId = Guid.NewGuid().ToString(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.Designer.cs new file mode 100644 index 00000000000..a8b394054b6 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.Designer.cs @@ -0,0 +1,2233 @@ +// +using System; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + [DbContext(typeof(ContentDbContext))] + [Migration("20241122105739_EES5625_CopyReleaseVersionFieldsToRelease")] + partial class EES5625_CopyReleaseVersionFieldsToRelease + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContactTelNo") + .HasColumnType("nvarchar(max)"); + + b.Property("TeamEmail") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TeamName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Contacts"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.FreeTextRank", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Rank") + .HasColumnType("int"); + + b.ToTable((string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("LegacyCreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("Resolved") + .HasColumnType("datetime2"); + + b.Property("ResolvedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ResolvedById"); + + b.ToTable("Comment"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentSectionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Locked") + .HasColumnType("datetime2"); + + b.Property("LockedById") + .IsConcurrencyToken() + .HasColumnType("uniqueidentifier"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentSectionId"); + + b.HasIndex("LockedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentBlock", (string)null); + + b.HasDiscriminator("Type").HasValue("ContentBlock"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Caption") + .HasColumnType("nvarchar(max)"); + + b.Property("Heading") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentSections"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestDraftVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("LatestDraftVersionId") + .IsUnique() + .HasFilter("[LatestDraftVersionId] IS NOT NULL"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("DataBlocks", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier") + .HasColumnName("DataBlockId"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("DataBlockVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("ExpectedImportedRows") + .HasColumnType("int"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("GeographicLevels") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ImportedRows") + .HasColumnType("int"); + + b.Property("LastProcessedRowIndex") + .HasColumnType("int"); + + b.Property("MetaFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("StagePercentageComplete") + .HasColumnType("int"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("TotalRows") + .HasColumnType("int"); + + b.Property("ZipFileId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId") + .IsUnique(); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("FileId"), new[] { "Status" }); + + b.HasIndex("MetaFileId") + .IsUnique(); + + b.HasIndex("ZipFileId"); + + b.ToTable("DataImports"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataImportId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("DataImportId"); + + b.ToTable("DataImportErrors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("EmbedBlocks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DataBlockId") + .IsUnique(); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("FeaturedTables"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentLength") + .HasColumnType("bigint"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileMeta") + .HasColumnType("nvarchar(max)"); + + b.Property("DataSetFileVersion") + .HasColumnType("int"); + + b.Property("Filename") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReplacedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReplacingId") + .HasColumnType("uniqueidentifier"); + + b.Property("RootPath") + .HasColumnType("uniqueidentifier"); + + b.Property("SourceId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReplacedById") + .IsUnique() + .HasFilter("[ReplacedById] IS NOT NULL"); + + b.HasIndex("ReplacingId") + .IsUnique() + .HasFilter("[ReplacingId] IS NOT NULL"); + + b.HasIndex("SourceId"); + + b.HasIndex("Type"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.ToTable("GlossaryEntries"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("GuidanceText") + .HasColumnType("nvarchar(max)"); + + b.Property("GuidanceTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Trend") + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("KeyStatistics"); + + b.UseTptMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("OwningPublicationSlug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OwningPublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("Methodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DisplayDate") + .HasColumnType("datetime2"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("MethodologyNotes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("MethodologyVersionId", "Slug"); + + b.ToTable("MethodologyRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("AlternativeSlug") + .HasColumnType("nvarchar(max)"); + + b.Property("AlternativeTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("PublishingStrategy") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ScheduledWithReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyId"); + + b.HasIndex("PreviousVersionId"); + + b.HasIndex("ScheduledWithReleaseVersionId"); + + b.ToTable("MethodologyVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Annexes") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("MethodologyVersionId"); + + b.ToTable("MethodologyVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Permalink", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataSetTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MigratedFromLegacy") + .HasColumnType("bit"); + + b.Property("PublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("SubjectId"); + + b.ToTable("Permalinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseSeries") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .IsRequired() + .HasMaxLength(160) + .HasColumnType("nvarchar(160)"); + + b.Property("SupersededById") + .HasColumnType("uniqueidentifier"); + + b.Property("ThemeId") + .HasColumnType("uniqueidentifier"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContactId"); + + b.HasIndex("LatestPublishedReleaseVersionId") + .IsUnique() + .HasFilter("[LatestPublishedReleaseVersionId] IS NOT NULL"); + + b.HasIndex("SupersededById"); + + b.HasIndex("ThemeId"); + + b.ToTable("Publications"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("Owner") + .HasColumnType("bit"); + + b.HasKey("PublicationId", "MethodologyId"); + + b.HasIndex("MethodologyId"); + + b.ToTable("PublicationMethodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("PublicationId", "Slug"); + + b.ToTable("PublicationRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("nvarchar(30)"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("nvarchar(5)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Year") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("PublicationId", "Year", "TimePeriodCoverage") + .IsUnique(); + + b.ToTable("Releases"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("FilterSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("IndicatorSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("PublicApiDataSetId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicApiDataSetVersion") + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("ReleaseVersionId", "FileId") + .IsUnique(); + + b.HasIndex("ReleaseVersionId", "PublicApiDataSetId", "PublicApiDataSetVersion") + .IsUnique() + .HasFilter("[PublicApiDataSetId] IS NOT NULL AND [PublicApiDataSetVersion] IS NOT NULL"); + + b.ToTable("ReleaseFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseRedirect", b => + { + b.Property("ReleaseId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("ReleaseId", "Slug"); + + b.ToTable("ReleaseRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("ReleaseStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataGuidance") + .HasColumnType("nvarchar(max)"); + + b.Property("NextReleaseDate") + .HasColumnType("nvarchar(max)"); + + b.Property("NotifiedOn") + .HasColumnType("datetime2"); + + b.Property("NotifySubscribers") + .HasColumnType("bit"); + + b.Property("PreReleaseAccessList") + .HasColumnType("nvarchar(max)"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublishScheduled") + .HasColumnType("datetime2"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("RelatedInformation") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseName") + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(6) + .HasColumnType("nvarchar(6)"); + + b.Property("Type") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("UpdatePublishedDate") + .HasColumnType("bit"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("ReleaseId"); + + b.HasIndex("Type"); + + b.HasIndex("PreviousVersionId", "Version"); + + b.ToTable("ReleaseVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Themes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("On") + .HasColumnType("datetime2"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("Update"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstName") + .HasColumnType("nvarchar(max)"); + + b.Property("LastName") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("DeletedById"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.ToTable("UserPublicationInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("UserId"); + + b.ToTable("UserPublicationRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EmailSent") + .HasColumnType("bit"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("UserReleaseInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UserId"); + + b.ToTable("UserReleaseRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Charts") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Charts"); + + b.Property("Heading") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Heading"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Query") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Query"); + + b.Property("Source") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Table") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Table"); + + b.HasDiscriminator().HasValue("DataBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("EmbedBlockId") + .HasColumnType("uniqueidentifier") + .HasColumnName("EmbedBlockId"); + + b.HasIndex("EmbedBlockId") + .IsUnique() + .HasFilter("[EmbedBlockId] IS NOT NULL"); + + b.HasDiscriminator().HasValue("EmbedBlockLink"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.HtmlBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("HtmlBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MarkDownBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("MarkDownBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.HasIndex("DataBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.ToTable("KeyStatisticsDataBlock", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("Statistic") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.ToTable("KeyStatisticsText", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", "ContentBlock") + .WithMany("Comments") + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "ResolvedBy") + .WithMany() + .HasForeignKey("ResolvedById"); + + b.Navigation("ContentBlock"); + + b.Navigation("CreatedBy"); + + b.Navigation("ResolvedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", "ContentSection") + .WithMany("Content") + .HasForeignKey("ContentSectionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "LockedBy") + .WithMany() + .HasForeignKey("LockedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentSection"); + + b.Navigation("LockedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Content") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestDraftVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestDraftVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestPublishedVersionId"); + + b.Navigation("LatestDraftVersion"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "ContentBlock") + .WithMany() + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("DataBlockVersions") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "FileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "MetaFile") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "MetaFileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ZipFile") + .WithMany() + .HasForeignKey("ZipFileId") + .OnDelete(DeleteBehavior.Restrict); + + b.Navigation("File"); + + b.Navigation("MetaFile"); + + b.Navigation("ZipFile"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "DataImport") + .WithMany("Errors") + .HasForeignKey("DataImportId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataImport"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", "DataBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("FeaturedTables") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedBy") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Replacing") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacingId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Source") + .WithMany() + .HasForeignKey("SourceId"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReplacedBy"); + + b.Navigation("Replacing"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("CreatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("KeyStatistics") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "LatestPublishedVersionId"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany("Notes") + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Versions") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ScheduledWithReleaseVersion") + .WithMany() + .HasForeignKey("ScheduledWithReleaseVersionId"); + + b.Navigation("CreatedBy"); + + b.Navigation("Methodology"); + + b.Navigation("PreviousVersion"); + + b.Navigation("ScheduledWithReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", null) + .WithOne("MethodologyContent") + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", "MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", "Contact") + .WithMany() + .HasForeignKey("ContactId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "LatestPublishedReleaseVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "LatestPublishedReleaseVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "SupersededBy") + .WithMany() + .HasForeignKey("SupersededById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", "Theme") + .WithMany("Publications") + .HasForeignKey("ThemeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ExternalMethodology", "ExternalMethodology", b1 => + { + b1.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b1.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.HasKey("PublicationId"); + + b1.ToTable("ExternalMethodology", (string)null); + + b1.WithOwner() + .HasForeignKey("PublicationId"); + }); + + b.Navigation("Contact"); + + b.Navigation("ExternalMethodology"); + + b.Navigation("LatestPublishedReleaseVersion"); + + b.Navigation("SupersededBy"); + + b.Navigation("Theme"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Publications") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Methodologies") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Methodology"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Releases") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") + .WithMany() + .HasForeignKey("ReleaseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Release"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("ReleaseStatuses") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("ReleaseVersions") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") + .WithMany("Versions") + .HasForeignKey("ReleaseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("PreviousVersion"); + + b.Navigation("Publication"); + + b.Navigation("Release"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Updates") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.User", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.Navigation("DeletedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("Publication"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", "EmbedBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", "EmbedBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("EmbedBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithMany() + .HasForeignKey("DataBlockId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Navigation("Comments"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Navigation("Content"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Navigation("Errors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Navigation("Publications"); + + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Navigation("MethodologyContent") + .IsRequired(); + + b.Navigation("Notes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Navigation("Methodologies"); + + b.Navigation("ReleaseVersions"); + + b.Navigation("Releases"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Navigation("Content"); + + b.Navigation("DataBlockVersions"); + + b.Navigation("FeaturedTables"); + + b.Navigation("KeyStatistics"); + + b.Navigation("ReleaseStatuses"); + + b.Navigation("Updates"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Navigation("Publications"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.cs new file mode 100644 index 00000000000..b859c9a6729 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.cs @@ -0,0 +1,158 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations; + +/// +[ExcludeFromCodeCoverage] +public partial class EES5625_CopyReleaseVersionFieldsToRelease : Migration +{ + private const string MigrationId = "20241122105739"; + + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + // Add the new columns, initially allowing for the values to be null + migrationBuilder.AddColumn( + name: "PublicationId", + table: "Releases", + type: "uniqueidentifier", + nullable: true); + + migrationBuilder.AddColumn( + name: "Slug", + table: "Releases", + type: "nvarchar(30)", + maxLength: 30, + nullable: true); + + migrationBuilder.AddColumn( + name: "TimePeriodCoverage", + table: "Releases", + type: "nvarchar(5)", + maxLength: 5, + nullable: true); + + migrationBuilder.AddColumn( + name: "Year", + table: "Releases", + type: "int", + nullable: true); + + // Copy attributes from ReleaseVersions to Releases based on the latest versions of each release + migrationBuilder.SqlFromFile( + MigrationConstants.ContentMigrationsPath, + $"{MigrationId}_{nameof(EES5625_CopyReleaseVersionFieldsToRelease)}.sql"); + + // Now that every row of Releases should have values, make the columns not nullable + migrationBuilder.AlterColumn( + name: "PublicationId", + table: "Releases", + type: "uniqueidentifier", + nullable: false, + oldClrType: typeof(Guid), + oldType: "uniqueidentifier", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "Slug", + table: "Releases", + type: "nvarchar(30)", + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(30)", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "TimePeriodCoverage", + table: "Releases", + type: "nvarchar(5)", + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(5)", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "Year", + table: "Releases", + type: "int", + nullable: false, + oldClrType: typeof(int), + oldType: "int", + oldNullable: true); + + // Recreate the foreign key constraint from ReleaseVersions to Publications + // to prevent multiple delete cascade paths + migrationBuilder.DropForeignKey( + name: "FK_ReleaseVersions_Publications_PublicationId", + table: "ReleaseVersions"); + + migrationBuilder.AddForeignKey( + name: "FK_ReleaseVersions_Publications_PublicationId", + table: "ReleaseVersions", + column: "PublicationId", + principalTable: "Publications", + principalColumn: "Id", + onDelete: ReferentialAction.NoAction); + + // Add a new foreign key constraint from Releases to Publications + migrationBuilder.AddForeignKey( + name: "FK_Releases_Publications_PublicationId", + table: "Releases", + column: "PublicationId", + principalTable: "Publications", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + + // Add a unique index on PublicationId / Year / TimePeriodCoverage + migrationBuilder.CreateIndex( + name: "IX_Releases_PublicationId_Year_TimePeriodCoverage", + table: "Releases", + columns: ["PublicationId", "Year", "TimePeriodCoverage"], + unique: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_Releases_Publications_PublicationId", + table: "Releases"); + + migrationBuilder.DropForeignKey( + name: "FK_ReleaseVersions_Publications_PublicationId", + table: "ReleaseVersions"); + + migrationBuilder.DropIndex( + name: "IX_Releases_PublicationId_Year_TimePeriodCoverage", + table: "Releases"); + + migrationBuilder.DropColumn( + name: "PublicationId", + table: "Releases"); + + migrationBuilder.DropColumn( + name: "Slug", + table: "Releases"); + + migrationBuilder.DropColumn( + name: "TimePeriodCoverage", + table: "Releases"); + + migrationBuilder.DropColumn( + name: "Year", + table: "Releases"); + + migrationBuilder.AddForeignKey( + name: "FK_ReleaseVersions_Publications_PublicationId", + table: "ReleaseVersions", + column: "PublicationId", + principalTable: "Publications", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.sql b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.sql new file mode 100644 index 00000000000..d53152afbae --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.sql @@ -0,0 +1,17 @@ +UPDATE Releases +SET Releases.PublicationId = ReleaseVersions.PublicationId, + Releases.Slug = ReleaseVersions.Slug, + Releases.Year = TRY_CAST(ReleaseVersions.ReleaseName AS INT), + Releases.TimePeriodCoverage = ReleaseVersions.TimePeriodCoverage, + Releases.Updated = SYSUTCDATETIME() +FROM Releases + INNER JOIN + ReleaseVersions + ON Releases.Id = ReleaseVersions.ReleaseId + INNER JOIN + (SELECT RV2.ReleaseId, MAX(RV2.Version) AS Version + FROM ReleaseVersions RV2 + WHERE RV2.Published IS NOT NULL + OR (RV2.Published IS NULL AND RV2.Version = 0) + GROUP BY ReleaseId) LatestVersion + ON ReleaseVersions.ReleaseId = LatestVersion.ReleaseId AND ReleaseVersions.Version = LatestVersion.Version diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs index 921c202c0f7..67bff7569b4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs @@ -894,11 +894,30 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Created") .HasColumnType("datetime2"); + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("nvarchar(30)"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("nvarchar(5)"); + b.Property("Updated") .HasColumnType("datetime2"); + b.Property("Year") + .HasColumnType("int"); + b.HasKey("Id"); + b.HasIndex("PublicationId", "Year", "TimePeriodCoverage") + .IsUnique(); + b.ToTable("Releases"); }); @@ -1877,6 +1896,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Publication"); }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Releases") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => { b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") @@ -1941,7 +1971,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") .WithMany("ReleaseVersions") .HasForeignKey("PublicationId") - .OnDelete(DeleteBehavior.Cascade) + .OnDelete(DeleteBehavior.NoAction) .IsRequired(); b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") @@ -2166,6 +2196,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Methodologies"); b.Navigation("ReleaseVersions"); + + b.Navigation("Releases"); }); modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseRequests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseRequests.cs index afa5128bc7f..d0d3a3b5420 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseRequests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseRequests.cs @@ -37,7 +37,7 @@ public record ReleaseUpdateRequest [Required] public TimeIdentifier TimePeriodCoverage { get; init; } - public string PreReleaseAccessList { get; init; } = String.Empty; + public string PreReleaseAccessList { get; init; } = string.Empty; public string Slug => SlugFromTitle(Title); @@ -45,4 +45,4 @@ public record ReleaseUpdateRequest [Range(1000, 9999)] public int Year { get; init; } -} \ No newline at end of file +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs index eb58d1ffe35..15d492a406f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs @@ -141,16 +141,26 @@ await _persistenceHelper.CheckEntityExists(releaseCreate.Publicatio await ValidateReleaseSlugUniqueToPublication(releaseCreate.Slug, releaseCreate.PublicationId)) .OnSuccess(async publication => { - var newReleaseVersion = new ReleaseVersion + var release = new Release { - Id = _guidGenerator.NewGuid(), - Release = new Release(), PublicationId = releaseCreate.PublicationId, - Slug = releaseCreate.Slug, + Year = releaseCreate.Year, TimePeriodCoverage = releaseCreate.TimePeriodCoverage, - ReleaseName = releaseCreate.Year.ToString(), + Slug = releaseCreate.Slug + }; + + var newReleaseVersion = new ReleaseVersion + { + Id = _guidGenerator.NewGuid(), + Release = release, Type = releaseCreate.Type, - ApprovalStatus = ReleaseApprovalStatus.Draft + ApprovalStatus = ReleaseApprovalStatus.Draft, + + // TODO Remove the following in EES-5659 + PublicationId = release.PublicationId, + TimePeriodCoverage = release.TimePeriodCoverage, + ReleaseName = release.Year.ToString(), + Slug = release.Slug }; if (releaseCreate.TemplateReleaseId.HasValue) @@ -499,7 +509,9 @@ public async Task> UpdateReleaseVersion( Guid releaseVersionId, ReleaseUpdateRequest request) { return await ReleaseUpdateRequestValidator.Validate(request) - .OnSuccess(async () => await CheckReleaseVersionExists(releaseVersionId)) + .OnSuccess(async () => + await _persistenceHelper.CheckEntityExists(releaseVersionId, + queryable => queryable.Include(rv => rv.Release))) .OnSuccess(_userService.CheckCanUpdateReleaseVersion) .OnSuccessDo(async releaseVersion => await ValidateReleaseSlugUniqueToPublication(request.Slug, @@ -509,10 +521,30 @@ await ValidateReleaseSlugUniqueToPublication(request.Slug, { return await _context.RequireTransaction(async () => { - releaseVersion.Slug = request.Slug; + if (releaseVersion.Version > 0) + { + var yearChanged = releaseVersion.Year != request.Year; + var timePeriodCoverageChanged = + releaseVersion.TimePeriodCoverage != request.TimePeriodCoverage; + + if (yearChanged || timePeriodCoverageChanged) + { + throw new ArgumentException( + $"Cannot update {nameof(request.Year)} or {nameof(request.TimePeriodCoverage)} for a release that has already been published", + nameof(request)); + } + } + + releaseVersion.Release.Year = request.Year; + releaseVersion.Release.TimePeriodCoverage = request.TimePeriodCoverage; + releaseVersion.Release.Slug = request.Slug; + + // TODO The following will be removed in EES-5659 + releaseVersion.ReleaseName = releaseVersion.Release.Year.ToString(); + releaseVersion.TimePeriodCoverage = releaseVersion.Release.TimePeriodCoverage; + releaseVersion.Slug = releaseVersion.Release.Slug; + releaseVersion.Type = request.Type; - releaseVersion.ReleaseName = request.Year.ToString(); - releaseVersion.TimePeriodCoverage = request.TimePeriodCoverage; releaseVersion.PreReleaseAccessList = request.PreReleaseAccessList; await _dataSetVersionService.UpdateVersionsForReleaseVersion( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/ReleaseFileControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/ReleaseFileControllerTests.cs index b7fe5d47dd4..ba6c99b8623 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/ReleaseFileControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/ReleaseFileControllerTests.cs @@ -199,8 +199,11 @@ public class StreamTests(TestApplicationFactory testApp) : ReleaseFileController [Fact] public async Task Success() { - ReleaseVersion releaseVersion = DataFixture.DefaultReleaseVersion() - .WithPublication(DataFixture.DefaultPublication()); + Publication publication = DataFixture.DefaultPublication() + .WithReleases(DataFixture.DefaultRelease(publishedVersions: 1) + .GenerateList(1)); + + var releaseVersion = publication.Releases.Single().Versions.Single(); await using var stream = "Test file".ToStream(); @@ -208,7 +211,7 @@ public async Task Success() await TestApp.AddTestData(context => { - context.ReleaseVersions.Add(releaseVersion); + context.Publications.Add(publication); }); var releaseFileService = new Mock(Strict); @@ -240,12 +243,15 @@ public class StreamFilesToZipTests(TestApplicationFactory testApp) [Fact] public async Task Success() { - ReleaseVersion releaseVersion = DataFixture.DefaultReleaseVersion() - .WithPublication(DataFixture.DefaultPublication()); + Publication publication = DataFixture.DefaultPublication() + .WithReleases(DataFixture.DefaultRelease(publishedVersions: 1) + .GenerateList(1)); + + var releaseVersion = publication.Releases.Single().Versions.Single(); await TestApp.AddTestData(context => { - context.ReleaseVersions.Add(releaseVersion); + context.Publications.Add(publication); }); var fileId1 = Guid.NewGuid(); @@ -281,12 +287,15 @@ await TestApp.AddTestData(context => [Fact] public async Task Success_NoFileIds() { - ReleaseVersion releaseVersion = DataFixture.DefaultReleaseVersion() - .WithPublication(DataFixture.DefaultPublication()); + Publication publication = DataFixture.DefaultPublication() + .WithReleases(DataFixture.DefaultRelease(publishedVersions: 1) + .GenerateList(1)); + + var releaseVersion = publication.Releases.Single().Versions.Single(); await TestApp.AddTestData(context => { - context.ReleaseVersions.Add(releaseVersion); + context.Publications.Add(publication); }); var releaseFileService = new Mock(Strict); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs index 131f750a9ce..36285a33ca6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs @@ -12,19 +12,24 @@ public static Generator DefaultMethodologyVersion(this DataF public static Generator WithDefaults(this Generator generator) => generator.ForInstance(d => d.SetDefaults()); + public static Generator WithAlternativeTitle( + this Generator generator, + string alternativeTitle) + => generator.ForInstance(d => d.SetAlternativeTitle(alternativeTitle)); + public static Generator WithApprovalStatus( - this Generator generator, + this Generator generator, MethodologyApprovalStatus approvalStatus) => generator.ForInstance(d => d.SetApprovalStatus(approvalStatus)); - + public static Generator WithApprovalStatuses( - this Generator generator, + this Generator generator, IEnumerable approvalStatuses) { - approvalStatuses.ForEach((status, index) => + approvalStatuses.ForEach((status, index) => generator.ForIndex(index, s => s.SetApprovalStatus(status))); - - return generator; + + return generator; } public static InstanceSetters SetDefaults(this InstanceSetters setters) @@ -32,10 +37,16 @@ public static InstanceSetters SetDefaults(this InstanceSette .SetDefault(p => p.Id) .SetDefault(p => p.Slug) .SetDefault(p => p.Title) - .SetDefault(p => p.Version); - + .SetDefault(p => p.Version) + .SetApprovalStatus(MethodologyApprovalStatus.Draft); + + public static InstanceSetters SetAlternativeTitle( + this InstanceSetters setters, + string alternativeTitle) + => setters.Set(mv => mv.AlternativeTitle, alternativeTitle); + public static InstanceSetters SetApprovalStatus( this InstanceSetters setters, - MethodologyApprovalStatus approvalStatus) + MethodologyApprovalStatus approvalStatus) => setters.Set(mv => mv.Status, approvalStatus); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs index c2cf89e1b65..989976bb1c4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs @@ -50,16 +50,6 @@ public static Generator WithReleases( Func> releases) => generator.ForInstance(s => s.SetReleases(releases.Invoke)); - public static Generator WithReleaseVersions( - this Generator generator, - IEnumerable releaseVersions) - => generator.ForInstance(s => s.SetReleaseVersions(releaseVersions)); - - public static Generator WithReleaseVersions( - this Generator generator, - Func> releaseVersions) - => generator.ForInstance(s => s.SetReleaseVersions(releaseVersions.Invoke)); - public static Generator WithContact( this Generator generator, Contact contact) @@ -106,7 +96,8 @@ public static InstanceSetters SetLatestPublishedReleaseVersionId( Guid? latestPublishedReleaseVersionId) => setters.Set(p => p.LatestPublishedReleaseVersionId, latestPublishedReleaseVersionId); - public static Generator WithThemes(this Generator generator, + public static Generator WithThemes( + this Generator generator, IEnumerable themes) { themes.ForEach((theme, index) => @@ -123,22 +114,34 @@ public static InstanceSetters SetReleases( private static InstanceSetters SetReleases( this InstanceSetters setters, Func> releases) - => setters.Set( - p => p.ReleaseVersions, + => setters.Set(p => p.Releases, (_, publication, context) => { - var releaseList = releases.Invoke(context).ToList(); + var list = releases.Invoke(context).ToList(); - releaseList.ForEach(release => + list.ForEach(release => { - publication.ReleaseSeries.Insert(0, new ReleaseSeriesItem - { - Id = Guid.NewGuid(), - ReleaseId = release.Id, - }); + release.Publication = publication; + release.PublicationId = publication.Id; }); - var releaseVersions = releaseList.SelectMany(r => r.Versions) + return list; + }) + .Set(p => p.ReleaseSeries, + (_, publication, context) => + publication.Releases + .OrderByDescending(r => r.Year) + .ThenByDescending(r => r.TimePeriodCoverage) + .Select(release => context.Fixture + .DefaultReleaseSeriesItem() + .WithReleaseId(release.Id) + .Generate()) + .ToList()) + .Set( + p => p.ReleaseVersions, + (_, publication, _) => + { + var releaseVersions = publication.Releases.SelectMany(r => r.Versions) .ToList(); releaseVersions.ForEach(releaseVersion => @@ -150,38 +153,46 @@ private static InstanceSetters SetReleases( return releaseVersions; } ) - .Set(p => p.LatestPublishedReleaseVersion, (_, publication, _) => - { - var publishedVersions = publication.ReleaseVersions - .Where(releaseVersion => releaseVersion.Published.HasValue) - .ToList(); - - if (publishedVersions.Count == 0) + .Set(p => p.LatestPublishedReleaseVersion, + (_, publication, _) => { - return null; - } + var releaseVersions = publication.Releases.SelectMany(r => r.Versions) + .ToList(); - if (publishedVersions.Count == 1) - { - return publishedVersions[0]; - } + var publishedVersions = releaseVersions + .Where(releaseVersion => releaseVersion.Published.HasValue) + .ToList(); + + if (publishedVersions.Count == 0) + { + return null; + } - return publishedVersions - .GroupBy(releaseVersion => releaseVersion.ReleaseId) - .Select(groupedReleases => - new - { - ReleaseId = groupedReleases.Key, - Version = groupedReleases.Max(releaseVersion => releaseVersion.Version) - }) - .Join(publishedVersions, - maxVersion => maxVersion, - releaseVersion => new { releaseVersion.ReleaseId, releaseVersion.Version }, - (_, release) => release) - .OrderByDescending(releaseVersion => releaseVersion.Year) - .ThenByDescending(releaseVersion => releaseVersion.TimePeriodCoverage) - .FirstOrDefault(); - }) + if (publishedVersions.Count == 1) + { + return publishedVersions[0]; + } + + return publishedVersions + .GroupBy(releaseVersion => releaseVersion.ReleaseId) + .Select(groupedReleases => + new + { + ReleaseId = groupedReleases.Key, + Version = groupedReleases.Max(releaseVersion => releaseVersion.Version) + }) + .Join(publishedVersions, + maxVersion => maxVersion, + releaseVersion => new + { + releaseVersion.ReleaseId, + releaseVersion.Version + }, + (_, releaseVersion) => releaseVersion) + .OrderByDescending(releaseVersion => releaseVersion.Release.Year) + .ThenByDescending(releaseVersion => releaseVersion.Release.TimePeriodCoverage) + .FirstOrDefault(); + }) .Set(p => p.LatestPublishedReleaseVersionId, (_, publication, _) => publication.LatestPublishedReleaseVersion?.Id); @@ -217,24 +228,26 @@ private static InstanceSetters SetExternalMethodology( private static InstanceSetters SetLegacyLinks( this InstanceSetters setters, - IEnumerable legacyLinks) - => setters.Set( - p => p.ReleaseSeries, - (_, publication, context) => - { - legacyLinks.ForEach(legacyLink => - { - publication.ReleaseSeries.Add(new() - { - Id = legacyLink.Id, - LegacyLinkDescription = legacyLink.LegacyLinkDescription, - LegacyLinkUrl = legacyLink.LegacyLinkUrl, - }); - }); + IEnumerable legacyLinks) => + setters.SetLegacyLinks(_ => legacyLinks); - return publication.ReleaseSeries; + private static InstanceSetters SetLegacyLinks( + this InstanceSetters setters, + Func> legacyLinks) => + setters.Set( + p => p.ReleaseSeries, + (_, publication, context) => + { + var list = legacyLinks.Invoke(context).ToList(); + + if (list.Any(rsi => !rsi.IsLegacyLink)) + { + throw new ArgumentException("List can only contain legacy links", nameof(legacyLinks)); } - ); + + return [.. publication.ReleaseSeries, .. list]; + } + ); public static InstanceSetters SetSupersededBy( this InstanceSetters setters, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs index 21a298eb679..35f6c2675b0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs @@ -1,12 +1,17 @@ using System; using System.Collections.Generic; using System.Linq; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; public static class ReleaseGeneratorExtensions { + private const int DefaultStartYear = 2000; + private const TimeIdentifier DefaultTimePeriodCoverage = TimeIdentifier.AcademicYear; + public static Generator DefaultRelease(this DataFixture fixture) => fixture.Generator().WithDefaults(); @@ -18,36 +23,45 @@ public static Generator DefaultRelease( { ReleaseVersion? previousVersion = null; return fixture.Generator() - .WithDefaults() - .WithVersions(releaseContext => fixture - .DefaultReleaseVersion() + .WithDefaults(year: year) + .WithVersions(fixture.DefaultReleaseVersion() .ForInstance(s => s - .Set(rv => rv.ReleaseName, - year != null ? year.ToString() : $"{2000 + releaseContext.FixtureTypeIndex}") - .Set(p => p.Slug, - (_, releaseVersion, _) => NamingUtils.SlugFromTitle(releaseVersion.YearTitle)) - .Set(rv => rv.Version, + .Set(p => p.Version, (_, _, context) => context.Index)) - .ForRange(..publishedVersions, s => s - .SetApprovalStatus(ReleaseApprovalStatus.Approved) - .Set(rv => rv.Published, (f, rel) => - rel.Version == 0 - ? f.Date.Past() - : f.Date.Between(previousVersion!.Published!.Value, DateTime.UtcNow))) + .ForRange(..publishedVersions, + s => s + .SetApprovalStatus(ReleaseApprovalStatus.Approved) + .Set(p => p.Published, + (f, releaseVersion) => + releaseVersion.Version == 0 + ? f.Date.Past() + : f.Date.Between(previousVersion!.Published!.Value, DateTime.UtcNow)) + .Set(rv => rv.PublishScheduled, (_, rv) => rv.Published!.Value.AsStartOfDayUtcForTimeZone())) .ForRange(1.., s => s .SetPreviousVersion(previousVersion)) .FinishWith(releaseVersion => previousVersion = releaseVersion) .Generate(draftVersion ? publishedVersions + 1 : publishedVersions)); } - public static Generator WithDefaults(this Generator generator) - => generator.ForInstance(s => s.SetDefaults()); + public static Generator WithDefaults( + this Generator generator, + int? year = null) + => generator.ForInstance(s => s.SetDefaults(year: year)); - public static InstanceSetters SetDefaults(this InstanceSetters setters) - => setters - .SetDefault(r => r.Id) - .Set(r => r.Created, f => f.Date.Past()) - .Set(r => r.Updated, (f, r) => f.Date.Soon(refDate: r.Created)); + public static Generator WithId( + this Generator generator, + Guid id) + => generator.ForInstance(s => s.SetId(id)); + + public static Generator WithPublication( + this Generator generator, + Publication publication) + => generator.ForInstance(s => s.SetPublication(publication)); + + public static Generator WithPublicationId( + this Generator generator, + Guid publicationId) + => generator.ForInstance(s => s.SetPublicationId(publicationId)); public static Generator WithVersions( this Generator generator, @@ -66,6 +80,11 @@ public static Generator WithCreated( return generator.ForInstance(s => s.SetCreated(created)); } + public static Generator WithTimePeriodCoverage( + this Generator generator, + TimeIdentifier timePeriodCoverage) + => generator.ForInstance(s => s.SetTimePeriodCoverage(timePeriodCoverage)); + public static Generator WithUpdated( this Generator generator, DateTime? updated = null) @@ -73,6 +92,99 @@ public static Generator WithUpdated( return generator.ForInstance(s => s.SetUpdated(updated)); } + public static Generator WithYear( + this Generator generator, + int year) + => generator.ForInstance(s => s.SetYear(year)); + + public static InstanceSetters SetDefaults( + this InstanceSetters setters, + int? year = null) + => setters + .SetDefault(p => p.Id) + .SetDefault(p => p.PublicationId) + .Set(p => p.Year, (_, _, context) => year ?? DefaultStartYear + context.Index) + .SetTimePeriodCoverage(DefaultTimePeriodCoverage) + .Set(p => p.Slug, (_, r) => NamingUtils.SlugFromTitle(r.YearTitle)) + .Set(p => p.Created, f => f.Date.Past()) + .Set(p => p.Updated, (f, r) => f.Date.Soon(refDate: r.Created)); + + public static InstanceSetters SetId( + this InstanceSetters setters, + Guid id) + => setters.Set(p => p.Id, id); + + public static InstanceSetters SetPublication( + this InstanceSetters setters, + Publication publication) + => setters + .Set(p => p.Publication, + (_, release) => + { + publication.Releases.Add(release); + + var releaseVersions = release.Versions; + + releaseVersions.ForEach(rv => + { + rv.Publication = publication; + rv.PublicationId = publication.Id; + }); + + publication.ReleaseVersions.AddRange(releaseVersions); + + return publication; + }) + .SetPublicationId(publication.Id) + .Set((_, release, context) => + { + publication.ReleaseSeries.Add(context.Fixture + .DefaultReleaseSeriesItem() + .WithReleaseId(release.Id) + .Generate()); + }) + .Set((_, _) => + { + var releaseVersions = publication.Releases.SelectMany(r => r.Versions) + .ToList(); + + var publishedVersions = releaseVersions + .Where(releaseVersion => releaseVersion.Published.HasValue) + .ToList(); + + if (publishedVersions.Count > 0) + { + publication.LatestPublishedReleaseVersion = publishedVersions.Count == 1 + ? publishedVersions[0] + : publishedVersions + .GroupBy(releaseVersion => releaseVersion.ReleaseId) + .Select(groupedReleases => + new + { + ReleaseId = groupedReleases.Key, + Version = groupedReleases.Max(releaseVersion => releaseVersion.Version) + }) + .Join(publishedVersions, + maxVersion => maxVersion, + releaseVersion => new + { + releaseVersion.ReleaseId, + releaseVersion.Version + }, + (_, releaseVersion) => releaseVersion) + .OrderByDescending(releaseVersion => releaseVersion.Release.Year) + .ThenByDescending(releaseVersion => releaseVersion.Release.TimePeriodCoverage) + .FirstOrDefault(); + } + + publication.LatestPublishedReleaseVersionId = publication.LatestPublishedReleaseVersion?.Id; + }); + + public static InstanceSetters SetPublicationId( + this InstanceSetters setters, + Guid publicationId) + => setters.Set(p => p.PublicationId, publicationId); + public static InstanceSetters SetVersions( this InstanceSetters setters, IEnumerable releaseVersions) @@ -82,7 +194,7 @@ private static InstanceSetters SetVersions( this InstanceSetters setters, Func> releaseVersions) => setters.Set( - r => r.Versions, + p => p.Versions, (_, release, context) => { var list = releaseVersions.Invoke(context).ToList(); @@ -91,6 +203,12 @@ private static InstanceSetters SetVersions( { releaseVersion.Release = release; releaseVersion.ReleaseId = release.Id; + + releaseVersion.Publication = release.Publication; + releaseVersion.PublicationId = release.PublicationId; + releaseVersion.ReleaseName = release.Year.ToString(); + releaseVersion.TimePeriodCoverage = release.TimePeriodCoverage; + releaseVersion.Slug = release.Slug; }); return list; @@ -100,10 +218,20 @@ private static InstanceSetters SetVersions( public static InstanceSetters SetCreated( this InstanceSetters setters, DateTime created) - => setters.Set(r => r.Created, created); + => setters.Set(p => p.Created, created); + + public static InstanceSetters SetTimePeriodCoverage( + this InstanceSetters setters, + TimeIdentifier timePeriodCoverage) + => setters.Set(p => p.TimePeriodCoverage, timePeriodCoverage); public static InstanceSetters SetUpdated( this InstanceSetters setters, DateTime? updated) - => setters.Set(r => r.Updated, updated); + => setters.Set(p => p.Updated, updated); + + public static InstanceSetters SetYear( + this InstanceSetters setters, + int year) + => setters.Set(p => p.Year, year); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseSeriesItemGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseSeriesItemGeneratorExtensions.cs new file mode 100644 index 00000000000..f7c91c5377d --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseSeriesItemGeneratorExtensions.cs @@ -0,0 +1,59 @@ +using System; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; + +namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; + +public static class ReleaseSeriesItemGeneratorExtensions +{ + public static Generator DefaultReleaseSeriesItem(this DataFixture fixture) + => fixture.Generator().WithDefaults(); + + public static Generator DefaultLegacyReleaseSeriesItem(this DataFixture fixture) + => fixture.Generator().WithLegacyDefaults(); + + public static Generator WithDefaults(this Generator generator) + => generator.ForInstance(s => s.SetDefaults()); + + public static Generator WithLegacyDefaults(this Generator generator) + => generator.ForInstance(s => s.SetLegacyDefaults()); + + public static Generator WithReleaseId( + this Generator generator, + Guid? releaseId) + => generator.ForInstance(s => s.SetReleaseId(releaseId)); + + public static Generator WithLegacyLinkUrl( + this Generator generator, + string? legacyLinkUrl) + => generator.ForInstance(s => s.SetLegacyLinkUrl(legacyLinkUrl)); + + public static Generator WithLegacyLinkDescription( + this Generator generator, + string? legacyLinkDescription) + => generator.ForInstance(s => s.SetLegacyLinkDescription(legacyLinkDescription)); + + public static InstanceSetters SetDefaults(this InstanceSetters setters) + => setters + .SetDefault(rsi => rsi.Id); + + public static InstanceSetters SetLegacyDefaults(this InstanceSetters setters) + => setters + .SetDefaults() + .SetDefault(rsi => rsi.LegacyLinkDescription) + .SetDefault(rsi => rsi.LegacyLinkUrl); + + public static InstanceSetters SetReleaseId( + this InstanceSetters instanceSetter, + Guid? releaseId) + => instanceSetter.Set(rsi => rsi.ReleaseId, releaseId); + + public static InstanceSetters SetLegacyLinkUrl( + this InstanceSetters instanceSetter, + string? legacyLinkUrl) + => instanceSetter.Set(rsi => rsi.LegacyLinkUrl, legacyLinkUrl); + + public static InstanceSetters SetLegacyLinkDescription( + this InstanceSetters instanceSetter, + string? legacyLinkDescription) + => instanceSetter.Set(rsi => rsi.LegacyLinkDescription, legacyLinkDescription); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseVersionGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseVersionGeneratorExtensions.cs index 05ee8a510b6..e06526cc718 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseVersionGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseVersionGeneratorExtensions.cs @@ -5,6 +5,7 @@ using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; @@ -31,30 +32,17 @@ public static Generator WithIds( return generator; } + [Obsolete("Provide relationship with Publication via Release. This will be removed in EES-5658/EES-5659")] public static Generator WithPublication( this Generator generator, Publication publication) => generator.ForInstance(s => s.SetPublication(publication)); - - public static Generator WithPublicationId( - this Generator generator, - Guid publicationId) - => generator.ForInstance(s => s.SetPublicationId(publicationId)); - - public static Generator WithPublications(this Generator generator, - IEnumerable publications) - { - publications.ForEach((publication, index) => - generator.ForIndex(index, releaseVersion => releaseVersion.SetPublication(publication))); - - return generator; - } public static Generator WithRelease( this Generator generator, Release release) => generator.ForInstance(s => s.SetRelease(release)); - + public static Generator WithReleaseId( this Generator generator, Guid releaseId) @@ -128,8 +116,7 @@ public static Generator WithYear( int year) => generator.ForInstance(releaseVersion => releaseVersion.SetYear(year)); - public static Generator WithSoftDeleted( - this Generator generator) + public static Generator WithSoftDeleted(this Generator generator) => generator.ForInstance(releaseVersion => releaseVersion.SetSoftDeleted()); public static Generator WithType( @@ -207,11 +194,13 @@ public static InstanceSetters SetDefaults(this InstanceSetters p.Id) .SetDefault(p => p.Slug) .SetDefault(p => p.DataGuidance) - .SetDefault(p => p.Type) + .Set(p => p.Type, + f => f.PickRandom(EnumUtil.GetEnums().Except([ReleaseType.ExperimentalStatistics]))) .SetDefault(p => p.PublicationId) .SetDefault(p => p.ReleaseId) .SetApprovalStatus(ReleaseApprovalStatus.Draft) .SetTimePeriodCoverage(TimeIdentifier.AcademicYear) + .SetDefault(p => p.PreReleaseAccessList) .Set(p => p.ReleaseName, (_, _, context) => $"{2000 + context.Index}") .Set(p => p.NextReleaseDate, (_, _, context) => new PartialDate { @@ -223,13 +212,21 @@ public static InstanceSetters SetId( Guid id) => setters.Set(releaseVersion => releaseVersion.Id, id); + [Obsolete("Set relationship with Publication via Release. This will be removed in EES-5658/EES-5659")] public static InstanceSetters SetPublication( this InstanceSetters setters, Publication publication) => setters - .Set(releaseVersion => releaseVersion.Publication, publication) + .Set(releaseVersion => releaseVersion.Publication, + (_, releaseVersion) => + { + publication.ReleaseVersions.Add(releaseVersion); + + return publication; + }) .SetPublicationId(publication.Id); + [Obsolete("Set relationship with Publication via Release. This will be removed in EES-5658/EES-5659")] public static InstanceSetters SetPublicationId( this InstanceSetters setters, Guid publicationId) @@ -238,7 +235,19 @@ public static InstanceSetters SetPublicationId( public static InstanceSetters SetRelease( this InstanceSetters setters, Release release) - => setters.Set(releaseVersion => releaseVersion.Release, release) + => setters.Set(releaseVersion => releaseVersion.Release, + (_, releaseVersion) => + { + release.Versions.Add(releaseVersion); + + releaseVersion.Publication = release.Publication; + releaseVersion.PublicationId = release.PublicationId; + releaseVersion.ReleaseName = release.Year.ToString(); + releaseVersion.TimePeriodCoverage = release.TimePeriodCoverage; + releaseVersion.Slug = release.Slug; + + return release; + }) .SetReleaseId(release.Id); public static InstanceSetters SetReleaseId( @@ -330,8 +339,7 @@ public static InstanceSetters SetYear( int year) => setters.Set(releaseVersion => releaseVersion.ReleaseName, year.ToString()); - public static InstanceSetters SetSoftDeleted( - this InstanceSetters setters) + public static InstanceSetters SetSoftDeleted(this InstanceSetters setters) => setters.Set(releaseVersion => releaseVersion.SoftDeleted, true); public static InstanceSetters SetType( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs index 37423f7ec04..a86f10300d4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs @@ -112,7 +112,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) ConfigureFile(modelBuilder); ConfigureContentBlock(modelBuilder); ConfigureContentSection(modelBuilder); - ConfigureRelease(modelBuilder); + ConfigureReleaseVersion(modelBuilder); ConfigureDataBlock(modelBuilder); ConfigureHtmlBlock(modelBuilder); ConfigureEmbedBlockLink(modelBuilder); @@ -131,7 +131,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) ConfigureDataBlockVersion(modelBuilder); // Apply model configuration for types which implement IEntityTypeConfiguration - + modelBuilder.ApplyConfigurationsFromAssembly(typeof(ContentDbContext).Assembly); new FreeTextRank.Config().Configure(modelBuilder.Entity()); } @@ -490,8 +490,14 @@ private static void ConfigureContentSection(ModelBuilder modelBuilder) }); } - private static void ConfigureRelease(ModelBuilder modelBuilder) + private static void ConfigureReleaseVersion(ModelBuilder modelBuilder) { + // TODO This will be removed in EES-5659. It's been added to prevent multiple delete cascade paths + modelBuilder.Entity() + .HasOne(rv => rv.Publication) + .WithMany(p => p.ReleaseVersions) + .OnDelete(DeleteBehavior.NoAction); + modelBuilder.Entity() .Property(rv => rv.TimePeriodCoverage) .HasConversion(new EnumToEnumValueConverter()) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Publication.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Publication.cs index 668c65171da..8386148e2c3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Publication.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Publication.cs @@ -17,6 +17,8 @@ public class Publication [MaxLength(160)] public string Summary { get; set; } = string.Empty; + public List Releases { get; set; } = []; + public List ReleaseVersions { get; set; } = []; public List Methodologies { get; set; } = []; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs index 9e375c7b69d..0e34b8b65a2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs @@ -1,7 +1,12 @@ #nullable enable using System; using System.Collections.Generic; +using GovUk.Education.ExploreEducationStatistics.Common.Converters; +using GovUk.Education.ExploreEducationStatistics.Common.Database; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; namespace GovUk.Education.ExploreEducationStatistics.Content.Model; @@ -9,9 +14,45 @@ public class Release : ICreatedUpdatedTimestamps { public Guid Id { get; set; } - public List Versions { get; set; } = new(); + public required Guid PublicationId { get; set; } + + public Publication Publication { get; set; } = null!; + + public required string Slug { get; set; } = string.Empty; + + public required TimeIdentifier TimePeriodCoverage { get; set; } + + public required int Year { get; set; } + + public List Versions { get; set; } = []; public DateTime Created { get; set; } public DateTime? Updated { get; set; } + + public string Title => + TimePeriodLabelFormatter.Format(Year, TimePeriodCoverage, TimePeriodLabelFormat.FullLabelBeforeYear); + + public string YearTitle => TimePeriodLabelFormatter.FormatYear(Year, TimePeriodCoverage); + + internal class Config : IEntityTypeConfiguration + { + public void Configure(EntityTypeBuilder builder) + { + builder.Property(m => m.Slug) + .HasMaxLength(30); + + builder.Property(m => m.TimePeriodCoverage) + .HasConversion(new EnumToEnumValueConverter()) + .HasMaxLength(5); + + builder.HasIndex(dsv => new + { + dsv.PublicationId, + dsv.Year, + dsv.TimePeriodCoverage + }) + .IsUnique(); + } + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs index 04781e471b9..d54156bf6b2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs @@ -65,7 +65,11 @@ public class ReleaseServiceTests private static readonly Release Release1 = new() { - Id = Guid.NewGuid() + Id = Guid.NewGuid(), + PublicationId = Guid.NewGuid(), + Year = 2018, + TimePeriodCoverage = AcademicYearQ1, + Slug = "2018-19-q1" }; private static readonly ReleaseVersion Release1V1 = new() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/TableBuilderControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/TableBuilderControllerTests.cs index 543db15ce85..d182167b4b8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/TableBuilderControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/TableBuilderControllerTests.cs @@ -26,6 +26,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Xunit; @@ -41,11 +42,10 @@ public class TableBuilderControllerTests(TestApplicationFactory testApp) { private static readonly DataFixture Fixture = new(); - private static readonly ReleaseVersion ReleaseVersion = Fixture - .DefaultReleaseVersion() - .WithPublication(Fixture.DefaultPublication().Generate()) - .WithPublished(DateTime.UtcNow.AddDays(-1)) - .Generate(); + private static readonly Release Release = Fixture.DefaultRelease(publishedVersions: 1) + .WithPublication(Fixture.DefaultPublication()); + + private static readonly ReleaseVersion ReleaseVersion = Release.Versions.Single(); private static readonly DataBlockParent DataBlockParent = Fixture .DefaultDataBlockParent() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Services/TableBuilderService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Services/TableBuilderService.cs index fe63e776dc7..a3c0c22295a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Services/TableBuilderService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Services/TableBuilderService.cs @@ -150,7 +150,7 @@ public async Task> QueryToCsvStream( CancellationToken cancellationToken = default) { return await FindLatestPublishedReleaseVersionId(query.SubjectId) - .OnSuccessVoid(releaseId => QueryToCsvStream(releaseId, query, stream, cancellationToken)); + .OnSuccessVoid(releaseVersionId => QueryToCsvStream(releaseVersionId, query, stream, cancellationToken)); } public async Task> QueryToCsvStream( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs index a51130b16ac..502026d9569 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs @@ -350,7 +350,8 @@ await AddTestData(context => .DefaultReleaseFile() .WithReleaseVersion(DataFixture .DefaultReleaseVersion() - .WithPublicationId(publicationId)) + .WithRelease(DataFixture.DefaultRelease() + .WithPublicationId(publicationId))) .WithFiles([ DataFixture .DefaultFile(FileType.Data) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs index 24c47a0b78e..6a2b6dd030a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs @@ -372,8 +372,11 @@ public async Task ReleaseFileInSameReleaseSeriesAsCurrentLiveVersion_ReturnsVali ReleaseVersion releaseAmendment = DataFixture .DefaultReleaseVersion() - .WithReleaseId(releaseVersion.ReleaseId) - .WithPublicationId(dataSet.PublicationId); + .WithReleaseId(releaseVersion.ReleaseId); + + // Need to set this to override the default value assigned by the test data generator, + // until ReleaseVersion.PublicationId is removed. + releaseAmendment.PublicationId = dataSet.PublicationId; var subjectId = Guid.NewGuid(); @@ -452,7 +455,8 @@ await AddTestData(context => .DefaultReleaseFile() .WithReleaseVersion(DataFixture .DefaultReleaseVersion() - .WithPublicationId(publicationId)) + .WithRelease(DataFixture.DefaultRelease() + .WithPublicationId(publicationId))) .WithFiles([ DataFixture .DefaultFile(FileType.Data) From da9b2df00c833517997361ada57fa04f8fd96749 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Fri, 22 Nov 2024 13:35:22 +0000 Subject: [PATCH 023/144] EES-5625 Add required validation on Type field in ReleaseCreateRequest and ReleaseUpdateRequest --- .../Requests/ReleaseRequests.cs | 9 ++++++--- .../Services/ReleaseService.cs | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseRequests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseRequests.cs index d0d3a3b5420..785fc3df42e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseRequests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseRequests.cs @@ -1,4 +1,5 @@ -using System; +#nullable enable +using System; using System.ComponentModel.DataAnnotations; using GovUk.Education.ExploreEducationStatistics.Common.Converters; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -13,7 +14,8 @@ public record ReleaseCreateRequest { public Guid PublicationId { get; set; } - [Required] public ReleaseType Type { get; set; } + [Required] + public ReleaseType? Type { get; init; } [Required] [JsonConverter(typeof(TimeIdentifierJsonConverter))] @@ -31,7 +33,8 @@ public record ReleaseCreateRequest public record ReleaseUpdateRequest { - [Required] public ReleaseType Type { get; init; } + [Required] + public ReleaseType? Type { get; init; } [JsonConverter(typeof(TimeIdentifierJsonConverter))] [Required] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs index 15d492a406f..abad7c72c1b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs @@ -153,7 +153,7 @@ await ValidateReleaseSlugUniqueToPublication(releaseCreate.Slug, releaseCreate.P { Id = _guidGenerator.NewGuid(), Release = release, - Type = releaseCreate.Type, + Type = releaseCreate.Type!.Value, ApprovalStatus = ReleaseApprovalStatus.Draft, // TODO Remove the following in EES-5659 @@ -544,7 +544,7 @@ await ValidateReleaseSlugUniqueToPublication(request.Slug, releaseVersion.TimePeriodCoverage = releaseVersion.Release.TimePeriodCoverage; releaseVersion.Slug = releaseVersion.Release.Slug; - releaseVersion.Type = request.Type; + releaseVersion.Type = request.Type!.Value; releaseVersion.PreReleaseAccessList = request.PreReleaseAccessList; await _dataSetVersionService.UpdateVersionsForReleaseVersion( From 6ba37f11407c533df3eab00d7fb86028ac1b7041 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 25 Nov 2024 10:13:16 +0000 Subject: [PATCH 024/144] EES-5625 No need to edit release type and year in the summary of the amended release --- .../bau/legacy_releases_reordering.robot | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot b/tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot index 2f72e0a872d..636fd0c172f 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot @@ -181,20 +181,6 @@ Return to Admin and create first amendment user navigates to admin dashboard Bau1 user creates amendment for release ${PUBLICATION_NAME} ${RELEASE_NAME} -Change the Release type and Academic year - user waits until page contains link Edit release summary - user clicks link Edit release summary - user waits until page finishes loading - user waits until h2 is visible Edit release summary - user checks page contains radio Official statistics in development - user enters text into element id:releaseSummaryForm-timePeriodCoverageStartYear 2024 - user clicks radio Official statistics in development - user clicks button Update release summary - user checks page contains element xpath://li/a[text()="Summary" and contains(@aria-current, 'page')] - user verifies release summary Academic year Q1 - ... 2024/25 - ... Official statistics in development - Navigate to 'Content' page for amendment user clicks link Content user waits until h2 is visible ${PUBLICATION_NAME} @@ -225,7 +211,7 @@ Navigate to publication page and verify the amended release Validate amended legacy releases user waits until page contains button Reorder releases - user checks table cell contains 1 1 Academic year Q1 2024/25 + user checks table cell contains 1 1 Academic year Q1 2022/23 user checks table cell contains 1 2 ${PUBLIC_AMENDED_RELEASE_LINK} user checks table cell contains 1 3 Latest release @@ -255,7 +241,7 @@ Delete legacy release Validate that deleted legacy release does not exist user waits until page contains button Reorder releases - user checks table cell contains 1 1 Academic year Q1 2024/25 + user checks table cell contains 1 1 Academic year Q1 2022/23 user checks table cell contains 1 2 ${PUBLIC_AMENDED_RELEASE_LINK} user checks table cell contains 1 3 Latest release From 68c44024a0bd7e9c86f60b8c6783313b263834ac Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Tue, 26 Nov 2024 12:53:02 +0000 Subject: [PATCH 025/144] EES-5625 Exclude soft deleted release versions in the inner query for determining the latest version, and in the outer versions query. --- ...41122105739_EES5625_CopyReleaseVersionFieldsToRelease.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.sql b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.sql index d53152afbae..f9eaeb6e151 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.sql +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241122105739_EES5625_CopyReleaseVersionFieldsToRelease.sql @@ -11,7 +11,8 @@ FROM Releases INNER JOIN (SELECT RV2.ReleaseId, MAX(RV2.Version) AS Version FROM ReleaseVersions RV2 - WHERE RV2.Published IS NOT NULL - OR (RV2.Published IS NULL AND RV2.Version = 0) + WHERE RV2.SoftDeleted = 0 + AND (RV2.Published IS NOT NULL OR (RV2.Published IS NULL AND RV2.Version = 0)) GROUP BY ReleaseId) LatestVersion ON ReleaseVersions.ReleaseId = LatestVersion.ReleaseId AND ReleaseVersions.Version = LatestVersion.Version +WHERE ReleaseVersions.SoftDeleted = 0 From e47d736d396a849b1c3c22a4ac851589b9466594 Mon Sep 17 00:00:00 2001 From: jack-hive <148866614+jack-hive@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:23:43 +0000 Subject: [PATCH 026/144] EES 5632 - Return the Release redirects from the `api/redirects` endpoint (#5391) * EES-5632 Adding release redirects to the Redirects Cache, and returning it in the `RedirectViewModel` in the `/redirects` endpoint * EES-5632 Adding integration tests * EES-5632 Addressing PR comments --- .../MethodologyApprovalServiceTests.cs | 5 +- .../Methodologies/MethodologyServiceTests.cs | 35 ++- .../Services/PublicationServiceTests.cs | 16 +- .../Controllers/RedirectsControllerTests.cs | 222 ++++++++++++++++++ .../MethodologyGeneratorExtensions.cs | 49 +++- .../MethodologyRedirectGeneratorExtensions.cs | 49 ++++ .../MethodologyVersionGeneratorExtensions.cs | 11 + .../PublicationRedirectGeneratorExtensions.cs | 49 ++++ .../ReleaseRedirectGeneratorExtensions.cs | 49 ++++ .../RedirectsService.cs | 12 +- .../RedirectViewModels.cs | 3 +- 11 files changed, 476 insertions(+), 24 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/RedirectsControllerTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyRedirectGeneratorExtensions.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationRedirectGeneratorExtensions.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseRedirectGeneratorExtensions.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs index 38a167d25c6..bf25fb5f09f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs @@ -680,7 +680,10 @@ public async Task UpdateApprovalStatus_ApprovingUsingImmediateStrategy() new List())); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) - .ReturnsAsync(new RedirectsViewModel(new List(), new List())); + .ReturnsAsync(new RedirectsViewModel( + Publications: [], + Methodologies: [], + Releases: [])); await using (var context = InMemoryApplicationDbContext(contentDbContextId)) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs index 20e83050f45..17499e9130b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs @@ -3599,7 +3599,10 @@ public async Task PublicationTitleOrSlugChanged() { var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) - .ReturnsAsync(new RedirectsViewModel(new List(), new List())); + .ReturnsAsync(new RedirectsViewModel( + Publications: [], + Methodologies: [], + Releases: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -3658,7 +3661,10 @@ public async Task PublicationTitleOrSlugChanged_NoMethodologyRedirectAsMethodolo { var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) - .ReturnsAsync(new RedirectsViewModel(new List(), new List())); + .ReturnsAsync(new RedirectsViewModel( + Publications: [], + Methodologies: [], + Releases: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -3738,7 +3744,10 @@ await contentDbContext.PublicationMethodologies.AddRangeAsync( { var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) - .ReturnsAsync(new RedirectsViewModel(new List(), new List())); + .ReturnsAsync(new RedirectsViewModel( + Publications: [], + Methodologies: [], + Releases: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -3837,7 +3846,10 @@ public async Task PublicationTitleOrSlugChanged_MethodologySlugIsAlternativeSlug { var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) - .ReturnsAsync(new RedirectsViewModel(new List(), new List())); + .ReturnsAsync(new RedirectsViewModel( + Publications: [], + Methodologies: [], + Releases: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -3904,7 +3916,10 @@ public async Task PublicationTitleOrSlugChanged_MethodologyIsLive() { var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) - .ReturnsAsync(new RedirectsViewModel(new List(), new List())); + .ReturnsAsync(new RedirectsViewModel( + Publications: [], + Methodologies: [], + Releases: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -3988,7 +4003,10 @@ public async Task { var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) - .ReturnsAsync(new RedirectsViewModel(new List(), new List())); + .ReturnsAsync(new RedirectsViewModel( + Publications: [], + Methodologies: [], + Releases: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -4091,7 +4109,10 @@ public async Task { var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) - .ReturnsAsync(new RedirectsViewModel(new List(), new List())); + .ReturnsAsync(new RedirectsViewModel( + Publications: [], + Methodologies: [], + Releases: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs index 0355e6fba5d..9c493f09b50 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs @@ -1127,7 +1127,9 @@ public async Task UpdatePublication_AlreadyPublished() redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - new List(), new List())); + Publications: [], + Methodologies: [], + Releases: [])); var publicationService = BuildPublicationService(context, methodologyService: methodologyService.Object, @@ -1270,7 +1272,9 @@ public async Task UpdatePublication_TitleChangesPublicationAndMethodologySlug() redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - new List(), new List())); + Publications: [], + Methodologies: [], + Releases: [])); var publicationService = BuildPublicationService(context, methodologyService: methodologyService.Object, @@ -1575,7 +1579,9 @@ public async Task UpdatePublication_CreateRedirectIfLiveSlugChanged() var redirectsCacheService = new Mock(Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - new List(), new List())); + Publications: [], + Methodologies: [], + Releases: [])); var publicationService = BuildPublicationService(context, methodologyService: methodologyService.Object, @@ -1672,7 +1678,9 @@ public async Task UpdatePublication_ChangeBackToPreviousLiveSlug() var redirectsCacheService = new Mock(Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - new List(), new List())); + Publications: [], + Methodologies: [], + Releases: [])); var publicationService = BuildPublicationService(context, methodologyService: methodologyService.Object, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/RedirectsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/RedirectsControllerTests.cs new file mode 100644 index 00000000000..7577fb5075c --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/RedirectsControllerTests.cs @@ -0,0 +1,222 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Content.Model; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; +using GovUk.Education.ExploreEducationStatistics.Content.Services.Cache; +using GovUk.Education.ExploreEducationStatistics.Content.ViewModels; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.Controllers; + +public abstract class RedirectsControllerTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) +{ + public class ListTests(TestApplicationFactory testApp) : RedirectsControllerTests(testApp) + { + public override async Task InitializeAsync() => await TestApp.StartAzurite(); + + [Fact] + public async Task RedirectsExist_Returns200WithRedirects() + { + var publicationRedirects = DataFixture.DefaultPublicationRedirect() + .WithPublication(DataFixture.DefaultPublication()) + .GenerateList(3); + + var releaseRedirects = DataFixture.DefaultReleaseRedirect() + .WithRelease(DataFixture.DefaultRelease()) + .GenerateList(3); + + var methodologyVersions = DataFixture.DefaultMethodologyVersion() + .ForIndex(3, s => s.SetPublished(DateTime.UtcNow)) + .GenerateList(4); + + Methodology methodology = DataFixture.DefaultMethodology() + .WithMethodologyVersions(methodologyVersions) + .WithLatestPublishedVersion(methodologyVersions.Last()); + + var methodologyRedirects = DataFixture.DefaultMethodologyRedirect() + .ForIndex(0, s => s.SetMethodologyVersion(methodologyVersions[0])) + .ForIndex(1, s => s.SetMethodologyVersion(methodologyVersions[1])) + .ForIndex(2, s => s.SetMethodologyVersion(methodologyVersions[2])) + .GenerateList(3); + + await TestApp.AddTestData(context => + { + context.PublicationRedirects.AddRange(publicationRedirects); + context.ReleaseRedirects.AddRange(releaseRedirects); + context.MethodologyRedirects.AddRange(methodologyRedirects); + }); + + var response = await ListRedirects(); + + var viewModel = response.AssertOk(); + + Assert.All( + publicationRedirects, + pr => Assert.Contains( + viewModel.Publications, + rvm => pr.Slug == rvm.FromSlug && pr.Publication.Slug == rvm.ToSlug)); + Assert.All( + releaseRedirects, + rr => Assert.Contains( + viewModel.Releases, + rvm => rr.Slug == rvm.FromSlug && rr.Release.Slug == rvm.ToSlug)); + Assert.All( + methodologyRedirects, + mr => Assert.Contains( + viewModel.Methodologies, + rvm => mr.Slug == rvm.FromSlug && mr.MethodologyVersion.Methodology.OwningPublicationSlug == rvm.ToSlug)); + } + + [Fact] + public async Task RedirectsExist_RedirectsAreCached() + { + var publicationRedirects = DataFixture.DefaultPublicationRedirect() + .WithPublication(DataFixture.DefaultPublication()) + .GenerateList(3); + + var releaseRedirects = DataFixture.DefaultReleaseRedirect() + .WithRelease(DataFixture.DefaultRelease()) + .GenerateList(3); + + var methodologyVersions = DataFixture.DefaultMethodologyVersion() + .ForIndex(3, s => s.SetPublished(DateTime.UtcNow)) + .GenerateList(4); + + Methodology methodology = DataFixture.DefaultMethodology() + .WithMethodologyVersions(methodologyVersions) + .WithLatestPublishedVersion(methodologyVersions.Last()); + + var methodologyRedirects = DataFixture.DefaultMethodologyRedirect() + .ForIndex(0, s => s.SetMethodologyVersion(methodologyVersions[0])) + .ForIndex(1, s => s.SetMethodologyVersion(methodologyVersions[1])) + .ForIndex(2, s => s.SetMethodologyVersion(methodologyVersions[2])) + .GenerateList(3); + + await TestApp.AddTestData(context => + { + context.PublicationRedirects.AddRange(publicationRedirects); + context.ReleaseRedirects.AddRange(releaseRedirects); + context.MethodologyRedirects.AddRange(methodologyRedirects); + }); + + var app = BuildApp(); + var client = app.CreateClient(); + + await ListRedirects(client); + + var blobCacheService = app.Services.GetRequiredService(); + + var cachedValue = await blobCacheService.GetItemAsync(new RedirectsCacheKey(), typeof(RedirectsViewModel)); + var cachedRedirectsViewModel = Assert.IsType(cachedValue); + + Assert.All( + publicationRedirects, + pr => Assert.Contains( + cachedRedirectsViewModel.Publications, + rvm => pr.Slug == rvm.FromSlug && pr.Publication.Slug == rvm.ToSlug)); + Assert.All( + releaseRedirects, + rr => Assert.Contains( + cachedRedirectsViewModel.Releases, + rvm => rr.Slug == rvm.FromSlug && rr.Release.Slug == rvm.ToSlug)); + Assert.All( + methodologyRedirects, + mr => Assert.Contains( + cachedRedirectsViewModel.Methodologies, + rvm => mr.Slug == rvm.FromSlug && mr.MethodologyVersion.Methodology.OwningPublicationSlug == rvm.ToSlug)); + } + + [Fact] + public async Task RedirectsExistInCache_Returns200WithRedirects() + { + var app = BuildApp(); + var client = app.CreateClient(); + + var blobCacheService = app.Services.GetRequiredService(); + + var cachedPublicationRedirects = new List() + { + new(FromSlug: "publication_fromSlug_1", ToSlug: "publication_toSlug_1"), + new(FromSlug: "publication_fromSlug_2", ToSlug: "publication_toSlug_2"), + new(FromSlug: "publication_fromSlug_3", ToSlug: "publication_toSlug_3") + }; + + var cachedReleaseRedirects = new List() + { + new(FromSlug: "release_fromSlug_1", ToSlug: "release_toSlug_1"), + new(FromSlug: "release_fromSlug_2", ToSlug: "release_toSlug_2"), + new(FromSlug: "release_fromSlug_3", ToSlug: "release_toSlug_3") + }; + + var cachedMethodologyRedirects = new List() + { + new(FromSlug: "methodology_fromSlug_1", ToSlug: "methodology_toSlug_1"), + new(FromSlug: "methodology_fromSlug_2", ToSlug: "methodology_toSlug_2"), + new(FromSlug: "methodology_fromSlug_3", ToSlug: "methodology_toSlug_3") + }; + + var cachedViewModel = new RedirectsViewModel( + Publications: cachedPublicationRedirects, + Releases: cachedReleaseRedirects, + Methodologies: cachedMethodologyRedirects); + + await blobCacheService.SetItemAsync(new RedirectsCacheKey(), cachedViewModel); + + var response = await ListRedirects(client); + + var viewModel = response.AssertOk(); + + Assert.All( + cachedPublicationRedirects, + cpr => Assert.Contains( + viewModel.Publications, + rvm => cpr.FromSlug == rvm.FromSlug && cpr.ToSlug == rvm.ToSlug)); + Assert.All( + cachedReleaseRedirects, + crr => Assert.Contains( + viewModel.Releases, + rvm => crr.FromSlug == rvm.FromSlug && crr.ToSlug == rvm.ToSlug)); + Assert.All( + cachedMethodologyRedirects, + cmr => Assert.Contains( + viewModel.Methodologies, + rvm => cmr.FromSlug == rvm.FromSlug && cmr.ToSlug == rvm.ToSlug)); + } + + [Fact] + public async Task NoRedirectsExist_Returns200WithNoRedirects() + { + var response = await ListRedirects(); + + var viewModel = response.AssertOk(); + + Assert.Empty(viewModel.Publications); + Assert.Empty(viewModel.Releases); + Assert.Empty(viewModel.Methodologies); + } + + private async Task ListRedirects( + HttpClient? client = null) + { + client ??= BuildApp().CreateClient(); + + return await client.GetAsync("/api/redirects"); + } + } + + private WebApplicationFactory BuildApp() + { + return TestApp + .WithAzurite(enabled: true); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyGeneratorExtensions.cs index eef5b92e9d3..12c6dc1bc79 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyGeneratorExtensions.cs @@ -18,32 +18,42 @@ public static InstanceSetters SetDefaults(this InstanceSetters m.Id) .SetDefault(m => m.OwningPublicationTitle) .SetDefault(m => m.OwningPublicationSlug); - + public static Generator WithOwningPublication( this Generator generator, Publication publication) => generator.ForInstance(s => s.SetOwningPublication(publication)); - + public static Generator WithAdoptingPublication( this Generator generator, Publication publication) => generator.ForInstance(s => s.SetAdoptingPublication(publication)); - + public static Generator WithMethodologyVersions( this Generator generator, IEnumerable methodologyVersions) => generator.ForInstance(s => s.SetMethodologyVersions(methodologyVersions)); - + public static Generator WithMethodologyVersions( this Generator generator, Func> methodologyVersions) => generator.ForInstance(s => s.SetMethodologyVersions(methodologyVersions.Invoke)); - + + public static Generator WithLatestPublishedVersion( + this Generator generator, + MethodologyVersion latestPublishedVersion) + => generator.ForInstance(s => s.SetLatestPublishedVersion(latestPublishedVersion)); + + public static Generator WithLatestPublishedVersionId( + this Generator generator, + Guid latestPublishedVersionId) + => generator.ForInstance(s => s.SetLatestPublishedVersionId(latestPublishedVersionId)); + public static InstanceSetters SetMethodologyVersions( this InstanceSetters setters, - IEnumerable methodologyVersions) + IEnumerable methodologyVersions) => setters.SetMethodologyVersions(_ => methodologyVersions); - + private static InstanceSetters SetMethodologyVersions( this InstanceSetters setters, Func> methodologyVersions) @@ -59,16 +69,35 @@ private static InstanceSetters SetMethodologyVersions( } ); + private static InstanceSetters SetLatestPublishedVersion( + this InstanceSetters setters, + MethodologyVersion latestPublishedVersion) + => setters.Set( + m => m.LatestPublishedVersion, + (_, methodology) => + { + latestPublishedVersion.Methodology = methodology; + + return latestPublishedVersion; + } + ) + .SetLatestPublishedVersionId(latestPublishedVersion.Id); + + private static InstanceSetters SetLatestPublishedVersionId( + this InstanceSetters setters, + Guid latestPublishedVersionId) + => setters.Set(m => m.LatestPublishedVersionId, latestPublishedVersionId); + public static InstanceSetters SetOwningPublication( this InstanceSetters setters, - Publication publication) + Publication publication) => setters.SetPublication(_ => publication, owner: true); public static InstanceSetters SetAdoptingPublication( this InstanceSetters setters, - Publication publication) + Publication publication) => setters.SetPublication(_ => publication, owner: false); - + private static InstanceSetters SetPublication( this InstanceSetters setters, Func publication, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyRedirectGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyRedirectGeneratorExtensions.cs new file mode 100644 index 00000000000..f81b516a742 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyRedirectGeneratorExtensions.cs @@ -0,0 +1,49 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; +using System; + +namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; + +public static class MethodologyRedirectGeneratorExtensions +{ + public static Generator DefaultMethodologyRedirect(this DataFixture fixture) + => fixture.Generator().WithDefaults(); + + public static Generator WithDefaults(this Generator generator) + => generator.ForInstance(d => d.SetDefaults()); + + public static InstanceSetters SetDefaults(this InstanceSetters setters) + => setters + .SetDefault(p => p.Slug) + .SetDefault(p => p.Created); + + public static Generator WithSlug( + this Generator generator, + string slug) + => generator.ForInstance(s => s.SetSlug(slug)); + + public static Generator WithMethodologyVersion( + this Generator generator, + MethodologyVersion methodologyVersion) + => generator.ForInstance(s => s.SetMethodologyVersion(methodologyVersion)); + + public static Generator WithMethodologyVersionId( + this Generator generator, + Guid methodologyVersionId) + => generator.ForInstance(s => s.SetMethodologyVersionId(methodologyVersionId)); + + public static InstanceSetters SetSlug( + this InstanceSetters setters, + string slug) + => setters.Set(mr => mr.Slug, slug); + + public static InstanceSetters SetMethodologyVersion( + this InstanceSetters setters, + MethodologyVersion methodologyVersion) + => setters.Set(mr => mr.MethodologyVersion, methodologyVersion) + .SetMethodologyVersionId(methodologyVersion.Id); + + public static InstanceSetters SetMethodologyVersionId( + this InstanceSetters setters, + Guid methodologyVersionId) + => setters.Set(mr => mr.MethodologyVersionId, methodologyVersionId); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs index 36285a33ca6..4c909e32735 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; @@ -32,6 +33,11 @@ public static Generator WithApprovalStatuses( return generator; } + public static Generator WithPublished( + this Generator generator, + DateTime published) + => generator.ForInstance(d => d.SetPublished(published)); + public static InstanceSetters SetDefaults(this InstanceSetters setters) => setters .SetDefault(p => p.Id) @@ -49,4 +55,9 @@ public static InstanceSetters SetApprovalStatus( this InstanceSetters setters, MethodologyApprovalStatus approvalStatus) => setters.Set(mv => mv.Status, approvalStatus); + + public static InstanceSetters SetPublished( + this InstanceSetters setters, + DateTime published) + => setters.Set(mv => mv.Published, published); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationRedirectGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationRedirectGeneratorExtensions.cs new file mode 100644 index 00000000000..2f002ddbbee --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationRedirectGeneratorExtensions.cs @@ -0,0 +1,49 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; +using System; + +namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; + +public static class PublicationRedirectGeneratorExtensions +{ + public static Generator DefaultPublicationRedirect(this DataFixture fixture) + => fixture.Generator().WithDefaults(); + + public static Generator WithDefaults(this Generator generator) + => generator.ForInstance(d => d.SetDefaults()); + + public static InstanceSetters SetDefaults(this InstanceSetters setters) + => setters + .SetDefault(p => p.Slug) + .SetDefault(p => p.Created); + + public static Generator WithSlug( + this Generator generator, + string slug) + => generator.ForInstance(s => s.SetSlug(slug)); + + public static Generator WithPublication( + this Generator generator, + Publication publication) + => generator.ForInstance(s => s.SetPublication(publication)); + + public static Generator WithPublicationId( + this Generator generator, + Guid publicationId) + => generator.ForInstance(s => s.SetPublicationId(publicationId)); + + public static InstanceSetters SetSlug( + this InstanceSetters setters, + string slug) + => setters.Set(pr => pr.Slug, slug); + + public static InstanceSetters SetPublication( + this InstanceSetters setters, + Publication publication) + => setters.Set(pr => pr.Publication, publication) + .SetPublicationId(publication.Id); + + public static InstanceSetters SetPublicationId( + this InstanceSetters setters, + Guid publicationId) + => setters.Set(pr => pr.PublicationId, publicationId); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseRedirectGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseRedirectGeneratorExtensions.cs new file mode 100644 index 00000000000..99d35b57b0c --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseRedirectGeneratorExtensions.cs @@ -0,0 +1,49 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; +using System; + +namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; + +public static class ReleaseRedirectGeneratorExtensions +{ + public static Generator DefaultReleaseRedirect(this DataFixture fixture) + => fixture.Generator().WithDefaults(); + + public static Generator WithDefaults(this Generator generator) + => generator.ForInstance(d => d.SetDefaults()); + + public static InstanceSetters SetDefaults(this InstanceSetters setters) + => setters + .SetDefault(p => p.Slug) + .SetDefault(p => p.Created); + + public static Generator WithSlug( + this Generator generator, + string slug) + => generator.ForInstance(s => s.SetSlug(slug)); + + public static Generator WithRelease( + this Generator generator, + Release release) + => generator.ForInstance(s => s.SetRelease(release)); + + public static Generator WithReleaseId( + this Generator generator, + Guid releaseId) + => generator.ForInstance(s => s.SetReleaseId(releaseId)); + + public static InstanceSetters SetSlug( + this InstanceSetters setters, + string slug) + => setters.Set(rr => rr.Slug, slug); + + public static InstanceSetters SetRelease( + this InstanceSetters setters, + Release release) + => setters.Set(rr => rr.Release, release) + .SetReleaseId(release.Id); + + public static InstanceSetters SetReleaseId( + this InstanceSetters setters, + Guid releaseId) + => setters.Set(rr => rr.ReleaseId, releaseId); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/RedirectsService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/RedirectsService.cs index 0588cfa70f7..10a392a8be3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/RedirectsService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/RedirectsService.cs @@ -64,8 +64,18 @@ public async Task> List() .Distinct() .ToList(); + var releaseRedirectViewModels = await _contentDbContext.ReleaseRedirects + .Where(rr => rr.Slug != rr.Release.Slug) // don't use redirects to the current live slug + .Distinct() + .Select(rr => new RedirectViewModel( + rr.Slug, + rr.Release.Slug + )) + .ToListAsync(); + return new RedirectsViewModel( publicationRedirectViewModels, - methodologyRedirectViewModels); + methodologyRedirectViewModels, + releaseRedirectViewModels); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/RedirectViewModels.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/RedirectViewModels.cs index b83b445e09a..e211a620400 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/RedirectViewModels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/RedirectViewModels.cs @@ -2,7 +2,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.ViewModels; public record RedirectsViewModel( List Publications, - List Methodologies); + List Methodologies, + List Releases); public record RedirectViewModel( string FromSlug, From 095cddea951293d9d30d1a4616ba688594e38194 Mon Sep 17 00:00:00 2001 From: "rian.thwaite" Date: Wed, 27 Nov 2024 13:05:01 +0000 Subject: [PATCH 027/144] EES-5149 remove check to hide hidden filter groups and fix tests appropriately --- .../__tests__/TimePeriodDataTable.test.tsx | 34 +- .../TimePeriodDataTable.test.tsx.snap | 73 ++++- .../utils/__tests__/mapTableToJson.test.ts | 303 ++++++++++++++++-- .../utils/__tests__/optimizeFilters.test.ts | 45 ++- .../table-tool/utils/optimizeFilters.ts | 10 +- 5 files changed, 392 insertions(+), 73 deletions(-) diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TimePeriodDataTable.test.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TimePeriodDataTable.test.tsx index 8660ec48b20..00d6402cd35 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TimePeriodDataTable.test.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/TimePeriodDataTable.test.tsx @@ -36,10 +36,10 @@ describe('TimePeriodDataTable', () => { const table = screen.getByRole('table'); - expect(table.querySelectorAll('thead tr')).toHaveLength(2); - expect(table.querySelectorAll('thead th')).toHaveLength(6); + expect(table.querySelectorAll('thead tr')).toHaveLength(3); + expect(table.querySelectorAll('thead th')).toHaveLength(7); expect(table.querySelectorAll('thead th[scope="colgroup"]')).toHaveLength( - 2, + 3, ); expect(table.querySelectorAll('thead th[scope="col"]')).toHaveLength(4); @@ -90,10 +90,10 @@ describe('TimePeriodDataTable', () => { const table = screen.getByRole('table'); - expect(table.querySelectorAll('thead tr')).toHaveLength(2); - expect(table.querySelectorAll('thead th')).toHaveLength(6); + expect(table.querySelectorAll('thead tr')).toHaveLength(3); + expect(table.querySelectorAll('thead th')).toHaveLength(7); expect(table.querySelectorAll('thead th[scope="colgroup"]')).toHaveLength( - 2, + 3, ); expect(table.querySelectorAll('thead th[scope="col"]')).toHaveLength(4); @@ -144,10 +144,10 @@ describe('TimePeriodDataTable', () => { const table = screen.getByRole('table'); - expect(table.querySelectorAll('thead tr')).toHaveLength(1); - expect(table.querySelectorAll('thead th')).toHaveLength(2); + expect(table.querySelectorAll('thead tr')).toHaveLength(2); + expect(table.querySelectorAll('thead th')).toHaveLength(3); expect(table.querySelectorAll('thead th[scope="colgroup"]')).toHaveLength( - 0, + 1, ); expect(table.querySelectorAll('thead th[scope="col"]')).toHaveLength(2); @@ -317,10 +317,10 @@ describe('TimePeriodDataTable', () => { const table = screen.getByRole('table'); - expect(table.querySelectorAll('thead tr')).toHaveLength(1); - expect(table.querySelectorAll('thead th')).toHaveLength(1); + expect(table.querySelectorAll('thead tr')).toHaveLength(2); + expect(table.querySelectorAll('thead th')).toHaveLength(2); expect(table.querySelectorAll('thead th[scope="colgroup"]')).toHaveLength( - 0, + 1, ); expect(table.querySelectorAll('thead th[scope="col"]')).toHaveLength(1); expect(table.querySelector('thead th[scope="col"]')).toHaveTextContent( @@ -1450,9 +1450,9 @@ describe('TimePeriodDataTable', () => { expect(table.querySelectorAll('thead th[scope="col"]')).toHaveLength(3); expect(table.querySelectorAll('tbody tr')).toHaveLength(6); - expect(table.querySelectorAll('tbody th')).toHaveLength(11); + expect(table.querySelectorAll('tbody th')).toHaveLength(13); expect(table.querySelectorAll('tbody th[scope="rowgroup"]')).toHaveLength( - 5, + 7, ); expect(table.querySelectorAll('tbody th[scope="row"]')).toHaveLength(6); @@ -1526,10 +1526,10 @@ describe('TimePeriodDataTable', () => { const table = screen.getByRole('table'); - expect(table.querySelectorAll('thead tr')).toHaveLength(2); - expect(table.querySelectorAll('thead th')).toHaveLength(6); + expect(table.querySelectorAll('thead tr')).toHaveLength(3); + expect(table.querySelectorAll('thead th')).toHaveLength(7); expect(table.querySelectorAll('thead th[scope="colgroup"]')).toHaveLength( - 2, + 3, ); expect(table.querySelectorAll('thead th[scope="col"]')).toHaveLength(4); diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/__snapshots__/TimePeriodDataTable.test.tsx.snap b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/__snapshots__/TimePeriodDataTable.test.tsx.snap index 34fb42825a1..3952e2f15c4 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/__snapshots__/TimePeriodDataTable.test.tsx.snap +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/__tests__/__snapshots__/TimePeriodDataTable.test.tsx.snap @@ -309,8 +309,17 @@ exports[`TimePeriodDataTable renders table with completely empty columns removed + + First language + + + Coventry + + First language + Croydon + + First language + + + Ethnic group major + + + + + Ethnic group major + + + + + Ethnic group major + + + + + Ethnic group major + + + { - test('returns the correct JSON for a table with one level of row and column headers', () => { + test.only('returns the correct JSON for a table with one level of row and column headers', () => { const result = mapTableToJson({ tableHeadersConfig: testTableWithOneLevelOfRowAndColHeadersConfig, subjectMeta: testTableWithOneLevelOfRowAndColHeaders.subjectMeta, @@ -66,7 +66,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ { - colSpan: 1, + colSpan: 2, rowSpan: 1, tag: 'td', }, @@ -82,6 +82,13 @@ describe('mapTableToJson', () => { expect(result.tbody).toEqual([ [ + { + rowSpan: 2, + colSpan: 1, + scope: 'rowgroup', + text: 'Region 1', + tag: 'th', + }, { rowSpan: 1, colSpan: 1, @@ -115,7 +122,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ { - colSpan: 2, + colSpan: 3, rowSpan: 1, tag: 'td', }, @@ -138,6 +145,13 @@ describe('mapTableToJson', () => { text: 'Indicator 1', tag: 'th', }, + { + rowSpan: 2, + colSpan: 1, + scope: 'rowgroup', + text: 'Region 1', + tag: 'th', + }, { rowSpan: 1, colSpan: 1, @@ -160,7 +174,7 @@ describe('mapTableToJson', () => { ]); }); - test('returns the correct JSON for a table with two levels of col headers and one level of row headers', () => { + test('returns the correct JSON for a table with three levels of col headers and one level of row headers', () => { const result = mapTableToJson({ tableHeadersConfig: testTableWithOneLevelOfRowsAndTwoLevelsOfColHeadersConfig, @@ -172,7 +186,7 @@ describe('mapTableToJson', () => { [ { colSpan: 1, - rowSpan: 2, + rowSpan: 3, tag: 'td', }, { @@ -183,6 +197,15 @@ describe('mapTableToJson', () => { tag: 'th', }, ], + [ + { + colSpan: 2, + rowSpan: 1, + scope: 'colgroup', + text: 'Region 1', + tag: 'th', + }, + ], [ { colSpan: 1, @@ -225,7 +248,16 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 2, rowSpan: 2, tag: 'td' }, + { colSpan: 2, rowSpan: 3, tag: 'td' }, + { + colSpan: 4, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Region 1', + }, + ], + [ { colSpan: 2, rowSpan: 1, scope: 'colgroup', tag: 'th', text: 'LA 1' }, { colSpan: 2, rowSpan: 1, scope: 'colgroup', tag: 'th', text: 'LA 2' }, ], @@ -307,7 +339,7 @@ describe('mapTableToJson', () => { ]); }); - test('returns the correct JSON for a table with three levels of row and column headers', () => { + test('returns the correct JSON for a table with three levels of row and four column headers', () => { const result = mapTableToJson({ tableHeadersConfig: testTableWithThreeLevelsOfRowAndColHeadersConfig, subjectMeta: testTableWithThreeLevelsOfRowAndColHeaders.subjectMeta, @@ -316,7 +348,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 3, rowSpan: 3, tag: 'td' }, + { colSpan: 3, rowSpan: 4, tag: 'td' }, { colSpan: 4, rowSpan: 1, @@ -332,6 +364,22 @@ describe('mapTableToJson', () => { text: 'Category 2 Filter 2', }, ], + [ + { + colSpan: 4, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + { + colSpan: 4, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + ], [ { colSpan: 2, @@ -1184,7 +1232,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 1, rowSpan: 2, tag: 'td' }, + { colSpan: 1, rowSpan: 3, tag: 'td' }, { colSpan: 2, rowSpan: 1, @@ -1200,6 +1248,22 @@ describe('mapTableToJson', () => { text: 'Category 1 Group 2', }, ], + [ + { + colSpan: 2, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + { + colSpan: 2, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + ], [ { colSpan: 1, @@ -1261,7 +1325,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 1, rowSpan: 3, tag: 'td' }, + { colSpan: 1, rowSpan: 4, tag: 'td' }, { colSpan: 2, rowSpan: 2, @@ -1286,6 +1350,22 @@ describe('mapTableToJson', () => { text: 'Category 1 Group 2 Filter 1', }, ], + [ + { + colSpan: 2, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + { + colSpan: 2, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + ], [ { colSpan: 1, @@ -1346,7 +1426,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 1, rowSpan: 3, tag: 'td' }, + { colSpan: 1, rowSpan: 4, tag: 'td' }, { colSpan: 4, rowSpan: 1, @@ -1371,6 +1451,22 @@ describe('mapTableToJson', () => { tag: 'th', }, ], + [ + { + colSpan: 2, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + { + colSpan: 2, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + ], [ { colSpan: 1, @@ -1444,7 +1540,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 1, rowSpan: 4, tag: 'td' }, + { colSpan: 1, rowSpan: 5, tag: 'td' }, { colSpan: 6, rowSpan: 1, @@ -1485,6 +1581,29 @@ describe('mapTableToJson', () => { tag: 'th', }, ], + [ + { + colSpan: 2, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + { + colSpan: 2, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + { + colSpan: 2, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + ], [ { colSpan: 1, @@ -1579,7 +1698,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 1, rowSpan: 3, tag: 'td' }, + { colSpan: 1, rowSpan: 4, tag: 'td' }, { colSpan: 4, rowSpan: 1, @@ -1588,6 +1707,15 @@ describe('mapTableToJson', () => { tag: 'th', }, ], + [ + { + colSpan: 4, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + ], [ { colSpan: 2, @@ -1677,7 +1805,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 1, rowSpan: 4, tag: 'td' }, + { colSpan: 1, rowSpan: 5, tag: 'td' }, { colSpan: 4, rowSpan: 1, @@ -1686,6 +1814,15 @@ describe('mapTableToJson', () => { tag: 'th', }, ], + [ + { + colSpan: 4, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + ], [ { colSpan: 2, @@ -1789,7 +1926,16 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 1, rowSpan: 3, tag: 'td' }, + { colSpan: 1, rowSpan: 4, tag: 'td' }, + { + colSpan: 3, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + ], + [ { colSpan: 2, rowSpan: 1, @@ -1866,8 +2012,8 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ { - colSpan: 1, - rowSpan: 4, + colSpan: 2, + rowSpan: 5, tag: 'td', }, { @@ -1878,6 +2024,15 @@ describe('mapTableToJson', () => { tag: 'th', }, ], + [ + { + colSpan: 3, + rowSpan: 1, + scope: 'colgroup', + tag: 'th', + text: 'Group 1', + }, + ], [ { colSpan: 2, @@ -1930,6 +2085,13 @@ describe('mapTableToJson', () => { expect(result.tbody).toEqual([ [ + { + colSpan: 1, + rowSpan: 2, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { text: 'Category 2 Group 1 Filter 1', rowSpan: 1, @@ -2084,7 +2246,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 2, rowSpan: 1, tag: 'td' }, + { colSpan: 3, rowSpan: 1, tag: 'td' }, { colSpan: 1, rowSpan: 1, @@ -2104,6 +2266,13 @@ describe('mapTableToJson', () => { tag: 'th', text: 'Category 1 Group 1', }, + { + colSpan: 1, + rowSpan: 2, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 1, @@ -2131,6 +2300,13 @@ describe('mapTableToJson', () => { tag: 'th', text: 'Category 1 Group 2', }, + { + colSpan: 1, + rowSpan: 2, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 1, @@ -2165,7 +2341,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 3, rowSpan: 1, tag: 'td' }, + { colSpan: 4, rowSpan: 1, tag: 'td' }, { colSpan: 1, rowSpan: 1, @@ -2185,6 +2361,13 @@ describe('mapTableToJson', () => { tag: 'th', text: 'Category 1 Group 1', }, + { + colSpan: 1, + rowSpan: 2, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 1, @@ -2219,6 +2402,13 @@ describe('mapTableToJson', () => { tag: 'th', text: 'Category 1 Group 2 Filter 1', }, + { + colSpan: 1, + rowSpan: 2, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 1, @@ -2252,7 +2442,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 3, rowSpan: 1, tag: 'td' }, + { colSpan: 4, rowSpan: 1, tag: 'td' }, { colSpan: 1, rowSpan: 1, @@ -2279,6 +2469,13 @@ describe('mapTableToJson', () => { tag: 'th', text: 'Category 1 Group 1', }, + { + colSpan: 1, + rowSpan: 2, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 1, @@ -2306,6 +2503,13 @@ describe('mapTableToJson', () => { tag: 'th', text: 'Category 1 Group 2', }, + { + colSpan: 1, + rowSpan: 2, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 1, @@ -2340,7 +2544,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 4, rowSpan: 1, tag: 'td' }, + { colSpan: 5, rowSpan: 1, tag: 'td' }, { colSpan: 1, rowSpan: 1, @@ -2367,6 +2571,13 @@ describe('mapTableToJson', () => { tag: 'th', text: 'Category 1 Group 1', }, + { + colSpan: 1, + rowSpan: 2, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 1, @@ -2401,6 +2612,13 @@ describe('mapTableToJson', () => { tag: 'th', text: 'Category 1 Group 2 Filter 1', }, + { + colSpan: 1, + rowSpan: 2, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 1, @@ -2434,7 +2652,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 3, rowSpan: 1, tag: 'td' }, + { colSpan: 4, rowSpan: 1, tag: 'td' }, { colSpan: 1, rowSpan: 1, @@ -2454,6 +2672,13 @@ describe('mapTableToJson', () => { tag: 'th', text: 'Indicator 1', }, + { + colSpan: 1, + rowSpan: 4, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 2, @@ -2522,7 +2747,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 4, rowSpan: 1, tag: 'td' }, + { colSpan: 5, rowSpan: 1, tag: 'td' }, { colSpan: 1, rowSpan: 1, @@ -2542,6 +2767,13 @@ describe('mapTableToJson', () => { tag: 'th', text: 'Indicator 1', }, + { + colSpan: 1, + rowSpan: 4, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 2, @@ -2622,7 +2854,7 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ - { colSpan: 3, rowSpan: 1, tag: 'td' }, + { colSpan: 4, rowSpan: 1, tag: 'td' }, { colSpan: 1, rowSpan: 1, @@ -2635,6 +2867,13 @@ describe('mapTableToJson', () => { expect(result.tbody).toEqual([ [ + { + colSpan: 1, + rowSpan: 3, + scope: 'rowgroup', + tag: 'th', + text: 'Group 1', + }, { colSpan: 1, rowSpan: 2, @@ -2699,11 +2938,16 @@ describe('mapTableToJson', () => { expect(result.thead).toEqual([ [ + { colSpan: 5, rowSpan: 2, tag: 'td' }, { - colSpan: 4, + colSpan: 2, rowSpan: 1, - tag: 'td', + scope: 'colgroup', + text: 'Group 1', + tag: 'th', }, + ], + [ { colSpan: 1, rowSpan: 1, @@ -2730,6 +2974,13 @@ describe('mapTableToJson', () => { colSpan: 1, tag: 'th', }, + { + text: 'Group 1', + rowSpan: 3, + scope: 'rowgroup', + colSpan: 1, + tag: 'th', + }, { text: 'Category 3 Group 1 Filter 1', rowSpan: 2, diff --git a/src/explore-education-statistics-common/src/modules/table-tool/utils/__tests__/optimizeFilters.test.ts b/src/explore-education-statistics-common/src/modules/table-tool/utils/__tests__/optimizeFilters.test.ts index c5a60c6bc21..49f85e14393 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/utils/__tests__/optimizeFilters.test.ts +++ b/src/explore-education-statistics-common/src/modules/table-tool/utils/__tests__/optimizeFilters.test.ts @@ -44,6 +44,12 @@ const testTimePeriod2 = new TimePeriodFilter({ code: 'AY', order: 1, }); +const testFilterGroup1 = new CategoryFilter({ + value: 'Filter Group 1 (level: 0)', + label: 'Filter Group 1', + group: undefined, + category: 'Category 1', +}); const testCategory1Group1Filter1 = new CategoryFilter({ value: 'filter-1', label: 'Filter 1', @@ -88,17 +94,6 @@ const testCategory3Group2Filter7 = new CategoryFilter({ }); describe('optimizeFilters', () => { - test('returns the filters unchanged when there are no groups to add or single headers to remove', () => { - const testFilters: Filter[] = [testLocationFilter1, testTimePeriod1]; - const testHeaderConfig: Filter[][] = [ - [testLocationFilter1, testLocationFilter2], - [testTimePeriod1, testTimePeriod2], - ]; - - const result = optimizeFilters(testFilters, testHeaderConfig); - expect(result).toEqual(testFilters); - }); - test('removes the last filter when there are zero filters in the last header array', () => { const testFilters: Filter[] = [testLocationFilter1, testTimePeriod1]; const testHeaderConfig: Filter[][] = [ @@ -107,7 +102,10 @@ describe('optimizeFilters', () => { ]; const result = optimizeFilters(testFilters, testHeaderConfig); - expect(result).toEqual([testLocationFilter1]); + expect(result).toEqual([ + new FilterGroup('North East', 0), + testLocationFilter1, + ]); }); test('removes the last filter when there is only one filter in the last header array', () => { @@ -118,7 +116,10 @@ describe('optimizeFilters', () => { ]; const result = optimizeFilters(testFilters, testHeaderConfig); - expect(result).toEqual([testLocationFilter1]); + expect(result).toEqual([ + new FilterGroup('North East', 0), + testLocationFilter1, + ]); }); test('adds FilterGroup when groups with different labels in 1 level are not `Default`', () => { @@ -249,7 +250,11 @@ describe('optimizeFilters', () => { ]; const result = optimizeFilters(testFilters, testHeaderConfig); - expect(result).toEqual([testCategory1Group1Filter1, testTimePeriod1]); + expect(result).toEqual([ + new FilterGroup('Filter Group 1', 0), + testCategory1Group1Filter1, + testTimePeriod1, + ]); }); test('does not add FilterGroups when groups across 1 level have same labels that are not `Default`', () => { @@ -261,7 +266,11 @@ describe('optimizeFilters', () => { ]; const result = optimizeFilters(testFilters, testHeaderConfig); - expect(result).toEqual([testCategory1Group1Filter1, testTimePeriod1]); + expect(result).toEqual([ + new FilterGroup('Filter Group 1', 0), + testCategory1Group1Filter1, + testTimePeriod1, + ]); }); test('does not add FilterGroups when groups across 2 adjacent levels have same labels that are not `Default`', () => { @@ -280,7 +289,9 @@ describe('optimizeFilters', () => { const result = optimizeFilters(testFilters, testHeaderConfig); expect(result).toEqual([ + new FilterGroup('Filter Group 1', 0), testCategory1Group1Filter2, + new FilterGroup('Filter Group 2', 1), testCategory3Group2Filter7, testTimePeriod2, ]); @@ -288,6 +299,7 @@ describe('optimizeFilters', () => { test('does not add FilterGroups when groups across 2 non-adjacent levels have same labels that are not `Default`', () => { const testFilters: Filter[] = [ + testFilterGroup1, testCategory1Group1Filter1, testTimePeriod1, testCategory3Group1Filter5, @@ -302,8 +314,11 @@ describe('optimizeFilters', () => { const result = optimizeFilters(testFilters, testHeaderConfig); expect(result).toEqual([ + testFilterGroup1, + new FilterGroup('Filter Group 1', 1), testCategory1Group1Filter1, testTimePeriod1, + new FilterGroup('Filter Group 1', 3), testCategory3Group1Filter5, ]); }); diff --git a/src/explore-education-statistics-common/src/modules/table-tool/utils/optimizeFilters.ts b/src/explore-education-statistics-common/src/modules/table-tool/utils/optimizeFilters.ts index 497678bda31..bf2e92d9243 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/utils/optimizeFilters.ts +++ b/src/explore-education-statistics-common/src/modules/table-tool/utils/optimizeFilters.ts @@ -35,15 +35,7 @@ export default function optimizeFilters( // Add additional filter groups to our filters if required. return optimizedFilters.flatMap((filter, index) => { - const firstSubGroup = headerConfig[index][0].group; - - // Don't bother showing a single group as this adds - // additional groups to a potentially crowded table. - const hasMultipleGroups = headerConfig[index].some( - header => header.group !== firstSubGroup, - ); - - return filter.group && filter.group !== 'Default' && hasMultipleGroups + return filter.group && filter.group !== 'Default' ? [new FilterGroup(filter.group, index), filter] : filter; }); From d6848e5c3bb065903a66c5cd6f4aabe99dba6e0a Mon Sep 17 00:00:00 2001 From: Stuart Bennett Date: Wed, 27 Nov 2024 15:42:11 +0000 Subject: [PATCH 028/144] EES-5541 tiny test fixes --- .../components/chart/__tests__/ChartBuilder.test.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBuilder.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBuilder.test.tsx index 4a282044a9c..1742849f54d 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBuilder.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartBuilder.test.tsx @@ -448,7 +448,7 @@ describe('ChartBuilder', () => { expect(handleUpdate).toHaveBeenCalledWith({ boundaryLevel: 1 }); }); - describe('data gorupings tab', () => { + describe('data groupings tab', () => { const testInitialChart: Chart = { type: 'map', boundaryLevel: 2, @@ -557,10 +557,11 @@ describe('ChartBuilder', () => { symbol: undefined, }, ], + position: 'bottom', }, }; - test('save chart with updates data groupings', async () => { + test('save chart with updated data groupings', async () => { const handleSubmit = jest.fn(); const { user } = render( From 28ea550de1df47ba101555874e05c64d9f61ee5d Mon Sep 17 00:00:00 2001 From: dfe-sdt Date: Thu, 28 Nov 2024 09:45:37 +0000 Subject: [PATCH 029/144] chore(tests): update test snapshots --- .../tests/snapshots/data_catalogue_snapshot.json | 2 +- .../tests/snapshots/find_statistics_snapshot.json | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json index 37fe8803cbe..71e1e83f697 100644 --- a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json +++ b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json @@ -1,5 +1,5 @@ { - "num_datasets": "947 data sets", + "num_datasets": "935 data sets", "themes": [ { "publications": [ diff --git a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json index 76a5794fb4f..fb0ac901e0c 100644 --- a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json +++ b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json @@ -9,7 +9,7 @@ { "publication_summary": "Attainment and retention data for A level and other qualifications by age 16-18 in England. Includes region, institution type, characteristics, and subject.", "publication_title": "A level and other 16 to 18 results", - "published": "18 Apr 2024", + "published": "28 Nov 2024", "release_type": "Accredited official statistics", "theme": "School and college outcomes and performance" }, @@ -30,7 +30,7 @@ { "publication_summary": "Apprenticeship starts, achievements and participation. Includes breakdowns by age, sex, ethnicity, subject, provider, geography etc.", "publication_title": "Apprenticeships", - "published": "10 Oct 2024", + "published": "28 Nov 2024", "release_type": "Accredited official statistics", "theme": "Further education" }, @@ -163,7 +163,7 @@ { "publication_summary": "Annual statistics on early years foundation stage profile assessments in England relating to the 7 areas of learning and the 17 early learning goals.", "publication_title": "Early years foundation stage profile results", - "published": "30 Nov 2023", + "published": "28 Nov 2024", "release_type": "Accredited official statistics", "theme": "Early years" }, @@ -177,7 +177,7 @@ { "publication_summary": "Annual data release on schools, pupils, teachers, qualifications gained, education expenditure, further education and higher education in the UK.", "publication_title": "Education and training statistics for the UK", - "published": "9 Nov 2023", + "published": "28 Nov 2024", "release_type": "Accredited official statistics", "theme": "UK education and training statistics" }, @@ -247,14 +247,14 @@ { "publication_summary": "Apprenticeship and adult 19+ Further Education and skills learning, including community learning, education and training, and basic skills (English and maths).", "publication_title": "Further education and skills", - "published": "9 Sep 2024", + "published": "28 Nov 2024", "release_type": "Accredited official statistics", "theme": "Further education" }, { "publication_summary": "Sustained employment and learning destinations, and earnings of further education learners in England. Annual statistics with demographic and other breakdowns.", "publication_title": "Further education outcomes", - "published": "30 Nov 2023", + "published": "28 Nov 2024", "release_type": "Official statistics", "theme": "Destination of pupils and students" }, From 7b30fd3cfa597513b175faa89eacdf248d4c15a1 Mon Sep 17 00:00:00 2001 From: "rian.thwaite" Date: Thu, 28 Nov 2024 09:59:59 +0000 Subject: [PATCH 030/144] remove .only from test --- .../modules/table-tool/utils/__tests__/mapTableToJson.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explore-education-statistics-common/src/modules/table-tool/utils/__tests__/mapTableToJson.test.ts b/src/explore-education-statistics-common/src/modules/table-tool/utils/__tests__/mapTableToJson.test.ts index 6ced952f44e..df7e0ed0bcf 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/utils/__tests__/mapTableToJson.test.ts +++ b/src/explore-education-statistics-common/src/modules/table-tool/utils/__tests__/mapTableToJson.test.ts @@ -56,7 +56,7 @@ import { } from '@common/modules/table-tool/utils/__data__/testTableDataWithDuplicateLabels'; describe('mapTableToJson', () => { - test.only('returns the correct JSON for a table with one level of row and column headers', () => { + test('returns the correct JSON for a table with one level of row and column headers', () => { const result = mapTableToJson({ tableHeadersConfig: testTableWithOneLevelOfRowAndColHeadersConfig, subjectMeta: testTableWithOneLevelOfRowAndColHeaders.subjectMeta, From d90d6f35a7805e657b9d2666cb7a3c497b3ce652 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 27 Nov 2024 13:12:47 +0000 Subject: [PATCH 031/144] EES-5694 - updated order of operations in changing ReleaseStatus value to mirror that of updating MethodologyStatus, where unused images are cleared up at the beginning of the process. Enabled the "forceDelete" option in RemoveUnsedImages() methods in case of future reordering. Added in missing database permission updates from EES-5631, that were preventing manual testing of this fix --- .../MethodologyApprovalServiceTests.cs | 8 +- .../Services/ReleaseApprovalServiceTests.cs | 48 +- ...ssToReleasesAndRedirectsTables.Designer.cs | 2233 +++++++++++++++++ ...ntentAccessToReleasesAndRedirectsTables.cs | 26 + .../MethodologyApprovalService.cs | 5 +- .../Services/ReleaseApprovalService.cs | 13 +- 6 files changed, 2308 insertions(+), 25 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241127130458_EES5631_GrantPublisherAndContentAccessToReleasesAndRedirectsTables.Designer.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241127130458_EES5631_GrantPublisherAndContentAccessToReleasesAndRedirectsTables.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs index bf25fb5f09f..0105556c036 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs @@ -204,12 +204,12 @@ public async Task UpdateApprovalStatus_ApprovingMethodologyWithUnusedImages() mock.GetContentBlocks(methodologyVersion.Id)) .ReturnsAsync(new List()); - imageService.Setup(mock => - mock.Delete(methodologyVersion.Id, new List + imageService.Setup(mock => mock.Delete( + methodologyVersion.Id, new List { imageFile1.File.Id, imageFile2.File.Id - }, false)) + }, true)) .ReturnsAsync(Unit.Instance); methodologyVersionRepository.Setup(mock => @@ -232,7 +232,7 @@ public async Task UpdateApprovalStatus_ApprovingMethodologyWithUnusedImages() { imageFile1.File.Id, imageFile2.File.Id - }, false), Times.Once); + }, true), Times.Once); VerifyAllMocks(contentService, imageService, methodologyVersionRepository); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseApprovalServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseApprovalServiceTests.cs index 9ce3ff4c514..8b59a627bbb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseApprovalServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseApprovalServiceTests.cs @@ -224,24 +224,30 @@ public async Task CreateReleaseStatus_Approved_FailsOnChecklistErrors() await context.SaveChangesAsync(); } + var contentService = new Mock(MockBehavior.Strict); var releaseChecklistService = new Mock(MockBehavior.Strict); + contentService.Setup(mock => + mock.GetContentBlocks(releaseVersion.Id)) + .ReturnsAsync(new List()); + + releaseChecklistService + .Setup(s => + s.GetErrors(It.Is(rv => rv.Id == releaseVersion.Id))) + .ReturnsAsync( + new List + { + new(DataFileImportsMustBeCompleted), + new(DataFileReplacementsMustBeCompleted) + } + ); + await using (var context = InMemoryApplicationDbContext(contextId)) { - releaseChecklistService - .Setup(s => - s.GetErrors(It.Is(rv => rv.Id == releaseVersion.Id))) - .ReturnsAsync( - new List - { - new(DataFileImportsMustBeCompleted), - new(DataFileReplacementsMustBeCompleted) - } - ); - var releaseService = BuildService( context, - releaseChecklistService: releaseChecklistService.Object); + releaseChecklistService: releaseChecklistService.Object, + contentService: contentService.Object); var result = await releaseService .CreateReleaseStatus( @@ -260,7 +266,7 @@ public async Task CreateReleaseStatus_Approved_FailsOnChecklistErrors() } ); - VerifyAllMocks(releaseChecklistService); + VerifyAllMocks(releaseChecklistService, contentService); result.AssertBadRequest( DataFileImportsMustBeCompleted, @@ -764,10 +770,18 @@ public async Task CreateReleaseStatus_Approved_Scheduled_FailsChangingToDraft() context.ReleaseVersions.Add(releaseVersion); await context.SaveChangesAsync(); } + + var contentService = new Mock(MockBehavior.Strict); + + contentService.Setup(mock => + mock.GetContentBlocks(releaseVersion.Id)) + .ReturnsAsync(new List()); await using (var context = InMemoryApplicationDbContext(contextId)) { - var releaseService = BuildService(context); + var releaseService = BuildService( + contentDbContext: context, + contentService: contentService.Object); var result = await releaseService .CreateReleaseStatus( @@ -777,6 +791,8 @@ public async Task CreateReleaseStatus_Approved_Scheduled_FailsChangingToDraft() ApprovalStatus = ReleaseApprovalStatus.Draft, } ); + + VerifyAllMocks(contentService); result.AssertBadRequest(PublishedReleaseCannotBeUnapproved); } @@ -1382,7 +1398,7 @@ public async Task CreateReleaseStatus_ReleaseHasUnusedImages() imageFile1.File.Id, imageFile2.File.Id }, - false)) + true)) .ReturnsAsync(Unit.Instance); userReleaseRoleService.Setup(mock => @@ -1422,7 +1438,7 @@ public async Task CreateReleaseStatus_ReleaseHasUnusedImages() imageFile1.File.Id, imageFile2.File.Id }, - false), + true), Times.Once); VerifyAllMocks(contentService, releaseFileService, userReleaseRoleService); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241127130458_EES5631_GrantPublisherAndContentAccessToReleasesAndRedirectsTables.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241127130458_EES5631_GrantPublisherAndContentAccessToReleasesAndRedirectsTables.Designer.cs new file mode 100644 index 00000000000..33ef522e74d --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241127130458_EES5631_GrantPublisherAndContentAccessToReleasesAndRedirectsTables.Designer.cs @@ -0,0 +1,2233 @@ +// +using System; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + [DbContext(typeof(ContentDbContext))] + [Migration("20241127130458_EES5631_GrantPublisherAndContentAccessToReleasesAndRedirectsTables")] + partial class EES5631_GrantPublisherAccessToReleasesAndReleaseRedirectsTables + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContactTelNo") + .HasColumnType("nvarchar(max)"); + + b.Property("TeamEmail") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TeamName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Contacts"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.FreeTextRank", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Rank") + .HasColumnType("int"); + + b.ToTable((string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("LegacyCreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("Resolved") + .HasColumnType("datetime2"); + + b.Property("ResolvedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ResolvedById"); + + b.ToTable("Comment"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentSectionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Locked") + .HasColumnType("datetime2"); + + b.Property("LockedById") + .IsConcurrencyToken() + .HasColumnType("uniqueidentifier"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentSectionId"); + + b.HasIndex("LockedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentBlock", (string)null); + + b.HasDiscriminator("Type").HasValue("ContentBlock"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Caption") + .HasColumnType("nvarchar(max)"); + + b.Property("Heading") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentSections"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestDraftVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("LatestDraftVersionId") + .IsUnique() + .HasFilter("[LatestDraftVersionId] IS NOT NULL"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("DataBlocks", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier") + .HasColumnName("DataBlockId"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("DataBlockVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("ExpectedImportedRows") + .HasColumnType("int"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("GeographicLevels") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ImportedRows") + .HasColumnType("int"); + + b.Property("LastProcessedRowIndex") + .HasColumnType("int"); + + b.Property("MetaFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("StagePercentageComplete") + .HasColumnType("int"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("TotalRows") + .HasColumnType("int"); + + b.Property("ZipFileId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId") + .IsUnique(); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("FileId"), new[] { "Status" }); + + b.HasIndex("MetaFileId") + .IsUnique(); + + b.HasIndex("ZipFileId"); + + b.ToTable("DataImports"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataImportId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("DataImportId"); + + b.ToTable("DataImportErrors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("EmbedBlocks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DataBlockId") + .IsUnique(); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("FeaturedTables"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentLength") + .HasColumnType("bigint"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileMeta") + .HasColumnType("nvarchar(max)"); + + b.Property("DataSetFileVersion") + .HasColumnType("int"); + + b.Property("Filename") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReplacedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReplacingId") + .HasColumnType("uniqueidentifier"); + + b.Property("RootPath") + .HasColumnType("uniqueidentifier"); + + b.Property("SourceId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReplacedById") + .IsUnique() + .HasFilter("[ReplacedById] IS NOT NULL"); + + b.HasIndex("ReplacingId") + .IsUnique() + .HasFilter("[ReplacingId] IS NOT NULL"); + + b.HasIndex("SourceId"); + + b.HasIndex("Type"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.ToTable("GlossaryEntries"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("GuidanceText") + .HasColumnType("nvarchar(max)"); + + b.Property("GuidanceTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Trend") + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("KeyStatistics"); + + b.UseTptMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("OwningPublicationSlug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OwningPublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("Methodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DisplayDate") + .HasColumnType("datetime2"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("MethodologyNotes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("MethodologyVersionId", "Slug"); + + b.ToTable("MethodologyRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("AlternativeSlug") + .HasColumnType("nvarchar(max)"); + + b.Property("AlternativeTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("PublishingStrategy") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ScheduledWithReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyId"); + + b.HasIndex("PreviousVersionId"); + + b.HasIndex("ScheduledWithReleaseVersionId"); + + b.ToTable("MethodologyVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Annexes") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("MethodologyVersionId"); + + b.ToTable("MethodologyVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Permalink", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataSetTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MigratedFromLegacy") + .HasColumnType("bit"); + + b.Property("PublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("SubjectId"); + + b.ToTable("Permalinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseSeries") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .IsRequired() + .HasMaxLength(160) + .HasColumnType("nvarchar(160)"); + + b.Property("SupersededById") + .HasColumnType("uniqueidentifier"); + + b.Property("ThemeId") + .HasColumnType("uniqueidentifier"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContactId"); + + b.HasIndex("LatestPublishedReleaseVersionId") + .IsUnique() + .HasFilter("[LatestPublishedReleaseVersionId] IS NOT NULL"); + + b.HasIndex("SupersededById"); + + b.HasIndex("ThemeId"); + + b.ToTable("Publications"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("Owner") + .HasColumnType("bit"); + + b.HasKey("PublicationId", "MethodologyId"); + + b.HasIndex("MethodologyId"); + + b.ToTable("PublicationMethodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("PublicationId", "Slug"); + + b.ToTable("PublicationRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("nvarchar(30)"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("nvarchar(5)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Year") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("PublicationId", "Year", "TimePeriodCoverage") + .IsUnique(); + + b.ToTable("Releases"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("FilterSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("IndicatorSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("PublicApiDataSetId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicApiDataSetVersion") + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("ReleaseVersionId", "FileId") + .IsUnique(); + + b.HasIndex("ReleaseVersionId", "PublicApiDataSetId", "PublicApiDataSetVersion") + .IsUnique() + .HasFilter("[PublicApiDataSetId] IS NOT NULL AND [PublicApiDataSetVersion] IS NOT NULL"); + + b.ToTable("ReleaseFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseRedirect", b => + { + b.Property("ReleaseId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("ReleaseId", "Slug"); + + b.ToTable("ReleaseRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("ReleaseStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataGuidance") + .HasColumnType("nvarchar(max)"); + + b.Property("NextReleaseDate") + .HasColumnType("nvarchar(max)"); + + b.Property("NotifiedOn") + .HasColumnType("datetime2"); + + b.Property("NotifySubscribers") + .HasColumnType("bit"); + + b.Property("PreReleaseAccessList") + .HasColumnType("nvarchar(max)"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublishScheduled") + .HasColumnType("datetime2"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("RelatedInformation") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseName") + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(6) + .HasColumnType("nvarchar(6)"); + + b.Property("Type") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("UpdatePublishedDate") + .HasColumnType("bit"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("ReleaseId"); + + b.HasIndex("Type"); + + b.HasIndex("PreviousVersionId", "Version"); + + b.ToTable("ReleaseVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Themes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("On") + .HasColumnType("datetime2"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("Update"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstName") + .HasColumnType("nvarchar(max)"); + + b.Property("LastName") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("DeletedById"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.ToTable("UserPublicationInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("UserId"); + + b.ToTable("UserPublicationRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EmailSent") + .HasColumnType("bit"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("UserReleaseInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UserId"); + + b.ToTable("UserReleaseRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Charts") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Charts"); + + b.Property("Heading") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Heading"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Query") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Query"); + + b.Property("Source") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Table") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Table"); + + b.HasDiscriminator().HasValue("DataBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("EmbedBlockId") + .HasColumnType("uniqueidentifier") + .HasColumnName("EmbedBlockId"); + + b.HasIndex("EmbedBlockId") + .IsUnique() + .HasFilter("[EmbedBlockId] IS NOT NULL"); + + b.HasDiscriminator().HasValue("EmbedBlockLink"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.HtmlBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("HtmlBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MarkDownBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("MarkDownBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.HasIndex("DataBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.ToTable("KeyStatisticsDataBlock", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("Statistic") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.ToTable("KeyStatisticsText", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", "ContentBlock") + .WithMany("Comments") + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "ResolvedBy") + .WithMany() + .HasForeignKey("ResolvedById"); + + b.Navigation("ContentBlock"); + + b.Navigation("CreatedBy"); + + b.Navigation("ResolvedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", "ContentSection") + .WithMany("Content") + .HasForeignKey("ContentSectionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "LockedBy") + .WithMany() + .HasForeignKey("LockedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentSection"); + + b.Navigation("LockedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Content") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestDraftVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestDraftVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestPublishedVersionId"); + + b.Navigation("LatestDraftVersion"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "ContentBlock") + .WithMany() + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("DataBlockVersions") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "FileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "MetaFile") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "MetaFileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ZipFile") + .WithMany() + .HasForeignKey("ZipFileId") + .OnDelete(DeleteBehavior.Restrict); + + b.Navigation("File"); + + b.Navigation("MetaFile"); + + b.Navigation("ZipFile"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "DataImport") + .WithMany("Errors") + .HasForeignKey("DataImportId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataImport"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", "DataBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("FeaturedTables") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedBy") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Replacing") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacingId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Source") + .WithMany() + .HasForeignKey("SourceId"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReplacedBy"); + + b.Navigation("Replacing"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("CreatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("KeyStatistics") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "LatestPublishedVersionId"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany("Notes") + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Versions") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ScheduledWithReleaseVersion") + .WithMany() + .HasForeignKey("ScheduledWithReleaseVersionId"); + + b.Navigation("CreatedBy"); + + b.Navigation("Methodology"); + + b.Navigation("PreviousVersion"); + + b.Navigation("ScheduledWithReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", null) + .WithOne("MethodologyContent") + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", "MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", "Contact") + .WithMany() + .HasForeignKey("ContactId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "LatestPublishedReleaseVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "LatestPublishedReleaseVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "SupersededBy") + .WithMany() + .HasForeignKey("SupersededById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", "Theme") + .WithMany("Publications") + .HasForeignKey("ThemeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ExternalMethodology", "ExternalMethodology", b1 => + { + b1.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b1.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.HasKey("PublicationId"); + + b1.ToTable("ExternalMethodology", (string)null); + + b1.WithOwner() + .HasForeignKey("PublicationId"); + }); + + b.Navigation("Contact"); + + b.Navigation("ExternalMethodology"); + + b.Navigation("LatestPublishedReleaseVersion"); + + b.Navigation("SupersededBy"); + + b.Navigation("Theme"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Publications") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Methodologies") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Methodology"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Releases") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") + .WithMany() + .HasForeignKey("ReleaseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Release"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("ReleaseStatuses") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("ReleaseVersions") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") + .WithMany("Versions") + .HasForeignKey("ReleaseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("PreviousVersion"); + + b.Navigation("Publication"); + + b.Navigation("Release"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Updates") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.User", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.Navigation("DeletedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("Publication"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", "EmbedBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", "EmbedBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("EmbedBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithMany() + .HasForeignKey("DataBlockId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Navigation("Comments"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Navigation("Content"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Navigation("Errors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Navigation("Publications"); + + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Navigation("MethodologyContent") + .IsRequired(); + + b.Navigation("Notes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Navigation("Methodologies"); + + b.Navigation("ReleaseVersions"); + + b.Navigation("Releases"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Navigation("Content"); + + b.Navigation("DataBlockVersions"); + + b.Navigation("FeaturedTables"); + + b.Navigation("KeyStatistics"); + + b.Navigation("ReleaseStatuses"); + + b.Navigation("Updates"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Navigation("Publications"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241127130458_EES5631_GrantPublisherAndContentAccessToReleasesAndRedirectsTables.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241127130458_EES5631_GrantPublisherAndContentAccessToReleasesAndRedirectsTables.cs new file mode 100644 index 00000000000..d8298fde338 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241127130458_EES5631_GrantPublisherAndContentAccessToReleasesAndRedirectsTables.cs @@ -0,0 +1,26 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + /// + public partial class EES5631_GrantPublisherAccessToReleasesAndReleaseRedirectsTables : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("GRANT SELECT ON dbo.Releases TO [content]"); + migrationBuilder.Sql("GRANT SELECT ON dbo.ReleaseRedirects TO [content]"); + + migrationBuilder.Sql("GRANT SELECT ON dbo.Releases TO [publisher]"); + migrationBuilder.Sql("GRANT SELECT ON dbo.ReleaseRedirects TO [publisher]"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Methodologies/MethodologyApprovalService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Methodologies/MethodologyApprovalService.cs index a36787b1f67..ff00aa516ad 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Methodologies/MethodologyApprovalService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Methodologies/MethodologyApprovalService.cs @@ -243,7 +243,10 @@ private async Task> RemoveUnusedImages(MethodologyVer if (unusedImages.Any()) { - return await _methodologyImageService.Delete(methodologyVersion.Id, unusedImages); + return await _methodologyImageService.Delete( + methodologyVersionId: methodologyVersion.Id, + fileIds: unusedImages, + forceDelete: true); } return Unit.Instance; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseApprovalService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseApprovalService.cs index 0ca79a36f1b..d5bf68a2d85 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseApprovalService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseApprovalService.cs @@ -110,6 +110,7 @@ public async Task> CreateReleaseStatus( .OnSuccessDo(releaseVersion => _userService.CheckCanUpdateReleaseVersionStatus(releaseVersion, request.ApprovalStatus)) .OnSuccessDo(() => ValidatePublishDate(request)) + .OnSuccessDo(() => RemoveUnusedImages(releaseVersionId)) .OnSuccess(async releaseVersion => { if (request.ApprovalStatus != ReleaseApprovalStatus.Approved && releaseVersion.Published.HasValue) @@ -141,7 +142,6 @@ public async Task> CreateReleaseStatus( return await ValidateReleaseWithChecklist(releaseVersion) - .OnSuccess(() => RemoveUnusedImages(releaseVersion.Id)) .OnSuccess(() => SendEmailNotificationsAndInvites(request, releaseVersion) .OnSuccess(() => NotifyPublisher(releasePublishingKey, request, oldStatus)) @@ -247,14 +247,16 @@ private async Task> SendPreReleaseUserInviteEmails(Re private async Task> RemoveUnusedImages(Guid releaseVersionId) { - return await _contentService.GetContentBlocks(releaseVersionId) + return await _contentService + .GetContentBlocks(releaseVersionId) .OnSuccess(async contentBlocks => { var contentImageIds = contentBlocks.SelectMany(contentBlock => HtmlImageUtil.GetReleaseImages(contentBlock.Body)) .Distinct(); - var imageFiles = await _releaseFileRepository.GetByFileType(releaseVersionId, types: FileType.Image); + var imageFiles = await _releaseFileRepository + .GetByFileType(releaseVersionId, types: FileType.Image); var unusedImages = imageFiles .Where(file => !contentImageIds.Contains(file.File.Id)) @@ -263,7 +265,10 @@ private async Task> RemoveUnusedImages(Guid releaseVe if (unusedImages.Any()) { - return await _releaseFileService.Delete(releaseVersionId, unusedImages); + return await _releaseFileService.Delete( + releaseVersionId: releaseVersionId, + fileIds: unusedImages, + forceDelete: true); } return Unit.Instance; From 3e12cd098c4c65a38866f4f853ded6f46f370b23 Mon Sep 17 00:00:00 2001 From: dfe-sdt Date: Thu, 28 Nov 2024 13:06:42 +0000 Subject: [PATCH 032/144] chore(tests): update test snapshots --- .../tests/snapshots/data_catalogue_snapshot.json | 2 +- .../tests/snapshots/find_statistics_snapshot.json | 7 ------- tests/robot-tests/tests/snapshots/table_tool_snapshot.json | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json index 71e1e83f697..eeeed40583d 100644 --- a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json +++ b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json @@ -1,5 +1,5 @@ { - "num_datasets": "935 data sets", + "num_datasets": "926 data sets", "themes": [ { "publications": [ diff --git a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json index fb0ac901e0c..45ca669aec5 100644 --- a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json +++ b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json @@ -300,13 +300,6 @@ "release_type": "Official statistics", "theme": "Teachers and school workforce" }, - { - "publication_summary": "Phonics screening check and key stage 1 attainment of pupils in England at national and LA level. This release includes pupil characteristic breakdowns.", - "publication_title": "Key stage 1 and phonics screening check attainment", - "published": "12 Oct 2023", - "release_type": "Accredited official statistics", - "theme": "School and college outcomes and performance" - }, { "publication_summary": "Key stage 2 attainment statistics by pupil characteristics, school characteristics, region and local authority.", "publication_title": "Key stage 2 attainment", diff --git a/tests/robot-tests/tests/snapshots/table_tool_snapshot.json b/tests/robot-tests/tests/snapshots/table_tool_snapshot.json index 7588be14b09..b1f29abb160 100644 --- a/tests/robot-tests/tests/snapshots/table_tool_snapshot.json +++ b/tests/robot-tests/tests/snapshots/table_tool_snapshot.json @@ -126,7 +126,6 @@ { "publications": [ "A level and other 16 to 18 results", - "Key stage 1 and phonics screening check attainment", "Key stage 2 attainment", "Key stage 2 attainment: National headlines", "Key stage 4 performance", From 3a9f4508fd541edcb31af0f1cf88955aed1529a7 Mon Sep 17 00:00:00 2001 From: "rian.thwaite" Date: Thu, 28 Nov 2024 16:18:02 +0000 Subject: [PATCH 033/144] EES-5695 Update schools homepage link --- .../src/pages/index.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/explore-education-statistics-frontend/src/pages/index.tsx b/src/explore-education-statistics-frontend/src/pages/index.tsx index 597a1b1b5f0..370f5e492b0 100644 --- a/src/explore-education-statistics-frontend/src/pages/index.tsx +++ b/src/explore-education-statistics-frontend/src/pages/index.tsx @@ -142,13 +142,12 @@ function HomePage() { educational organisations and governors in England.

    - - Schools financial benchmarking + + Financial Benchmarking and Insights Tool

    - Compare your school's income and expenditure with other schools - in England. + Compare your school's expenditure with other schools in England.

    From db0f04f4588df62567ed6aca07df9c06be3e8070 Mon Sep 17 00:00:00 2001 From: "rian.thwaite" Date: Mon, 2 Dec 2024 09:43:00 +0000 Subject: [PATCH 034/144] EES-5695 run format --- src/explore-education-statistics-frontend/src/pages/index.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/explore-education-statistics-frontend/src/pages/index.tsx b/src/explore-education-statistics-frontend/src/pages/index.tsx index 370f5e492b0..deccc97b267 100644 --- a/src/explore-education-statistics-frontend/src/pages/index.tsx +++ b/src/explore-education-statistics-frontend/src/pages/index.tsx @@ -147,7 +147,8 @@ function HomePage() {

    - Compare your school's expenditure with other schools in England. + Compare your school's expenditure with other schools in + England.

    From ccbf93f6cf3e751cb219fcfbf22a1dbae47756f3 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 25 Nov 2024 15:21:09 +0000 Subject: [PATCH 035/144] EES-5564 - added error handling when fetching data set details of changelog results --- .../data/ReleaseApiDataSetChangelogPage.tsx | 92 ++++++++++--------- .../ReleaseApiDataSetChangelogPage.test.tsx | 30 ++++++ 2 files changed, 81 insertions(+), 41 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetChangelogPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetChangelogPage.tsx index 142aea148d6..3b7dd9ca93f 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetChangelogPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetChangelogPage.tsx @@ -19,6 +19,7 @@ import Tag from '@common/components/Tag'; import { useQuery } from '@tanstack/react-query'; import { generatePath, useParams } from 'react-router-dom'; import React, { useEffect } from 'react'; +import WarningMessage from '@common/components/WarningMessage'; export default function ReleaseApiDataSetChangelogPage() { const { dataSetId, dataSetVersionId, releaseId, publicationId } = @@ -28,11 +29,14 @@ export default function ReleaseApiDataSetChangelogPage() { data: dataSet, isLoading: isLoadingDataSet, refetch: refetchDataSet, + isError: errorFetchingDataSet, } = useQuery(apiDataSetQueries.get(dataSetId)); - const { data: changes, isLoading: isLoadingChanges } = useQuery( - apiDataSetVersionQueries.getChanges(dataSetVersionId), - ); + const { + data: changes, + isLoading: isLoadingChanges, + isError: errorFetchingChanges, + } = useQuery(apiDataSetVersionQueries.getChanges(dataSetVersionId)); const isDraft = dataSet?.draftVersion?.id === dataSetVersionId; @@ -74,49 +78,55 @@ export default function ReleaseApiDataSetChangelogPage() { > Back to API data set details + -
    -
    - API data set changelog -

    {dataSet?.title}

    -
    -
    + {!errorFetchingDataSet && !errorFetchingChanges ? ( + <> +
    +
    + API data set changelog +

    {dataSet?.title}

    +
    +
    + + {`${ + isDraft ? 'Draft' : 'Published' + } v${dataSetVersion?.version}`} + {`${dataSetVersion?.type} update`} + - - {`${ - isDraft ? 'Draft' : 'Published' - } v${dataSetVersion?.version}`} - {`${dataSetVersion?.type} update`} - + {isDraft && showForm ? ( + + ) : ( + <> +

    Public guidance notes

    +

    + {dataSetVersion?.notes || + 'No notes have been added for this API data set.'} +

    + {isDraft && ( + + )} + + )} - {isDraft && showForm ? ( - - ) : ( - <> -

    Public guidance notes

    -

    - {dataSetVersion?.notes || - 'No notes have been added for this API data set.'} -

    - {isDraft && ( - + {changes && dataSetVersion && ( + )} - )} - - {changes && dataSetVersion && ( - + ) : ( + Could not load changelog )}     diff --git a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx index b17bd2c5a11..5343ad5a6ce 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx @@ -194,6 +194,36 @@ describe('ReleaseApiDataSetChangelogPage', () => { }); }); + test('renders warning if unable to fetch data set', async () => { + apiDataSetService.getDataSet.mockRejectedValue( + new Error('Unable to fetch changes'), + ); + apiDataSetVersionService.getChanges.mockResolvedValue(testChanges); + + renderPage('draft-version-id'); + + expect(screen.queryByText('Data set title')).not.toBeInTheDocument(); + + expect( + await screen.findByText('Could not load changelog'), + ).toBeInTheDocument(); + }); + + test('renders warning if unable to fetch changes', async () => { + apiDataSetService.getDataSet.mockResolvedValue(testDataSet); + apiDataSetVersionService.getChanges.mockRejectedValue( + new Error('Unable to fetch changes'), + ); + + renderPage('draft-version-id'); + + expect(screen.queryByText('Data set title')).not.toBeInTheDocument(); + + expect( + await screen.findByText('Could not load changelog'), + ).toBeInTheDocument(); + }); + function renderPage(dataSetVersionId: string) { return render( From 549a29635024ac49859abe697cf4caae1e43b19c Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 2 Dec 2024 16:03:32 +0000 Subject: [PATCH 036/144] EES-5564 - responding to PR comments. Replacing error flag usage with null checks after loading is complete. Made display of overarching data set details independent from whether or not the changelog is fetched successfully. --- .../data/ReleaseApiDataSetChangelogPage.tsx | 27 +++++++++---------- .../ReleaseApiDataSetChangelogPage.test.tsx | 8 +++--- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetChangelogPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetChangelogPage.tsx index 3b7dd9ca93f..b6b43f29120 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetChangelogPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetChangelogPage.tsx @@ -29,14 +29,11 @@ export default function ReleaseApiDataSetChangelogPage() { data: dataSet, isLoading: isLoadingDataSet, refetch: refetchDataSet, - isError: errorFetchingDataSet, } = useQuery(apiDataSetQueries.get(dataSetId)); - const { - data: changes, - isLoading: isLoadingChanges, - isError: errorFetchingChanges, - } = useQuery(apiDataSetVersionQueries.getChanges(dataSetVersionId)); + const { data: changes, isLoading: isLoadingChanges } = useQuery( + apiDataSetVersionQueries.getChanges(dataSetVersionId), + ); const isDraft = dataSet?.draftVersion?.id === dataSetVersionId; @@ -80,26 +77,26 @@ export default function ReleaseApiDataSetChangelogPage() { - {!errorFetchingDataSet && !errorFetchingChanges ? ( + {dataSet && dataSetVersion ? ( <>
    API data set changelog -

    {dataSet?.title}

    +

    {dataSet.title}

    {`${ isDraft ? 'Draft' : 'Published' - } v${dataSetVersion?.version}`} + } v${dataSetVersion.version}`} {`${dataSetVersion?.type} update`} + >{`${dataSetVersion.type} update`} {isDraft && showForm ? ( ) : ( @@ -117,16 +114,18 @@ export default function ReleaseApiDataSetChangelogPage() { )} - {changes && dataSetVersion && ( + {changes ? ( + ) : ( + Could not load changelog )} ) : ( - Could not load changelog + Could not load API data set )}     diff --git a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx index 5343ad5a6ce..cca943b6f5f 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx @@ -205,7 +205,7 @@ describe('ReleaseApiDataSetChangelogPage', () => { expect(screen.queryByText('Data set title')).not.toBeInTheDocument(); expect( - await screen.findByText('Could not load changelog'), + await screen.findByText('Could not load API data set'), ).toBeInTheDocument(); }); @@ -217,11 +217,9 @@ describe('ReleaseApiDataSetChangelogPage', () => { renderPage('draft-version-id'); - expect(screen.queryByText('Data set title')).not.toBeInTheDocument(); + expect(await screen.findByText('Data set title')).toBeInTheDocument(); - expect( - await screen.findByText('Could not load changelog'), - ).toBeInTheDocument(); + expect(screen.getByText('Could not load changelog')).toBeInTheDocument(); }); function renderPage(dataSetVersionId: string) { From 3fcdf911ced5931926aad60b2e27389b1450df47 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 25 Nov 2024 13:50:02 +0000 Subject: [PATCH 037/144] EES-5558 - got "Data and files" link to close modal and navigate to the Data Uploads tab --- .../data/components/ApiDataSetCreateModal.tsx | 8 +++--- .../__tests__/ApiDataSetCreateModal.test.tsx | 26 ++++++++++++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx index 1480114545b..3a4421289c0 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx @@ -14,7 +14,8 @@ import WarningMessage from '@common/components/WarningMessage'; import useToggle from '@common/hooks/useToggle'; import { useQuery } from '@tanstack/react-query'; import React, { ReactNode } from 'react'; -import { generatePath } from 'react-router-dom'; +import { generatePath, useHistory } from 'react-router-dom'; +import releaseDataPageTabIds from '@admin/pages/release/data/utils/releaseDataPageTabIds'; interface Props { buttonText?: ReactNode | string; @@ -65,10 +66,11 @@ export default function ApiDataSetCreateModal({ No API data sets can be created as there are no candidate data files available. New candidate data files can be uploaded in the{' '} (releaseDataRoute.path, { + to={`${generatePath(releaseDataRoute.path, { publicationId, releaseId, - })} + })}#${releaseDataPageTabIds.dataUploads}`} + onClick={toggleOpen.off} > Data and files {' '} diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx index bd7a2a44f79..f634fcb14e0 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx @@ -29,6 +29,8 @@ describe('ApiDataSetCreateModal', () => { ]; test('renders warning message in modal when no candidates', async () => { + const history = createMemoryHistory(); + apiDataSetCandidateService.listCandidates.mockResolvedValue([]); const { user } = render( @@ -37,19 +39,37 @@ describe('ApiDataSetCreateModal', () => { releaseId="release-id" onSubmit={noop} />, + { history }, ); + expect(await screen.findByText('Create API data set')).toBeInTheDocument(); + await user.click( await screen.findByRole('button', { name: 'Create API data set' }), ); + expect(screen.getByRole('button', { name: 'Close' })).toBeInTheDocument(); + expect( - await screen.findByText( + screen.getByText( /No API data sets can be created as there are no candidate data files available/, ), ).toBeInTheDocument(); - expect(screen.getByRole('button', { name: 'Close' })).toBeInTheDocument(); + expect( + await screen.findByRole('link', { name: 'Data and files' }), + ).toBeInTheDocument(); + + await user.click(screen.getByRole('link', { name: 'Data and files' })); + + expect( + screen.queryByRole('link', { name: 'Data and files' }), + ).not.toBeInTheDocument(); + + expect(history.location.pathname).toEqual( + '/publication/publication-id/release/release-id/data', + ); + expect(history.location.hash).toEqual('#data-uploads'); }); test('renders form in modal when there are candidates', async () => { @@ -98,7 +118,6 @@ describe('ApiDataSetCreateModal', () => { previousReleaseIds: [], }); - const history = createMemoryHistory(); const handleSubmit = jest.fn(); const { user } = render( @@ -107,7 +126,6 @@ describe('ApiDataSetCreateModal', () => { releaseId="release-id" onSubmit={handleSubmit} />, - { history }, ); expect(await screen.findByText('Create API data set')).toBeInTheDocument(); From 029fc499e4cb6a024047345656e495eb024666f5 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 2 Dec 2024 16:20:59 +0000 Subject: [PATCH 038/144] EES-5558 - responding to PR comments. Removing navigation check from tests as it is not necessary to test. --- .../data/components/ApiDataSetCreateModal.tsx | 2 +- .../__tests__/ApiDataSetCreateModal.test.tsx | 25 +++++++------------ 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx index 3a4421289c0..e7bb9376434 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetCreateModal.tsx @@ -14,7 +14,7 @@ import WarningMessage from '@common/components/WarningMessage'; import useToggle from '@common/hooks/useToggle'; import { useQuery } from '@tanstack/react-query'; import React, { ReactNode } from 'react'; -import { generatePath, useHistory } from 'react-router-dom'; +import { generatePath } from 'react-router-dom'; import releaseDataPageTabIds from '@admin/pages/release/data/utils/releaseDataPageTabIds'; interface Props { diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx index f634fcb14e0..463d1c81232 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ApiDataSetCreateModal.test.tsx @@ -4,7 +4,7 @@ import _apiDataSetCandidateService, { } from '@admin/services/apiDataSetCandidateService'; import _apiDataSetService from '@admin/services/apiDataSetService'; import baseRender from '@common-test/render'; -import { screen, waitFor } from '@testing-library/react'; +import { screen, waitFor, within } from '@testing-library/react'; import { createMemoryHistory, History } from 'history'; import noop from 'lodash/noop'; import { ReactElement } from 'react'; @@ -29,8 +29,6 @@ describe('ApiDataSetCreateModal', () => { ]; test('renders warning message in modal when no candidates', async () => { - const history = createMemoryHistory(); - apiDataSetCandidateService.listCandidates.mockResolvedValue([]); const { user } = render( @@ -39,7 +37,6 @@ describe('ApiDataSetCreateModal', () => { releaseId="release-id" onSubmit={noop} />, - { history }, ); expect(await screen.findByText('Create API data set')).toBeInTheDocument(); @@ -50,26 +47,22 @@ describe('ApiDataSetCreateModal', () => { expect(screen.getByRole('button', { name: 'Close' })).toBeInTheDocument(); + const modal = within(screen.getByRole('dialog')); + expect( - screen.getByText( + modal.getByText( /No API data sets can be created as there are no candidate data files available/, ), ).toBeInTheDocument(); - expect( - await screen.findByRole('link', { name: 'Data and files' }), - ).toBeInTheDocument(); + expect(modal.getByRole('link', { name: 'Data and files' })).toHaveAttribute( + 'href', + '/publication/publication-id/release/release-id/data#data-uploads', + ); await user.click(screen.getByRole('link', { name: 'Data and files' })); - expect( - screen.queryByRole('link', { name: 'Data and files' }), - ).not.toBeInTheDocument(); - - expect(history.location.pathname).toEqual( - '/publication/publication-id/release/release-id/data', - ); - expect(history.location.hash).toEqual('#data-uploads'); + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); }); test('renders form in modal when there are candidates', async () => { From 963d6e558165bf96191a37a87cb020b849a11c1a Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 2 Dec 2024 21:03:52 +0000 Subject: [PATCH 039/144] EES-5660 - responding to PR comments. Replacing explicit Tuple instances with named tuples. --- .../Extensions/EnumerableExtensionsTests.cs | 28 +++++++++---------- .../Extensions/EnumerableExtensions.cs | 10 +++---- ...rateDraftDataSetVersionFolderNamesTests.cs | 10 +++---- .../DataSetVersionPathResolverTests.cs | 8 +++--- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs index 0ccf2a08f0a..d70fa90b556 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs @@ -474,12 +474,12 @@ public void TwoLists_Cartesian() List list1 = [1, 2]; List list2 = ["3", "4"]; - List> expected = + List<(int, string)> expected = [ - new Tuple(1, "3"), - new Tuple(1, "4"), - new Tuple(2, "3"), - new Tuple(2, "4") + new(1, "3"), + new(1, "4"), + new(2, "3"), + new(2, "4") ]; var actual = list1.Cartesian(list2); @@ -507,16 +507,16 @@ public void ThreeLists_Cartesian() List list2 = ["3", "4"]; List list3 = ['5', '6']; - List> expected = + List<(int, string, char)> expected = [ - new Tuple(1, "3", '5'), - new Tuple(1, "3", '6'), - new Tuple(1, "4", '5'), - new Tuple(1, "4", '6'), - new Tuple(2, "3", '5'), - new Tuple(2, "3", '6'), - new Tuple(2, "4", '5'), - new Tuple(2, "4", '6'), + new(1, "3", '5'), + new(1, "3", '6'), + new(1, "4", '5'), + new(1, "4", '6'), + new(2, "3", '5'), + new(2, "3", '6'), + new(2, "4", '5'), + new(2, "4", '6'), ]; var actual = list1.Cartesian(list2, list3); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs index 130bb844d3b..5ff99b81aff 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs @@ -320,18 +320,18 @@ public static IOrderedEnumerable NaturalThenBy( return source.ThenBy(keySelector, comparison.WithNaturalSort()); } - public static List> Cartesian( + public static List<(T1, T2)> Cartesian( this IEnumerable list1, IEnumerable? list2) { return list2 == null ? [] : list1 - .Join(list2, _ => true, _ => true, (t1, t2) => new Tuple(t1, t2)) + .Join(list2, _ => true, _ => true, (t1, t2) => (t1, t2)) .ToList(); } - public static List> Cartesian( + public static List<(T1, T2, T3)> Cartesian( this IEnumerable list1, IEnumerable? list2, IEnumerable? list3) @@ -339,9 +339,9 @@ public static List> Cartesian( return list2 == null || list3 == null ? [] : list1 - .Join(list2, _ => true, _ => true, (t1, t2) => new Tuple(t1, t2)) + .Join(list2, _ => true, _ => true, (t1, t2) => (t1, t2)) .Join(list3, _ => true, _ => true, - (tuple, t3) => new Tuple(tuple.Item1, tuple.Item2, t3)) + (tuple, t3) => (tuple.t1, tuple.t2, t3)) .ToList(); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs index 63c0806d5e6..5a2f0463548 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Migrations/EES5660_MigrateDraftDataSetVersionFolderNamesTests.cs @@ -18,19 +18,19 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Migra public class EES5660_MigrateDraftDataSetVersionFolderNamesTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { - public static readonly TheoryData> + public static readonly TheoryData<(DataSetVersionStatus, DataSetVersionType)> PrivateDataSetVersionStatusAndTypes = new(DataSetVersionAuthExtensions .PrivateStatuses .Cartesian(EnumUtil.GetEnums())); - public static readonly TheoryData> + public static readonly TheoryData<(DataSetVersionStatus, DataSetVersionType)> PublicDataSetVersionStatusAndTypes = new(DataSetVersionAuthExtensions .PublicStatuses .Cartesian(EnumUtil.GetEnums())); [Theory] [MemberData(nameof(PrivateDataSetVersionStatusAndTypes))] - public async Task Success_NonMigratedDraft(Tuple statusAndType) + public async Task Success_NonMigratedDraft((DataSetVersionStatus, DataSetVersionType) statusAndType) { var (status, type) = statusAndType; @@ -66,7 +66,7 @@ public async Task Success_NonMigratedDraft(Tuple statusAndType) + public async Task Success_AlreadyMigratedDraft((DataSetVersionStatus, DataSetVersionType) statusAndType) { var (status, type) = statusAndType; @@ -133,7 +133,7 @@ public async Task Failure_DraftFolderAndVersionedFolderExist() [Theory] [MemberData(nameof(PublicDataSetVersionStatusAndTypes))] - public async Task Success_PublicVersionsNotMigrated(Tuple statusAndType) + public async Task Success_PublicVersionsNotMigrated((DataSetVersionStatus, DataSetVersionType) statusAndType) { var (status, type) = statusAndType; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs index 14fcd86c36a..aad25a6c7df 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Tests/DataSetVersionPathResolverTests.cs @@ -48,10 +48,10 @@ public class PathTests : DataSetVersionPathResolverTests public static readonly TheoryData GetEnvironmentNames = new(EnvironmentNames); - public static readonly TheoryData> GetEnvironmentNamesAndPublicStatuses = + public static readonly TheoryData<(string, DataSetVersionStatus)> GetEnvironmentNamesAndPublicStatuses = new(EnvironmentNames.Cartesian(DataSetVersionAuthExtensions.PublicStatuses)); - public static readonly TheoryData> GetEnvironmentNamesAndPrivateStatuses = + public static readonly TheoryData<(string, DataSetVersionStatus)> GetEnvironmentNamesAndPrivateStatuses = new(EnvironmentNames.Cartesian(DataSetVersionAuthExtensions.PrivateStatuses)); [Fact] @@ -125,7 +125,7 @@ public void ProductionEnv_ValidBasePath() [Theory] [MemberData(nameof(GetEnvironmentNamesAndPublicStatuses))] - public void ValidDirectoryPath_PublicVersion(Tuple environmentNameAndStatus) + public void ValidDirectoryPath_PublicVersion((string, DataSetVersionStatus) environmentNameAndStatus) { var (environmentName, status) = environmentNameAndStatus; @@ -153,7 +153,7 @@ public void ValidDirectoryPath_PublicVersion(Tuple [Theory] [MemberData(nameof(GetEnvironmentNamesAndPrivateStatuses))] - public void ValidDirectoryPath_PrivateVersion(Tuple environmentNameAndStatus) + public void ValidDirectoryPath_PrivateVersion((string, DataSetVersionStatus) environmentNameAndStatus) { var (environmentName, status) = environmentNameAndStatus; From 1710beec4db994fc0b30c0ad1fb6c7107ff30856 Mon Sep 17 00:00:00 2001 From: Nusrath Moh Date: Tue, 3 Dec 2024 09:48:14 +0000 Subject: [PATCH 040/144] EES-5711-Fixing a UI test failure in general_public suite --- tests/robot-tests/tests/general_public/miscellaneous.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/robot-tests/tests/general_public/miscellaneous.robot b/tests/robot-tests/tests/general_public/miscellaneous.robot index 4df3159f03a..ff353239daf 100644 --- a/tests/robot-tests/tests/general_public/miscellaneous.robot +++ b/tests/robot-tests/tests/general_public/miscellaneous.robot @@ -37,7 +37,7 @@ Validate homepage user checks page contains element link:Statistics at DfE user checks page contains element link:Compare school and college performance user checks page contains element link:Get information about schools - user checks page contains element link:Schools financial benchmarking + user checks page contains element link:Financial Benchmarking and Insights Tool user checks page contains element xpath://h2[text()="Contact us"] From 8a8163b11b9de4ec3dc800e610456703096ccf48 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 2 Dec 2024 21:06:26 +0000 Subject: [PATCH 041/144] EES-5660 - replaced contraction of "Fileshare" with separate "File share" to fit in with standard Azure terminology --- .../application/public-api/publicApiApp.bicep | 6 +++--- .../public-api/publicApiDataProcessor.bicep | 4 ++-- .../public-api/publicApiStorage.bicep | 4 ++-- .../shared/containerAppEnvironment.bicep | 4 ++-- .../components/appServiceSlotConfig.bicep | 4 ++-- .../public-api/components/functionApp.bicep | 4 ++-- infrastructure/templates/public-api/main.bicep | 2 +- infrastructure/templates/public-api/types.bicep | 4 ++-- infrastructure/templates/template.json | 16 ++++++++-------- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep b/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep index fd48d5be7cd..5bfc5d1ca52 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep @@ -78,15 +78,15 @@ module apiContainerAppModule '../../components/containerApp.bicep' = { vnetName: resourceNames.existingResources.vNet volumeMounts: [ { - volumeName: 'public-api-fileshare-mount' + volumeName: 'public-api-file-share-mount' mountPath: dataFilesFileShareMountPath } ] volumes: [ { - name: 'public-api-fileshare-mount' + name: 'public-api-file-share-mount' storageType: 'AzureFile' - storageName: resourceNames.publicApi.publicApiFileshare + storageName: resourceNames.publicApi.publicApiFileShare } ] appSettings: [ diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index d21319d2836..88be79535df 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -106,10 +106,10 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { App__MetaInsertBatchSize: 1000 } azureFileShares: [{ - storageName: resourceNames.publicApi.publicApiFileshare + storageName: resourceNames.publicApi.publicApiFileShare storageAccountKey: publicApiStorageAccount.listKeys().keys[0].value storageAccountName: resourceNames.publicApi.publicApiStorageAccount - fileShareName: resourceNames.publicApi.publicApiFileshare + fileShareName: resourceNames.publicApi.publicApiFileShare mountPath: publicApiDataFileShareMountPath }] storageFirewallRules: storageFirewallRules diff --git a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep index 72b918256c6..ce25740625f 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep @@ -49,7 +49,7 @@ module publicApiStorageAccountModule '../../components/storageAccount.bicep' = { module dataFilesFileShareModule '../../components/fileShare.bicep' = { name: 'fileShareDeploy' params: { - fileShareName: resourceNames.publicApi.publicApiFileshare + fileShareName: resourceNames.publicApi.publicApiFileShare fileShareQuota: publicApiDataFileShareQuota storageAccountName: publicApiStorageAccountModule.outputs.storageAccountName fileShareAccessTier: 'TransactionOptimized' @@ -59,5 +59,5 @@ module dataFilesFileShareModule '../../components/fileShare.bicep' = { output storageAccountName string = publicApiStorageAccountModule.outputs.storageAccountName output connectionStringSecretName string = publicApiStorageAccountModule.outputs.connectionStringSecretName output accessKeySecretName string = publicApiStorageAccountModule.outputs.accessKeySecretName -output publicApiDataFileShareName string = resourceNames.publicApi.publicApiFileshare +output publicApiDataFileShareName string = resourceNames.publicApi.publicApiFileShare output publicApiStorageAccountName string = resourceNames.publicApi.publicApiStorageAccount diff --git a/infrastructure/templates/public-api/application/shared/containerAppEnvironment.bicep b/infrastructure/templates/public-api/application/shared/containerAppEnvironment.bicep index b45b6510813..6222b2ea4cf 100644 --- a/infrastructure/templates/public-api/application/shared/containerAppEnvironment.bicep +++ b/infrastructure/templates/public-api/application/shared/containerAppEnvironment.bicep @@ -36,10 +36,10 @@ module containerAppEnvironmentModule '../../components/containerAppEnvironment.b tagValues: tagValues azureFileStorages: [ { - storageName: resourceNames.publicApi.publicApiFileshare + storageName: resourceNames.publicApi.publicApiFileShare storageAccountName: resourceNames.publicApi.publicApiStorageAccount storageAccountKey: publicApiStorageAccount.listKeys().keys[0].value - fileShareName: resourceNames.publicApi.publicApiFileshare + fileShareName: resourceNames.publicApi.publicApiFileShare accessMode: 'ReadWrite' } ] diff --git a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep index da455e60ba5..32bcff1f6e1 100644 --- a/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep +++ b/infrastructure/templates/public-api/components/appServiceSlotConfig.bicep @@ -1,4 +1,4 @@ -import { AzureFileshareMount } from '../types.bicep' +import { AzureFileShareMount } from '../types.bicep' @description('Specifies the Web / Function App name that these settings belong to') param appName string @@ -25,7 +25,7 @@ param existingStagingAppSettings object param existingProductionAppSettings object @description('Specifies additional Azure Storage Accounts to make available to the staging slot') -param azureFileShares AzureFileshareMount[] = [] +param azureFileShares AzureFileShareMount[] = [] @description('Set specific appsettings to be slot specific values') resource functionSlotConfig 'Microsoft.Web/sites/config@2023-12-01' = { diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index e3e5d7564bf..9807d5e75bd 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -1,4 +1,4 @@ -import { FirewallRule, AzureFileshareMount, EntraIdAuthentication } from '../types.bicep' +import { FirewallRule, AzureFileShareMount, EntraIdAuthentication } from '../types.bicep' @description('Specifies the location for all resources.') param location string @@ -71,7 +71,7 @@ param healthCheck { }? @description('Specifies additional Azure Storage Accounts to make available to this Function App') -param azureFileShares AzureFileshareMount[] = [] +param azureFileShares AzureFileShareMount[] = [] @description('Specifies firewall rules for the various storage accounts in use by the Function App') param storageFirewallRules FirewallRule[] = [] diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 70d5e67086e..d3df11ca2b1 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -149,7 +149,7 @@ var resourceNames = { dataProcessorPlan: '${publicApiResourcePrefix}-${abbreviations.webServerFarms}-${abbreviations.webSitesFunctions}-processor' dataProcessorStorageAccountsPrefix: '${subscription}eessaprocessor' docsApp: '${publicApiResourcePrefix}-${abbreviations.staticWebApps}-docs' - publicApiFileshare: '${publicApiResourcePrefix}-fs-data' + publicApiFileShare: '${publicApiResourcePrefix}-fs-data' publicApiStorageAccount: '${replace(publicApiResourcePrefix, '-', '')}${abbreviations.storageStorageAccounts}' } } diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index f116d46127b..fb5199aafa5 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -36,7 +36,7 @@ type ResourceNames = { dataProcessorStorageAccountsPrefix: string docsApp: string publicApiStorageAccount: string - publicApiFileshare: string + publicApiFileShare: string } } @@ -47,7 +47,7 @@ type FirewallRule = { } @export() -type AzureFileshareMount = { +type AzureFileShareMount = { storageName: string storageAccountKey: string storageAccountName: string diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index 6e690b8a75e..d3c9abd0cb5 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -1198,8 +1198,8 @@ "[concat('Microsoft.Web/sites/', variables('contentAppName'))]", "[concat('Microsoft.Web/sites/', variables('dataAppName'))]" ], - "publicDataFileshareMountPath": "/data/public-api-data", - "publicDataFileshareName": "[concat(parameters('subscription'), '-ees-papi-fs-data')]", + "publicDataFileShareMountPath": "/data/public-api-data", + "publicDataFileShareName": "[concat(parameters('subscription'), '-ees-papi-fs-data')]", "publicDataStorageAccountName": "[concat(parameters('subscription'), 'eespapisa')]" }, "resources": [ @@ -3286,7 +3286,7 @@ "App:NotifierStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-notifications')).secretUriWithVersion, ')')]", "App:PublicStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-public')).secretUriWithVersion, ')')]", "App:PublisherStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-publisher')).secretUriWithVersion, ')')]", - "DataFiles:BasePath": "[parameters('publicDataFileshareMountPath')]", + "DataFiles:BasePath": "[parameters('publicDataFileShareMountPath')]", "PublicDataDbExists": "[parameters('publicDataDbExists')]" } }, @@ -3298,16 +3298,16 @@ "apiVersion": "2019-08-01", "dependsOn": [ "[resourceId('Microsoft.Web/sites', variables('publisherAppName'))]", - "[variables('publicDataFileshareMountPath')]", - "[variables('publicDataFileshareName')]", + "[variables('publicDataFileShareMountPath')]", + "[variables('publicDataFileShareName')]", "[variables('publicDataStorageAccountName')]" ], "properties": { - "[variables('publicDataFileshareName')]": { + "[variables('publicDataFileShareName')]": { "type": "AzureFiles", "accountName": "[variables('publicDataStorageAccountName')]", - "shareName": "[variables('publicDataFileshareName')]", - "mountPath": "[variables('publicDataFileshareMountPath')]", + "shareName": "[variables('publicDataFileShareName')]", + "mountPath": "[variables('publicDataFileShareMountPath')]", "protocol": "Smb" } } From 61cb8ca152c3e4aec8246b6f1e8824754fb67219 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Tue, 3 Dec 2024 12:25:15 +0000 Subject: [PATCH 042/144] Various tidy up and clarifications across FE style guide --- styleguides/styleguide-frontend.md | 108 +++++++++++++++++------------ 1 file changed, 65 insertions(+), 43 deletions(-) diff --git a/styleguides/styleguide-frontend.md b/styleguides/styleguide-frontend.md index 7988ff25df6..1280d41ce2d 100644 --- a/styleguides/styleguide-frontend.md +++ b/styleguides/styleguide-frontend.md @@ -148,7 +148,7 @@ onB -#### 2.4.2 Context API conventions e.g. use\*Context hook +#### 2.4.2 Context API conventions e.g. `use\*Context` hook - Create an exported custom hook called `use*Context` which uses `Context.Consumer`. - Create an exported Provider component which uses `Context.Provider`. @@ -288,7 +288,7 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m -#### 5.1.3 Determining whether or not to leave particular test cases out as they're covered by Robot tests +#### 5.1.3 Determining whether to leave particular test cases out as they're covered by Robot tests - Currently, UI tests only really test happy path. - Frontend unit tests tend to test happy and unhappy paths as well. @@ -299,8 +299,10 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m #### 5.1.4 Use of snapshot tests -- Prefer to not use snapshot tests - they tend to let mistakes through as they rely on people checking them properly (which is hard). -- If snapshot is small in scope and there's a valid use case (e.g. some HTML), then a snapshot might be a suitable tool - up to developer. +- Prefer to not use snapshot tests - they tend to let mistakes through as they rely on people checking + them properly (which is hard). +- If snapshot is small in scope and there's a valid use case (e.g. some HTML), then a snapshot might + be a suitable tool - up to developer. @@ -310,15 +312,21 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m #### 5.2.1 Test case naming -- The general  style is 'action does something'. -- The name should start lowercased. +- The general naming convention is `{action} {subject?} {expectation or condition?}` e.g. + - `renders correctly` + - `renders button if user is logged in` + - `clicking the button calls x service` + - `submitting form successfully` + - `submitting form with invalid values shows errors` +- The name should start **lowercased** unless there is a specific reason for it not to. -#### 5.2.2 testid naming +#### 5.2.2 Test ID (`data-testid`) naming - Use hyphens -- We're not too precious about this as sometimes it makes sense to put a string (e.g. a name) inside of the test ID for ease +- We're not too precious about this as sometimes it makes sense to put a string (e.g. a name) inside + of the test ID for ease @@ -338,7 +346,7 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m -#### 5.3.1 getByRole vs getByText vs getByLabel +#### 5.3.1 `getByRole` vs `getByText` vs `getByLabel` - `getByRole` is good for most things and should be used as much as possible. - `getByLabel` is good for form inputs. @@ -351,8 +359,14 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m #### 5.3.2 Waiting for UI changes - UI changes on load or following user interaction usually have to be waited for. -- Use `findBy*` where applicable, e.g. `expect(await screen.findByText('find me')).toBeInTheDocument()`. -- When using `waitFor` don't use `getByRole` as it causes performance degradation. Use a faster selector like `getByText` or `getByLabel` in `waitFor` and keep assertions minimal inside. Follow up after `waitFor` with any `getByRoles`. +- Use `findBy*` where applicable e.g. + ```tsx + expect(await screen.findByText('find me')).toBeInTheDocument() + ``` +- When using `waitFor` don't use `getByRole` as it causes performance degradation. Use a faster + selector like `getByText` or `getByLabel` in `waitFor` and keep assertions minimal inside. + + Follow up after `waitFor` with any `getByRole` selectors. @@ -360,11 +374,14 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m -#### 5.4.1 userEvents +#### 5.4.1 Using `userEvent` library -- Use `await` with all `userEvents`. -- Invoke `userEvent.setup()` before the component is rendered (this is a recent change to Testing Library so most tests still use `userEvent` directly), e.g. - ``` +- Don't use `fireEvent` unless it's necessary. Use `userEvent` library instead. +- Use `await` with all `userEvent` methods. +- Invoke `userEvent.setup()` before the component is rendered (this is a recent change to Testing + Library so most tests still use `userEvent` directly), e.g. + + ```tsx const user = userEvent.setup(); render(); await user.click(screen.getByRole('button', {name: 'click me'})); @@ -372,13 +389,17 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m -#### 5.4.2 Using jest.useFakeTimers with userEvents - -- Using `jest.useFakeTimers()` with `userEvents` can cause tests to timeout. -- To fix this invoke `userEvent` with one of the following as appropriate: - - `const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime });` - - `const user = userEvent.setup({ delay: null });` +#### 5.4.2 Using `jest.useFakeTimers` with `userEvent` +- Using `jest.useFakeTimers()` with `userEvent` can cause tests to timeout. +- To fix this invoke `userEvent` with one of following: + ```tsx + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }); + ``` + + ```tsx + const user = userEvent.setup({ delay: null }); + ``` ### 5.5 Test data @@ -402,8 +423,8 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m #### 5.5.3 What to use for test data values -- Values don't have to realistic. Prefer to use generic defaults like 'Publication title 1' following structure: - `{type} {property} {number}`. +- Values don't have to realistic. Prefer to use generic defaults like 'Publication title 1' following + structure: `{type} {property} {number}`. - This doesn't need to hard and fast rule and people can use their discretion. @@ -412,17 +433,17 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m -#### 5.6.1 toEqual vs individual assertions +#### 5.6.1 `toEqual` vs individual assertions - Prefer to use toEqual when comparing objects (instead assertions on each property). -#### 5.6.2 toBeInTheDocument vs toBeVisible +#### 5.6.2 `toBeInTheDocument` vs `toBeVisible` -- Use toBeInTheDocument by default -- Use toBeVisible when you actually need to test visibility e.g. transparent or display: none stuff -- toBeVisible incurs extra style checks during assertion (which is overkill) +- Use `toBeInTheDocument` by default +- Use `toBeVisible` when you actually need to test visibility e.g. transparent or `display: none` stuff +- `toBeVisible` incurs extra style checks during assertion (which is overkill) @@ -430,7 +451,8 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m - Have a check that the error summary message links back to the input that has the error - Check that the error message is rendered next to the field -- We plan to create a custom Jest assertion that handles this for us  at some point e.g. `expect().toHaveFormError()`. This would roll up any required assertions into one. +- We plan to create a custom Jest assertion that handles this for us at some point e.g. + `expect().toHaveFormError()`. This would roll up any required assertions into one. @@ -438,17 +460,17 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m - Length check - make sure there are correct number of checkboxes or radios. - Check each radio or checkbox state as part of a list of assertions -- getByLabelText to get the input and assert that it's one of the radios - -``` -const radios = within( - screen.getByRole('group', { name: 'Change page view' }), -).getAllByRole('radio'); - -expect(radios).toHaveLength(3); - -expect(radios[0]).toEqual(screen.getByLabelText('Edit content')); -``` +- `getByLabelText` to get the input and assert that it's one of the radios + + ```tsx + const radios = within( + screen.getByRole('group', { name: 'Change page view' }), + ).getAllByRole('radio'); + + expect(radios).toHaveLength(3); + + expect(radios[0]).toEqual(screen.getByLabelText('Edit content')); + ``` @@ -462,7 +484,7 @@ expect(radios[0]).toEqual(screen.getByLabelText('Edit content')); #### 5.6.6 Asserting mocks have been called - Check that the mock hasn't been called before an event happens. -- Check that the mock has been called with right values after event happens. +- Check that the mock has been called with right values after event happens. - Check number of calls explicitly if required, but don't need to in most cases - up to developer. @@ -483,12 +505,12 @@ expect(radios[0]).toEqual(screen.getByLabelText('Edit content')); #### 5.7.2 Setting up Jest mocks - We currently do a lot of the following: - ``` + ```tsx import _theService from 'theService'; const theService = _theService as jest.Mocked ``` - However, we should switch to using jest.mocked. - ``` + ```tsx import _theService from 'theService' const theService = jest.mocked(_theService); ``` From fa9199f3bf2925694c6df18f73a077588aa2127e Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Tue, 3 Dec 2024 12:48:43 +0000 Subject: [PATCH 043/144] Add section on using logic + loops in tests to FE style guide --- styleguides/styleguide-frontend.md | 51 ++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/styleguides/styleguide-frontend.md b/styleguides/styleguide-frontend.md index 1280d41ce2d..ac18a98e92d 100644 --- a/styleguides/styleguide-frontend.md +++ b/styleguides/styleguide-frontend.md @@ -304,6 +304,57 @@ Work is ongoing to convert existing forms to RHF. New RHF form components that m - If snapshot is small in scope and there's a valid use case (e.g. some HTML), then a snapshot might be a suitable tool - up to developer. + + +#### 5.1.5 Avoid logic and loops in tests + +- Prefer to avoid loops and logic in tests - keep things as simple as possible! +- It may be tempting to abstract some repeated assertions on some items, however, it's usually + simpler, clearer and more precise to just copy and paste the assertions across your items. +- For example, instead of doing something like the following: + + ```tsx + // ❌ Don't do this + + const expectedOptions: SelectOption[] = [ + { value: 'value-1', label: 'Value 1' }, + { value: 'value-2', label: 'Value 2' }, + ]; + + const options = screen.getAllByRole('option'); + + options.forEach((option, index) => { + if (index === 0) { + expect(option).toHaveValue(''); + expect(option).toHaveTextContent('Choose option'); + } else { + expect(option).toHaveValue(expectedOptions[index].value); + expect(option).toHaveTextContent(expectedOptions[index].label); + } + }); + ``` + + Simplify to the following instead: + + ```tsx + // ✅ Do this + + const options = screen.getAllByRole('option'); + + expect(options[0]).toHaveValue(''); + expect(options[0]).toHaveTextContent('Choose option'); + + expect(options[1]).toHaveValue('value-1'); + expect(options[1]).toHaveTextContent('Value 1'); + + expect(options[2]).toHaveValue('value-2'); + expect(options[2]).toHaveTextContent('Value 2'); + ``` +- If a test needs to use loops or logic, it should be for a specific case where a copy and paste + approach would result in reduced clarity by the sheer number of assertions. Typically, this would + be where there are many options, or where we need to test **many permutations** within a broad + range of constraints. + ### 5.2 Naming and syntax From 6a37c29ccb73bc36bf651bb5048a3728f3f16110 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 2 Dec 2024 11:35:33 +0000 Subject: [PATCH 044/144] EES-5656 Remove ReleaseVersionRepository method ListLatestReleaseVersions --- .../UserManagementServicePermissionTests.cs | 11 +- .../Services/UserManagementServiceTests.cs | 215 +++++++++--------- .../UserRoleServicePermissionTests.cs | 27 +-- .../Services/UserRoleServiceTests.cs | 140 ++++++------ .../UserManagementController.cs | 2 +- .../Services/Interfaces/IUserRoleService.cs | 8 +- .../Services/UserManagementService.cs | 39 ++-- .../Services/UserRoleService.cs | 61 ++--- .../ReleaseVersionRepositoryTests.cs | 163 ++----------- .../Interfaces/IReleaseVersionRepository.cs | 10 +- .../Repository/ReleaseVersionRepository.cs | 11 +- 11 files changed, 267 insertions(+), 420 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServicePermissionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServicePermissionTests.cs index 01892a66e1c..4625f8a3d45 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServicePermissionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServicePermissionTests.cs @@ -19,12 +19,10 @@ using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.PermissionTestUtils; using static Moq.MockBehavior; -using IReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces.IReleaseVersionRepository; -using ReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.ReleaseVersionRepository; namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services { - public class UserManagementServicePermissionTest + public class UserManagementServicePermissionTests { [Fact] public async Task ListAllUsers() @@ -130,7 +128,7 @@ public class DeleteUserTests public async Task Success() { await using var contentDbContext = DbUtils.InMemoryApplicationDbContext(); - contentDbContext.Users.Add(new User { Email = "ees-test.user@education.gov.uk"} ); + contentDbContext.Users.Add(new User { Email = "ees-test.user@education.gov.uk" }); await contentDbContext.SaveChangesAsync(); await PolicyCheckBuilder() @@ -154,14 +152,12 @@ private static UserManagementService SetupUserManagementService( UsersAndRolesDbContext? usersAndRolesDbContext = null, IPersistenceHelper? usersAndRolesPersistenceHelper = null, IEmailTemplateService? emailTemplateService = null, - IReleaseVersionRepository? releaseVersionRepository = null, IUserRoleService? userRoleService = null, IUserService? userService = null, IUserInviteRepository? userInviteRepository = null, IUserReleaseInviteRepository? userReleaseInviteRepository = null, IUserPublicationInviteRepository? userPublicationInviteRepository = null, - UserManager? userManager = null, - bool enableDeletion = false) + UserManager? userManager = null) { contentDbContext ??= InMemoryApplicationDbContext(); usersAndRolesDbContext ??= InMemoryUserAndRolesDbContext(); @@ -171,7 +167,6 @@ private static UserManagementService SetupUserManagementService( contentDbContext, usersAndRolesPersistenceHelper ?? new PersistenceHelper(usersAndRolesDbContext), emailTemplateService ?? Mock.Of(Strict), - releaseVersionRepository ?? new ReleaseVersionRepository(contentDbContext), userRoleService ?? Mock.Of(Strict), userService ?? AlwaysTrueUserService().Object, userInviteRepository ?? new UserInviteRepository(usersAndRolesDbContext), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServiceTests.cs index 03b6c94a90f..4106921753b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServiceTests.cs @@ -9,12 +9,15 @@ using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Admin.Validators; using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using Microsoft.AspNetCore.Identity; using Moq; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.DbUtils; @@ -22,13 +25,13 @@ using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; using static Moq.MockBehavior; -using IReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces.IReleaseVersionRepository; -using ReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.ReleaseVersionRepository; namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services { public class UserManagementServiceTests { + private readonly DataFixture _dataFixture = new(); + private static readonly Guid CreatedById = Guid.NewGuid(); [Fact] @@ -212,15 +215,15 @@ public async Task InviteUser() var usersAndRolesDbContextId = Guid.NewGuid().ToString(); await using (var usersAndRolesDbContext = InMemoryUserAndRolesDbContext(usersAndRolesDbContextId)) { - await usersAndRolesDbContext.AddRangeAsync(role); + usersAndRolesDbContext.Roles.Add(role); await usersAndRolesDbContext.SaveChangesAsync(); } - var releaseVersion = new ReleaseVersion - { - Publication = new Publication(), - }; - var publication = new Publication(); + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); var emailTemplateService = new Mock(Strict); @@ -240,8 +243,7 @@ public async Task InviteUser() var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - contentDbContext.ReleaseVersions.AddRange(releaseVersion); - contentDbContext.Publications.AddRange(publication); + contentDbContext.Publications.Add(publication); await contentDbContext.SaveChangesAsync(); } @@ -259,7 +261,7 @@ public async Task InviteUser() RoleId = role.Id, UserReleaseRoles = ListOf(new UserReleaseRoleCreateRequest { - ReleaseId = releaseVersion.Id, + ReleaseId = release.Id, ReleaseRole = ReleaseRole.Approver, }), UserPublicationRoles = ListOf(new UserPublicationRoleCreateRequest @@ -285,7 +287,7 @@ public async Task InviteUser() Assert.Equal("test@test.com", userInvite.Email); Assert.False(userInvite.Accepted); Assert.Equal(role.Id, userInvite.RoleId); - Assert.InRange(DateTime.UtcNow.Subtract(userInvite.Created).Milliseconds, 0, 1500); + userInvite.Created.AssertUtcNow(); Assert.Equal(CreatedById.ToString(), userInvite.CreatedById); } @@ -298,7 +300,7 @@ public async Task InviteUser() Assert.Equal(releaseVersion.Id, userReleaseInvite.ReleaseVersionId); Assert.Equal(ReleaseRole.Approver, userReleaseInvite.Role); Assert.True(userReleaseInvite.EmailSent); - Assert.InRange(DateTime.UtcNow.Subtract(userReleaseInvite.Created).Milliseconds, 0, 1500); + userReleaseInvite.Created.AssertUtcNow(); Assert.Equal(CreatedById, userReleaseInvite.CreatedById); var userPublicationInvites = contentDbContext.UserPublicationInvites @@ -307,7 +309,7 @@ public async Task InviteUser() Assert.Equal("test@test.com", userPublicationInvite.Email); Assert.Equal(publication.Id, userPublicationInvite.PublicationId); Assert.Equal(PublicationRole.Owner, userPublicationInvite.Role); - Assert.InRange(DateTime.UtcNow.Subtract(userPublicationInvite.Created).Milliseconds, 0, 1500); + userPublicationInvite.Created.AssertUtcNow(); Assert.Equal(CreatedById, userPublicationInvite.CreatedById); } } @@ -332,11 +334,11 @@ public async Task InviteUser_OptionalCreatedDate() await usersAndRolesDbContext.SaveChangesAsync(); } - var releaseVersion = new ReleaseVersion - { - Publication = new Publication(), - }; - var publication = new Publication(); + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); var emailTemplateService = new Mock(Strict); @@ -356,8 +358,7 @@ public async Task InviteUser_OptionalCreatedDate() var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - contentDbContext.ReleaseVersions.AddRange(releaseVersion); - contentDbContext.Publications.AddRange(publication); + contentDbContext.Publications.Add(publication); await contentDbContext.SaveChangesAsync(); } @@ -375,7 +376,7 @@ public async Task InviteUser_OptionalCreatedDate() RoleId = role.Id, UserReleaseRoles = ListOf(new UserReleaseRoleCreateRequest { - ReleaseId = releaseVersion.Id, + ReleaseId = release.Id, ReleaseRole = ReleaseRole.Approver, }), UserPublicationRoles = ListOf(new UserPublicationRoleCreateRequest @@ -442,20 +443,21 @@ public async Task InviteUser_MultipleReleaseAndPublicationRoles() var usersAndRolesDbContextId = Guid.NewGuid().ToString(); await using (var usersAndRolesDbContext = InMemoryUserAndRolesDbContext(usersAndRolesDbContextId)) { - await usersAndRolesDbContext.AddRangeAsync(role); + usersAndRolesDbContext.Roles.Add(role); await usersAndRolesDbContext.SaveChangesAsync(); } - var releaseVersion1 = new ReleaseVersion - { - Publication = new Publication(), - }; - var releaseVersion2 = new ReleaseVersion - { - Publication = new Publication(), - }; - var publication1 = new Publication(); - var publication2 = new Publication(); + var publications = _dataFixture.DefaultPublication() + .ForRange(..2, + s => s.SetReleases( + [_dataFixture.DefaultRelease(publishedVersions: 1)])) + .GenerateList(4); + + var release1 = publications[0].Releases.Single(); + var releaseVersion1 = release1.Versions.Single(); + + var release2 = publications[1].Releases.Single(); + var releaseVersion2 = release2.Versions.Single(); var emailTemplateService = new Mock(Strict); @@ -470,17 +472,16 @@ public async Task InviteUser_MultipleReleaseAndPublicationRoles() && invites[1].Role == ReleaseRole.Contributor), It.Is>(invites => invites.Count == 2 - && invites[0].PublicationId == publication1.Id + && invites[0].PublicationId == publications[2].Id && invites[0].Role == PublicationRole.Owner - && invites[1].PublicationId == publication2.Id + && invites[1].PublicationId == publications[3].Id && invites[1].Role == PublicationRole.Approver))) .Returns(Unit.Instance); var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - contentDbContext.ReleaseVersions.AddRange(releaseVersion1, releaseVersion2); - contentDbContext.Publications.AddRange(publication1, publication2); + contentDbContext.Publications.AddRange(publications); await contentDbContext.SaveChangesAsync(); } @@ -496,27 +497,32 @@ public async Task InviteUser_MultipleReleaseAndPublicationRoles() { Email = "test@test.com", RoleId = role.Id, - UserReleaseRoles = ListOf(new UserReleaseRoleCreateRequest + UserReleaseRoles = + [ + new UserReleaseRoleCreateRequest { - ReleaseId = releaseVersion1.Id, + ReleaseId = release1.Id, ReleaseRole = ReleaseRole.Approver, }, new UserReleaseRoleCreateRequest { - ReleaseId = releaseVersion2.Id, + ReleaseId = release2.Id, ReleaseRole = ReleaseRole.Contributor, - }), - UserPublicationRoles = ListOf( + } + ], + UserPublicationRoles = + [ new UserPublicationRoleCreateRequest { - PublicationId = publication1.Id, + PublicationId = publications[2].Id, PublicationRole = PublicationRole.Owner, }, new UserPublicationRoleCreateRequest { - PublicationId = publication2.Id, + PublicationId = publications[3].Id, PublicationRole = PublicationRole.Approver, - }) + } + ] }; var result = await service.InviteUser(inviteRequest); @@ -535,7 +541,7 @@ public async Task InviteUser_MultipleReleaseAndPublicationRoles() Assert.Equal("test@test.com", userInvite.Email); Assert.False(userInvite.Accepted); Assert.Equal(role.Id, userInvite.RoleId); - Assert.InRange(DateTime.UtcNow.Subtract(userInvite.Created).Milliseconds, 0, 1500); + userInvite.Created.AssertUtcNow(); Assert.Equal(CreatedById.ToString(), userInvite.CreatedById); } @@ -550,14 +556,14 @@ public async Task InviteUser_MultipleReleaseAndPublicationRoles() Assert.Equal(releaseVersion1.Id, userReleaseInvites[0].ReleaseVersionId); Assert.Equal(ReleaseRole.Approver, userReleaseInvites[0].Role); Assert.True(userReleaseInvites[0].EmailSent); - Assert.InRange(DateTime.UtcNow.Subtract(userReleaseInvites[0].Created).Milliseconds, 0, 1500); + userReleaseInvites[0].Created.AssertUtcNow(); Assert.Equal(CreatedById, userReleaseInvites[0].CreatedById); Assert.Equal("test@test.com", userReleaseInvites[1].Email); Assert.Equal(releaseVersion2.Id, userReleaseInvites[1].ReleaseVersionId); Assert.Equal(ReleaseRole.Contributor, userReleaseInvites[1].Role); Assert.True(userReleaseInvites[1].EmailSent); - Assert.InRange(DateTime.UtcNow.Subtract(userReleaseInvites[1].Created).Milliseconds, 0, 1500); + userReleaseInvites[1].Created.AssertUtcNow(); Assert.Equal(CreatedById, userReleaseInvites[1].CreatedById); var userPublicationInvites = contentDbContext.UserPublicationInvites @@ -565,15 +571,15 @@ public async Task InviteUser_MultipleReleaseAndPublicationRoles() Assert.Equal(2, userPublicationInvites.Count); Assert.Equal("test@test.com", userPublicationInvites[0].Email); - Assert.Equal(publication1.Id, userPublicationInvites[0].PublicationId); + Assert.Equal(publications[2].Id, userPublicationInvites[0].PublicationId); Assert.Equal(PublicationRole.Owner, userPublicationInvites[0].Role); - Assert.InRange(DateTime.UtcNow.Subtract(userPublicationInvites[0].Created).Milliseconds, 0, 1500); + userPublicationInvites[0].Created.AssertUtcNow(); Assert.Equal(CreatedById, userPublicationInvites[0].CreatedById); Assert.Equal("test@test.com", userPublicationInvites[1].Email); - Assert.Equal(publication2.Id, userPublicationInvites[1].PublicationId); + Assert.Equal(publications[3].Id, userPublicationInvites[1].PublicationId); Assert.Equal(PublicationRole.Approver, userPublicationInvites[1].Role); - Assert.InRange(DateTime.UtcNow.Subtract(userPublicationInvites[1].Created).Milliseconds, 0, 1500); + userPublicationInvites[1].Created.AssertUtcNow(); Assert.Equal(CreatedById, userPublicationInvites[1].CreatedById); } } @@ -591,17 +597,10 @@ public async Task InviteUser_NoUserReleaseRolesAndUserPublicationRoles() var usersAndRolesDbContextId = Guid.NewGuid().ToString(); await using (var usersAndRolesDbContext = InMemoryUserAndRolesDbContext(usersAndRolesDbContextId)) { - await usersAndRolesDbContext.AddRangeAsync(role); + usersAndRolesDbContext.Roles.Add(role); await usersAndRolesDbContext.SaveChangesAsync(); } - var contentDbContextId = Guid.NewGuid().ToString(); - await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) - { - contentDbContext.ReleaseVersions.AddRange(new ReleaseVersion()); - await contentDbContext.SaveChangesAsync(); - } - var emailTemplateService = new Mock(Strict); emailTemplateService.Setup(mock => @@ -611,8 +610,9 @@ public async Task InviteUser_NoUserReleaseRolesAndUserPublicationRoles() new List())) .Returns(Unit.Instance); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var userAndRolesDbContext = InMemoryUserAndRolesDbContext(usersAndRolesDbContextId)) - await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + await using (var contentDbContext = InMemoryApplicationDbContext()) { var service = SetupUserManagementService( contentDbContext: contentDbContext, @@ -623,8 +623,8 @@ public async Task InviteUser_NoUserReleaseRolesAndUserPublicationRoles() { Email = "test@test.com", RoleId = role.Id, - UserReleaseRoles = new List(), - UserPublicationRoles = new List() + UserReleaseRoles = [], + UserPublicationRoles = [] }; var result = await service.InviteUser(inviteRequest); @@ -662,7 +662,7 @@ public async Task InviteUser_UserAlreadyExists() var usersAndRolesDbContextId = Guid.NewGuid().ToString(); await using (var usersAndRolesDbContext = InMemoryUserAndRolesDbContext(usersAndRolesDbContextId)) { - await usersAndRolesDbContext.AddRangeAsync(user); + usersAndRolesDbContext.Users.Add(user); await usersAndRolesDbContext.SaveChangesAsync(); } @@ -674,14 +674,13 @@ public async Task InviteUser_UserAlreadyExists() { Email = "test@test.com", RoleId = Guid.NewGuid().ToString(), - UserReleaseRoles = new List(), - UserPublicationRoles = new List() + UserReleaseRoles = [], + UserPublicationRoles = [] }; var result = await service.InviteUser(inviteRequest); - var actionResult = result.AssertLeft(); - actionResult.AssertValidationProblem(ValidationErrorMessages.UserAlreadyExists); + result.AssertBadRequest(ValidationErrorMessages.UserAlreadyExists); } } @@ -701,29 +700,18 @@ public async Task InviteUser_UserAlreadyInvitedAndExpired() await AssertExistingUserInviteOverridden(inviteCreatedDate); } - private static async Task AssertExistingUserInviteOverridden(DateTime inviteCreatedDate) + private async Task AssertExistingUserInviteOverridden(DateTime inviteCreatedDate) { - var publication1 = new Publication - { - Id = Guid.NewGuid() - }; + var (publication1, publication2) = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]) + .Generate(2) + .ToTuple2(); - var releaseVersion1 = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication1 - }; + var release1 = publication1.Releases.Single(); + var releaseVersion1 = release1.Versions.Single(); - var publication2 = new Publication - { - Id = Guid.NewGuid() - }; - - var releaseVersion2 = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication2 - }; + var release2 = publication2.Releases.Single(); + var releaseVersion2 = release2.Versions.Single(); var role1 = new IdentityRole { @@ -758,8 +746,6 @@ private static async Task AssertExistingUserInviteOverridden(DateTime inviteCrea var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - contentDbContext.ReleaseVersions.AddRange(releaseVersion1, releaseVersion2); - contentDbContext.Publications.AddRange(publication1, publication2); contentDbContext.UserReleaseInvites.AddRange( @@ -813,16 +799,22 @@ private static async Task AssertExistingUserInviteOverridden(DateTime inviteCrea { Email = "test@test.com", RoleId = role2.Id, - UserReleaseRoles = ListOf(new UserReleaseRoleCreateRequest - { - ReleaseId = releaseVersion2.Id, - ReleaseRole = ReleaseRole.PrereleaseViewer - }), - UserPublicationRoles = ListOf(new UserPublicationRoleCreateRequest - { - PublicationId = publication2.Id, - PublicationRole = PublicationRole.Owner - }), + UserReleaseRoles = + [ + new UserReleaseRoleCreateRequest + { + ReleaseId = release2.Id, + ReleaseRole = ReleaseRole.PrereleaseViewer + } + ], + UserPublicationRoles = + [ + new UserPublicationRoleCreateRequest + { + PublicationId = publication2.Id, + PublicationRole = PublicationRole.Owner + } + ] }; var result = await service.InviteUser(inviteRequest); @@ -841,7 +833,7 @@ private static async Task AssertExistingUserInviteOverridden(DateTime inviteCrea Assert.Equal("test@test.com", userInvite.Email); Assert.False(userInvite.Accepted); Assert.Equal(role2.Id, userInvite.RoleId); - Assert.InRange(DateTime.UtcNow.Subtract(userInvite.Created).Milliseconds, 0, 1500); + userInvite.Created.AssertUtcNow(); Assert.Equal(CreatedById.ToString(), userInvite.CreatedById); } @@ -854,7 +846,7 @@ private static async Task AssertExistingUserInviteOverridden(DateTime inviteCrea Assert.Equal(releaseVersion2.Id, userReleaseInvite.ReleaseVersionId); Assert.Equal(ReleaseRole.PrereleaseViewer, userReleaseInvite.Role); Assert.True(userReleaseInvite.EmailSent); - Assert.InRange(DateTime.UtcNow.Subtract(userReleaseInvite.Created).Milliseconds, 0, 1500); + userReleaseInvite.Created.AssertUtcNow(); Assert.Equal(CreatedById, userReleaseInvite.CreatedById); var userPublicationInvites = contentDbContext.UserPublicationInvites @@ -863,7 +855,7 @@ private static async Task AssertExistingUserInviteOverridden(DateTime inviteCrea Assert.Equal("test@test.com", userPublicationInvite.Email); Assert.Equal(publication2.Id, userPublicationInvite.PublicationId); Assert.Equal(PublicationRole.Owner, userPublicationInvite.Role); - Assert.InRange(DateTime.UtcNow.Subtract(userPublicationInvite.Created).Milliseconds, 0, 1500); + userPublicationInvite.Created.AssertUtcNow(); Assert.Equal(CreatedById, userPublicationInvite.CreatedById); } } @@ -877,14 +869,13 @@ public async Task InviteUser_InvalidUserRole() { Email = "test@test.com", RoleId = Guid.NewGuid().ToString(), - UserReleaseRoles = new List(), - UserPublicationRoles = new List() + UserReleaseRoles = [], + UserPublicationRoles = [] }; var result = await service.InviteUser(inviteRequest); - var actionResult = result.AssertLeft(); - actionResult.AssertValidationProblem(ValidationErrorMessages.InvalidUserRole); + result.AssertBadRequest(ValidationErrorMessages.InvalidUserRole); } [Fact] @@ -1010,10 +1001,10 @@ await usersAndRolesDbContext.UserInvites.AddRangeAsync( [Fact] public async Task CancelInvite_InviteNotFound() { - var service = SetupUserManagementService(); - var result = await service.CancelInvite("test@test.com"); - var actionResult = result.AssertLeft(); - actionResult.AssertValidationProblem(ValidationErrorMessages.InviteNotFound); + var service = SetupUserManagementService(); + var result = await service.CancelInvite("test@test.com"); + var actionResult = result.AssertLeft(); + actionResult.AssertValidationProblem(ValidationErrorMessages.InviteNotFound); } [Fact] @@ -1144,7 +1135,6 @@ private static UserManagementService SetupUserManagementService( UsersAndRolesDbContext? usersAndRolesDbContext = null, IPersistenceHelper? usersAndRolesPersistenceHelper = null, IEmailTemplateService? emailTemplateService = null, - IReleaseVersionRepository? releaseVersionRepository = null, IUserRoleService? userRoleService = null, IUserService? userService = null, IUserInviteRepository? userInviteRepository = null, @@ -1160,7 +1150,6 @@ private static UserManagementService SetupUserManagementService( contentDbContext, usersAndRolesPersistenceHelper ?? new PersistenceHelper(usersAndRolesDbContext), emailTemplateService ?? Mock.Of(Strict), - releaseVersionRepository ?? new ReleaseVersionRepository(contentDbContext), userRoleService ?? Mock.Of(Strict), userService ?? AlwaysTrueUserService(CreatedById).Object, userInviteRepository ?? new UserInviteRepository(usersAndRolesDbContext), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserRoleServicePermissionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserRoleServicePermissionTests.cs index 6712fc14372..1ed9381ee0a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserRoleServicePermissionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserRoleServicePermissionTests.cs @@ -1,6 +1,7 @@ #nullable enable using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Database; using GovUk.Education.ExploreEducationStatistics.Admin.Models; @@ -8,9 +9,11 @@ using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using Microsoft.AspNetCore.Identity; using Moq; using static GovUk.Education.ExploreEducationStatistics.Admin.Security.SecurityPolicies; @@ -25,6 +28,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services { public class UserRoleServicePermissionTests { + private readonly DataFixture _dataFixture = new(); + private readonly Publication _publication = new() { Id = Guid.NewGuid(), @@ -57,16 +62,10 @@ await PolicyCheckBuilder() [Fact] public async Task AddReleaseRole() { - var userId = Guid.NewGuid(); - var releaseVersion = new ReleaseVersion(); - var publication = new Publication - { - Id = Guid.NewGuid(), - ReleaseVersions = new List - { - releaseVersion, - } - }; + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); await PolicyCheckBuilder() .SetupResourceCheckToFailWithMatcher>( @@ -77,7 +76,7 @@ await PolicyCheckBuilder() var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - await contentDbContext.AddRangeAsync(publication); + contentDbContext.Publications.Add(publication); await contentDbContext.SaveChangesAsync(); } @@ -85,8 +84,10 @@ await PolicyCheckBuilder() { var service = SetupUserRoleService(contentDbContext: contentDbContext, userService: userService.Object); - return await service.AddReleaseRole(userId: userId, - releaseVersionId: releaseVersion.Id, + + return await service.AddReleaseRole( + userId: Guid.NewGuid(), + releaseId: release.Id, Contributor); } }); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserRoleServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserRoleServiceTests.cs index f18cb9bf4a9..ba091979bae 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserRoleServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserRoleServiceTests.cs @@ -10,9 +10,11 @@ using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using Microsoft.AspNetCore.Identity; using Microsoft.EntityFrameworkCore; using Moq; @@ -33,6 +35,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services { public class UserRoleServiceTests { + private readonly DataFixture _dataFixture = new(); + private readonly User _user = new() { Id = Guid.NewGuid(), @@ -744,10 +748,11 @@ public async Task AddReleaseRole() Id = userId.ToString() }; - var releaseVersion = new ReleaseVersion - { - Publication = new Publication() - }; + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); var userAndRolesDbContextId = Guid.NewGuid().ToString(); var contentDbContextId = Guid.NewGuid().ToString(); @@ -755,10 +760,10 @@ public async Task AddReleaseRole() await using (var userAndRolesDbContext = InMemoryUserAndRolesDbContext(userAndRolesDbContextId)) await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - await userAndRolesDbContext.AddAsync(user); + userAndRolesDbContext.Users.Add(user); await userAndRolesDbContext.SaveChangesAsync(); - contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.Publications.Add(publication); await contentDbContext.SaveChangesAsync(); } @@ -774,7 +779,7 @@ public async Task AddReleaseRole() userManager .Setup(s => s.GetRolesAsync(ItIsUser(user))) - .ReturnsAsync(ListOf(RoleNames.Analyst)); + .ReturnsAsync([RoleNames.Analyst]); await using (var userAndRolesDbContext = InMemoryUserAndRolesDbContext(userAndRolesDbContextId)) await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) @@ -784,19 +789,14 @@ public async Task AddReleaseRole() emailTemplateService: emailTemplateService.Object, userManager: userManager.Object); - var result = await service.AddReleaseRole(userId: userId, - releaseVersionId: releaseVersion.Id, + var result = await service.AddReleaseRole( + userId: userId, + releaseId: release.Id, Contributor); VerifyAllMocks(emailTemplateService, userManager); result.AssertRight(); - - emailTemplateService.Verify(mock => - mock.SendReleaseRoleEmail(user.Email, - It.Is(p => p.Id == releaseVersion.Id), - Contributor), - Times.Once); } await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) @@ -825,10 +825,11 @@ public async Task AddReleaseRole_UserAlreadyHasReleaseRole() Id = userId.ToString() }; - var releaseVersion = new ReleaseVersion - { - Publication = new Publication() - }; + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); var userReleaseRole = new UserReleaseRole { @@ -844,10 +845,10 @@ public async Task AddReleaseRole_UserAlreadyHasReleaseRole() await using (var userAndRolesDbContext = InMemoryUserAndRolesDbContext(userAndRolesDbContextId)) await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - await userAndRolesDbContext.AddAsync(user); + userAndRolesDbContext.Users.Add(user); await userAndRolesDbContext.SaveChangesAsync(); - contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.Publications.Add(publication); contentDbContext.UserReleaseRoles.Add(userReleaseRole); await contentDbContext.SaveChangesAsync(); } @@ -858,8 +859,9 @@ public async Task AddReleaseRole_UserAlreadyHasReleaseRole() var service = SetupUserRoleService(usersAndRolesDbContext: userAndRolesDbContext, contentDbContext: contentDbContext); - var result = await service.AddReleaseRole(userId: userId, - releaseVersionId: releaseVersion.Id, + var result = await service.AddReleaseRole( + userId: userId, + releaseId: release.Id, Contributor); result.AssertBadRequest(UserAlreadyHasResourceRole); @@ -884,28 +886,28 @@ public async Task AddReleaseRole_UserAlreadyHasReleaseRole() [Fact] public async Task AddReleaseRole_NoUser() { - var releaseVersion = new ReleaseVersion - { - Publication = new Publication() - }; + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); - var userAndRolesDbContextId = Guid.NewGuid().ToString(); var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.Publications.Add(publication); await contentDbContext.SaveChangesAsync(); } - await using (var userAndRolesDbContext = InMemoryUserAndRolesDbContext(userAndRolesDbContextId)) + await using (var userAndRolesDbContext = InMemoryUserAndRolesDbContext()) await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { var service = SetupUserRoleService(usersAndRolesDbContext: userAndRolesDbContext, contentDbContext: contentDbContext); - var result = await service.AddReleaseRole(userId: Guid.NewGuid(), - releaseVersionId: releaseVersion.Id, + var result = await service.AddReleaseRole( + userId: Guid.NewGuid(), + releaseId: release.Id, Contributor); result.AssertNotFound(); @@ -936,7 +938,7 @@ public async Task AddReleaseRole_NoRelease() await using (var userAndRolesDbContext = InMemoryUserAndRolesDbContext(userAndRolesDbContextId)) { - await userAndRolesDbContext.AddAsync(user); + userAndRolesDbContext.Users.Add(user); await userAndRolesDbContext.SaveChangesAsync(); } @@ -946,7 +948,10 @@ public async Task AddReleaseRole_NoRelease() var service = SetupUserRoleService(usersAndRolesDbContext: userAndRolesDbContext, contentDbContext: contentDbContext); - var result = await service.AddReleaseRole(userId, Guid.NewGuid(), Contributor); + var result = await service.AddReleaseRole( + userId: userId, + releaseId: Guid.NewGuid(), + Contributor); result.AssertNotFound(); } @@ -971,10 +976,11 @@ public async Task AddReleaseRole_AssignedAnalystGlobalRole() Id = userId.ToString() }; - var releaseVersion = new ReleaseVersion - { - Publication = new Publication() - }; + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); var userAndRolesDbContextId = Guid.NewGuid().ToString(); var contentDbContextId = Guid.NewGuid().ToString(); @@ -982,10 +988,10 @@ public async Task AddReleaseRole_AssignedAnalystGlobalRole() await using (var userAndRolesDbContext = InMemoryUserAndRolesDbContext(userAndRolesDbContextId)) await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - await userAndRolesDbContext.AddAsync(user); + userAndRolesDbContext.Users.Add(user); await userAndRolesDbContext.SaveChangesAsync(); - contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.Publications.Add(publication); await contentDbContext.SaveChangesAsync(); } @@ -1001,7 +1007,7 @@ public async Task AddReleaseRole_AssignedAnalystGlobalRole() userManager .Setup(s => s.GetRolesAsync(ItIsUser(user))) - .ReturnsAsync(new List()); + .ReturnsAsync([]); userManager .Setup(s => s.AddToRoleAsync(ItIsUser(user), RoleNames.Analyst)) @@ -1016,15 +1022,11 @@ public async Task AddReleaseRole_AssignedAnalystGlobalRole() emailTemplateService: emailTemplateService.Object, userManager: userManager.Object); - var result = await service.AddReleaseRole(userId: userId, - releaseVersionId: releaseVersion.Id, + var result = await service.AddReleaseRole( + userId: userId, + releaseId: release.Id, Contributor); - emailTemplateService.Verify(mock => - mock.SendReleaseRoleEmail(user.Email, - It.Is(p => p.Id == releaseVersion.Id), - Contributor), - Times.Once); VerifyAllMocks(emailTemplateService, userManager); result.AssertRight(); @@ -1056,10 +1058,11 @@ public async Task AddReleaseRole_HasLowerGlobalRoleThanAnalyst() Id = userId.ToString() }; - var releaseVersion = new ReleaseVersion - { - Publication = new Publication() - }; + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); var userAndRolesDbContextId = Guid.NewGuid().ToString(); var contentDbContextId = Guid.NewGuid().ToString(); @@ -1067,10 +1070,10 @@ public async Task AddReleaseRole_HasLowerGlobalRoleThanAnalyst() await using (var userAndRolesDbContext = InMemoryUserAndRolesDbContext(userAndRolesDbContextId)) await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - await userAndRolesDbContext.AddAsync(user); + userAndRolesDbContext.Users.Add(user); await userAndRolesDbContext.SaveChangesAsync(); - contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.Publications.Add(publication); await contentDbContext.SaveChangesAsync(); } @@ -1108,18 +1111,13 @@ public async Task AddReleaseRole_HasLowerGlobalRoleThanAnalyst() emailTemplateService: emailTemplateService.Object, userManager: userManager.Object); - var result = await service.AddReleaseRole(userId: userId, - releaseVersionId: releaseVersion.Id, + var result = await service.AddReleaseRole( + userId: userId, + releaseId: release.Id, Contributor); VerifyAllMocks(emailTemplateService, userManager); - emailTemplateService.Verify(mock => - mock.SendReleaseRoleEmail(user.Email, - It.Is(p => p.Id == releaseVersion.Id), - Contributor), - Times.Once); - result.AssertRight(); } @@ -1149,10 +1147,11 @@ public async Task AddReleaseRole_HasHigherGlobalRoleThanAnalyst() Id = userId.ToString() }; - var releaseVersion = new ReleaseVersion - { - Publication = new Publication() - }; + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); var userAndRolesDbContextId = Guid.NewGuid().ToString(); var contentDbContextId = Guid.NewGuid().ToString(); @@ -1160,10 +1159,10 @@ public async Task AddReleaseRole_HasHigherGlobalRoleThanAnalyst() await using (var userAndRolesDbContext = InMemoryUserAndRolesDbContext(userAndRolesDbContextId)) await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - await userAndRolesDbContext.AddAsync(user); + userAndRolesDbContext.Users.Add(user); await userAndRolesDbContext.SaveChangesAsync(); - contentDbContext.ReleaseVersions.Add(releaseVersion); + contentDbContext.Publications.Add(publication); await contentDbContext.SaveChangesAsync(); } @@ -1192,8 +1191,9 @@ public async Task AddReleaseRole_HasHigherGlobalRoleThanAnalyst() emailTemplateService: emailTemplateService.Object, userManager: userManager.Object); - var result = await service.AddReleaseRole(userId: userId, - releaseVersionId: releaseVersion.Id, + var result = await service.AddReleaseRole( + userId: userId, + releaseId: release.Id, Contributor); emailTemplateService.Verify(mock => diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/UserManagement/UserManagementController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/UserManagement/UserManagementController.cs index 71257cbdd4e..7a3b05162bf 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/UserManagement/UserManagementController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/UserManagement/UserManagementController.cs @@ -68,7 +68,7 @@ public async Task> AddPublicationRole(Guid userId, UserPublic public async Task> AddReleaseRole(Guid userId, UserReleaseRoleCreateRequest request) { return await _userRoleService - .AddReleaseRole(userId, request.ReleaseId, request.ReleaseRole) + .AddReleaseRole(userId: userId, releaseId: request.ReleaseId, request.ReleaseRole) .HandleFailuresOrOk(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IUserRoleService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IUserRoleService.cs index 802871c1a99..77ae0b9ec25 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IUserRoleService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IUserRoleService.cs @@ -15,8 +15,9 @@ public interface IUserRoleService Task> AddPublicationRole(Guid userId, Guid publicationId, PublicationRole role); - Task> AddReleaseRole(Guid userId, - Guid releaseVersionId, + Task> AddReleaseRole( + Guid userId, + Guid releaseId, ReleaseRole role); Task> UpgradeToGlobalRoleIfRequired(string globalRoleNameToSet, Guid userId); @@ -31,7 +32,8 @@ Task> AddReleaseRole(Guid userId, Task>> GetPublicationRolesForUser(Guid userId); - Task>> GetPublicationRolesForPublication(Guid publicationId); + Task>> GetPublicationRolesForPublication( + Guid publicationId); Task>> GetReleaseRoles(Guid userId); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs index a85acdbd5f5..7a0fe3e13ee 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs @@ -14,14 +14,13 @@ using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Predicates; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using static GovUk.Education.ExploreEducationStatistics.Admin.Models.GlobalRoles; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationUtils; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationErrorMessages; -using IReleaseVersionRepository = - GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces.IReleaseVersionRepository; namespace GovUk.Education.ExploreEducationStatistics.Admin.Services { @@ -31,7 +30,6 @@ public class UserManagementService : IUserManagementService private readonly ContentDbContext _contentDbContext; private readonly IPersistenceHelper _usersAndRolesPersistenceHelper; private readonly IEmailTemplateService _emailTemplateService; - private readonly IReleaseVersionRepository _releaseVersionRepository; private readonly IUserRoleService _userRoleService; private readonly IUserService _userService; private readonly IUserInviteRepository _userInviteRepository; @@ -44,7 +42,6 @@ public UserManagementService( ContentDbContext contentDbContext, IPersistenceHelper usersAndRolesPersistenceHelper, IEmailTemplateService emailTemplateService, - IReleaseVersionRepository releaseVersionRepository, IUserRoleService userRoleService, IUserService userService, IUserInviteRepository userInviteRepository, @@ -56,7 +53,6 @@ public UserManagementService( _contentDbContext = contentDbContext; _usersAndRolesPersistenceHelper = usersAndRolesPersistenceHelper; _emailTemplateService = emailTemplateService; - _releaseVersionRepository = releaseVersionRepository; _userRoleService = userRoleService; _userService = userService; _userInviteRepository = userInviteRepository; @@ -109,26 +105,13 @@ public async Task>> ListReleases() { return await _userService .CheckCanManageAllUsers() - .OnSuccess(async () => - { - var releaseVersions = await _releaseVersionRepository.ListLatestReleaseVersions(); - return await releaseVersions - .ToAsyncEnumerable() - .SelectAwait(async releaseVersion => - { - var publicationTitle = await _contentDbContext.Publications - .Where(p => p.Id == releaseVersion.PublicationId) - .Select(p => p.Title) - .FirstAsync(); - - return new IdTitleViewModel - { - Id = releaseVersion.Id, - Title = $"{publicationTitle} - {releaseVersion.Title}" - }; - }) - .ToListAsync(); - }); + .OnSuccess(async () => await _contentDbContext.Releases + .Select(r => new IdTitleViewModel + { + Id = r.Id, + Title = $"{r.Publication.Title} - {r.Title}" + }) + .ToListAsync()); } public async Task>> ListRoles() @@ -324,8 +307,12 @@ public async Task> InviteUser(UserInviteCreateR foreach (var userReleaseRole in request.UserReleaseRoles) { + var latestReleaseVersion = await _contentDbContext.ReleaseVersions + .LatestReleaseVersion(releaseId: userReleaseRole.ReleaseId) + .SingleAsync(); + await _userReleaseInviteRepository.Create( - releaseVersionId: userReleaseRole.ReleaseId, + releaseVersionId: latestReleaseVersion!.Id, email: sanitisedEmail, releaseRole: userReleaseRole.ReleaseRole, emailSent: true, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserRoleService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserRoleService.cs index c220c0977d0..1521a0c1534 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserRoleService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserRoleService.cs @@ -14,6 +14,7 @@ using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Predicates; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; @@ -110,35 +111,36 @@ await _userPublicationRoleRepository.Create( }); } - public async Task> AddReleaseRole(Guid userId, Guid releaseVersionId, ReleaseRole role) + public async Task> AddReleaseRole( + Guid userId, + Guid releaseId, + ReleaseRole role) { - return await _contentPersistenceHelper - .CheckEntityExists(releaseVersionId, query => query - .Include(rv => rv.Publication)) - .OnSuccess(releaseVersion => - _userService.CheckCanUpdateReleaseRole(releaseVersion.Publication, role) - .OnSuccess(async () => - { - return await _usersAndRolesPersistenceHelper - .CheckEntityExists(userId.ToString()) - .OnSuccessDo(_ => ValidateReleaseRoleCanBeAdded(userId: userId, - releaseVersionId: releaseVersionId, - role)) - .OnSuccess(async user => - { - await _userReleaseRoleRepository.Create( - userId: userId, - releaseVersionId: releaseVersion.Id, - role, - createdById: _userService.GetUserId()); - - var globalRole = GetAssociatedGlobalRoleNameForReleaseRole(role); - await UpgradeToGlobalRoleIfRequired(globalRole, user); - - return _emailTemplateService.SendReleaseRoleEmail(user.Email, releaseVersion, role); - }); - }) - ); + return await _contentDbContext.ReleaseVersions + .Include(rv => rv.Release) + .ThenInclude(r => r.Publication) + .LatestReleaseVersion(releaseId: releaseId) + .SingleOrNotFoundAsync() + .OnSuccessDo(rv => _userService.CheckCanUpdateReleaseRole(rv!.Release.Publication, role)) + .OnSuccessDo(rv => ValidateReleaseRoleCanBeAdded(userId: userId, + releaseVersionId: rv!.Id, + role)) + .OnSuccessCombineWith(_ => _usersAndRolesDbContext.Users + .SingleOrNotFoundAsync(u => u.Id == userId.ToString())) + .OnSuccess(async tuple => + { + var (releaseVersion, user) = tuple; + await _userReleaseRoleRepository.Create( + userId: userId, + releaseVersionId: releaseVersion!.Id, + role, + createdById: _userService.GetUserId()); + + var globalRole = GetAssociatedGlobalRoleNameForReleaseRole(role); + await UpgradeToGlobalRoleIfRequired(globalRole, user); + + return _emailTemplateService.SendReleaseRoleEmail(user.Email!, releaseVersion, role); + }); } private async Task SetExclusiveGlobalRole(string? globalRoleNameToSet, ApplicationUser user) @@ -526,7 +528,8 @@ private async Task> ValidatePublicationRoleCanBeAdded return Unit.Instance; } - private async Task> ValidateReleaseRoleCanBeAdded(Guid userId, + private async Task> ValidateReleaseRoleCanBeAdded( + Guid userId, Guid releaseVersionId, ReleaseRole role) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs index a33fa22be7b..8cf258824ce 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs @@ -393,11 +393,12 @@ public async Task Success() var result = await repository.ListLatestPublishedReleaseVersionIds(publications[0].Id); // Expect the result to contain the highest published version of each release for the specified publication - AssertIdsAreEqualIgnoringOrder(new[] - { - publications[0].ReleaseVersions[2].Id, - publications[0].ReleaseVersions[5].Id - }, result); + AssertIdsAreEqualIgnoringOrder( + [ + publications[0].ReleaseVersions[2].Id, + publications[0].ReleaseVersions[5].Id + ], + result); } [Fact] @@ -479,11 +480,12 @@ public async Task Success() var result = await repository.ListLatestPublishedReleaseVersions(publications[0].Id); // Expect the result to contain the highest published version of each release for the specified publication - AssertIdsAreEqualIgnoringOrder(new[] - { - publications[0].ReleaseVersions[2].Id, - publications[0].ReleaseVersions[5].Id - }, result); + AssertIdsAreEqualIgnoringOrder( + [ + publications[0].ReleaseVersions[2].Id, + publications[0].ReleaseVersions[5].Id + ], + result); } [Fact] @@ -567,12 +569,13 @@ public async Task Success() var result = await repository.ListLatestReleaseVersionIds(publications[0].Id); // Expect the result to contain the highest version of each release for the specified publication - AssertIdsAreEqualIgnoringOrder(new[] - { - publications[0].ReleaseVersions[0].Id, - publications[0].ReleaseVersions[3].Id, - publications[0].ReleaseVersions[5].Id - }, result); + AssertIdsAreEqualIgnoringOrder( + [ + publications[0].ReleaseVersions[0].Id, + publications[0].ReleaseVersions[3].Id, + publications[0].ReleaseVersions[5].Id + ], + result); } [Fact] @@ -611,136 +614,16 @@ public async Task PublicationDoesNotExist_ReturnsEmpty() } } - public class ListLatestReleaseVersionsTests : ReleaseVersionRepositoryTests - { - [Fact] - public async Task AllPublications_Success() - { - var publications = _dataFixture - .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - var result = await repository.ListLatestReleaseVersions(); - - // Expect the result to contain the highest version of each release for each publication - AssertIdsAreEqualIgnoringOrder(new[] - { - publications[0].ReleaseVersions[0].Id, - publications[0].ReleaseVersions[3].Id, - publications[0].ReleaseVersions[5].Id, - publications[1].ReleaseVersions[0].Id, - publications[1].ReleaseVersions[3].Id, - publications[1].ReleaseVersions[5].Id - }, result); - } - - [Fact] - public async Task AllPublications_PublicationsHaveNoReleaseVersions_ReturnsEmpty() - { - var publications = _dataFixture - .DefaultPublication() - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestReleaseVersions()); - } - - [Fact] - public async Task AllPublications_NoPublicationsExist_ReturnsEmpty() - { - await using var contentDbContext = InMemoryContentDbContext(); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestReleaseVersions()); - } - - [Fact] - public async Task SpecificPublication_Success() - { - var publications = _dataFixture - .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - var result = await repository.ListLatestReleaseVersions(publications[0].Id); - - // Expect the result to contain the highest version of each release for the specified publication - AssertIdsAreEqualIgnoringOrder(new[] - { - publications[0].ReleaseVersions[0].Id, - publications[0].ReleaseVersions[3].Id, - publications[0].ReleaseVersions[5].Id - }, result); - } - - [Fact] - public async Task SpecificPublication_PublicationHasNoReleaseVersions_ReturnsEmpty() - { - var publications = _dataFixture - .DefaultPublication() - // Index 0 has no release versions - // Index 1 has a published release version - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestReleaseVersions(publications[0].Id)); - } - - [Fact] - public async Task SpecificPublication_PublicationDoesNotExist_ReturnsEmpty() - { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); - - var contextId = await AddTestData(publication); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestReleaseVersions(Guid.NewGuid())); - } - } - - private static void AssertIdsAreEqualIgnoringOrder(IReadOnlyCollection expectedIds, + private static void AssertIdsAreEqualIgnoringOrder( + IReadOnlyCollection expectedIds, IReadOnlyCollection actualReleaseVersions) { Assert.Equal(expectedIds.Count, actualReleaseVersions.Count); Assert.True(SequencesAreEqualIgnoringOrder(expectedIds, actualReleaseVersions.Select(rv => rv.Id))); } - private static void AssertIdsAreEqualIgnoringOrder(IReadOnlyCollection expectedIds, + private static void AssertIdsAreEqualIgnoringOrder( + IReadOnlyCollection expectedIds, IReadOnlyCollection actualIds) { Assert.Equal(expectedIds.Count, actualIds.Count); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs index 4c5656e94ab..45486db4b32 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs @@ -8,7 +8,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.In public interface IReleaseVersionRepository { - Task GetPublishedDate(Guid releaseVersionId, + Task GetPublishedDate( + Guid releaseVersionId, DateTime actualPublishedDate); /// @@ -101,13 +102,6 @@ Task> ListLatestReleaseVersionIds( Guid publicationId, CancellationToken cancellationToken = default); - /// - /// Retrieves the latest versions of all releases in reverse chronological order. - /// - /// A to observe while waiting for the task to complete. - /// A collection of the latest versions of all releases. - Task> ListLatestReleaseVersions(CancellationToken cancellationToken = default); - /// /// Retrieves the latest versions of all releases associated with a given publication in reverse chronological order. /// diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs index 378190ab617..dac9d53b4f1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs @@ -20,7 +20,8 @@ public ReleaseVersionRepository(ContentDbContext contentDbContext) _contentDbContext = contentDbContext; } - public async Task GetPublishedDate(Guid releaseVersionId, + public async Task GetPublishedDate( + Guid releaseVersionId, DateTime actualPublishedDate) { var releaseVersion = await _contentDbContext.ReleaseVersions @@ -138,14 +139,6 @@ public async Task> ListLatestReleaseVersionIds( .ToListAsync(cancellationToken: cancellationToken); } - public async Task> ListLatestReleaseVersions(CancellationToken cancellationToken = default) - { - return (await _contentDbContext.ReleaseVersions.LatestReleaseVersions() - .ToListAsync(cancellationToken: cancellationToken)) - .OrderByReverseChronologicalOrder() - .ToList(); - } - public async Task> ListLatestReleaseVersions( Guid publicationId, CancellationToken cancellationToken = default) From 24b97a18d83d5b98be565cfccd6f267dbf45461c Mon Sep 17 00:00:00 2001 From: dfe-sdt Date: Thu, 5 Dec 2024 09:44:49 +0000 Subject: [PATCH 045/144] chore(tests): update test snapshots --- .../tests/snapshots/data_catalogue_snapshot.json | 2 +- .../tests/snapshots/find_statistics_snapshot.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json index eeeed40583d..3308500e031 100644 --- a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json +++ b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json @@ -1,5 +1,5 @@ { - "num_datasets": "926 data sets", + "num_datasets": "924 data sets", "themes": [ { "publications": [ diff --git a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json index 45ca669aec5..7705e194c2f 100644 --- a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json +++ b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json @@ -289,7 +289,7 @@ { "publication_summary": "Annual national and provider level statistics on new entrants to Initial Teacher Training (ITT) in England and their characteristics.", "publication_title": "Initial Teacher Training Census", - "published": "7 Dec 2023", + "published": "5 Dec 2024", "release_type": "Official statistics", "theme": "Teachers and school workforce" }, @@ -324,8 +324,8 @@ { "publication_summary": "GCSE results of pupils at the end of KS4 attending state-funded schools in England at national & LA level. This release includes pupil characteristic breakdowns", "publication_title": "Key stage 4 performance", - "published": "1 Feb 2024", - "release_type": "Accredited official statistics", + "published": "5 Dec 2024", + "release_type": "Official statistics", "theme": "School and college outcomes and performance" }, { @@ -527,7 +527,7 @@ { "publication_summary": "Pupil attendance and absence data including termly national statistics and fortnightly statistics in development derived from DfE\u2019s regular attendance data", "publication_title": "Pupil attendance in schools", - "published": "21 Nov 2024", + "published": "5 Dec 2024", "release_type": "Official statistics in development", "theme": "Pupils and schools" }, From e1975b0b24502d203a551893ac7ea03193a810be Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 10:05:29 +0000 Subject: [PATCH 046/144] EES-5685 - added many additional details to alerts being sent through to Slack, including colour-coding based on severity, alert conditions and links to alerts and the affected resources --- .../templates/logic-app-template.json | 185 +++++++++++++++++- infrastructure/templates/template.json | 6 + 2 files changed, 183 insertions(+), 8 deletions(-) diff --git a/infrastructure/templates/logic-app-template.json b/infrastructure/templates/logic-app-template.json index 38ab6f9cd79..5d45e64bb63 100644 --- a/infrastructure/templates/logic-app-template.json +++ b/infrastructure/templates/logic-app-template.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", - "parameters":{ + "parameters": { "logicAppName": { "type": "string" }, @@ -10,9 +10,15 @@ }, "slackAppToken": { "type": "securestring" + }, + "subscription": { + "type": "string" + }, + "resourceGroup": { + "type": "string" } }, - "resources":[ + "resources": [ { "type": "Microsoft.Logic/workflows", "apiVersion": "2017-07-01", @@ -34,7 +40,36 @@ "data": { "properties": { "alertContext": { - "properties": {}, + "properties": { + "condition": { + "properties": { + "allOf": { + "properties": { + "metricName": { + "type": "string" + }, + "timeAggregation": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "threshold": { + "type": "string" + }, + "metricValue": { + "type": "number" + }, + "type": "object" + } + }, + "windowStartTime": "string", + "windowEndTime": "string", + "type": "array" + } + }, + "type": "object" + }, "type": "object" }, "essentials": { @@ -97,17 +132,151 @@ } }, "actions": { - "HTTP_Webhook": { + "Initialize severity variable": { + "runAfter": {}, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "severity", + "type": "string", + "value": "@{triggerBody()?['data']?['essentials']?['severity']}" + } + ] + } + }, + "Initialize monitorCondition variable": { + "runAfter": {}, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "monitorCondition", + "type": "string", + "value": "@{triggerBody()?['data']?['essentials']?['monitorCondition']}" + } + ] + } + }, + "Initialize outsideThresholdMessage variable": { + "runAfter": {}, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "outsideThresholdMessage", + "type": "string", + "value": "The @{triggerBody()?['data']?['alertContext']?['condition']?['allOf']?[0]?['timeAggregation']} of @{triggerBody()?['data']?['alertContext']?['condition']?['allOf']?[0]?['metricName']} is at @{triggerBody()?['data']?['alertContext']?['condition']?['allOf']?[0]?['metricValue']}, which is @{triggerBody()?['data']?['alertContext']?['condition']?['allOf']?[0]?['operator']} the threshold of @{triggerBody()?['data']?['alertContext']?['condition']?['allOf']?[0]?['threshold']}" + } + ] + } + }, + "Initialize withinThresholdMessage variable": { "runAfter": {}, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "withinThresholdMessage", + "type": "string", + "value": "The @{triggerBody()?['data']?['alertContext']?['condition']?['allOf']?[0]?['timeAggregation']} of @{triggerBody()?['data']?['alertContext']?['condition']?['allOf']?[0]?['metricName']} is at @{triggerBody()?['data']?['alertContext']?['condition']?['allOf']?[0]?['metricValue']}, and is no longer @{triggerBody()?['data']?['alertContext']?['condition']?['allOf']?[0]?['operator']} the threshold of @{triggerBody()?['data']?['alertContext']?['condition']?['allOf']?[0]?['threshold']}" + } + ] + } + }, + "Initialize severityDisplay variable": { + "runAfter": { + "Initialize severity variable": [ + "Succeeded" + ] + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "severityDisplay", + "type": "string", + "value": "@{if(equals(variables('severity'), 'Sev0'), 'CRITICAL!', '')}@{if(equals(variables('severity'), 'Sev1'), 'Error!', '')}@{if(equals(variables('severity'), 'Sev2'), 'Warning', '')}@{if(equals(variables('severity'), 'Sev3'), 'Information', '')}@{if(equals(variables('severity'), 'Sev4'), 'Verbose', '')}" + } + ] + } + }, + "Initialize severityColour variable": { + "runAfter": { + "Initialize severity variable": [ + "Succeeded" + ] + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "severityColour", + "type": "string", + "value": "@{if(equals(variables('severity'), 'Sev0'), 'danger', '')}@{if(equals(variables('severity'), 'Sev1'), 'danger', '')}@{if(equals(variables('severity'), 'Sev2'), 'warning', '')}@{if(equals(variables('severity'), 'Sev3'), 'good', '')}@{if(equals(variables('severity'), 'Sev4'), 'good', '')}" + } + ] + } + }, + "HTTP_Webhook": { + "runAfter": { + "Initialize monitorCondition variable": [ + "Succeeded" + ], + "Initialize severityColour variable": [ + "Succeeded" + ], + "Initialize severityDisplay variable": [ + "Succeeded" + ], + "Initialize withinThresholdMessage variable": [ + "Succeeded" + ], + "Initialize outsideThresholdMessage variable": [ + "Succeeded" + ] + }, "type": "HttpWebhook", "inputs": { "subscribe": { + "method": "POST", + "uri": "https://slack.com/api/chat.postMessage", "body": { "channel": "[parameters('slackAlertsChannel')]", - "text": "Alert @{triggerBody()?['data']?['essentials']?['monitorCondition']}!\n@{triggerBody()?['data']?['essentials']?['alertRule']}\n@{triggerBody()?['data']?['essentials']?['description']}" + "text": "*Alert @{variables('monitorCondition')}!*\n@{triggerBody()?['data']?['essentials']?['alertRule']}\n@{triggerBody()?['data']?['essentials']?['description']}\n\n@{if(equals(variables('monitorCondition'), 'Resolved'), variables('withinThresholdMessage'), variables('outsideThresholdMessage'))}\n\n", + "attachments": [ + { + "color": "@{if(equals(variables('monitorCondition'), 'Resolved'), 'good', variables('severityColour'))}", + "fields": [ + { + "title": "Alert", + "value": "@{triggerBody()?['data']?['essentials']?['alertRule']}", + "short": true + }, + { + "title": "Severity", + "value": "@{if(equals(variables('monitorCondition'), 'Resolved'), 'Resolved', variables('severityDisplay'))}", + "short": true + }, + { + "title": "@{variables('monitorCondition')} at", + "value": "@{if(equals(variables('monitorCondition'), 'Resolved'), formatDateTime(triggerBody()?['data']?['essentials']?['resolvedDateTime'], 'dd/MM/yyyy h:mm:sstt'), formatDateTime(triggerBody()?['data']?['essentials']?['firedDateTime'], 'dd/MM/yyyy h:mm:sstt'))}", + "short": false + }, + { + "title": "Window start", + "value": "@{if(equals(triggerBody()?['data']?['alertContext']?['condition']?['windowStartTime'], null), '', formatDateTime(triggerBody()?['data']?['alertContext']?['condition']?['windowStartTime'], 'dd/MM/yyyy h:mm:sstt'))}", + "short": true + }, + { + "title": "Window end", + "value": "@{if(equals(triggerBody()?['data']?['alertContext']?['condition']?['windowEndTime'], null), '', formatDateTime(triggerBody()?['data']?['alertContext']?['condition']?['windowEndTime'], 'dd/MM/yyyy h:mm:sstt'))}", + "short": true + } + ] + } + ] }, - "method": "POST", - "uri": "https://slack.com/api/chat.postMessage", "headers": { "Content-Type": "application/json", "Authorization": "[concat('Bearer ', parameters('slackAppToken'))]" @@ -123,4 +292,4 @@ } } ] -} \ No newline at end of file +} diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index d3c9abd0cb5..e61d96b884b 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -4034,6 +4034,12 @@ }, "secretName": "ees-alerts-slackapptoken" } + }, + "subscription": { + "value": "[parameters('subscription')]" + }, + "resourceGroup": { + "value": "[resourceGroup().name]" } } }, From 935b161fff313c83659cb35b9431aab4adee237a Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 5 Dec 2024 11:42:24 +0000 Subject: [PATCH 047/144] EES-5717 set number of decimal places on chart minor axis --- .../Model/Chart/ChartAxisConfiguration.cs | 1 + .../chart/ChartAxisConfiguration.tsx | 22 ++++++++++++++----- .../__tests__/ChartAxisConfiguration.test.tsx | 1 + .../charts/components/HorizontalBarBlock.tsx | 5 ++++- .../charts/components/LineChartBlock.tsx | 5 ++++- .../charts/components/VerticalBarBlock.tsx | 5 ++++- .../src/modules/charts/types/chart.ts | 1 + .../charts/util/getMinorAxisDecimalPlaces.ts | 4 ++++ 8 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Chart/ChartAxisConfiguration.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Chart/ChartAxisConfiguration.cs index 8bc8266b8c5..337dec84286 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Chart/ChartAxisConfiguration.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Chart/ChartAxisConfiguration.cs @@ -28,6 +28,7 @@ public class ChartAxisConfiguration public bool Visible = true; public string Title = null!; public string Unit = null!; + public int? DecimalPlaces; public bool ShowGrid = true; public AxisLabel Label = null!; diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartAxisConfiguration.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartAxisConfiguration.tsx index c671e0d1304..23240174f83 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartAxisConfiguration.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartAxisConfiguration.tsx @@ -278,6 +278,7 @@ const ChartAxisConfiguration = ({ min: Yup.number(), visible: Yup.boolean(), unit: Yup.string(), + decimalPlaces: Yup.string(), labelText: Yup.string(), labelWidth: Yup.number().positive('Label width must be positive'), }); @@ -464,6 +465,7 @@ const ChartAxisConfiguration = ({ labelRotated: values?.label?.rotated ?? false, labelText: values?.label?.text ?? '', labelWidth: values?.label?.width ?? undefined, + decimalPlaces: values?.decimalPlaces ?? undefined, referenceLines: values?.referenceLines.map(line => { return { @@ -592,12 +594,20 @@ const ChartAxisConfiguration = ({ name="visible" label="Show axis" conditional={ - - label="Displayed unit" - name="unit" - hint="Leave blank to set default from metadata" - width={10} - /> + <> + + label="Displayed unit" + name="unit" + hint="Leave blank to set default from metadata" + width={10} + /> + + label="Displayed decimal places" + name="decimalPlaces" + hint="Leave blank to set default from metadata" + width={10} + /> + } /> )} diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartAxisConfiguration.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartAxisConfiguration.test.tsx index 9192b3b5083..4eccfd5afb1 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartAxisConfiguration.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/__tests__/ChartAxisConfiguration.test.tsx @@ -100,6 +100,7 @@ describe('ChartAxisConfiguration', () => { tickSpacing: 1, type: 'minor', unit: '', + decimalPlaces: undefined, visible: true, label: { text: '', width: 100, rotated: false }, max: undefined, diff --git a/src/explore-education-statistics-common/src/modules/charts/components/HorizontalBarBlock.tsx b/src/explore-education-statistics-common/src/modules/charts/components/HorizontalBarBlock.tsx index 779b51b3204..1508d8fef76 100644 --- a/src/explore-education-statistics-common/src/modules/charts/components/HorizontalBarBlock.tsx +++ b/src/explore-education-statistics-common/src/modules/charts/components/HorizontalBarBlock.tsx @@ -103,7 +103,10 @@ const HorizontalBarBlock = ({ meta, }); - const minorAxisDecimals = getMinorAxisDecimalPlaces(dataSetCategoryConfigs); + const minorAxisDecimals = getMinorAxisDecimalPlaces( + dataSetCategoryConfigs, + axes.minor.decimalPlaces, + ); const minorAxisUnit = axes.minor.unit || getUnit(dataSetCategoryConfigs); const chartHasNegativeValues = (parseNumber(minorDomainTicks.domain?.[0]) ?? 0) < 0; diff --git a/src/explore-education-statistics-common/src/modules/charts/components/LineChartBlock.tsx b/src/explore-education-statistics-common/src/modules/charts/components/LineChartBlock.tsx index 2988a26d92f..6cce1a6c2fa 100644 --- a/src/explore-education-statistics-common/src/modules/charts/components/LineChartBlock.tsx +++ b/src/explore-education-statistics-common/src/modules/charts/components/LineChartBlock.tsx @@ -90,7 +90,10 @@ const LineChartBlock = ({ legendItems: legend.items, meta, }); - const minorAxisDecimals = getMinorAxisDecimalPlaces(dataSetCategoryConfigs); + const minorAxisDecimals = getMinorAxisDecimalPlaces( + dataSetCategoryConfigs, + axes.minor.decimalPlaces, + ); const minorAxisUnit = axes.minor.unit || getUnit(dataSetCategoryConfigs); const yAxisWidth = getMinorAxisSize({ dataSetCategories, diff --git a/src/explore-education-statistics-common/src/modules/charts/components/VerticalBarBlock.tsx b/src/explore-education-statistics-common/src/modules/charts/components/VerticalBarBlock.tsx index cd8d9d59653..658345387c3 100644 --- a/src/explore-education-statistics-common/src/modules/charts/components/VerticalBarBlock.tsx +++ b/src/explore-education-statistics-common/src/modules/charts/components/VerticalBarBlock.tsx @@ -109,7 +109,10 @@ const VerticalBarBlock = ({ legendItems: legend.items, meta, }); - const minorAxisDecimals = getMinorAxisDecimalPlaces(dataSetCategoryConfigs); + const minorAxisDecimals = getMinorAxisDecimalPlaces( + dataSetCategoryConfigs, + axes.minor.decimalPlaces, + ); const minorAxisUnit = axes.minor.unit || getUnit(dataSetCategoryConfigs); const yAxisWidth = getMinorAxisSize({ dataSetCategories, diff --git a/src/explore-education-statistics-common/src/modules/charts/types/chart.ts b/src/explore-education-statistics-common/src/modules/charts/types/chart.ts index 3e23bf25703..66771b464a6 100644 --- a/src/explore-education-statistics-common/src/modules/charts/types/chart.ts +++ b/src/explore-education-statistics-common/src/modules/charts/types/chart.ts @@ -67,6 +67,7 @@ export interface AxisConfiguration { referenceLines: ReferenceLine[]; visible?: boolean; unit?: string; + decimalPlaces?: number; showGrid?: boolean; label?: Label; size?: number; diff --git a/src/explore-education-statistics-common/src/modules/charts/util/getMinorAxisDecimalPlaces.ts b/src/explore-education-statistics-common/src/modules/charts/util/getMinorAxisDecimalPlaces.ts index 56befa4b1ba..f9c421cd849 100644 --- a/src/explore-education-statistics-common/src/modules/charts/util/getMinorAxisDecimalPlaces.ts +++ b/src/explore-education-statistics-common/src/modules/charts/util/getMinorAxisDecimalPlaces.ts @@ -16,7 +16,11 @@ import { DataSetCategoryConfig } from '@common/modules/charts/util/getDataSetCat */ export default function getMinorAxisDecimalPlaces( categoryDataSets: DataSetCategoryConfig[], + decimalPlaces?: number, ): number | undefined { + if (decimalPlaces || decimalPlaces === 0) { + return decimalPlaces; + } return categoryDataSets.reduce((acc, { dataSet }) => { if (typeof dataSet.indicator.decimalPlaces === 'undefined') { return acc; From 700441cd492e81eea84331b27d3c3b792d754995 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 12:13:03 +0000 Subject: [PATCH 048/144] EES-5705 - calculating final test failure count from final run attempt so as to not mask flaky failures in final run from the final merged report, which can mark a test as successful if it passed in any other run. --- tests/robot-tests/run_tests.py | 5 ++- tests/robot-tests/tests/libs/slack.py | 44 +++++++++++++++++++-------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/tests/robot-tests/run_tests.py b/tests/robot-tests/run_tests.py index 0f7ce02fdb4..6e603e8b29c 100755 --- a/tests/robot-tests/run_tests.py +++ b/tests/robot-tests/run_tests.py @@ -127,10 +127,14 @@ def run(): logger.info(f"Running Robot tests with {max_run_attempts} maximum run attempts") + test_run_results_folder = "" + try: # Run tests while test_run_index < max_run_attempts: try: + test_run_results_folder = f"{main_results_folder}{os.sep}run-{test_run_index + 1}" + # Ensure all SeleniumLibrary elements and keywords are updated to use a brand new # Selenium instance for every test (re)run. if test_run_index > 0: @@ -142,7 +146,6 @@ def run(): _clear_files_before_next_test_run_attempt(rerunning_failed_suites) # Create a folder to contain this test run attempt's outputs and reports. - test_run_results_folder = f"{main_results_folder}{os.sep}run-{test_run_index + 1}" os.makedirs(test_run_results_folder) if not Path(f"{main_results_folder}/downloads").exists(): diff --git a/tests/robot-tests/tests/libs/slack.py b/tests/robot-tests/tests/libs/slack.py index ad776bb6bc5..e322c12ca0d 100644 --- a/tests/robot-tests/tests/libs/slack.py +++ b/tests/robot-tests/tests/libs/slack.py @@ -23,27 +23,43 @@ def __init__(self): self.client = WebClient(token=self.slack_app_token) def _build_test_results_attachments(self, env: str, suites_ran: str, suites_failed: [], number_of_test_runs: int): - with open(f"{PATH}{os.sep}output.xml", "rb") as report: - contents = report.read() - try: - soup = BeautifulSoup(contents, features="xml") - tests = soup.find("total").find("stat") - failed_tests = int(tests["fail"]) - passed_tests = int(tests["pass"]) - skipped_tests = int(tests["skip"]) + # Gather high-level passing test count from the merged report from all run attempts. + with open(f"{PATH}{os.sep}output.xml", "rb") as report: + contents = report.read() + soup = BeautifulSoup(contents, features="xml") + tests = soup.find("total").find("stat") + passed_tests = int(tests["pass"]) + skipped_tests = int(tests["skip"]) + failed_tests_in_merged_report = int(tests["fail"]) + + # Gather final failing test counts from the final run attempt's report. We do this so as to most + # accurately report what was failing right at the end of the process. It's not safe to do this from + # the merged report because failures in one run attempt that then succeed in another attempt will be + # marked as PASSED during the merge, and subsequent test following a failure are only reported as + # SKIPPED, thus potentially suggesting that no tests were actually left as failing at the end. + with open(f"{PATH}{os.sep}run-{number_of_test_runs}{os.sep}output.xml", "rb") as report: + contents = report.read() + soup = BeautifulSoup(contents, features="xml") + tests = soup.find("total").find("stat") + failed_tests_in_final_run = int(tests["fail"]) except AttributeError as e: raise Exception("Error parsing the XML report") from e - total_tests_count = passed_tests + failed_tests + skipped_tests + total_tests_count = passed_tests + failed_tests_in_merged_report + skipped_tests + + # Whilst genuine bugs that fail a step every time will always show as FAILED in the merged test report AND + # the final run report, this is not always true of flaky / intermittent test failures, which can be identified + # in part by an unequal number of failures in the merged test report versus that of the final run attempt. + flaky_test_failures_likely = failed_tests_in_merged_report != failed_tests_in_final_run blocks = [ { "type": "header", "text": { "type": "plain_text", - "text": f"{':warning:' if failed_tests > 0 else ':white_check_mark:'} All results", + "text": f"{':warning:' if failed_tests_in_final_run > 0 else ':white_check_mark:'} All results", }, }, { @@ -51,11 +67,15 @@ def _build_test_results_attachments(self, env: str, suites_ran: str, suites_fail "fields": [ {"type": "mrkdwn", "text": f"*Environment*\n{env}"}, {"type": "mrkdwn", "text": f"*Suite*\n{suites_ran.replace('tests/', '')}"}, - {"type": "mrkdwn", "text": f"*Total runs*\n{number_of_test_runs}"}, + {"type": "mrkdwn", "text": f"*Total run attempts*\n{number_of_test_runs}"}, {"type": "mrkdwn", "text": f"*Total test cases*\n{total_tests_count}"}, {"type": "mrkdwn", "text": f"*Passed test cases*\n{passed_tests}"}, - {"type": "mrkdwn", "text": f"*Failed test cases*\n{failed_tests}"}, + {"type": "mrkdwn", "text": f"*Failed test cases*\n{failed_tests_in_final_run}"}, {"type": "mrkdwn", "text": f"*Skipped test cases*\n{skipped_tests}"}, + { + "type": "mrkdwn", + "text": f"*Flaky failures likely*\n{'Yes' if flaky_test_failures_likely else 'Unknown'}", + }, ], }, ] From 6c34884cdf1a62fb391ab06c960e8f5bd3e1f01d Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 12:33:18 +0000 Subject: [PATCH 049/144] EES-5705 - broke UI test suite jobs into yml template for easier maintenance --- azure-pipelines-run-ui-test-suite.yml | 62 ++++++ azure-pipelines-ui-tests.yml | 288 +++----------------------- 2 files changed, 96 insertions(+), 254 deletions(-) create mode 100644 azure-pipelines-run-ui-test-suite.yml diff --git a/azure-pipelines-run-ui-test-suite.yml b/azure-pipelines-run-ui-test-suite.yml new file mode 100644 index 00000000000..68ef1b6e0b8 --- /dev/null +++ b/azure-pipelines-run-ui-test-suite.yml @@ -0,0 +1,62 @@ +parameters: + - name: jobName + type: string + - name: displayName + type: string + - name: testFolder + type: string + - name: artifactName + type: string + +jobs: + - job: ${{ parameters.jobName }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: 160 + cancelTimeoutInMinutes: 10 + condition: succeededOrFailed() + pool: ees-ubuntu2204-large + workspace: + clean: all + steps: + - checkout: self + clean: true + fetchDepth: 5 + fetchTags: false + + - task: UsePythonVersion@0 + displayName: Use Python 3.10 + timeoutInMinutes: 5 + inputs: + versionSpec: 3.10 + + - task: AzureKeyVault@2 + displayName: Azure Key Vault - s101d01-kv-ees-01 + inputs: + azureSubscription: $(SPN_NAME) + KeyVaultName: s101d01-kv-ees-01 + SecretsFilter: ees-test-ADMIN-PASSWORD,ees-test-ANALYST-PASSWORD,ees-test-expiredinvite-password,ees-test-NOINVITE-PASSWORD,ees-test-PENDINGINVITE-PASSWORD,ees-alerts-slackapptoken + RunAsPreJob: true + + - task: PythonScript@0 + displayName: Run tests + condition: succeededOrFailed() + inputs: + scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py + arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --noinvite-pass '"$(ees-test-NOINVITE-PASSWORD)"' --pendinginvite-pass '"$(ees-test-PENDINGINVITE-PASSWORD)"' --env "dev" --file "${{ parameters.testFolder }}" --processes 4 --rerun-attempts 3 + # The magic incantation '"$(variable)"'was added by Mark to resolve an issue with Analyst password that contained ampersands. + workingDirectory: tests/robot-tests + env: + SLACK_APP_TOKEN: $(ees-alerts-slackapptoken) + + - task: PublishTestResults@2 + displayName: Publish test results + inputs: + testResultsFiles: tests/robot-tests/test-results/xunit.xml + failTaskOnFailedTests: true + + - task: PublishPipelineArtifact@1 + displayName: Publish pipeline artifact + condition: succeededOrFailed() + inputs: + path: tests/robot-tests/test-results/ + artifactName: ${{ parameters.artifactName }} \ No newline at end of file diff --git a/azure-pipelines-ui-tests.yml b/azure-pipelines-ui-tests.yml index 753533ecb45..a316e1fd2d0 100644 --- a/azure-pipelines-ui-tests.yml +++ b/azure-pipelines-ui-tests.yml @@ -17,261 +17,41 @@ resources: ref: refs/heads/dev jobs: -- job: Public - displayName: Public suite - Robot UI tests - timeoutInMinutes: 160 - cancelTimeoutInMinutes: 10 - condition: succeededOrFailed() - pool: ees-ubuntu2204-large - workspace: - clean: all - steps: - - checkout: self - clean: true - fetchDepth: 5 - fetchTags: false +- template: azure-pipelines-run-ui-test-suite.yml + parameters: + jobName: Public + displayName: Public suite - Robot UI tests + testFolder: tests/general_public + artifactName: test-results-public + +- template: azure-pipelines-run-ui-test-suite.yml + parameters: + jobName: PublishAndAmend + displayName: Publish release and amend suites - Robot UI tests + testFolder: tests/admin_and_public_2 + artifactName: test-results-admin-and-public-2 + +- template: azure-pipelines-run-ui-test-suite.yml + parameters: + jobName: Admin + displayName: Admin suites - Robot UI tests + testFolder: tests/admin + artifactName: test-results-admin + +- template: azure-pipelines-run-ui-test-suite.yml + parameters: + jobName: AdminAndPublic + displayName: Admin & public suites - Robot UI tests + testFolder: tests/admin_and_public + artifactName: test-results-admin-public + +- template: azure-pipelines-run-ui-test-suite.yml + parameters: + jobName: PublicAPI + displayName: Public API suite - Robot UI tests + testFolder: tests/public_api + artifactName: test-results-admin-public-api - - task: UsePythonVersion@0 - displayName: Use Python 3.10 - timeoutInMinutes: 5 - inputs: - versionSpec: 3.10 - - - task: AzureKeyVault@2 - displayName: Azure Key Vault - s101d01-kv-ees-01 - inputs: - azureSubscription: $(SPN_NAME) - KeyVaultName: s101d01-kv-ees-01 - SecretsFilter: ees-alerts-slackapptoken - RunAsPreJob: true - - - task: PythonScript@0 - displayName: Public UI tests - inputs: - scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py - arguments: --admin-pass "test" --analyst-pass "test" --expiredinvite-pass "test" --noinvite-pass "test" --pendinginvite-pass "test" --env "dev" --file "tests/general_public" --processes 4 --rerun-attempts 3 - workingDirectory: tests/robot-tests - env: - SLACK_APP_TOKEN: $(ees-alerts-slackapptoken) - - - task: PublishTestResults@2 - displayName: Publish Test Results - condition: succeededOrFailed() - inputs: - testResultsFiles: tests/robot-tests/test-results/xunit.xml - failTaskOnFailedTests: true - - - task: PublishPipelineArtifact@1 - displayName: Publish Test Pipeline Artifact - condition: succeededOrFailed() - inputs: - path: tests/robot-tests/test-results/ - artifactName: test-results-public - -- job: PublishAndAmend - displayName: Publish release and amend suites - Robot UI tests - timeoutInMinutes: 160 - cancelTimeoutInMinutes: 10 - condition: succeededOrFailed() - pool: ees-ubuntu2204-large - workspace: - clean: all - steps: - - checkout: self - clean: true - fetchDepth: 5 - fetchTags: false - - - task: UsePythonVersion@0 - displayName: Use Python 3.10 - timeoutInMinutes: 5 - inputs: - versionSpec: 3.10 - - - task: AzureKeyVault@2 - displayName: Azure Key Vault - s101d01-kv-ees-01 - inputs: - azureSubscription: $(SPN_NAME) - KeyVaultName: s101d01-kv-ees-01 - SecretsFilter: ees-test-ADMIN-PASSWORD,ees-test-ANALYST-PASSWORD,ees-test-expiredinvite-password,ees-test-NOINVITE-PASSWORD,ees-test-PENDINGINVITE-PASSWORD,ees-alerts-slackapptoken - RunAsPreJob: true - - - task: PythonScript@0 - displayName: Publish release and amend UI tests - condition: succeededOrFailed() - inputs: - scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py - arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --noinvite-pass '"$(ees-test-NOINVITE-PASSWORD)"' --pendinginvite-pass '"$(ees-test-PENDINGINVITE-PASSWORD)"' --env "dev" --file "tests/admin_and_public_2" --processes 4 --rerun-attempts 3 - # The magic incantation '"$(variable)"'was added by Mark to resolve an issue with Analyst password that contained ampersands. - workingDirectory: tests/robot-tests - env: - SLACK_APP_TOKEN: $(ees-alerts-slackapptoken) - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: tests/robot-tests/test-results/xunit.xml - failTaskOnFailedTests: true - - - task: PublishPipelineArtifact@1 - displayName: Publish Pipeline Artifact - condition: succeededOrFailed() - inputs: - path: tests/robot-tests/test-results/ - artifactName: test-results-admin-and-public-2 - - -- job: Admin - displayName: Admin suites - Robot UI tests - timeoutInMinutes: 160 - cancelTimeoutInMinutes: 10 - condition: succeededOrFailed() - pool: ees-ubuntu2204-large - workspace: - clean: all - steps: - - checkout: self - clean: true - fetchDepth: 5 - fetchTags: false - - - task: UsePythonVersion@0 - displayName: Use Python 3.10 - timeoutInMinutes: 5 - inputs: - versionSpec: 3.10 - - - task: AzureKeyVault@2 - displayName: Azure Key Vault - s101d01-kv-ees-01 - inputs: - azureSubscription: $(SPN_NAME) - KeyVaultName: s101d01-kv-ees-01 - SecretsFilter: ees-test-ADMIN-PASSWORD,ees-test-ANALYST-PASSWORD,ees-test-expiredinvite-password,ees-test-NOINVITE-PASSWORD,ees-test-PENDINGINVITE-PASSWORD,ees-alerts-slackapptoken - RunAsPreJob: true - - - task: PythonScript@0 - displayName: Admin UI tests - condition: succeededOrFailed() - inputs: - scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py - arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --noinvite-pass '"$(ees-test-NOINVITE-PASSWORD)"' --pendinginvite-pass '"$(ees-test-PENDINGINVITE-PASSWORD)"' --env "dev" --file "tests/admin" --processes 4 --rerun-attempts 3 - workingDirectory: tests/robot-tests - env: - SLACK_APP_TOKEN: $(ees-alerts-slackapptoken) - - - task: PublishTestResults@2 - displayName: Publish Test Results - condition: succeededOrFailed() - inputs: - testResultsFiles: tests/robot-tests/test-results/xunit.xml - failTaskOnFailedTests: true - - - task: PublishPipelineArtifact@1 - displayName: Publish Test Pipeline Artifact - condition: succeededOrFailed() - inputs: - path: tests/robot-tests/test-results/ - artifactName: test-results-admin - -- job: AdminAndPublic - displayName: Admin & public suites - Robot UI tests - timeoutInMinutes: 160 - cancelTimeoutInMinutes: 10 - condition: succeededOrFailed() - pool: ees-ubuntu2204-large - workspace: - clean: all - steps: - - checkout: self - clean: true - fetchDepth: 5 - fetchTags: false - - - task: UsePythonVersion@0 - displayName: Use Python 3.10 - timeoutInMinutes: 5 - inputs: - versionSpec: 3.10 - - - task: AzureKeyVault@2 - displayName: Azure Key Vault - s101d01-kv-ees-01 - inputs: - azureSubscription: $(SPN_NAME) - KeyVaultName: s101d01-kv-ees-01 - SecretsFilter: ees-test-ADMIN-PASSWORD,ees-test-ANALYST-PASSWORD,ees-test-expiredinvite-password,ees-test-NOINVITE-PASSWORD,ees-test-PENDINGINVITE-PASSWORD,ees-alerts-slackapptoken - RunAsPreJob: true - - - task: PythonScript@0 - displayName: Admin public UI tests - inputs: - scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py - arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --noinvite-pass '"$(ees-test-NOINVITE-PASSWORD)"' --pendinginvite-pass '"$(ees-test-PENDINGINVITE-PASSWORD)"' --env "dev" --file "tests/admin_and_public" --processes 4 --rerun-attempts 3 - workingDirectory: tests/robot-tests - env: - SLACK_APP_TOKEN: $(ees-alerts-slackapptoken) - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: tests/robot-tests/test-results/xunit.xml - failTaskOnFailedTests: true - - - task: PublishPipelineArtifact@1 - displayName: Publish Test Pipeline Artifact - condition: succeededOrFailed() - inputs: - path: tests/robot-tests/test-results/ - artifactName: test-results-admin-public - -- job: PublicAPI - displayName: Public API suite - Robot UI tests - timeoutInMinutes: 160 - cancelTimeoutInMinutes: 10 - condition: succeededOrFailed() - pool: ees-ubuntu2204-large - workspace: - clean: all - steps: - - checkout: self - clean: true - fetchDepth: 5 - fetchTags: false - - - task: UsePythonVersion@0 - displayName: Use Python 3.10 - timeoutInMinutes: 5 - inputs: - versionSpec: 3.10 - - - task: AzureKeyVault@2 - displayName: Azure Key Vault - s101d01-kv-ees-01 - inputs: - azureSubscription: $(SPN_NAME) - KeyVaultName: s101d01-kv-ees-01 - SecretsFilter: ees-test-ADMIN-PASSWORD,ees-test-ANALYST-PASSWORD,ees-test-expiredinvite-password,ees-test-NOINVITE-PASSWORD,ees-test-PENDINGINVITE-PASSWORD,ees-alerts-slackapptoken - RunAsPreJob: true - - - task: PythonScript@0 - displayName: Public API UI tests - inputs: - scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py - arguments: --admin-pass '"$(ees-test-ADMIN-PASSWORD)"' --analyst-pass '"$(ees-test-ANALYST-PASSWORD)"' --expiredinvite-pass '"$(ees-test-expiredinvite-password)"' --noinvite-pass '"$(ees-test-NOINVITE-PASSWORD)"' --pendinginvite-pass '"$(ees-test-PENDINGINVITE-PASSWORD)"' --env "dev" --file "tests/public_api" --processes 4 --rerun-attempts 3 - workingDirectory: tests/robot-tests - env: - SLACK_APP_TOKEN: $(ees-alerts-slackapptoken) - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: tests/robot-tests/test-results/xunit.xml - failTaskOnFailedTests: true - - - task: PublishPipelineArtifact@1 - displayName: Publish Test Pipeline Artifact - condition: succeededOrFailed() - inputs: - path: tests/robot-tests/test-results/ - artifactName: test-results-admin-public-api #- job: PublicPlaywrightUItest # displayName: Public suite - Playwright UI tests From 1c7aa987b4c204df950b99a13671b30db4dcdf29 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 13:03:05 +0000 Subject: [PATCH 050/144] EES-5705 - removed SKIP merging behaviour in CheckForAtLeastOnePassingRunPrerebotModifier.py as this is covering up failures from previous runs with skipped tests in later test runs which were skipped due to fail-fast behaviour --- ...heckForAtLeastOnePassingRunPrerebotModifier.py | 6 ------ tests/robot-tests/tests/libs/slack.py | 15 ++++++++++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/robot-tests/report-modifiers/CheckForAtLeastOnePassingRunPrerebotModifier.py b/tests/robot-tests/report-modifiers/CheckForAtLeastOnePassingRunPrerebotModifier.py index 8d39ba80862..13b666042ff 100644 --- a/tests/robot-tests/report-modifiers/CheckForAtLeastOnePassingRunPrerebotModifier.py +++ b/tests/robot-tests/report-modifiers/CheckForAtLeastOnePassingRunPrerebotModifier.py @@ -16,9 +16,3 @@ def visit_test(self, test): ) test.status = "PASS" test.message = f'Marking test "{test}" as PASS because it passed in at least one of the test runs. Previous message is {test.message}' - elif "SKIP" in test.message: - self.logger.info( - f'CheckForAtLeastOnePassingRunPrerebotModifier - marking test "{test}" as SKIPPED because it was skipped in the latter run.' - ) - test.status = "SKIP" - test.message = f'Marking test "{test}" as SKIP because it was skipped in the latter test run. Previous message is {test.message}' diff --git a/tests/robot-tests/tests/libs/slack.py b/tests/robot-tests/tests/libs/slack.py index e322c12ca0d..c26eb7872bb 100644 --- a/tests/robot-tests/tests/libs/slack.py +++ b/tests/robot-tests/tests/libs/slack.py @@ -51,8 +51,17 @@ def _build_test_results_attachments(self, env: str, suites_ran: str, suites_fail # Whilst genuine bugs that fail a step every time will always show as FAILED in the merged test report AND # the final run report, this is not always true of flaky / intermittent test failures, which can be identified - # in part by an unequal number of failures in the merged test report versus that of the final run attempt. - flaky_test_failures_likely = failed_tests_in_merged_report != failed_tests_in_final_run + # in part by an unequal number of failures in the merged test report versus that of the final run attempt. It can + # also be identified by the fact that the tests finally all passed but the number of attempts was greater than 1. + flaky_test_failures_likely = ( + failed_tests_in_merged_report != failed_tests_in_final_run + or failed_tests_in_final_run == 0 + and number_of_test_runs > 1 + ) + + flaky_test_message = ( + "Yes" if flaky_test_failures_likely else "No" if failed_tests_in_final_run == 0 else "Unknown" + ) blocks = [ { @@ -74,7 +83,7 @@ def _build_test_results_attachments(self, env: str, suites_ran: str, suites_fail {"type": "mrkdwn", "text": f"*Skipped test cases*\n{skipped_tests}"}, { "type": "mrkdwn", - "text": f"*Flaky failures likely*\n{'Yes' if flaky_test_failures_likely else 'Unknown'}", + "text": f"*Flaky failures likely*\n{flaky_test_message}", }, ], }, From 4c023b614afb99951d1773f3e04f872fed40907c Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 13:13:50 +0000 Subject: [PATCH 051/144] EES-5705 - raising run_tests.py script failure if any suites are failing by the end of the runs, to allow the pipeline to report the jobs as failed if suites are left failing --- tests/robot-tests/run_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/robot-tests/run_tests.py b/tests/robot-tests/run_tests.py index 6e603e8b29c..d02ddd7b460 100755 --- a/tests/robot-tests/run_tests.py +++ b/tests/robot-tests/run_tests.py @@ -207,6 +207,9 @@ def run(): time.sleep(5) slack_service.send_test_report(args.env, args.tests, failing_suites, number_of_test_runs) + if len(failing_suites) > 0: + sys.exit(1) + except Exception as ex: if args.enable_slack_notifications: slack_service = SlackService() From a8a40f7142fe2ac3baf3696b8fb8b8881c94e3c2 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 13:45:52 +0000 Subject: [PATCH 052/144] EES-5724 - failing fast when a deploy fails to be published --- .../robot-tests/tests/libs/admin-utilities.py | 4 ++-- tests/robot-tests/tests/libs/slack.py | 19 ++++++++----------- .../public_api_cancel_and_removal.robot | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/tests/robot-tests/tests/libs/admin-utilities.py b/tests/robot-tests/tests/libs/admin-utilities.py index d707aaf9af2..509e47f71aa 100644 --- a/tests/robot-tests/tests/libs/admin-utilities.py +++ b/tests/robot-tests/tests/libs/admin-utilities.py @@ -109,12 +109,12 @@ def user_waits_for_release_process_status_to_be(status, timeout): try: sl().driver.find_element(By.ID, f"release-process-status-Failed") raise_assertion_error("Release process status FAILED!") - except BaseException: + except NoSuchElementException: pass try: sl().driver.find_element(By.ID, f"release-process-status-{status}") return - except BaseException: + except NoSuchElementException: sl().reload_page() # Necessary if release previously scheduled time.sleep(3) raise_assertion_error(f"Release process status wasn't {status} after {timeout} seconds!") diff --git a/tests/robot-tests/tests/libs/slack.py b/tests/robot-tests/tests/libs/slack.py index c26eb7872bb..f51166a801d 100644 --- a/tests/robot-tests/tests/libs/slack.py +++ b/tests/robot-tests/tests/libs/slack.py @@ -53,14 +53,14 @@ def _build_test_results_attachments(self, env: str, suites_ran: str, suites_fail # the final run report, this is not always true of flaky / intermittent test failures, which can be identified # in part by an unequal number of failures in the merged test report versus that of the final run attempt. It can # also be identified by the fact that the tests finally all passed but the number of attempts was greater than 1. - flaky_test_failures_likely = ( - failed_tests_in_merged_report != failed_tests_in_final_run - or failed_tests_in_final_run == 0 - and number_of_test_runs > 1 - ) - flaky_test_message = ( - "Yes" if flaky_test_failures_likely else "No" if failed_tests_in_final_run == 0 else "Unknown" + "Yes" + if failed_tests_in_final_run == 0 and number_of_test_runs > 1 + else "No" + if failed_tests_in_final_run == 0 and number_of_test_runs == 1 + else "Likely" + if failed_tests_in_merged_report != failed_tests_in_final_run + else "No" ) blocks = [ @@ -81,10 +81,7 @@ def _build_test_results_attachments(self, env: str, suites_ran: str, suites_fail {"type": "mrkdwn", "text": f"*Passed test cases*\n{passed_tests}"}, {"type": "mrkdwn", "text": f"*Failed test cases*\n{failed_tests_in_final_run}"}, {"type": "mrkdwn", "text": f"*Skipped test cases*\n{skipped_tests}"}, - { - "type": "mrkdwn", - "text": f"*Flaky failures likely*\n{flaky_test_message}", - }, + {"type": "mrkdwn", "text": f"*Flaky tests?*\n{flaky_test_message}"}, ], }, ] diff --git a/tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot b/tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot index 1c0209a2957..9cd8da9d911 100644 --- a/tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot +++ b/tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot @@ -99,7 +99,7 @@ Click on 'cancel' button while attempting to remove draft API data set user clicks button Cancel ${modal} user waits until h2 is visible API data sets -Verify the contents inside the 'Draft API data sets' table +Verify the contents inside the 'Draft API data sets' table again user waits until h3 is visible Draft API data sets user checks table column heading contains 1 1 Draft version testid:draft-api-data-sets From 8ec68facbeff8ee7b58f7447a87882cd3008f0c7 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 13:46:48 +0000 Subject: [PATCH 053/144] EES-5724 - failing fast when a deploy fails to be published --- tests/robot-tests/tests/libs/slack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/robot-tests/tests/libs/slack.py b/tests/robot-tests/tests/libs/slack.py index f51166a801d..2c8c40f0988 100644 --- a/tests/robot-tests/tests/libs/slack.py +++ b/tests/robot-tests/tests/libs/slack.py @@ -54,13 +54,13 @@ def _build_test_results_attachments(self, env: str, suites_ran: str, suites_fail # in part by an unequal number of failures in the merged test report versus that of the final run attempt. It can # also be identified by the fact that the tests finally all passed but the number of attempts was greater than 1. flaky_test_message = ( - "Yes" + "Definitely" if failed_tests_in_final_run == 0 and number_of_test_runs > 1 - else "No" + else "Definitely not" if failed_tests_in_final_run == 0 and number_of_test_runs == 1 else "Likely" if failed_tests_in_merged_report != failed_tests_in_final_run - else "No" + else "Unlikely" ) blocks = [ From f387ffef10203bde63d82ade2e052e9cba364fe8 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 3 Dec 2024 14:49:14 +0000 Subject: [PATCH 054/144] EES-5685 - added sample set of availability and health alerts for Public API components --- .../public-api/publicApiDataProcessor.bicep | 22 +++++++ .../public-api/publicApiStorage.bicep | 10 +++ .../public-api/ci/azure-pipelines.yml | 5 ++ .../ci/jobs/deploy-infrastructure.yml | 2 + .../public-api/ci/tasks/deploy-bicep.yml | 6 +- .../components/alerts/sites/healthAlert.bicep | 39 +++++++++++ .../components/alerts/staticMetricAlert.bicep | 66 +++++++++++++++++++ .../storageAccount/availabilityAlert.bicep | 26 ++++++++ .../public-api/components/alerts/types.bicep | 43 ++++++++++++ .../public-api/components/functionApp.bicep | 3 + .../templates/public-api/main.bicep | 4 ++ 11 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 infrastructure/templates/public-api/components/alerts/sites/healthAlert.bicep create mode 100644 infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep create mode 100644 infrastructure/templates/public-api/components/alerts/storageAccount/availabilityAlert.bicep create mode 100644 infrastructure/templates/public-api/components/alerts/types.bicep diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 88be79535df..f9422066bc9 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -21,6 +21,8 @@ param dataProcessorAppRegistrationClientId string @description('Public API Storage : Firewall rules.') param storageFirewallRules FirewallRule[] = [] +param deployAlerts bool = false + @description('Specifies a set of tags with which to tag the resource in Azure.') param tagValues object @@ -117,6 +119,26 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { } } +module storageAvailabilityAlerts '../../components/alerts/storageAccount/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}StorageAvailabilityAlert' + params: { + storageAccountNames: [ + dataProcessorFunctionAppModule.outputs.managementStorageAccountName + dataProcessorFunctionAppModule.outputs.slot1StorageAccountName + dataProcessorFunctionAppModule.outputs.slot2StorageAccountName + ] + alertsGroupName: resourceNames.existingResources.alertsGroup + } +} + +module functionAppHealthAlert '../../components/alerts/sites/healthAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}SitesHealthCheckAlertDeploy' + params: { + resourceNames: [resourceNames.publicApi.dataProcessor] + alertsGroupName: resourceNames.existingResources.alertsGroup + } +} + output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath diff --git a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep index ce25740625f..5125005843b 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep @@ -14,6 +14,8 @@ param storageFirewallRules FirewallRule[] = [] @description('Specifies a set of tags with which to tag the resource in Azure.') param tagValues object +param deployAlerts bool = false + resource vNet 'Microsoft.Network/virtualNetworks@2023-11-01' existing = { name: resourceNames.existingResources.vNet } @@ -56,6 +58,14 @@ module dataFilesFileShareModule '../../components/fileShare.bicep' = { } } +module availabilityAlert '../../components/alerts/storageAccount/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.publicApiStorageAccount}AvailabilityAlertDeploy' + params: { + storageAccountNames: [resourceNames.publicApi.publicApiStorageAccount] + alertsGroupName: resourceNames.existingResources.alertsGroup + } +} + output storageAccountName string = publicApiStorageAccountModule.outputs.storageAccountName output connectionStringSecretName string = publicApiStorageAccountModule.outputs.connectionStringSecretName output accessKeySecretName string = publicApiStorageAccountModule.outputs.accessKeySecretName diff --git a/infrastructure/templates/public-api/ci/azure-pipelines.yml b/infrastructure/templates/public-api/ci/azure-pipelines.yml index 175f98d62e6..c22b2ea1060 100644 --- a/infrastructure/templates/public-api/ci/azure-pipelines.yml +++ b/infrastructure/templates/public-api/ci/azure-pipelines.yml @@ -7,6 +7,9 @@ parameters: - name: updatePsqlFlexibleServer displayName: Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys. default: false + - name: deployAlerts + displayName: Should we deploy alerts with this deploy? + default: false - name: forceDeployToEnvironment displayName: Set to either dev or test to force a deploy to that environment from the chosen branch. type: string @@ -42,6 +45,8 @@ variables: value: ${{ parameters.deployContainerApp }} - name: updatePsqlFlexibleServer value: ${{ parameters.updatePsqlFlexibleServer }} + - name: deployAlerts + value: ${{ parameters.deployAlerts }} pool: vmImage: $(vmImageName) diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml b/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml index 6851c86dec0..4deeb28a247 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml @@ -37,6 +37,7 @@ jobs: parameterFile: $(paramFile) deployContainerApp: true updatePsqlFlexibleServer: false + deployAlerts: false dataProcessorExists: true - task: AzureCLI@2 @@ -63,6 +64,7 @@ jobs: parameterFile: $(paramFile) deployContainerApp: $(deployContainerApp) updatePsqlFlexibleServer: $(updatePsqlFlexibleServer) + deployAlerts: $(deployAlerts) dataProcessorExists: $(dataProcessorExists) # - template: ../tasks/assign-app-role-to-service-principal.yml diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index add810c13cc..fa5ceeaa331 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -13,13 +13,12 @@ parameters: type: string - name: deployContainerApp type: string - default: true - name: updatePsqlFlexibleServer type: string - default: false + - name: deployAlerts + type: string - name: dataProcessorExists type: string - default: true steps: - task: AzureCLI@2 @@ -48,6 +47,7 @@ steps: dockerImagesTag='$(resources.pipeline.MainBuild.runName)' \ deployContainerApp=${{ parameters.deployContainerApp }} \ updatePsqlFlexibleServer=${{ parameters.updatePsqlFlexibleServer }} \ + deployAlerts=${{ parameters.deployAlerts }} \ dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \ dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \ apiAppRegistrationClientId='$(apiAppRegistrationClientId)' diff --git a/infrastructure/templates/public-api/components/alerts/sites/healthAlert.bicep b/infrastructure/templates/public-api/components/alerts/sites/healthAlert.bicep new file mode 100644 index 00000000000..61baf6e6db8 --- /dev/null +++ b/infrastructure/templates/public-api/components/alerts/sites/healthAlert.bicep @@ -0,0 +1,39 @@ +import { EvaluationFrequency, WindowSize, Severity } from '../types.bicep' + +@description('Name of the resource that this alert is being applied to.') +param resourceNames string[] + +@description('Type of the resource that this alert is being applied to.') +param resourceType + | 'Microsoft.Web/sites' + | 'Microsoft.Web/sites/slots' = 'Microsoft.Web/sites' + +@description('The evaluation frequency.') +param evaluationFrequency EvaluationFrequency = 'PT1M' + +@description('The window size.') +param windowSize WindowSize = 'PT5M' + +@description('Name of the Alerts Group used to send alert messages.') +param alertsGroupName string + +param severity Severity = 'Critical' + +module metricAlertModule '../staticMetricAlert.bicep' = [for name in resourceNames: { + name: '${replace(name, '/', '-')}UnhealthyAlertModule' + params: { + alertName: '${replace(name, '/', '-')}Unhealthy' + alertsGroupName: alertsGroupName + resourceIds: [resourceId(resourceType, name)] + resourceType: resourceType + query: { + metric: 'HealthCheckStatus' + aggregation: 'Minimum' + operator: 'LessThan' + threshold: 100 + } + evaluationFrequency: evaluationFrequency + windowSize: windowSize + severity: severity + } +}] diff --git a/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep b/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep new file mode 100644 index 00000000000..6a1e07a0ffc --- /dev/null +++ b/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep @@ -0,0 +1,66 @@ +import { EvaluationFrequency, MetricName, MetricOperator, ResourceType, TimeAggregation, WindowSize, Severity, severityMapping } from 'types.bicep' + +@description('Name of the alert.') +param alertName string + +@description('Ids of the resources that this alert is being applied to.') +param resourceIds string[] + +@description('Type of the resource that this alert is being applied to.') +param resourceType ResourceType + +@description('The query being used to test if the alert should be fired.') +param query { + metric: MetricName + aggregation: TimeAggregation + operator: MetricOperator + threshold: int +} + +@description('The evaluation frequency.') +param evaluationFrequency EvaluationFrequency = 'PT1M' + +@description('The window size.') +param windowSize WindowSize = 'PT5M' + +@description('The alert severity.') +param severity Severity = 'Error' + +@description('Name of the Alerts Group used to send alert messages.') +param alertsGroupName string + +var severityLevel = severityMapping[severity] + +resource alertsActionGroup 'Microsoft.Insights/actionGroups@2023-01-01' existing = { + name: alertsGroupName +} + +resource metricAlertRule 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: alertName + location: 'Global' + properties: { + enabled: true + scopes: resourceIds + severity: severityLevel + evaluationFrequency: evaluationFrequency + windowSize: windowSize + criteria: { + 'odata.type': length(resourceIds) > 1 ? 'Microsoft.Azure.Monitor.MultipleResourceMultipleMetricCriteria' : 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + allOf: [{ + criterionType: 'StaticThresholdCriterion' + name: 'Metric1' + metricName: query.metric + metricNamespace: resourceType + timeAggregation: query.aggregation + operator: query.operator + threshold: query.threshold + skipMetricValidation: false + }] + } + actions: [ + { + actionGroupId: alertsActionGroup.id + } + ] + } +} diff --git a/infrastructure/templates/public-api/components/alerts/storageAccount/availabilityAlert.bicep b/infrastructure/templates/public-api/components/alerts/storageAccount/availabilityAlert.bicep new file mode 100644 index 00000000000..4adf3d56dd0 --- /dev/null +++ b/infrastructure/templates/public-api/components/alerts/storageAccount/availabilityAlert.bicep @@ -0,0 +1,26 @@ +import { Severity } from '../types.bicep' + +param storageAccountNames string[] + +param alertsGroupName string + +param severity Severity = 'Critical' + +module alerts '../staticMetricAlert.bicep' = [for name in storageAccountNames: { + name: '${name}StorageAvailabilityAlertModule' + params: { + alertName: '${name}Availability' + resourceIds: [resourceId('Microsoft.Storage/storageAccounts', name)] + resourceType: 'Microsoft.Storage/storageAccounts' + query: { + metric: 'availability' + aggregation: 'Average' + operator: 'LessThan' + threshold: 100 + } + evaluationFrequency: 'PT1M' + windowSize: 'PT5M' + severity: severity + alertsGroupName: alertsGroupName + } +}] diff --git a/infrastructure/templates/public-api/components/alerts/types.bicep b/infrastructure/templates/public-api/components/alerts/types.bicep new file mode 100644 index 00000000000..25c7d87ceb4 --- /dev/null +++ b/infrastructure/templates/public-api/components/alerts/types.bicep @@ -0,0 +1,43 @@ +@export() +type EvaluationFrequency = 'PT1M' + +@export() +type WindowSize = 'PT5M' + +@export() +type MetricOperator = 'GreaterThan' | 'LessThan' + +@export() +type TimeAggregation = + | 'Total' + | 'Minimum' + | 'Average' + +@export() +type Severity = 'Critical' | 'Error' | 'Warning' | 'Informational' | 'Verbose' + +@export() +var severityMapping = { + Critical: 0 + Error: 1 + Warning: 2 + Informational: 3 + Verbose: 4 +} + +@export() +type ResourceType = + | 'Microsoft.Web/sites' + | 'Microsoft.Web/sites/slots' + | 'Microsoft.DBforPostgreSQL/flexibleServers' + | 'Microsoft.Sql/servers/databases' + | 'Microsoft.Storage/storageAccounts' + +@export() +type MetricName = + | 'blocked_by_firewall' + | 'connection_failed' + | 'connections_failed' + | 'cpu_percent' + | 'HealthCheckStatus' + | 'availability' diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 9807d5e75bd..72c98040f8d 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -431,3 +431,6 @@ module privateEndpointModule 'privateEndpoint.bicep' = if (privateEndpointSubnet } output functionAppName string = functionApp.name +output managementStorageAccountName string = sharedStorageAccountName +output slot1StorageAccountName string = slot1StorageAccountName +output slot2StorageAccountName string = slot2StorageAccountName diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index d3df11ca2b1..d34ce279319 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -75,6 +75,8 @@ param deployContainerApp bool = true @description('Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys.') param updatePsqlFlexibleServer bool = false +param deployAlerts bool = false + @description('Public URLs of other components in the service.') param publicUrls { contentApi: string @@ -183,6 +185,7 @@ module publicApiStorageModule 'application/public-api/publicApiStorage.bicep' = resourceNames: resourceNames publicApiDataFileShareQuota: publicApiDataFileShareQuota storageFirewallRules: storageFirewallRules + deployAlerts: deployAlerts tagValues: tagValues } } @@ -370,6 +373,7 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId storageFirewallRules: storageFirewallRules dataProcessorFunctionAppExists: dataProcessorFunctionAppExists + deployAlerts: deployAlerts tagValues: tagValues } dependsOn: [ From a60b2dd2b8c1f0d45d38563b9613f7b54470497e Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 4 Dec 2024 15:08:47 +0000 Subject: [PATCH 055/144] EES-5685 - reworked some conventions of existing alert code and alert resource and module naming patterns. Removed health alert setup code from functionApp.bicep in favour of administring at a higher level --- .../public-api/publicApiDataProcessor.bicep | 24 +++--- .../public-api/publicApiStorage.bicep | 14 ++-- .../application/shared/appGateway.bicep | 12 +++ .../appGateways/backendPoolHealth.bicep | 33 +++++++++ .../components/alerts/sites/healthAlert.bicep | 11 ++- .../components/alerts/staticMetricAlert.bicep | 4 + .../availabilityAlert.bicep | 17 +++-- .../public-api/components/alerts/types.bicep | 9 ++- .../public-api/components/functionApp.bicep | 74 +------------------ .../templates/public-api/main.bicep | 2 +- 10 files changed, 99 insertions(+), 101 deletions(-) create mode 100644 infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep rename infrastructure/templates/public-api/components/alerts/{storageAccount => storageAccounts}/availabilityAlert.bicep (50%) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index f9422066bc9..30e0213ec0e 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -6,14 +6,11 @@ param resourceNames ResourceNames @description('Specifies the location for all resources.') param location string -@description('Alert metric name prefix') -param metricsNamePrefix string - @description('The Application Insights key that is associated with this resource') param applicationInsightsKey string @description('Specifies whether or not the Data Processor Function App already exists.') -param dataProcessorFunctionAppExists bool = false +param dataProcessorFunctionAppExists bool @description('Specifies the Application (Client) Id of a pre-existing App Registration used to represent the Data Processor Function App.') param dataProcessorAppRegistrationClientId string @@ -21,7 +18,8 @@ param dataProcessorAppRegistrationClientId string @description('Public API Storage : Firewall rules.') param storageFirewallRules FirewallRule[] = [] -param deployAlerts bool = false +@description('Whether to create or update Azure Monitor alerts during this deploy') +param deployAlerts bool @description('Specifies a set of tags with which to tag the resource in Azure.') param tagValues object @@ -70,7 +68,6 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { functionAppName: resourceNames.publicApi.dataProcessor appServicePlanName: resourceNames.publicApi.dataProcessor storageAccountsNamePrefix: resourceNames.publicApi.dataProcessorStorageAccountsPrefix - alertsGroupName: resourceNames.existingResources.alertsGroup location: location applicationInsightsKey: applicationInsightsKey subnetId: outboundVnetSubnet.id @@ -100,10 +97,7 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { family: 'EP' } preWarmedInstanceCount: 1 - healthCheck: { - path: '/api/HealthCheck' - unhealthyMetricName: '${metricsNamePrefix}Unhealthy' - } + healthCheckPath: '/api/HealthCheck' appSettings: { App__MetaInsertBatchSize: 1000 } @@ -119,23 +113,25 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { } } -module storageAvailabilityAlerts '../../components/alerts/storageAccount/availabilityAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.dataProcessor}StorageAvailabilityAlert' +module storageAvailabilityAlerts '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}AvailabilityAlert' params: { - storageAccountNames: [ + resourceNames: [ dataProcessorFunctionAppModule.outputs.managementStorageAccountName dataProcessorFunctionAppModule.outputs.slot1StorageAccountName dataProcessorFunctionAppModule.outputs.slot2StorageAccountName ] alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues } } module functionAppHealthAlert '../../components/alerts/sites/healthAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.dataProcessor}SitesHealthCheckAlertDeploy' + name: '${resourceNames.publicApi.dataProcessor}HealthDeploy' params: { resourceNames: [resourceNames.publicApi.dataProcessor] alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues } } diff --git a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep index 5125005843b..cfa4d568048 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep @@ -6,15 +6,16 @@ param resourceNames ResourceNames param location string @description('Public API Storage : Size of the file share in GB.') -param publicApiDataFileShareQuota int = 1 +param publicApiDataFileShareQuota int @description('Public API Storage : Firewall rules.') -param storageFirewallRules FirewallRule[] = [] +param storageFirewallRules FirewallRule[] @description('Specifies a set of tags with which to tag the resource in Azure.') param tagValues object -param deployAlerts bool = false +@description('Whether to create or update Azure Monitor alerts during this deploy') +param deployAlerts bool resource vNet 'Microsoft.Network/virtualNetworks@2023-11-01' existing = { name: resourceNames.existingResources.vNet @@ -58,11 +59,12 @@ module dataFilesFileShareModule '../../components/fileShare.bicep' = { } } -module availabilityAlert '../../components/alerts/storageAccount/availabilityAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.publicApiStorageAccount}AvailabilityAlertDeploy' +module availabilityAlert '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.publicApiStorageAccount}AvailabilityDeploy' params: { - storageAccountNames: [resourceNames.publicApi.publicApiStorageAccount] + resourceNames: [resourceNames.publicApi.publicApiStorageAccount] alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues } } diff --git a/infrastructure/templates/public-api/application/shared/appGateway.bicep b/infrastructure/templates/public-api/application/shared/appGateway.bicep index 9926990eb6a..2d85993b272 100644 --- a/infrastructure/templates/public-api/application/shared/appGateway.bicep +++ b/infrastructure/templates/public-api/application/shared/appGateway.bicep @@ -24,6 +24,9 @@ param routes AppGatewayRoute[] @description('Rules for how the App Gateway should rewrite URLs') param rewrites AppGatewayRewriteSet[] +@description('Whether to create or update Azure Monitor alerts during this deploy') +param deployAlerts bool + @description('Tags for the resources') param tagValues object @@ -51,3 +54,12 @@ module appGatewayModule '../../components/appGateway.bicep' = { tagValues: tagValues } } + +module backendPoolsHealthAlert '../../components/alerts/appGateways/backendPoolHealth.bicep' = if (deployAlerts) { + name: '${resourceNames.sharedResources.appGateway}BackedPoolsHealthDeploy' + params: { + resourceNames: [resourceNames.sharedResources.appGateway] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} diff --git a/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep b/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep new file mode 100644 index 00000000000..440aa4f4fad --- /dev/null +++ b/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep @@ -0,0 +1,33 @@ +import { Severity } from '../types.bicep' + +@description('Names of the resources that these alerts are being applied to.') +param resourceNames string[] + +@description('The alert severity.') +param severity Severity = 'Critical' + +@description('Name of the Alerts Group used to send alert messages.') +param alertsGroupName string + +@description('Specifies a set of tags with which to tag the resource in Azure.') +param tagValues object + +module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: { + name: '${name}BackendHealthAlertModule' + params: { + alertName: '${name}-backend-pool-health' + resourceIds: [resourceId('Microsoft.Network/applicationGateways', name)] + resourceType: 'Microsoft.Network/applicationGateways' + query: { + metric: 'UnhealthyHostCount' + aggregation: 'Average' + operator: 'GreaterThan' + threshold: 0 + } + evaluationFrequency: 'PT1M' + windowSize: 'PT5M' + severity: severity + alertsGroupName: alertsGroupName + tagValues: tagValues + } +}] diff --git a/infrastructure/templates/public-api/components/alerts/sites/healthAlert.bicep b/infrastructure/templates/public-api/components/alerts/sites/healthAlert.bicep index 61baf6e6db8..f7af74839dd 100644 --- a/infrastructure/templates/public-api/components/alerts/sites/healthAlert.bicep +++ b/infrastructure/templates/public-api/components/alerts/sites/healthAlert.bicep @@ -1,6 +1,6 @@ import { EvaluationFrequency, WindowSize, Severity } from '../types.bicep' -@description('Name of the resource that this alert is being applied to.') +@description('Names of the resources that these alerts are being applied to.') param resourceNames string[] @description('Type of the resource that this alert is being applied to.') @@ -17,12 +17,16 @@ param windowSize WindowSize = 'PT5M' @description('Name of the Alerts Group used to send alert messages.') param alertsGroupName string +@description('Severity level of the alert.') param severity Severity = 'Critical' +@description('Tags with which to tag the resource in Azure.') +param tagValues object + module metricAlertModule '../staticMetricAlert.bicep' = [for name in resourceNames: { - name: '${replace(name, '/', '-')}UnhealthyAlertModule' + name: '${replace(name, '/', '-')}HealthAlertModule' params: { - alertName: '${replace(name, '/', '-')}Unhealthy' + alertName: '${replace(name, '/', '-')}-health' alertsGroupName: alertsGroupName resourceIds: [resourceId(resourceType, name)] resourceType: resourceType @@ -35,5 +39,6 @@ module metricAlertModule '../staticMetricAlert.bicep' = [for name in resourceNam evaluationFrequency: evaluationFrequency windowSize: windowSize severity: severity + tagValues: tagValues } }] diff --git a/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep b/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep index 6a1e07a0ffc..d3f33eac260 100644 --- a/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep +++ b/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep @@ -29,6 +29,9 @@ param severity Severity = 'Error' @description('Name of the Alerts Group used to send alert messages.') param alertsGroupName string +@description('Tags with which to tag the resource in Azure.') +param tagValues object + var severityLevel = severityMapping[severity] resource alertsActionGroup 'Microsoft.Insights/actionGroups@2023-01-01' existing = { @@ -63,4 +66,5 @@ resource metricAlertRule 'Microsoft.Insights/metricAlerts@2018-03-01' = { } ] } + tags: tagValues } diff --git a/infrastructure/templates/public-api/components/alerts/storageAccount/availabilityAlert.bicep b/infrastructure/templates/public-api/components/alerts/storageAccounts/availabilityAlert.bicep similarity index 50% rename from infrastructure/templates/public-api/components/alerts/storageAccount/availabilityAlert.bicep rename to infrastructure/templates/public-api/components/alerts/storageAccounts/availabilityAlert.bicep index 4adf3d56dd0..d71ce969f80 100644 --- a/infrastructure/templates/public-api/components/alerts/storageAccount/availabilityAlert.bicep +++ b/infrastructure/templates/public-api/components/alerts/storageAccounts/availabilityAlert.bicep @@ -1,15 +1,21 @@ import { Severity } from '../types.bicep' -param storageAccountNames string[] +@description('Names of the resources that these alerts are being applied to.') +param resourceNames string[] +@description('The alert severity.') +param severity Severity = 'Critical' + +@description('Name of the Alerts Group used to send alert messages.') param alertsGroupName string -param severity Severity = 'Critical' +@description('Tags with which to tag the resource in Azure.') +param tagValues object -module alerts '../staticMetricAlert.bicep' = [for name in storageAccountNames: { - name: '${name}StorageAvailabilityAlertModule' +module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: { + name: '${name}AvailabilityAlertModule' params: { - alertName: '${name}Availability' + alertName: '${name}-availability' resourceIds: [resourceId('Microsoft.Storage/storageAccounts', name)] resourceType: 'Microsoft.Storage/storageAccounts' query: { @@ -22,5 +28,6 @@ module alerts '../staticMetricAlert.bicep' = [for name in storageAccountNames: { windowSize: 'PT5M' severity: severity alertsGroupName: alertsGroupName + tagValues: tagValues } }] diff --git a/infrastructure/templates/public-api/components/alerts/types.bicep b/infrastructure/templates/public-api/components/alerts/types.bicep index 25c7d87ceb4..d7f83ed13ce 100644 --- a/infrastructure/templates/public-api/components/alerts/types.bicep +++ b/infrastructure/templates/public-api/components/alerts/types.bicep @@ -14,7 +14,12 @@ type TimeAggregation = | 'Average' @export() -type Severity = 'Critical' | 'Error' | 'Warning' | 'Informational' | 'Verbose' +type Severity = + | 'Critical' + | 'Error' + | 'Warning' + | 'Informational' + | 'Verbose' @export() var severityMapping = { @@ -32,6 +37,7 @@ type ResourceType = | 'Microsoft.DBforPostgreSQL/flexibleServers' | 'Microsoft.Sql/servers/databases' | 'Microsoft.Storage/storageAccounts' + | 'Microsoft.Network/applicationGateways' @export() type MetricName = @@ -41,3 +47,4 @@ type MetricName = | 'cpu_percent' | 'HealthCheckStatus' | 'availability' + | 'UnhealthyHostCount' diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 72c98040f8d..4e274ad4cdf 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -12,9 +12,6 @@ param appServicePlanName string @description('Specifies the name prefix for all storage accounts') param storageAccountsNamePrefix string -@description('Specifies the name of an alerts group for reporting metric alerts') -param alertsGroupName string - @description('Function App Plan : operating system') param appServicePlanOS 'Windows' | 'Linux' = 'Linux' @@ -64,11 +61,8 @@ param preWarmedInstanceCount int? @description('Specifies whether or not the Function App will always be on and not idle after periods of no traffic - must be compatible with the chosen hosting plan') param alwaysOn bool? -@description('Specifies configuration for setting up automatic health checks and metric alerts') -param healthCheck { - path: string - unhealthyMetricName: string -}? +@description('Specifies an optional URL for Azure to use to monitor the health of this resource') +param healthCheckPath string? @description('Specifies additional Azure Storage Accounts to make available to this Function App') param azureFileShares AzureFileShareMount[] = [] @@ -190,7 +184,7 @@ var commonSiteProperties = { reserved: reserved siteConfig: { alwaysOn: alwaysOn ?? null - healthCheckPath: healthCheck != null ? healthCheck!.path : null + healthCheckPath: healthCheckPath preWarmedInstanceCount: preWarmedInstanceCount ?? null netFrameworkVersion: '8.0' linuxFxVersion: appServicePlanOS == 'Linux' ? 'DOTNET-ISOLATED|8.0' : null @@ -232,68 +226,6 @@ module azureAuthentication 'siteAzureAuthentication.bicep' = if (entraIdAuthenti } } -resource alertsActionGroup 'Microsoft.Insights/actionGroups@2023-01-01' existing = { - name: alertsGroupName -} - -var commonUnhealthyMetricAlertRuleProperties = { - enabled: true - severity: 1 - evaluationFrequency: 'PT1M' - windowSize: 'PT5M' - criteria: { - 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' - allOf: [ - { - name: 'Metric1' - criterionType: 'StaticThresholdCriterion' - metricName: 'HealthCheckStatus' - timeAggregation: 'Minimum' - operator: 'LessThan' - threshold: 100 - skipMetricValidation: false - } - ] - } - actions: [ - { - actionGroupId: alertsActionGroup.id - } - ] -} - -resource functionAppUnhealthyMetricAlertRule 'Microsoft.Insights/metricAlerts@2018-03-01' = if (healthCheck != null) { - name: healthCheck!.unhealthyMetricName - location: 'Global' - properties: union(commonUnhealthyMetricAlertRuleProperties, { - scopes: [functionApp.id] - criteria: { - allOf: [union( - commonUnhealthyMetricAlertRuleProperties.criteria.allOf[0], - { - metricNamespace: 'Microsoft.Web/sites' - } - )] - } - }) -} - -resource stagingSlotUnhealthyMetricAlertRule 'Microsoft.Insights/metricAlerts@2018-03-01' = if (healthCheck != null) { - name: '${healthCheck!.unhealthyMetricName}Staging' - location: 'Global' - properties: union(commonUnhealthyMetricAlertRuleProperties, { - scopes: [stagingSlot.id] - criteria: { - allOf: [union( - commonUnhealthyMetricAlertRuleProperties.criteria.allOf[0], - { - metricNamespace: 'Microsoft.Web/sites/slots' - } - )] - } - }) -} - // Allow Key Vault references passed as secure appsettings to be resolved by the Function App and its deployment slots. // Where the staging slot's managed identity differs from the main slot's managed identity, add its id to the list. var keyVaultPrincipalIds = userAssignedManagedIdentityParams != null diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index d34ce279319..c91a9cf7037 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -359,6 +359,7 @@ module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContai ] } ] + deployAlerts: deployAlerts tagValues: tagValues } } @@ -368,7 +369,6 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' params: { location: location resourceNames: resourceNames - metricsNamePrefix: '${subscription}PublicDataProcessor' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId storageFirewallRules: storageFirewallRules From 8d3cb5e684a4fcbd45ec18f91252699f08042f36 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 4 Dec 2024 15:45:15 +0000 Subject: [PATCH 056/144] EES-5685 - added health alerts for Container Apps and PSQL Flexible Server --- .../application/public-api/publicApiApp.bicep | 12 +++++++ .../shared/postgreSqlFlexibleServer.bicep | 12 +++++++ .../appGateways/backendPoolHealth.bicep | 2 +- .../alerts/containerApps/restarts.bicep | 33 +++++++++++++++++++ .../flexibleServers/databaseAlive.bicep | 33 +++++++++++++++++++ .../public-api/components/alerts/types.bicep | 10 ++++-- .../templates/public-api/main.bicep | 2 ++ 7 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 infrastructure/templates/public-api/components/alerts/containerApps/restarts.bicep create mode 100644 infrastructure/templates/public-api/components/alerts/flexibleServers/databaseAlive.bicep diff --git a/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep b/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep index 5bfc5d1ca52..414920edf6e 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiApp.bicep @@ -30,6 +30,9 @@ param apiAppRegistrationClientId string @description('Specifies the Application Insights connection string for this Container App to use for its monitoring.') param appInsightsConnectionString string +@description('Whether to create or update Azure Monitor alerts during this deploy') +param deployAlerts bool + @description('Specifies a set of tags with which to tag the resource in Azure.') param tagValues object @@ -146,6 +149,15 @@ module apiContainerAppModule '../../components/containerApp.bicep' = { } } +module containerAppRestartsAlert '../../components/alerts/containerApps/restarts.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.apiApp}RestartsDeploy' + params: { + resourceNames: [resourceNames.publicApi.apiApp] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} + output containerAppFqdn string = apiContainerAppModule.outputs.containerAppFqdn output containerAppName string = apiContainerAppModule.outputs.containerAppName output healthProbePath string = '/health' diff --git a/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep b/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep index 26a15e4ac14..2f9469be5d8 100644 --- a/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep +++ b/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep @@ -31,6 +31,9 @@ param privateEndpointSubnetId string @description('An array of Entra ID admin principal names for this resource') param entraIdAdminPrincipals PrincipalNameAndId[] = [] +@description('Whether to create or update Azure Monitor alerts during this deploy') +param deployAlerts bool + @description('Specifies a set of tags with which to tag the resource in Azure.') param tagValues object @@ -70,6 +73,15 @@ resource maxPreparedTransactionsConfig 'Microsoft.DBforPostgreSQL/flexibleServer ] } +module databaseAliveAlert '../../components/alerts/flexibleServers/databaseAlive.bicep' = if (deployAlerts) { + name: '${resourceNames.sharedResources.postgreSqlFlexibleServer}DbAliveDeploy' + params: { + resourceNames: [resourceNames.sharedResources.postgreSqlFlexibleServer] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} + var managedIdentityConnectionStringTemplate = postgreSqlServerModule.outputs.managedIdentityConnectionStringTemplate var dataProcessorPsqlConnectionStringSecretKey = 'ees-publicapi-data-processor-connectionstring-publicdatadb' diff --git a/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep b/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep index 440aa4f4fad..069e01ba904 100644 --- a/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep +++ b/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep @@ -9,7 +9,7 @@ param severity Severity = 'Critical' @description('Name of the Alerts Group used to send alert messages.') param alertsGroupName string -@description('Specifies a set of tags with which to tag the resource in Azure.') +@description('Tags with which to tag the resource in Azure.') param tagValues object module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: { diff --git a/infrastructure/templates/public-api/components/alerts/containerApps/restarts.bicep b/infrastructure/templates/public-api/components/alerts/containerApps/restarts.bicep new file mode 100644 index 00000000000..a43630be58f --- /dev/null +++ b/infrastructure/templates/public-api/components/alerts/containerApps/restarts.bicep @@ -0,0 +1,33 @@ +import { Severity } from '../types.bicep' + +@description('Names of the resources that these alerts are being applied to.') +param resourceNames string[] + +@description('The alert severity.') +param severity Severity = 'Warning' + +@description('Name of the Alerts Group used to send alert messages.') +param alertsGroupName string + +@description('Tags with which to tag the resource in Azure.') +param tagValues object + +module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: { + name: '${name}RestartsAlertModule' + params: { + alertName: '${name}-restarts' + resourceIds: [resourceId('Microsoft.App/containerApps', name)] + resourceType: 'Microsoft.App/containerApps' + query: { + metric: 'RestartCount' + aggregation: 'Average' + operator: 'GreaterThan' + threshold: 0 + } + evaluationFrequency: 'PT1M' + windowSize: 'PT5M' + severity: severity + alertsGroupName: alertsGroupName + tagValues: tagValues + } +}] diff --git a/infrastructure/templates/public-api/components/alerts/flexibleServers/databaseAlive.bicep b/infrastructure/templates/public-api/components/alerts/flexibleServers/databaseAlive.bicep new file mode 100644 index 00000000000..21358742e02 --- /dev/null +++ b/infrastructure/templates/public-api/components/alerts/flexibleServers/databaseAlive.bicep @@ -0,0 +1,33 @@ +import { Severity } from '../types.bicep' + +@description('Names of the resources that these alerts are being applied to.') +param resourceNames string[] + +@description('The alert severity.') +param severity Severity = 'Critical' + +@description('Name of the Alerts Group used to send alert messages.') +param alertsGroupName string + +@description('Tags with which to tag the resource in Azure.') +param tagValues object + +module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: { + name: '${name}DbAliveAlertModule' + params: { + alertName: '${name}-database-alive' + resourceIds: [resourceId('Microsoft.DBforPostgreSQL/flexibleServers', name)] + resourceType: 'Microsoft.DBforPostgreSQL/flexibleServers' + query: { + metric: 'is_db_alive' + aggregation: 'Minimum' + operator: 'LessThan' + threshold: 1 + } + evaluationFrequency: 'PT1M' + windowSize: 'PT5M' + severity: severity + alertsGroupName: alertsGroupName + tagValues: tagValues + } +}] diff --git a/infrastructure/templates/public-api/components/alerts/types.bicep b/infrastructure/templates/public-api/components/alerts/types.bicep index d7f83ed13ce..ec21e1977e1 100644 --- a/infrastructure/templates/public-api/components/alerts/types.bicep +++ b/infrastructure/templates/public-api/components/alerts/types.bicep @@ -5,7 +5,10 @@ type EvaluationFrequency = 'PT1M' type WindowSize = 'PT5M' @export() -type MetricOperator = 'GreaterThan' | 'LessThan' +type MetricOperator = + | 'GreaterOrLessThan' + | 'GreaterThan' + | 'LessThan' @export() type TimeAggregation = @@ -38,13 +41,16 @@ type ResourceType = | 'Microsoft.Sql/servers/databases' | 'Microsoft.Storage/storageAccounts' | 'Microsoft.Network/applicationGateways' + | 'Microsoft.App/containerApps' @export() type MetricName = + | 'availability' | 'blocked_by_firewall' | 'connection_failed' | 'connections_failed' | 'cpu_percent' | 'HealthCheckStatus' - | 'availability' + | 'is_db_alive' + | 'RestartCount' | 'UnhealthyHostCount' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index c91a9cf7037..92c0cafd7e0 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -221,6 +221,7 @@ module postgreSqlServerModule 'application/shared/postgreSqlFlexibleServer.bicep firewallRules: postgreSqlFirewallRules sku: postgreSqlSkuName storageSizeGB: postgreSqlStorageSizeGB + deployAlerts: deployAlerts tagValues: tagValues } dependsOn: [ @@ -265,6 +266,7 @@ module apiAppModule 'application/public-api/publicApiApp.bicep' = if (deployCont publicSiteUrl: publicUrls.publicSite dockerImagesTag: dockerImagesTag appInsightsConnectionString: appInsightsModule.outputs.appInsightsConnectionString + deployAlerts: deployAlerts tagValues: tagValues } dependsOn: [ From 6818915468d85cdffe2173ee25e58d2f79bf5e26 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 4 Dec 2024 16:28:16 +0000 Subject: [PATCH 057/144] EES-5685 - added availability alerts for fileServices --- .../public-api/publicApiDataProcessor.bicep | 23 ++++++++++--- .../public-api/publicApiStorage.bicep | 11 ++++++- .../fileServices/availabilityAlert.bicep | 33 +++++++++++++++++++ .../components/alerts/staticMetricAlert.bicep | 4 +-- .../public-api/components/alerts/types.bicep | 22 +++++++++---- 5 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 infrastructure/templates/public-api/components/alerts/fileServices/availabilityAlert.bicep diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 30e0213ec0e..29a9a450634 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -113,8 +113,17 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { } } -module storageAvailabilityAlerts '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.dataProcessor}AvailabilityAlert' +module functionAppHealthAlert '../../components/alerts/sites/healthAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}HealthDeploy' + params: { + resourceNames: [resourceNames.publicApi.dataProcessor] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} + +module storageAccountAvailabilityAlerts '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}StorageAvailabilityDeploy' params: { resourceNames: [ dataProcessorFunctionAppModule.outputs.managementStorageAccountName @@ -126,10 +135,14 @@ module storageAvailabilityAlerts '../../components/alerts/storageAccounts/availa } } -module functionAppHealthAlert '../../components/alerts/sites/healthAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.dataProcessor}HealthDeploy' +module fileServiceAvailabilityAlerts '../../components/alerts/fileServices/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}FsAvailabilityDeploy' params: { - resourceNames: [resourceNames.publicApi.dataProcessor] + resourceNames: [ + dataProcessorFunctionAppModule.outputs.managementStorageAccountName + dataProcessorFunctionAppModule.outputs.slot1StorageAccountName + dataProcessorFunctionAppModule.outputs.slot2StorageAccountName + ] alertsGroupName: resourceNames.existingResources.alertsGroup tagValues: tagValues } diff --git a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep index cfa4d568048..744d1217aa7 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep @@ -59,7 +59,7 @@ module dataFilesFileShareModule '../../components/fileShare.bicep' = { } } -module availabilityAlert '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) { +module storageAccountAvailabilityAlert '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) { name: '${resourceNames.publicApi.publicApiStorageAccount}AvailabilityDeploy' params: { resourceNames: [resourceNames.publicApi.publicApiStorageAccount] @@ -68,6 +68,15 @@ module availabilityAlert '../../components/alerts/storageAccounts/availabilityAl } } +module fileServiceAvailabilityAlert '../../components/alerts/fileServices/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.publicApiStorageAccount}FsAvailabilityDeploy' + params: { + resourceNames: [resourceNames.publicApi.publicApiStorageAccount] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} + output storageAccountName string = publicApiStorageAccountModule.outputs.storageAccountName output connectionStringSecretName string = publicApiStorageAccountModule.outputs.connectionStringSecretName output accessKeySecretName string = publicApiStorageAccountModule.outputs.accessKeySecretName diff --git a/infrastructure/templates/public-api/components/alerts/fileServices/availabilityAlert.bicep b/infrastructure/templates/public-api/components/alerts/fileServices/availabilityAlert.bicep new file mode 100644 index 00000000000..a4f4e3c7714 --- /dev/null +++ b/infrastructure/templates/public-api/components/alerts/fileServices/availabilityAlert.bicep @@ -0,0 +1,33 @@ +import { Severity } from '../types.bicep' + +@description('Names of the resources that these alerts are being applied to.') +param resourceNames string[] + +@description('The alert severity.') +param severity Severity = 'Critical' + +@description('Name of the Alerts Group used to send alert messages.') +param alertsGroupName string + +@description('Tags with which to tag the resource in Azure.') +param tagValues object + +module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: { + name: '${name}FsAvailabilityAlertModule' + params: { + alertName: '${name}-fileservice-availability' + resourceIds: [resourceId('Microsoft.Storage/storageAccounts/fileServices', name, 'default')] + resourceType: 'Microsoft.Storage/storageAccounts/fileServices' + query: { + metric: 'availability' + aggregation: 'Average' + operator: 'LessThan' + threshold: 100 + } + evaluationFrequency: 'PT1M' + windowSize: 'PT5M' + severity: severity + alertsGroupName: alertsGroupName + tagValues: tagValues + } +}] diff --git a/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep b/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep index d3f33eac260..324df78ee6b 100644 --- a/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep +++ b/infrastructure/templates/public-api/components/alerts/staticMetricAlert.bicep @@ -1,4 +1,4 @@ -import { EvaluationFrequency, MetricName, MetricOperator, ResourceType, TimeAggregation, WindowSize, Severity, severityMapping } from 'types.bicep' +import { EvaluationFrequency, MetricName, StaticMetricOperator, ResourceType, TimeAggregation, WindowSize, Severity, severityMapping } from 'types.bicep' @description('Name of the alert.') param alertName string @@ -13,7 +13,7 @@ param resourceType ResourceType param query { metric: MetricName aggregation: TimeAggregation - operator: MetricOperator + operator: StaticMetricOperator threshold: int } diff --git a/infrastructure/templates/public-api/components/alerts/types.bicep b/infrastructure/templates/public-api/components/alerts/types.bicep index ec21e1977e1..af92b51467f 100644 --- a/infrastructure/templates/public-api/components/alerts/types.bicep +++ b/infrastructure/templates/public-api/components/alerts/types.bicep @@ -5,10 +5,19 @@ type EvaluationFrequency = 'PT1M' type WindowSize = 'PT5M' @export() -type MetricOperator = - | 'GreaterOrLessThan' +type DynamicMetricOperator = +| 'GreaterOrLessThan' +| 'GreaterThan' + | 'LessThan' + + +@export() +type StaticMetricOperator = + | 'Equals' | 'GreaterThan' + | 'GreaterThanOrEqual' | 'LessThan' + | 'LessThanOrEqual' @export() type TimeAggregation = @@ -35,13 +44,14 @@ var severityMapping = { @export() type ResourceType = - | 'Microsoft.Web/sites' - | 'Microsoft.Web/sites/slots' + | 'Microsoft.App/containerApps' | 'Microsoft.DBforPostgreSQL/flexibleServers' + | 'Microsoft.Network/applicationGateways' | 'Microsoft.Sql/servers/databases' | 'Microsoft.Storage/storageAccounts' - | 'Microsoft.Network/applicationGateways' - | 'Microsoft.App/containerApps' + | 'Microsoft.Storage/storageAccounts/fileServices' + | 'Microsoft.Web/sites' + | 'Microsoft.Web/sites/slots' @export() type MetricName = From 2de2fd2433c366c24db380227fc73856bb1d16fc Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 10:23:37 +0000 Subject: [PATCH 058/144] EES-5685 - tidy-up before raising PR --- .../templates/public-api/ci/azure-pipelines.yml | 2 +- .../templates/public-api/components/alerts/types.bicep | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/infrastructure/templates/public-api/ci/azure-pipelines.yml b/infrastructure/templates/public-api/ci/azure-pipelines.yml index c22b2ea1060..f3d3a984229 100644 --- a/infrastructure/templates/public-api/ci/azure-pipelines.yml +++ b/infrastructure/templates/public-api/ci/azure-pipelines.yml @@ -8,7 +8,7 @@ parameters: displayName: Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys. default: false - name: deployAlerts - displayName: Should we deploy alerts with this deploy? + displayName: Whether to create or update Azure Monitor alerts during this deploy. default: false - name: forceDeployToEnvironment displayName: Set to either dev or test to force a deploy to that environment from the chosen branch. diff --git a/infrastructure/templates/public-api/components/alerts/types.bicep b/infrastructure/templates/public-api/components/alerts/types.bicep index af92b51467f..c767d9d5f06 100644 --- a/infrastructure/templates/public-api/components/alerts/types.bicep +++ b/infrastructure/templates/public-api/components/alerts/types.bicep @@ -6,8 +6,8 @@ type WindowSize = 'PT5M' @export() type DynamicMetricOperator = -| 'GreaterOrLessThan' -| 'GreaterThan' + | 'GreaterOrLessThan' + | 'GreaterThan' | 'LessThan' @@ -21,9 +21,11 @@ type StaticMetricOperator = @export() type TimeAggregation = - | 'Total' - | 'Minimum' | 'Average' + | 'Count' + | 'Maximum' + | 'Minimum' + | 'Total' @export() type Severity = From eac866a129f328adba7a35c28739e743f440e9e2 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 14:02:06 +0000 Subject: [PATCH 059/144] EES-5685 - responding to PR comments. Updating timeAggregation on some alerts to produce a more useful report --- .../templates/public-api/application/shared/appGateway.bicep | 2 +- .../components/alerts/appGateways/backendPoolHealth.bicep | 2 +- .../public-api/components/alerts/containerApps/restarts.bicep | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/infrastructure/templates/public-api/application/shared/appGateway.bicep b/infrastructure/templates/public-api/application/shared/appGateway.bicep index 2d85993b272..d93258d1402 100644 --- a/infrastructure/templates/public-api/application/shared/appGateway.bicep +++ b/infrastructure/templates/public-api/application/shared/appGateway.bicep @@ -56,7 +56,7 @@ module appGatewayModule '../../components/appGateway.bicep' = { } module backendPoolsHealthAlert '../../components/alerts/appGateways/backendPoolHealth.bicep' = if (deployAlerts) { - name: '${resourceNames.sharedResources.appGateway}BackedPoolsHealthDeploy' + name: '${resourceNames.sharedResources.appGateway}BackendPoolsHealthDeploy' params: { resourceNames: [resourceNames.sharedResources.appGateway] alertsGroupName: resourceNames.existingResources.alertsGroup diff --git a/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep b/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep index 069e01ba904..3fabc48a219 100644 --- a/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep +++ b/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep @@ -20,7 +20,7 @@ module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: { resourceType: 'Microsoft.Network/applicationGateways' query: { metric: 'UnhealthyHostCount' - aggregation: 'Average' + aggregation: 'Total' operator: 'GreaterThan' threshold: 0 } diff --git a/infrastructure/templates/public-api/components/alerts/containerApps/restarts.bicep b/infrastructure/templates/public-api/components/alerts/containerApps/restarts.bicep index a43630be58f..fb79c183cfd 100644 --- a/infrastructure/templates/public-api/components/alerts/containerApps/restarts.bicep +++ b/infrastructure/templates/public-api/components/alerts/containerApps/restarts.bicep @@ -20,7 +20,7 @@ module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: { resourceType: 'Microsoft.App/containerApps' query: { metric: 'RestartCount' - aggregation: 'Average' + aggregation: 'Total' operator: 'GreaterThan' threshold: 0 } From 417faccbc157a6d5d0355c98b11456fa64fc76f8 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 14:40:32 +0000 Subject: [PATCH 060/144] EES-5705 - tidy-up for PR --- tests/robot-tests/run_tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/robot-tests/run_tests.py b/tests/robot-tests/run_tests.py index d02ddd7b460..f0d1d6b99d6 100755 --- a/tests/robot-tests/run_tests.py +++ b/tests/robot-tests/run_tests.py @@ -127,8 +127,6 @@ def run(): logger.info(f"Running Robot tests with {max_run_attempts} maximum run attempts") - test_run_results_folder = "" - try: # Run tests while test_run_index < max_run_attempts: From ba4da9da726f5d172c0f52e398530fb2a1638f54 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 15:15:48 +0000 Subject: [PATCH 061/144] EES-XXXX - fixed misplaced variable definitions in "dependsOn" array --- infrastructure/templates/template.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index d3c9abd0cb5..dc3c461e975 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -3297,10 +3297,7 @@ "location": "[resourceGroup().location]", "apiVersion": "2019-08-01", "dependsOn": [ - "[resourceId('Microsoft.Web/sites', variables('publisherAppName'))]", - "[variables('publicDataFileShareMountPath')]", - "[variables('publicDataFileShareName')]", - "[variables('publicDataStorageAccountName')]" + "[resourceId('Microsoft.Web/sites', variables('publisherAppName'))]" ], "properties": { "[variables('publicDataFileShareName')]": { From 9a7999c131672586c356b41e3425288ae20ccd2e Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Mon, 2 Dec 2024 11:00:49 +0000 Subject: [PATCH 062/144] EES-5583: Add endpoint to return a zip archive upload plan. --- .../Controllers/Api/ReleasesController.cs | 20 +- .../Models/DataFileInfo.cs | 2 +- .../Services/DataArchiveValidationService.cs | 104 ++++---- .../IDataArchiveValidationService.cs | 11 +- .../Interfaces/IReleaseDataFileService.cs | 18 +- .../Services/ReleaseDataFileService.cs | 224 ++++++++++++------ .../Validators/ValidationMessages.cs | 30 ++- .../ViewModels/ArchiveDataSetFileViewModel.cs | 23 ++ .../BlobContainers.cs | 3 +- .../Services/BlobStorageService.cs | 42 ++-- .../Services/FileStoragePathUtils.cs | 4 +- .../Interfaces/IBlobStorageService.cs | 16 +- .../Services/PrivateReleaseFileBlobService.cs | 2 +- .../Services/PublicReleaseFileBlobService.cs | 2 +- 14 files changed, 333 insertions(+), 168 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ArchiveDataSetFileViewModel.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs index f192797c8fa..331b6667e50 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs @@ -136,15 +136,27 @@ public async Task> UploadDataSetAsZip(Guid releaseVer .HandleFailuresOrOk(); } - [HttpPost("release/{releaseVersionId:guid}/bulk-zip-data")] + [HttpPost("release/{releaseVersionId:guid}/upload-bulk-zip-data")] [DisableRequestSizeLimit] [RequestFormLimits(ValueLengthLimit = int.MaxValue, MultipartBodyLengthLimit = int.MaxValue)] - public async Task>> UploadDataSetsAsBulkZip( + public async Task>> UploadBulkZipDataSetsToTempStorage( Guid releaseVersionId, - IFormFile zipFile) + IFormFile zipFile, + CancellationToken cancellationToken) + { + return await _releaseDataFileService + .ValidateAndUploadBulkZip(releaseVersionId, zipFile, cancellationToken) + .HandleFailuresOrOk(); + } + + [HttpPost("release/{releaseVersionId:guid}/import-bulk-zip-data")] + public async Task>> ImportBulkZipDataSetsFromTempStorage( + Guid releaseVersionId, + List dataSetFiles, + CancellationToken cancellationToken) { return await _releaseDataFileService - .UploadAsBulkZip(releaseVersionId: releaseVersionId, bulkZipFormFile: zipFile) + .SaveDataSetsFromTemporaryBlobStorage(releaseVersionId, dataSetFiles, cancellationToken) .HandleFailuresOrOk(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Models/DataFileInfo.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Models/DataFileInfo.cs index 5a4e0eeffe0..e044a931505 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Models/DataFileInfo.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Models/DataFileInfo.cs @@ -1,9 +1,9 @@ #nullable enable -using System; using GovUk.Education.ExploreEducationStatistics.Common.Converters; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model; using Newtonsoft.Json; +using System; namespace GovUk.Education.ExploreEducationStatistics.Admin.Models { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs index d0ca5865233..71153aea010 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs @@ -1,9 +1,4 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.IO.Compression; -using System.Linq; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Models; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Admin.Validators; @@ -15,7 +10,11 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using static GovUk.Education.ExploreEducationStatistics.Common.Model.FileType; +using System; +using System.Collections.Generic; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; namespace GovUk.Education.ExploreEducationStatistics.Admin.Services { @@ -89,16 +88,10 @@ public async Task> ValidateDataArchiveF return archiveDataSet; } - public async Task>> ValidateBulkDataArchiveFile( + public async Task>> ValidateBulkDataArchiveFiles( Guid releaseVersionId, IFormFile zipFile) { - var errors = await IsValidZipFile(zipFile); - if (errors.Count > 0) - { - return Common.Validators.ValidationUtils.ValidationResult(errors); - } - await using var stream = zipFile.OpenReadStream(); using var archive = new ZipArchive(stream); @@ -109,26 +102,27 @@ public async Task>> ValidateBulkDa if (dataSetNamesFile == null) { return Common.Validators.ValidationUtils.ValidationResult(new ErrorViewModel - { - Code = ValidationMessages.BulkDataZipMustContainDatasetNamesCsv.Code, - Message = ValidationMessages.BulkDataZipMustContainDatasetNamesCsv.Message, - }); + { + Code = ValidationMessages.BulkDataZipMustContainDatasetNamesCsv.Code, + Message = ValidationMessages.BulkDataZipMustContainDatasetNamesCsv.Message, + }); } unprocessedArchiveFiles.Remove(dataSetNamesFile); List headers; - await using (var dataSetNamesStream = dataSetNamesFile.Open()) { + await using (var dataSetNamesStream = dataSetNamesFile.Open()) + { headers = await CsvUtils.GetCsvHeaders(dataSetNamesStream); } if (headers is not ["file_name", "dataset_name"]) { return Common.Validators.ValidationUtils.ValidationResult(new ErrorViewModel - { - Code = ValidationMessages.DatasetNamesCsvIncorrectHeaders.Code, - Message = ValidationMessages.DatasetNamesCsvIncorrectHeaders.Message, - }); + { + Code = ValidationMessages.DatasetNamesCsvIncorrectHeaders.Code, + Message = ValidationMessages.DatasetNamesCsvIncorrectHeaders.Message, + }); } var fileNameIndex = headers[0] == "file_name" ? 0 : 1; @@ -149,6 +143,8 @@ public async Task>> ValidateBulkDa dataSetNamesCsvEntries.Add((BaseFilename: filename, Title: datasetName)); } + var errors = new List(); + dataSetNamesCsvEntries .Select(entry => entry.BaseFilename) .Where(baseFilename => baseFilename.EndsWith(".csv")) @@ -158,7 +154,7 @@ public async Task>> ValidateBulkDa errors.Add(ValidationMessages.GenerateErrorDatasetNamesCsvFilenamesShouldNotEndDotCsv(baseFilename)); }); - // Check for duplicate data set titles - because the bulk zip itself main contain duplicates! + // Check for duplicate data set titles - because the bulk zip itself may contain duplicates! dataSetNamesCsvEntries .GroupBy(entry => entry.Title) .Where(group => group.Count() > 1) @@ -170,9 +166,9 @@ public async Task>> ValidateBulkDa .GenerateErrorDataSetTitleShouldBeUnique(duplicateTitle)); }); - // Check for duplicate data set filenames - because the bulk zip itself main contain duplicates! + // Check for duplicate data set filenames - because the bulk zip itself may contain duplicates! dataSetNamesCsvEntries - .GroupBy(entry => entry.BaseFilename) + .GroupBy(entry => entry.BaseFilename) .Where(group => group.Count() > 1) .Select(group => group.Key) .ToList() @@ -188,7 +184,7 @@ public async Task>> ValidateBulkDa } var dataSetFiles = new List(); - foreach(var entry in dataSetNamesCsvEntries) + foreach (var entry in dataSetNamesCsvEntries) { var dataFile = archive.GetEntry($"{entry.BaseFilename}.csv"); var metaFile = archive.GetEntry($"{entry.BaseFilename}.meta.csv"); @@ -204,7 +200,7 @@ public async Task>> ValidateBulkDa if (metaFile == null) { - errors.Add(ValidationMessages.GenerateErrorFileNotFoundInZip($"{entry.BaseFilename}.meta.csv", Metadata)); + errors.Add(ValidationMessages.GenerateErrorFileNotFoundInZip($"{entry.BaseFilename}.meta.csv", FileType.Metadata)); } else { @@ -213,16 +209,17 @@ public async Task>> ValidateBulkDa if (dataFile != null && metaFile != null) { - // We replace files with the same title. If there is no releaseFile with the same title, - // it's a new data set. - var releaseFileToBeReplaced = contentDbContext.ReleaseFiles - .Include(rf => rf.File) - .SingleOrDefault(rf => - rf.ReleaseVersionId == releaseVersionId - && rf.File.Type == FileType.Data - && rf.Name == entry.Title); - - var dataArchiveFile = new ArchiveDataSetFile( + try + { + // We replace files with the same title. If there is no releaseFile with the same title, it's a new data set. + var releaseFileToBeReplaced = await contentDbContext.ReleaseFiles + .Include(rf => rf.File) + .SingleOrDefaultAsync(rf => + rf.ReleaseVersionId == releaseVersionId + && rf.File.Type == FileType.Data + && rf.Name == entry.Title); + + var dataArchiveFile = new ArchiveDataSetFile( entry.Title, dataFile.FullName, metaFile.FullName, @@ -230,17 +227,22 @@ public async Task>> ValidateBulkDa metaFile.Length, releaseFileToBeReplaced?.File); - await using (var dataFileStream = dataFile.Open()) - await using (var metaFileStream = metaFile.Open()) + await using (var dataFileStream = dataFile.Open()) + await using (var metaFileStream = metaFile.Open()) + { + errors.AddRange(await fileUploadsValidatorService.ValidateDataSetFilesForUpload( + releaseVersionId, + dataArchiveFile, + dataFileStream: dataFileStream, + metaFileStream: metaFileStream)); + } + + dataSetFiles.Add(dataArchiveFile); + } + catch (InvalidOperationException) { - errors.AddRange(await fileUploadsValidatorService.ValidateDataSetFilesForUpload( - releaseVersionId, - dataArchiveFile, - dataFileStream: dataFileStream, - metaFileStream: metaFileStream)); + errors.Add(ValidationMessages.GenerateErrorDataReplacementAlreadyInProgress()); } - - dataSetFiles.Add(dataArchiveFile); } } @@ -261,14 +263,20 @@ public async Task>> ValidateBulkDa return dataSetFiles; } - private async Task> IsValidZipFile(IFormFile zipFile) + public async Task> IsValidZipFile(IFormFile zipFile) { List errors = []; + if (zipFile is null) + { + errors.Add(ValidationMessages.GenerateErrorFileIsNull()); + return errors; + } + if (zipFile.FileName.Length > MaxFilenameSize) { errors.Add(ValidationMessages.GenerateErrorFilenameTooLong( - zipFile.FileName, MaxFilenameSize)); + zipFile.FileName, MaxFilenameSize)); } if (!zipFile.FileName.ToLower().EndsWith(".zip")) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IDataArchiveValidationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IDataArchiveValidationService.cs index f1f3007f8c1..181620d6b87 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IDataArchiveValidationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IDataArchiveValidationService.cs @@ -1,12 +1,13 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Models; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using GovUk.Education.ExploreEducationStatistics.Content.Model; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; namespace GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces { @@ -18,8 +19,10 @@ Task> ValidateDataArchiveFile( IFormFile zipFile, File? replacingFile = null); - Task>> ValidateBulkDataArchiveFile( + Task>> ValidateBulkDataArchiveFiles( Guid releaseVersionId, IFormFile zipFile); + + Task> IsValidZipFile(IFormFile zipFile); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReleaseDataFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReleaseDataFileService.cs index fe71e7d4868..08bce86cb69 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReleaseDataFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Interfaces/IReleaseDataFileService.cs @@ -1,11 +1,13 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Models; +using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; using GovUk.Education.ExploreEducationStatistics.Common.Model; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; namespace GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces { @@ -44,8 +46,14 @@ Task> UploadAsZip( string? dataSetTitle, Guid? replacingFileId); - Task>> UploadAsBulkZip( + Task>> ValidateAndUploadBulkZip( + Guid releaseVersionId, + IFormFile zipFile, + CancellationToken cancellationToken); + + Task>> SaveDataSetsFromTemporaryBlobStorage( Guid releaseVersionId, - IFormFile bulkZipFormFile); + List archiveDataSetFiles, + CancellationToken cancellationToken); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs index 30286f4414b..8da1a64fc67 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs @@ -1,13 +1,11 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Models; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Security; +using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Services; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Common.Utils; @@ -18,10 +16,15 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using System; +using System.Collections.Generic; +using System.IO.Compression; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationErrorMessages; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationUtils; using static GovUk.Education.ExploreEducationStatistics.Common.BlobContainers; -using static GovUk.Education.ExploreEducationStatistics.Common.Model.FileType; using IReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IReleaseVersionRepository; using Unit = GovUk.Education.ExploreEducationStatistics.Common.Model.Unit; @@ -75,7 +78,7 @@ public async Task> Delete(Guid releaseVersionId, Guid fileId, bool forceDelete = false) { - return await Delete(releaseVersionId, [ fileId ], forceDelete: forceDelete); + return await Delete(releaseVersionId, [fileId], forceDelete: forceDelete); } public async Task> Delete(Guid releaseVersionId, @@ -277,7 +280,8 @@ public async Task> Upload(Guid releaseVersion return newDataSetTitle; }) - .OnSuccess(async tuple => { + .OnSuccess(async tuple => + { var (replacingFile, newDataSetTitle) = tuple; @@ -309,7 +313,7 @@ await _releaseVersionRepository subjectId: subjectId, filename: metaFormFile.FileName.ToLower(), contentLength: metaFormFile.Length, - type: Metadata, + type: FileType.Metadata, createdById: _userService.GetUserId()); await UploadFileToStorage(dataFile, dataFormFile); @@ -367,7 +371,7 @@ public async Task> UploadAsZip(Guid releaseVe ContentLength = zipFormFile.Length, ContentType = zipFormFile.ContentType, Filename = zipFormFile.FileName.ToLower(), - Type = DataZip, + Type = FileType.DataZip, }; _contentDbContext.Files.Add(zipFile); await _contentDbContext.SaveChangesAsync(); @@ -401,7 +405,7 @@ public async Task> UploadAsZip(Guid releaseVe subjectId: subjectId, filename: archiveDataSet.MetaFilename, contentLength: archiveDataSet.MetaFileSize, - type: Metadata, + type: FileType.Metadata, createdById: _userService.GetUserId(), source: zipFile); @@ -428,89 +432,138 @@ public async Task> UploadAsZip(Guid releaseVe }); } - public async Task>> UploadAsBulkZip( + public async Task>> ValidateAndUploadBulkZip( Guid releaseVersionId, - IFormFile bulkZipFormFile) + IFormFile zipFile, + CancellationToken cancellationToken) { return await _persistenceHelper .CheckEntityExists(releaseVersionId) .OnSuccess(_userService.CheckCanUpdateReleaseVersion) - .OnSuccess(_ => _dataArchiveValidationService.ValidateBulkDataArchiveFile( - releaseVersionId, bulkZipFormFile) - .OnSuccess(async dataArchiveFiles => + .OnSuccessVoid(async _ => { - var bulkZipFile = new File + var errors = await _dataArchiveValidationService.IsValidZipFile(zipFile); + if (errors.Count > 0) { - CreatedById = _userService.GetUserId(), - RootPath = releaseVersionId, - ContentLength = bulkZipFormFile.Length, - ContentType = bulkZipFormFile.ContentType, - Filename = bulkZipFormFile.FileName.ToLower(), - Type = BulkDataZip, - }; - _contentDbContext.Files.Add(bulkZipFile); - await _contentDbContext.SaveChangesAsync(); + return new Either(Common.Validators.ValidationUtils.ValidationResult(errors)); + } + + return Unit.Instance; + }) + .OnSuccess(async _ => await _dataArchiveValidationService.ValidateBulkDataArchiveFiles(releaseVersionId, zipFile)) + .OnSuccess(async dataSetFiles => + { + await using var stream = zipFile.OpenReadStream(); + var archive = new ZipArchive(stream); - // each data/meta file pair are extracted to blob storage by _dataImportService.Import - await UploadFileToStorage(bulkZipFile, bulkZipFormFile); + var viewModels = new List(); - var results = new List(); + foreach (var file in dataSetFiles) + { + var dataFileId = await UploadFileToTempStorage(releaseVersionId, archive.GetEntry(file.DataFilename)!, FileType.Data, cancellationToken); + var metaFileId = await UploadFileToTempStorage(releaseVersionId, archive.GetEntry(file.MetaFilename)!, FileType.Metadata, cancellationToken); - foreach (var archiveFile in dataArchiveFiles) + viewModels.Add(new ArchiveDataSetFileViewModel { - var subjectId = await _releaseVersionRepository - .CreateStatisticsDbReleaseAndSubjectHierarchy(releaseVersionId); + Title = file.Title, + DataFilename = file.DataFilename, + MetaFilename = file.MetaFilename, + DataFileId = dataFileId, + MetaFileId = metaFileId, + DataFileSize = file.DataFileSize, + MetaFileSize = file.MetaFileSize, + ReplacingFileId = file.ReplacingFile?.Id, + }); + } - var releaseDataFileOrder = await GetNextDataFileOrder( - releaseVersionId, archiveFile.ReplacingFile); + return viewModels; + }); + } - var dataFile = await _releaseDataFileRepository.Create( - releaseVersionId: releaseVersionId, - subjectId: subjectId, - filename: archiveFile.DataFilename, - contentLength: archiveFile.DataFileSize, - type: FileType.Data, - createdById: _userService.GetUserId(), - name: archiveFile.Title, - replacingDataFile: archiveFile.ReplacingFile, - source: bulkZipFile, - order: releaseDataFileOrder); + public async Task>> SaveDataSetsFromTemporaryBlobStorage( + Guid releaseVersionId, + List archiveDataSetFiles, + CancellationToken cancellationToken) + { + return await _persistenceHelper + .CheckEntityExists(releaseVersionId) + .OnSuccess(_userService.CheckCanUpdateReleaseVersion) + .OnSuccess(() => archiveDataSetFiles + .Select(importRequest => ValidateTempDataSetFileExistence(releaseVersionId, importRequest)) + .OnSuccessAll()) + .OnSuccess(async _ => + { + var releaseFiles = new List(); - var dataReleaseFile = await _contentDbContext.ReleaseFiles - .Include(rf => rf.File) - .SingleAsync(rf => - rf.ReleaseVersionId == releaseVersionId - && rf.FileId == dataFile.Id); + foreach (var archiveDataSetFile in archiveDataSetFiles) + { + var subjectId = await _releaseVersionRepository.CreateStatisticsDbReleaseAndSubjectHierarchy(releaseVersionId); - var metaFile = await _releaseDataFileRepository.Create( - releaseVersionId: releaseVersionId, - subjectId: subjectId, - filename: archiveFile.MetaFilename, - contentLength: archiveFile.MetaFileSize, - type: Metadata, - createdById: _userService.GetUserId(), - source: bulkZipFile); + var replacingFile = archiveDataSetFile.ReplacingFileId is null + ? null + : await _contentDbContext.Files.FirstAsync(f => f.Id == archiveDataSetFile.ReplacingFileId); - var dataImport = await _dataImportService.Import( - subjectId: subjectId, - dataFile: dataFile, - metaFile: metaFile, - sourceZipFile: bulkZipFile); + var releaseDataFileOrder = await GetNextDataFileOrder(releaseVersionId, replacingFile); - var permissions = await _userService.GetDataFilePermissions(dataFile); + var dataFile = await _releaseDataFileRepository.Create( + releaseVersionId: releaseVersionId, + subjectId: subjectId, + filename: archiveDataSetFile.DataFilename, + contentLength: archiveDataSetFile.DataFileSize, + type: FileType.Data, + createdById: _userService.GetUserId(), + name: archiveDataSetFile.Title, + replacingDataFile: replacingFile, + order: releaseDataFileOrder); + + var sourceDataFilePath = FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Data, archiveDataSetFile.DataFileId); + var destinationDataFilePath = FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Data, dataFile.Id); // Same path, but a new ID has been generated by the creation step above + + var dataReleaseFile = await _contentDbContext.ReleaseFiles + .Include(rf => rf.File) + .SingleAsync(rf => + rf.ReleaseVersionId == releaseVersionId + && rf.FileId == dataFile.Id); + + releaseFiles.Add(dataReleaseFile); + + var metaFile = await _releaseDataFileRepository.Create( + releaseVersionId: releaseVersionId, + subjectId: subjectId, + filename: archiveDataSetFile.MetaFilename, + contentLength: archiveDataSetFile.MetaFileSize, + type: FileType.Metadata, + createdById: _userService.GetUserId()); + + var sourceMetaFilePath = FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Metadata, archiveDataSetFile.MetaFileId); + var destinationMetaFilePath = FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Metadata, metaFile.Id); // Same path, but a new ID has been generated by the creation step above + + await _dataImportService.Import( + subjectId: subjectId, + dataFile: dataFile, + metaFile: metaFile); + + await _privateBlobStorageService.MoveBlob(PrivateReleaseTempFiles, sourceDataFilePath, destinationDataFilePath, PrivateReleaseFiles); + await _privateBlobStorageService.MoveBlob(PrivateReleaseTempFiles, sourceMetaFilePath, destinationMetaFilePath, PrivateReleaseFiles); + } - results.Add( - BuildDataFileViewModel( - dataReleaseFile: dataReleaseFile, - metaFile: metaFile, - archiveFile.Title, - dataImport.TotalRows, - dataImport.Status, - permissions)); - } + return await BuildDataFileViewModels(releaseFiles); + }); + } + + private async Task> ValidateTempDataSetFileExistence( + Guid releaseVersionId, + ArchiveDataSetFileViewModel fileImportRequest) + { + var dataBlobExists = await _privateBlobStorageService.CheckBlobExists(PrivateReleaseTempFiles, $"{FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Data, fileImportRequest.DataFileId)}"); + var metaBlobExists = await _privateBlobStorageService.CheckBlobExists(PrivateReleaseTempFiles, $"{FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Metadata, fileImportRequest.MetaFileId)}"); + + if (!dataBlobExists || !metaBlobExists) + { + throw new Exception("Unable to locate temporary files at the locations specified"); + } - return results; - })); + return Unit.Instance; } private async Task BuildDataFileViewModel(ReleaseFile releaseFile) @@ -636,14 +689,33 @@ await _privateBlobStorageService.UploadFile( ); } + private async Task UploadFileToTempStorage( + Guid releaseVersionId, + ZipArchiveEntry archiveEntry, + FileType fileType, + CancellationToken cancellationToken) + { + var fileId = Guid.NewGuid(); + var path = $"{FileStoragePathUtils.FilesPath(releaseVersionId, fileType)}{fileId}"; + + await _privateBlobStorageService.UploadStream( + containerName: PrivateReleaseTempFiles, + path: path, + stream: archiveEntry.Open(), + contentType: ContentTypes.Csv, + cancellationToken: cancellationToken); + + return fileId; + } + private async Task GetAssociatedMetaFile(Guid releaseVersionId, File dataFile) { return await _contentDbContext.ReleaseFiles .Include(rf => rf.File) .Where(rf => rf.ReleaseVersionId == releaseVersionId - && rf.File.Type == Metadata - && rf.File.SubjectId == dataFile.SubjectId) + && rf.File.Type == FileType.Metadata + && rf.File.SubjectId == dataFile.SubjectId) .Select(rf => rf.File) .SingleAsync(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs index 68a5d90c604..46f26173e4b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs @@ -1,9 +1,9 @@ #nullable enable -using System.Collections.Generic; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using System.Collections.Generic; namespace GovUk.Education.ExploreEducationStatistics.Admin.Validators; @@ -177,6 +177,34 @@ public static ErrorViewModel GenerateErrorDatasetNamesCsvFilenamesShouldBeUnique Message: "Inside dataset_names.csv, file_name cell entries should not end in '.csv' i.e. should be 'filename' not 'filename.csv'. Filename found with extension: '{0}'." ); + public static readonly LocalizableMessage FileIsNull = new( + Code: nameof(FileIsNull), + Message: "No file provided." + ); + + public static ErrorViewModel GenerateErrorFileIsNull() + { + return new ErrorViewModel + { + Code = FileIsNull.Code, + Message = FileIsNull.Message, + }; + } + + public static readonly LocalizableMessage DataReplacementAlreadyInProgress = new( + Code: nameof(DataReplacementAlreadyInProgress), + Message: "Data replacement already in progress" + ); + + public static ErrorViewModel GenerateErrorDataReplacementAlreadyInProgress() + { + return new ErrorViewModel + { + Code = DataReplacementAlreadyInProgress.Code, + Message = DataReplacementAlreadyInProgress.Message, + }; + } + public static ErrorViewModel GenerateErrorDatasetNamesCsvFilenamesShouldNotEndDotCsv(string filename) { return new ErrorViewModel diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ArchiveDataSetFileViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ArchiveDataSetFileViewModel.cs new file mode 100644 index 00000000000..d7826d13742 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ArchiveDataSetFileViewModel.cs @@ -0,0 +1,23 @@ +#nullable enable +using System; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; + +public record ArchiveDataSetFileViewModel +{ + public string Title { get; set; } = string.Empty; + + public string DataFilename { get; set; } = string.Empty; + + public Guid DataFileId { get; set; } + + public long DataFileSize { get; set; } + + public string MetaFilename { get; set; } = string.Empty; + + public Guid MetaFileId { get; set; } + + public long MetaFileSize { get; set; } + + public Guid? ReplacingFileId { get; set; } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/BlobContainers.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/BlobContainers.cs index 5ef238e86bb..1569c9d91c1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/BlobContainers.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/BlobContainers.cs @@ -1,4 +1,4 @@ -namespace GovUk.Education.ExploreEducationStatistics.Common +namespace GovUk.Education.ExploreEducationStatistics.Common { public interface IBlobContainer { @@ -9,6 +9,7 @@ public interface IBlobContainer public static class BlobContainers { public static readonly IBlobContainer PrivateReleaseFiles = new BlobContainer("releases"); + public static readonly IBlobContainer PrivateReleaseTempFiles = new BlobContainer("releases-temp"); public static readonly IBlobContainer PublicReleaseFiles = new BlobContainer("downloads"); public static readonly IBlobContainer PrivateContent = new PrivateBlobContainer("cache"); public static readonly IBlobContainer PublicContent = new PublicBlobContainer("cache"); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs index 5abd806a12e..9a82c518df9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/BlobStorageService.cs @@ -1,10 +1,4 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.IO; -using System.Net.Mime; -using System.Threading; -using System.Threading.Tasks; using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; @@ -21,6 +15,12 @@ using Microsoft.Azure.Storage.DataMovement; using Microsoft.Extensions.Logging; using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Mime; +using System.Threading; +using System.Threading.Tasks; using BlobInfo = GovUk.Education.ExploreEducationStatistics.Common.Model.BlobInfo; using BlobProperties = Azure.Storage.Blobs.Models.BlobProperties; using CopyStatus = Azure.Storage.Blobs.Models.CopyStatus; @@ -223,27 +223,33 @@ await blob.UploadAsync( } public async Task MoveBlob( - IBlobContainer containerName, + IBlobContainer sourceContainer, string sourcePath, - string destinationPath) + string destinationPath, + IBlobContainer? destinationContainer = null) { - var blobContainer = await GetBlobContainer(containerName); + var sourceContainerClient = await GetBlobContainer(sourceContainer); + var destinationContainerClient = destinationContainer is not null + ? await GetBlobContainer(destinationContainer) + : sourceContainerClient; - var destinationBlob = blobContainer.GetBlobClient(destinationPath); - if (await destinationBlob.ExistsAsync()) + var sourceBlob = sourceContainerClient.GetBlobClient(sourcePath); + if (!await sourceBlob.ExistsAsync()) { _logger.LogWarning( - "Destination already exists while moving blob. Source: '{source}' Destination: '{destination}'", - sourcePath, destinationPath); + "Source blob not found while moving blob. Source: '{Source}' Destination: '{Destination}'", + sourcePath, + destinationPath); return false; } - var sourceBlob = blobContainer.GetBlobClient(sourcePath); - if (!await sourceBlob.ExistsAsync()) + var destinationBlob = destinationContainerClient.GetBlobClient(destinationPath); + if (await destinationBlob.ExistsAsync()) { _logger.LogWarning( - "Source blob not found while moving blob. Source: '{source}' Destination: '{destination}'", - sourcePath, destinationPath); + "Destination already exists while moving blob. Source: '{Source}' Destination: '{Destination}'", + sourcePath, + destinationPath); return false; } @@ -263,7 +269,7 @@ public async Task MoveBlob( while (destinationProperties.CopyStatus == CopyStatus.Pending) { await Task.Delay(1000); - _logger.LogInformation("Copy progress: {progress}", destinationProperties.CopyProgress); + _logger.LogInformation("Copy progress: {Progress}", destinationProperties.CopyProgress); destinationProperties = await destinationBlob.GetPropertiesAsync(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/FileStoragePathUtils.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/FileStoragePathUtils.cs index 3b1c6bee4a5..2c1740207a0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/FileStoragePathUtils.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/FileStoragePathUtils.cs @@ -99,10 +99,10 @@ public static string PublicContentReleaseSubjectsPath(string publicationSlug, st return $"{PublicContentReleaseParentPath(publicationSlug, releaseSlug)}/subjects.json"; } - public static string FilesPath(Guid rootPath, FileType type) + public static string FilesPath(Guid rootPath, FileType type, Guid? filePath = null) { var typeFolder = (type == Metadata ? Data : type).GetEnumLabel(); - return $"{rootPath}/{typeFolder}/"; + return $"{rootPath}/{typeFolder}/{filePath}"; } private static string AppendPathSeparator(string? segment = null) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/Interfaces/IBlobStorageService.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/Interfaces/IBlobStorageService.cs index c567ec07ba5..79e9f3d2d8e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/Interfaces/IBlobStorageService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/Interfaces/IBlobStorageService.cs @@ -1,15 +1,15 @@ #nullable enable +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.Storage.DataMovement; +using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using GovUk.Education.ExploreEducationStatistics.Common.Model; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Azure.Storage.DataMovement; -using Newtonsoft.Json; namespace GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces { @@ -36,7 +36,11 @@ public Task DeleteBlobs( public Task DeleteBlob(IBlobContainer containerName, string path); - public Task MoveBlob(IBlobContainer containerName, string sourcePath, string destinationPath); + public Task MoveBlob( + IBlobContainer sourceContainer, + string sourcePath, + string destinationPath, + IBlobContainer? destinationContainer = null); public Task UploadFile( IBlobContainer containerName, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Services/PrivateReleaseFileBlobService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Services/PrivateReleaseFileBlobService.cs index a2e9b12519a..da21dc1943a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Services/PrivateReleaseFileBlobService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Services/PrivateReleaseFileBlobService.cs @@ -46,7 +46,7 @@ public Task DeleteFile(ReleaseFile releaseFile) public Task MoveBlob(ReleaseFile releaseFile, string destinationPath) { return _privateBlobStorageService.MoveBlob( - containerName: PrivateReleaseFiles, + sourceContainer: PrivateReleaseFiles, sourcePath: releaseFile.Path(), destinationPath: destinationPath ); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Services/PublicReleaseFileBlobService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Services/PublicReleaseFileBlobService.cs index 5771fd82f85..1436b680ab1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Services/PublicReleaseFileBlobService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Services/PublicReleaseFileBlobService.cs @@ -46,7 +46,7 @@ public Task DeleteFile(ReleaseFile releaseFile) public Task MoveBlob(ReleaseFile releaseFile, string destinationPath) { return _publicBlobStorageService.MoveBlob( - containerName: PublicReleaseFiles, + sourceContainer: PublicReleaseFiles, sourcePath: releaseFile.PublicPath(), destinationPath: destinationPath); } From d2cc7b47fc96ebce84c506aef619b11dc12acc5f Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Mon, 2 Dec 2024 11:01:57 +0000 Subject: [PATCH 063/144] EES-5584: Add modal to display file upload/replacement plan. --- .../release/data/ReleaseDataFilePage.tsx | 11 +++ .../components/BulkZipUploadModalConfirm.tsx | 51 ++++++++++++++ .../data/components/DataFileUploadForm.tsx | 3 + .../components/ReleaseDataUploadsSection.tsx | 70 ++++++++++++++----- .../src/services/releaseDataFileService.ts | 28 ++++++-- 5 files changed, 140 insertions(+), 23 deletions(-) create mode 100644 src/explore-education-statistics-admin/src/pages/release/data/components/BulkZipUploadModalConfirm.tsx diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataFilePage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataFilePage.tsx index eabe7adef74..1fbbd215c72 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataFilePage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataFilePage.tsx @@ -15,6 +15,8 @@ import { generatePath, RouteComponentProps } from 'react-router'; import LoadingSpinner from '@common/components/LoadingSpinner'; import Yup from '@common/validation/yup'; import Button from '@common/components/Button'; +import { useQueryClient } from '@tanstack/react-query'; +import releaseDataFileQueries from '@admin/queries/releaseDataFileQueries'; interface FormValues { title: string; @@ -31,11 +33,20 @@ export default function ReleaseDataFilePage({ [releaseId, fileId], ); + const queryClient = useQueryClient(); + const handleSubmit = async (values: FormValues) => { await releaseDataFileService.updateFile(releaseId, fileId, { title: values.title, }); + queryClient.removeQueries({ + queryKey: releaseDataFileQueries.list(releaseId).queryKey, + }); + await queryClient.invalidateQueries({ + queryKey: releaseDataFileQueries.list._def, + }); + history.push( generatePath(releaseDataRoute.path, { publicationId, diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/BulkZipUploadModalConfirm.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/BulkZipUploadModalConfirm.tsx new file mode 100644 index 00000000000..fd5b252d451 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/BulkZipUploadModalConfirm.tsx @@ -0,0 +1,51 @@ +import ModalConfirm from '@common/components/ModalConfirm'; +import WarningMessage from '@common/components/WarningMessage'; +import { ArchiveDataSetFile } from 'src/services/releaseDataFileService'; +import React from 'react'; + +interface Props { + bulkUploadPlan: ArchiveDataSetFile[]; + onConfirm: (bulkUploadPlan: ArchiveDataSetFile[]) => void; + onCancel: () => void; +} + +export default function BulkZipUploadModalConfirm({ + bulkUploadPlan, + onConfirm, + onCancel, +}: Props) { + return ( + onConfirm(bulkUploadPlan)} + onExit={onCancel} + onCancel={onCancel} + > + + + + + + + + + + {bulkUploadPlan.map(archiveDataSet => ( + + + + + + ))} + +
    Dataset nameData fileAdditional information
    {archiveDataSet.title}{archiveDataSet.dataFilename} + {archiveDataSet.replacingFileId && ( + + Upload will initiate a file replacement + + )} +
    +     + ); +} diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx index aa8da292515..62380b403f0 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx @@ -80,6 +80,8 @@ function baseErrorMappings( 'DatasetNamesCsvFilenamesShouldBeUnique', FileNotFoundInZip: 'FileNotFoundInZip', ZipContainsUnusedFiles: 'ZipContainsUnusedFiles', + DataReplacementAlreadyInProgress: + 'Data replacement already in progress', }, }), ]; @@ -218,6 +220,7 @@ export default function DataFileUploadForm({ > {({ formState, reset, getValues }) => { const uploadType = getValues('uploadType'); + return (
    diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx index 89f58586370..8238ffb61de 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx @@ -17,6 +17,7 @@ import { } from '@admin/routes/releaseRoutes'; import permissionService from '@admin/services/permissionService'; import releaseDataFileService, { + ArchiveDataSetFile, DataFile, DataFileImportStatus, DeleteDataFilePlan, @@ -32,7 +33,7 @@ import DataUploadCancelButton from '@admin/pages/release/data/components/DataUpl import React, { useCallback, useEffect, useState } from 'react'; import { generatePath } from 'react-router'; import { useQuery } from '@tanstack/react-query'; -import { reverse, uniqBy } from 'lodash'; +import BulkZipUploadModalConfirm from './BulkZipUploadModalConfirm'; interface Props { publicationId: string; @@ -55,17 +56,18 @@ const ReleaseDataUploadsSection = ({ const [deleteDataFile, setDeleteDataFile] = useState(); const [activeFileIds, setActiveFileIds] = useState(); const [dataFiles, setDataFiles] = useState([]); + const [bulkUploadPlan, setBulkUploadPlan] = useState(); - const { data: initialDataFiles = [], isLoading } = useQuery( - releaseDataFileQueries.list(releaseId), - ); + const { + data: initialDataFiles = [], + isLoading, + refetch: refetchDataFiles, + } = useQuery(releaseDataFileQueries.list(releaseId)); // Store the data files on state so we can reliably update them // when the permissions/status change. useEffect(() => { - if (!isLoading) { - setDataFiles(initialDataFiles); - } + setDataFiles(initialDataFiles); }, [initialDataFiles, isLoading, setDataFiles]); useEffect(() => { @@ -85,6 +87,26 @@ const ReleaseDataUploadsSection = ({ ); }; + const confirmBulkUploadPlan = useCallback( + async (archiveDataSetFiles: ArchiveDataSetFile[]) => { + const newFiles = await releaseDataFileService.importBulkZipDataFile( + releaseId, + archiveDataSetFiles, + ); + + setBulkUploadPlan(undefined); + setActiveFileIds([...dataFiles, ...newFiles].map(file => file.id)); + refetchDataFiles(); + }, + [ + releaseId, + setBulkUploadPlan, + setActiveFileIds, + dataFiles, + refetchDataFiles, + ], + ); + const handleStatusChange = async ( dataFile: DataFile, { totalRows, status }: DataFileImportStatus, @@ -126,6 +148,7 @@ const ReleaseDataUploadsSection = ({ metadataFile: values.metadataFile as File, }), ); + refetchDataFiles(); break; } case 'zip': { @@ -138,26 +161,25 @@ const ReleaseDataUploadsSection = ({ zipFile: values.zipFile as File, }), ); + refetchDataFiles(); break; } - case 'bulkZip': - newFiles.push( - ...(await releaseDataFileService.uploadBulkZipDataFile( + case 'bulkZip': { + const uploadPlan = + await releaseDataFileService.getUploadBulkZipDataFilePlan( releaseId, - values.bulkZipFile as File, - )), - ); + values.bulkZipFile!, + ); + + setBulkUploadPlan(uploadPlan); break; + } default: break; } setActiveFileIds(newFiles.map(file => file.id)); - setDataFiles(currentDataFiles => - // double reverse keeps *new* files as uniq item and at the end of the list - reverse(uniqBy(reverse([...currentDataFiles, ...newFiles]), 'title')), - ); }, - [releaseId], + [releaseId, refetchDataFiles], ); return ( @@ -191,7 +213,17 @@ const ReleaseDataUploadsSection = ({ {canUpdateRelease ? ( - + <> + + {bulkUploadPlan === undefined || + bulkUploadPlan.length === 0 ? null : ( + setBulkUploadPlan(undefined)} + /> + )} + ) : ( This release has been approved, and can no longer be updated. diff --git a/src/explore-education-statistics-admin/src/services/releaseDataFileService.ts b/src/explore-education-statistics-admin/src/services/releaseDataFileService.ts index 2a1c496e809..354352c2f0b 100644 --- a/src/explore-education-statistics-admin/src/services/releaseDataFileService.ts +++ b/src/explore-education-statistics-admin/src/services/releaseDataFileService.ts @@ -65,6 +65,17 @@ export type UploadZipDataFileRequest = zipFile: File; }; +export type ArchiveDataSetFile = { + title: string; + dataFilename: string; + dataFileId: string; + dataFileSize: number; + metaFilename: string; + metaFileId: string; + metaFileSize: number; + replacingFileId?: string; +}; + export interface DataFileUpdateRequest { title: string; } @@ -156,17 +167,26 @@ const releaseDataFileService = { return mapFile(file); }, - async uploadBulkZipDataFile( + async getUploadBulkZipDataFilePlan( releaseId: string, zipFile: File, - ): Promise { + ): Promise { const data = new FormData(); data.append('zipFile', zipFile); - const files = await client.post( - `/release/${releaseId}/bulk-zip-data`, + return client.post( + `/release/${releaseId}/upload-bulk-zip-data`, data, ); + }, + async importBulkZipDataFile( + releaseId: string, + dataSetFiles: ArchiveDataSetFile[], + ): Promise { + const files = await client.post( + `/release/${releaseId}/import-bulk-zip-data`, + dataSetFiles, + ); return files.map(file => mapFile(file)); }, From 02915092a18db65917df1a512c9f65b2a98b3cc5 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Tue, 3 Dec 2024 17:39:50 +0000 Subject: [PATCH 064/144] EES-5583/4: Update backend and frontend tests. --- .../Api/ReleasesControllerTests.cs | 66 ++++ .../DataArchiveValidationServiceTests.cs | 256 +++++---------- .../MethodologyImageServiceTests.cs | 8 +- .../ReleaseDataFileServicePermissionTests.cs | 31 +- .../Services/ReleaseDataFileServiceTests.cs | 293 ++++-------------- .../Services/ReleaseFileServiceTests.cs | 16 +- .../Services/ReleaseImageServiceTests.cs | 8 +- .../Services/BlobStorageServiceTests.cs | 78 ++++- .../Utils/MockFormTestUtils.cs | 26 +- .../components/ReleaseDataUploadsSection.tsx | 6 +- .../__tests__/DataFileUploadForm.test.tsx | 19 ++ .../ReleaseDataUploadsSection.test.tsx | 260 +++++----------- 12 files changed, 432 insertions(+), 635 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/ReleasesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/ReleasesControllerTests.cs index d91043be350..fced1e589a3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/ReleasesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/ReleasesControllerTests.cs @@ -448,6 +448,72 @@ public async Task CreateReleaseAmendment() result.AssertOkResult(amendmentCreatedResponse); } + [Fact] + public async Task UploadBulkZipDataSetsToTempStorage() + { + // Arrange + var dataSetFiles = new List(); + + var releaseDataFileService = new Mock(Strict); + + releaseDataFileService + .Setup(s => s.ValidateAndUploadBulkZip( + It.IsAny(), + It.IsAny(), + default)) + .ReturnsAsync(dataSetFiles); + + var controller = BuildController(releaseDataFileService: releaseDataFileService.Object); + + // Act + var result = await controller.UploadBulkZipDataSetsToTempStorage( + Guid.NewGuid(), + MockFile("bulk.zip"), + default); + + // Assert + VerifyAllMocks(releaseDataFileService); + + result.AssertOkResult(dataSetFiles); + } + + [Fact] + public async Task ImportBulkZipDataSetsFromTempStorage() + { + // Arrange + var dataFileInfo = new List + { + new() { FileName = "one.csv", Name = "Data set title", Size = "1024" }, + }; + + var importRequests = new List + { + new(){ DataFileId = Guid.NewGuid(), MetaFileId = Guid.NewGuid(), Title = "Data set title", DataFilename = "one.csv", MetaFilename = "one.meta.csv", DataFileSize = 1024, MetaFileSize = 128 } + }; + + var releaseDataFileService = new Mock(Strict); + + releaseDataFileService + .Setup(s => s.SaveDataSetsFromTemporaryBlobStorage( + It.IsAny(), + It.IsAny>(), + default)) + .ReturnsAsync(dataFileInfo); + + var controller = BuildController(releaseDataFileService: releaseDataFileService.Object); + + // Act + var result = await controller.ImportBulkZipDataSetsFromTempStorage( + Guid.NewGuid(), + importRequests, + default); + + // Assert + VerifyAllMocks(releaseDataFileService); + + result.AssertOkResult(dataFileInfo); + } + private static IFormFile MockFile(string fileName) { var fileMock = new Mock(Strict); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs index 6917b2b33a3..9faafa5ccdd 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs @@ -1,8 +1,4 @@ #nullable enable -using System; -using System.IO; -using System.Reflection; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Models; using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; @@ -14,9 +10,13 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using Microsoft.AspNetCore.Http; using Moq; +using System; +using System.IO; +using System.Threading.Tasks; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions.ValidationProblemViewModelTestExtensions; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockFormTestUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; using static Moq.MockBehavior; -using File = System.IO.File; using ValidationMessages = GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationMessages; namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services @@ -24,7 +24,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services public class DataArchiveValidationServiceTests { [Fact] - public async Task ValidateDataArchiveFile_Success() + public async Task ValidateDataArchiveFile_Success_ReturnsArchiveSummary() { var releaseVersionId = Guid.NewGuid(); var archive = CreateFormFileFromResource("data-zip-valid.zip"); @@ -63,44 +63,7 @@ public async Task ValidateDataArchiveFile_Success() } [Fact] - public async Task ValidateDataArchiveFile_ZipFilenameTooLong_DoesNotEndInDotZip() - { - var fileTypeService = new Mock(Strict); - - var fileName = - "LoremipsumdolorsitametconsecteturadipiscingelitInsitametelitaccumsanbibendumlacusutmattismaurisCrasvehiculaaccumsaneratidelementumaugueposuereatNuncege.zipp"; - var archive = CreateFormFileFromResource("data-zip-valid.zip", fileName); - - fileTypeService - .Setup(s => s.IsValidZipFile(archive)) - .ReturnsAsync(() => true); - - var contentDbContextId = Guid.NewGuid().ToString(); - await using (var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId)) - { - var service = SetupDataArchiveValidationService( - contentDbContext: contentDbContext, - fileTypeService: fileTypeService.Object); - - var result = await service.ValidateDataArchiveFile( - Guid.NewGuid(), - "Data set title", - archive); - - result - .AssertLeft() - .AssertBadRequestWithValidationErrors([ - ValidationMessages.GenerateErrorFilenameTooLong( - fileName, DataArchiveValidationService.MaxFilenameSize), - ValidationMessages.GenerateErrorZipFilenameMustEndDotZip(fileName), - ]); - } - - VerifyAllMocks(fileTypeService); - } - - [Fact] - public async Task ValidateDataArchiveFile_DataZipShouldContainTwoFiles() + public async Task ValidateDataArchiveFile_ArchiveContainsOnlyOneFile_ReturnsValidationError() { var fileTypeService = new Mock(Strict); @@ -137,20 +100,18 @@ public async Task ValidateDataArchiveFile_DataZipShouldContainTwoFiles() } [Fact] - public async Task ValidateBulkDataArchiveFile_Success() + public async Task ValidateBulkDataArchiveFiles_Success_ReturnsArchiveSummary() { var releaseVersionId = Guid.NewGuid(); var archive = CreateFormFileFromResource("bulk-data-zip-valid.zip"); var fileUploadsValidatorService = new Mock(Strict); - var fileTypeService = new Mock(Strict); var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId)) { var service = SetupDataArchiveValidationService( contentDbContext: contentDbContext, - fileTypeService: fileTypeService.Object, fileUploadsValidatorService: fileUploadsValidatorService.Object); fileUploadsValidatorService.Setup(mock => mock.ValidateDataSetFilesForUpload( @@ -160,11 +121,7 @@ public async Task ValidateBulkDataArchiveFile_Success() It.IsAny())) .ReturnsAsync([]); - fileTypeService - .Setup(s => s.IsValidZipFile(archive)) - .ReturnsAsync(true); - - var result = await service.ValidateBulkDataArchiveFile( + var result = await service.ValidateBulkDataArchiveFiles( releaseVersionId, archive); @@ -187,17 +144,16 @@ public async Task ValidateBulkDataArchiveFile_Success() Assert.Null(archiveDataSets[1].ReplacingFile); } - VerifyAllMocks(fileUploadsValidatorService, fileTypeService); + VerifyAllMocks(fileUploadsValidatorService); } [Fact] - public async Task ValidateBulkDataArchiveFile_Replacement_Success() + public async Task ValidateBulkDataArchiveFiles_WithFileReplacement_ReturnsArchiveSummary() { var releaseVersionId = Guid.NewGuid(); var archive = CreateFormFileFromResource("bulk-data-zip-valid.zip"); var fileUploadsValidatorService = new Mock(Strict); - var fileTypeService = new Mock(Strict); var releaseFile = new ReleaseFile { @@ -226,7 +182,6 @@ public async Task ValidateBulkDataArchiveFile_Replacement_Success() { var service = SetupDataArchiveValidationService( contentDbContext: contentDbContext, - fileTypeService: fileTypeService.Object, fileUploadsValidatorService: fileUploadsValidatorService.Object); fileUploadsValidatorService.Setup(mock => mock.ValidateDataSetFilesForUpload( @@ -236,11 +191,7 @@ public async Task ValidateBulkDataArchiveFile_Replacement_Success() It.IsAny())) .ReturnsAsync([]); - fileTypeService - .Setup(s => s.IsValidZipFile(archive)) - .ReturnsAsync(true); - - var result = await service.ValidateBulkDataArchiveFile( + var result = await service.ValidateBulkDataArchiveFiles( releaseVersionId, archive); @@ -263,67 +214,22 @@ public async Task ValidateBulkDataArchiveFile_Replacement_Success() Assert.Null(archiveDataSets[1].ReplacingFile); } - VerifyAllMocks(fileUploadsValidatorService, fileTypeService); - } - - [Fact] - public async Task ValidateBulkDataArchiveFile_Fail_IsValidZipFile() - { - var releaseVersionId = Guid.NewGuid(); - var fileTypeService = new Mock(Strict); - - var longFilename = - "loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongfilename.csv"; - var archive = CreateFormFileFromResource("test-data.csv", longFilename); - - fileTypeService - .Setup(s => s.IsValidZipFile(archive)) - .ReturnsAsync(false); - - var contentDbContextId = Guid.NewGuid().ToString(); - await using (var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId)) - { - var service = SetupDataArchiveValidationService( - contentDbContext: contentDbContext, - fileTypeService: fileTypeService.Object); - - var result = await service.ValidateBulkDataArchiveFile( - releaseVersionId, - archive); - - result - .AssertLeft() - .AssertBadRequestWithValidationErrors([ - ValidationMessages.GenerateErrorFilenameTooLong(longFilename, - FileUploadsValidatorService.MaxFilenameSize), - ValidationMessages.GenerateErrorZipFilenameMustEndDotZip(longFilename), - ValidationMessages.GenerateErrorMustBeZipFile(longFilename), - ]); - } - - VerifyAllMocks(fileTypeService); + VerifyAllMocks(fileUploadsValidatorService); } [Fact] - public async Task ValidateBulkDataArchiveFile_Fail_NoDatasetNamesCsv() + public async Task ValidateBulkDataArchiveFiles_IndexFileMissing_ReturnsValidationError() { var releaseVersionId = Guid.NewGuid(); var archive = CreateFormFileFromResource("bulk-data-zip-invalid-no-datasetnames-csv.zip"); - var fileTypeService = new Mock(Strict); - - fileTypeService - .Setup(s => s.IsValidZipFile(archive)) - .ReturnsAsync(true); - var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId)) { var service = SetupDataArchiveValidationService( - contentDbContext: contentDbContext, - fileTypeService: fileTypeService.Object); + contentDbContext: contentDbContext); - var result = await service.ValidateBulkDataArchiveFile( + var result = await service.ValidateBulkDataArchiveFiles( releaseVersionId, archive); @@ -337,30 +243,22 @@ public async Task ValidateBulkDataArchiveFile_Fail_NoDatasetNamesCsv() } ]); } - - VerifyAllMocks(fileTypeService); } [Fact] - public async Task ValidateBulkDataArchiveFile_Fail_DatasetNamesCsvIncorrectHeaders() + public async Task ValidateBulkDataArchiveFiles_IndexFileHasIncorrectHeaders_ReturnsValidationError() { var releaseVersionId = Guid.NewGuid(); - var fileTypeService = new Mock(Strict); var archive = CreateFormFileFromResource("bulk-data-zip-invalid-datasetnames-headers.zip"); - fileTypeService - .Setup(s => s.IsValidZipFile(archive)) - .ReturnsAsync(true); - var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId)) { var service = SetupDataArchiveValidationService( - contentDbContext: contentDbContext, - fileTypeService: fileTypeService.Object); + contentDbContext: contentDbContext); - var result = await service.ValidateBulkDataArchiveFile( + var result = await service.ValidateBulkDataArchiveFiles( releaseVersionId, archive); @@ -374,30 +272,21 @@ public async Task ValidateBulkDataArchiveFile_Fail_DatasetNamesCsvIncorrectHeade }, ]); } - - VerifyAllMocks(fileTypeService); } [Fact] - public async Task ValidateBulkDataArchiveFile_Fail_FilesNotFoundInZip() + public async Task ValidateBulkDataArchiveFiles_ReferencedFilesNotFoundInArchive_ReturnsValidationError() { var releaseVersionId = Guid.NewGuid(); var archive = CreateFormFileFromResource("bulk-data-zip-invalid-files-not-found.zip"); - var fileTypeService = new Mock(Strict); - - fileTypeService - .Setup(s => s.IsValidZipFile(archive)) - .ReturnsAsync(true); - var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId)) { var service = SetupDataArchiveValidationService( - contentDbContext: contentDbContext, - fileTypeService: fileTypeService.Object); + contentDbContext: contentDbContext); - var result = await service.ValidateBulkDataArchiveFile( + var result = await service.ValidateBulkDataArchiveFiles( releaseVersionId, archive); @@ -408,30 +297,21 @@ public async Task ValidateBulkDataArchiveFile_Fail_FilesNotFoundInZip() ValidationMessages.GenerateErrorFileNotFoundInZip("two.csv", FileType.Data), ]); } - - VerifyAllMocks(fileTypeService); } [Fact] - public async Task ValidateBulkDataArchiveFile_Fail_DuplicateDataSetTitlesAndFilenames() + public async Task ValidateBulkDataArchiveFiles_DuplicateDataSetTitlesAndFileNames_ReturnsValidationError() { var releaseVersionId = Guid.NewGuid(); var archive = CreateFormFileFromResource("bulk-data-zip-invalid-duplicate-names.zip"); - var fileTypeService = new Mock(Strict); - - fileTypeService - .Setup(s => s.IsValidZipFile(archive)) - .ReturnsAsync(true); - var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId)) { var service = SetupDataArchiveValidationService( - contentDbContext: contentDbContext, - fileTypeService: fileTypeService.Object); + contentDbContext: contentDbContext); - var result = await service.ValidateBulkDataArchiveFile( + var result = await service.ValidateBulkDataArchiveFiles( releaseVersionId, archive); @@ -442,29 +322,20 @@ public async Task ValidateBulkDataArchiveFile_Fail_DuplicateDataSetTitlesAndFile ValidationMessages.GenerateErrorDatasetNamesCsvFilenamesShouldBeUnique("one"), ]); } - - VerifyAllMocks(fileTypeService); } [Fact] - public async Task ValidateBulkDataArchiveFile_Fail_DataSetNamesCsvFilesnamesShouldNotEndDotCsv() + public async Task ValidateBulkDataArchiveFiles_Fail_DataSetNamesCsvFilesnamesShouldNotEndDotCsv() { var releaseVersionId = Guid.NewGuid(); - var fileTypeService = new Mock(Strict); - var archive = CreateFormFileFromResource("bulk-data-zip-invalid-filename-contains-extension.zip"); - fileTypeService - .Setup(s => s.IsValidZipFile(archive)) - .ReturnsAsync(true); - var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId)) { var service = SetupDataArchiveValidationService( - contentDbContext: contentDbContext, - fileTypeService: fileTypeService.Object); - var result = await service.ValidateBulkDataArchiveFile( + contentDbContext: contentDbContext); + var result = await service.ValidateBulkDataArchiveFiles( releaseVersionId, archive); @@ -474,18 +345,15 @@ public async Task ValidateBulkDataArchiveFile_Fail_DataSetNamesCsvFilesnamesShou ValidationMessages.GenerateErrorDatasetNamesCsvFilenamesShouldNotEndDotCsv("one.csv") ]); } - - VerifyAllMocks(fileTypeService); } [Fact] - public async Task ValidateBulkDataArchiveFile_Fail_UnusedFilesInZip() + public async Task ValidateBulkDataArchiveFiles_IndexFileMissingReferenceToArchiveFile_ReturnsValidationError() { var releaseVersionId = Guid.NewGuid(); var archive = CreateFormFileFromResource("bulk-data-zip-invalid-unused-files.zip"); var fileUploadsValidatorService = new Mock(Strict); - var fileTypeService = new Mock(Strict); fileUploadsValidatorService.Setup(mock => mock.ValidateDataSetFilesForUpload( releaseVersionId, @@ -494,19 +362,14 @@ public async Task ValidateBulkDataArchiveFile_Fail_UnusedFilesInZip() It.IsAny())) .ReturnsAsync([]); - fileTypeService - .Setup(s => s.IsValidZipFile(archive)) - .ReturnsAsync(true); - var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId)) { var service = SetupDataArchiveValidationService( contentDbContext: contentDbContext, - fileTypeService: fileTypeService.Object, fileUploadsValidatorService: fileUploadsValidatorService.Object); - var result = await service.ValidateBulkDataArchiveFile( + var result = await service.ValidateBulkDataArchiveFiles( releaseVersionId, archive); @@ -517,29 +380,60 @@ public async Task ValidateBulkDataArchiveFile_Fail_UnusedFilesInZip() ]); } - VerifyAllMocks(fileUploadsValidatorService, fileTypeService); + VerifyAllMocks(fileUploadsValidatorService); } - private static IFormFile CreateFormFileFromResource(string fileName, string? newFileName = null) + [Fact] + public async Task IsValidZipFile_NoFileProvided_ReturnsValidationError() { - var filePath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, - "Resources" + Path.DirectorySeparatorChar + fileName); + // Arrange + var contentDbContextId = Guid.NewGuid().ToString(); + await using var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId); + + var service = SetupDataArchiveValidationService(contentDbContext: contentDbContext); + + IFormFile? archive = null; + + // Act + var result = await service.IsValidZipFile(archive!); - return CreateFormFileFromResourceWithPath(filePath, newFileName ?? fileName); + // Assert + AssertHasErrors(result, [ + ValidationMessages.GenerateErrorFileIsNull(), + ]); } - private static IFormFile CreateFormFileFromResourceWithPath(string filePath, string fileName) + [Fact] + public async Task IsValidZipFile_InvalidFileNameAndType_ReturnsValidationErrors() { - var formFile = new Mock(); - formFile - .Setup(f => f.OpenReadStream()) - .Returns(() => File.OpenRead(filePath)); + // Arrange + var contentDbContextId = Guid.NewGuid().ToString(); + await using var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId); + + + var longFilename = "loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongfilename.csv"; + var archive = CreateFormFileFromResource("test-data.csv", longFilename); - formFile - .Setup(f => f.FileName) - .Returns(() => fileName); + var fileTypeService = new Mock(Strict); + fileTypeService + .Setup(s => s.IsValidZipFile(archive)) + .ReturnsAsync(false); - return formFile.Object; + var service = SetupDataArchiveValidationService( + contentDbContext: contentDbContext, + fileTypeService: fileTypeService.Object); + + // Act + var result = await service.IsValidZipFile(archive); + + // Assert + AssertHasErrors(result, [ + ValidationMessages.GenerateErrorFilenameTooLong(longFilename, DataArchiveValidationService.MaxFilenameSize), + ValidationMessages.GenerateErrorZipFilenameMustEndDotZip(longFilename), + ValidationMessages.GenerateErrorMustBeZipFile(longFilename), + ]); + + VerifyAllMocks(fileTypeService); } private static DataArchiveValidationService SetupDataArchiveValidationService( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyImageServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyImageServiceTests.cs index f08631159d0..be12dcb553e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyImageServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyImageServiceTests.cs @@ -1,6 +1,4 @@ #nullable enable -using System; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Methodologies; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Methodologies; @@ -18,6 +16,8 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using Microsoft.EntityFrameworkCore; using Moq; +using System; +using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.DbUtils; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationErrorMessages; using static GovUk.Education.ExploreEducationStatistics.Common.BlobContainers; @@ -714,7 +714,7 @@ public async Task Upload() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateMethodologyFiles, It.Is(path => - path.Contains(FilesPath(methodologyVersion.Id, Image))), + path.Contains(FilesPath(methodologyVersion.Id, Image, null))), formFile )).Returns(Task.CompletedTask); @@ -740,7 +740,7 @@ public async Task Upload() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateMethodologyFiles, It.Is(path => - path.Contains(FilesPath(methodologyVersion.Id, Image))), + path.Contains(FilesPath(methodologyVersion.Id, Image, null))), formFile ), Times.Once); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServicePermissionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServicePermissionTests.cs index 52d47d98bd4..54c6dddf040 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServicePermissionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServicePermissionTests.cs @@ -1,10 +1,8 @@ #nullable enable -using System; -using System.Collections.Generic; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Security; using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils; @@ -17,6 +15,9 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; using Microsoft.AspNetCore.Http; using Moq; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Admin.Security.SecurityPolicies; using static GovUk.Education.ExploreEducationStatistics.Common.Model.FileType; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.PermissionTestUtils; @@ -190,7 +191,24 @@ await PolicyCheckBuilder() } [Fact] - public async Task UploadAsBulkZip() + public async Task ValidateAndUploadBulkZip() + { + await PolicyCheckBuilder() + .SetupResourceCheckToFail(_releaseVersion, CanUpdateSpecificRelease) + .AssertForbidden( + userService => + { + var service = SetupReleaseDataFileService(userService: userService.Object); + return service.ValidateAndUploadBulkZip( + releaseVersionId: _releaseVersion.Id, + zipFile: new Mock().Object, + default); + } + ); + } + + [Fact] + public async Task SaveDataSetsFromTemporaryBlobStorage() { await PolicyCheckBuilder() .SetupResourceCheckToFail(_releaseVersion, CanUpdateSpecificRelease) @@ -198,9 +216,10 @@ await PolicyCheckBuilder() userService => { var service = SetupReleaseDataFileService(userService: userService.Object); - return service.UploadAsBulkZip( + return service.SaveDataSetsFromTemporaryBlobStorage( releaseVersionId: _releaseVersion.Id, - bulkZipFormFile: new Mock().Object); + archiveDataSetFiles: new Mock>().Object, + default); } ); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServiceTests.cs index 9769665c676..be5c4778261 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServiceTests.cs @@ -2,6 +2,7 @@ using GovUk.Education.ExploreEducationStatistics.Admin.Models; using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Admin.Validators; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; @@ -9,6 +10,7 @@ using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils; using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; @@ -23,6 +25,7 @@ using Moq; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.DbUtils; @@ -1839,14 +1842,14 @@ public async Task Upload() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), dataFormFile )).Returns(Task.CompletedTask); privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), metaFormFile )).Returns(Task.CompletedTask); @@ -2011,14 +2014,14 @@ public async Task Upload_Replacing() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), dataFormFile )).Returns(Task.CompletedTask); privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), metaFormFile )).Returns(Task.CompletedTask); @@ -2221,14 +2224,14 @@ public async Task Upload_Order() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), dataFormFile )).Returns(Task.CompletedTask); privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), metaFormFile )).Returns(Task.CompletedTask); @@ -2355,7 +2358,7 @@ public async Task UploadAsZip() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, DataZip))), + path.Contains(FilesPath(releaseVersion.Id, DataZip, null))), zipFormFile )).Returns(Task.CompletedTask); @@ -2520,7 +2523,7 @@ public async Task UploadAsZip_Order() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, DataZip))), + path.Contains(FilesPath(releaseVersion.Id, DataZip, null))), zipFormFile )).Returns(Task.CompletedTask); @@ -2673,7 +2676,7 @@ public async Task UploadAsZip_Replacing() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, DataZip))), + path.Contains(FilesPath(releaseVersion.Id, DataZip, null))), zipFormFile )).Returns(Task.CompletedTask); @@ -2780,10 +2783,9 @@ public async Task UploadAsZip_Replacing() } [Fact] - public async Task UploadAsBulkZip() + public async Task ValidateAndUploadBulkZip_InvalidZipFile_ReturnsValidationResult() { - const string zipFileName = "test-data-archive.zip"; - + // Arrange var releaseVersion = new ReleaseVersion { Id = Guid.NewGuid(), @@ -2800,7 +2802,6 @@ public async Task UploadAsBulkZip() }; var contentDbContextId = Guid.NewGuid().ToString(); - var statisticsDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { @@ -2809,144 +2810,35 @@ public async Task UploadAsBulkZip() await contentDbContext.SaveChangesAsync(); } - var zipFormFile = CreateFormFileMock(zipFileName, "application/zip").Object; - - var archiveFile1 = new ArchiveDataSetFile("One", "one.csv", "one.meta.csv"); - var archiveFile2 = new ArchiveDataSetFile("Two", "two.csv", "two.meta.csv"); + var archive = CreateFormFileFromResource("bulk-data-zip-invalid-unused-files.zip"); - var privateBlobStorageService = new Mock(Strict); var dataArchiveValidationService = new Mock(Strict); - var fileUploadsValidatorService = new Mock(Strict); - var dataImportService = new Mock(Strict); + + dataArchiveValidationService + .Setup(s => s.IsValidZipFile(archive)) + .ReturnsAsync([new ErrorViewModel { + Code = ValidationMessages.ZipContainsUnusedFiles.Code, + }]); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) - await using (var statisticsDbContext = InMemoryStatisticsDbContext(statisticsDbContextId)) { - dataArchiveValidationService - .Setup(s => s.ValidateBulkDataArchiveFile( - releaseVersion.Id, - zipFormFile)) - .ReturnsAsync(new List - { - archiveFile1, archiveFile2, - }); - - dataImportService - .Setup(s => s.Import( - It.IsAny(), - It.Is(file => file.Type == FileType.Data && file.Filename == "one.csv"), - It.Is(file => file.Type == Metadata && file.Filename == "one.meta.csv"), - It.Is(file => file.Type == BulkDataZip && file.Filename == zipFileName))) - .ReturnsAsync(new DataImport - { - Status = QUEUED - }); - dataImportService - .Setup(s => s.Import( - It.IsAny(), - It.Is(file => file.Type == FileType.Data && file.Filename == "two.csv"), - It.Is(file => file.Type == Metadata && file.Filename == "two.meta.csv"), - It.Is(file => file.Type == BulkDataZip && file.Filename == zipFileName))) - .ReturnsAsync(new DataImport - { - Status = QUEUED - }); - - privateBlobStorageService.Setup(mock => - mock.UploadFile(PrivateReleaseFiles, - It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, BulkDataZip))), - zipFormFile - )).Returns(Task.CompletedTask); - var service = SetupReleaseDataFileService( contentDbContext: contentDbContext, - statisticsDbContext: statisticsDbContext, - privateBlobStorageService: privateBlobStorageService.Object, - dataImportService: dataImportService.Object, - dataArchiveValidationService: dataArchiveValidationService.Object, - fileUploadsValidatorService: fileUploadsValidatorService.Object + dataArchiveValidationService: dataArchiveValidationService.Object ); - var result = (await service.UploadAsBulkZip( - releaseVersionId: releaseVersion.Id, - bulkZipFormFile: zipFormFile)) - .AssertRight(); - - MockUtils.VerifyAllMocks(privateBlobStorageService, - dataArchiveValidationService, - fileUploadsValidatorService, - dataImportService); - - Assert.Equal(2, result.Count); - - Assert.Equal("One", result[0].Name); - Assert.Equal("one.csv", result[0].FileName); - Assert.Equal("one.meta.csv", result[0].MetaFileName); - - Assert.Equal("Two", result[1].Name); - Assert.Equal("two.csv", result[1].FileName); - Assert.Equal("two.meta.csv", result[1].MetaFileName); - } - - await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) - { - var files = contentDbContext.Files.ToList(); - - Assert.Equal(5, files.Count); - - var zipFile = files - .Single(f => f.Filename == zipFileName); - Assert.Equal(10240, zipFile.ContentLength); - Assert.Equal("application/zip", zipFile.ContentType); - Assert.Equal(BulkDataZip, zipFile.Type); - - var dataFile1 = files - .Single(f => f.Filename == archiveFile1.DataFilename); - Assert.Equal(1048576, dataFile1.ContentLength); - Assert.Equal("text/csv", dataFile1.ContentType); - Assert.Equal(FileType.Data, dataFile1.Type); - Assert.Equal(zipFile.Id, dataFile1.SourceId); - - var metaFile1 = files - .Single(f => f.Filename == archiveFile1.MetaFilename); - Assert.Equal(1024, metaFile1.ContentLength); - Assert.Equal("text/csv", metaFile1.ContentType); - Assert.Equal(Metadata, metaFile1.Type); - - var dataFile2 = files - .Single(f => f.Filename == archiveFile2.DataFilename); - Assert.Equal(1048576, dataFile2.ContentLength); - Assert.Equal("text/csv", dataFile2.ContentType); - Assert.Equal(FileType.Data, dataFile2.Type); - Assert.Equal(zipFile.Id, dataFile2.SourceId); - - var metaFile2 = files - .Single(f => f.Filename == archiveFile2.MetaFilename); - Assert.Equal(1024, metaFile2.ContentLength); - Assert.Equal("text/csv", metaFile2.ContentType); - Assert.Equal(Metadata, metaFile2.Type); + // Act & Assert + (await service + .ValidateAndUploadBulkZip(releaseVersion.Id, archive, default)) + .AssertBadRequestWithValidationProblem(); - var releaseFiles = contentDbContext.ReleaseFiles.ToList(); - - Assert.Equal(4, releaseFiles.Count); - - Assert.NotNull(releaseFiles.SingleOrDefault(rf => - rf.ReleaseVersionId == releaseVersion.Id && rf.FileId == dataFile1.Id)); - Assert.NotNull(releaseFiles.SingleOrDefault(rf => - rf.ReleaseVersionId == releaseVersion.Id && rf.FileId == metaFile1.Id)); - Assert.NotNull(releaseFiles.SingleOrDefault(rf => - rf.ReleaseVersionId == releaseVersion.Id && rf.FileId == dataFile2.Id)); - Assert.NotNull(releaseFiles.SingleOrDefault(rf => - rf.ReleaseVersionId == releaseVersion.Id && rf.FileId == metaFile2.Id)); + MockUtils.VerifyAllMocks(dataArchiveValidationService); } } [Fact] - public async Task UploadAsBulkZip_Order() + public async Task ValidateAndUploadBulkZip() { - const string zipFileName = "test-data-archive.zip"; - var releaseVersion = new ReleaseVersion { Id = Guid.NewGuid(), @@ -2962,150 +2854,79 @@ public async Task UploadAsBulkZip_Order() } }; - var releaseFiles = new List - { - new() - { - Name = "Original one", - ReleaseVersion = releaseVersion, - File = new File { Type = FileType.Data }, - Order = 0, - }, - new() - { - Name = "Original two", - ReleaseVersion = releaseVersion, - File = new File { Type = FileType.Data }, - Order = 1, - }, - new() - { - Name = "Original three", - ReleaseVersion = releaseVersion, - File = new File { Type = FileType.Data }, - Order = 3, - }, - new() // Ancillary files should be ignored - { - ReleaseVersion = releaseVersion, - File = new File { Type = FileType.Ancillary }, - Order = 5, - }, - }; var contentDbContextId = Guid.NewGuid().ToString(); - var statisticsDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { contentDbContext.ReleaseVersions.Add(releaseVersion); - contentDbContext.ReleaseFiles.AddRange(releaseFiles); await contentDbContext.SaveChangesAsync(); } - var zipFormFile = CreateFormFileMock(zipFileName, "application/zip").Object; + var archive = CreateFormFileFromResource("bulk-data-zip-valid.zip"); var archiveFile1 = new ArchiveDataSetFile("One", "one.csv", "one.meta.csv"); var archiveFile2 = new ArchiveDataSetFile("Two", "two.csv", "two.meta.csv"); - var privateBlobStorageService = new Mock(Strict); var dataArchiveValidationService = new Mock(Strict); var fileUploadsValidatorService = new Mock(Strict); - var dataImportService = new Mock(Strict); + var privateBlobStorageService = new Mock(Strict); + + dataArchiveValidationService + .Setup(s => s.IsValidZipFile(archive)) + .ReturnsAsync([]); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) - await using (var statisticsDbContext = InMemoryStatisticsDbContext(statisticsDbContextId)) { dataArchiveValidationService - .Setup(s => s.ValidateBulkDataArchiveFile( + .Setup(s => s.ValidateBulkDataArchiveFiles( releaseVersion.Id, - zipFormFile)) + archive)) .ReturnsAsync(new List { archiveFile1, archiveFile2, }); - dataImportService - .Setup(s => s.Import( - It.IsAny(), - It.Is(file => file.Type == FileType.Data && file.Filename == "one.csv"), - It.Is(file => file.Type == Metadata && file.Filename == "one.meta.csv"), - It.Is(file => file.Type == BulkDataZip && file.Filename == zipFileName))) - .ReturnsAsync(new DataImport - { - Status = QUEUED - }); - dataImportService - .Setup(s => s.Import( - It.IsAny(), - It.Is(file => file.Type == FileType.Data && file.Filename == "two.csv"), - It.Is(file => file.Type == Metadata && file.Filename == "two.meta.csv"), - It.Is(file => file.Type == BulkDataZip && file.Filename == zipFileName))) - .ReturnsAsync(new DataImport - { - Status = QUEUED - }); - privateBlobStorageService.Setup(mock => - mock.UploadFile(PrivateReleaseFiles, - It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, BulkDataZip))), - zipFormFile + mock.UploadStream( + PrivateReleaseTempFiles, + It.IsAny(), + It.IsAny(), + ContentTypes.Csv, + null, + default )).Returns(Task.CompletedTask); var service = SetupReleaseDataFileService( contentDbContext: contentDbContext, - statisticsDbContext: statisticsDbContext, - privateBlobStorageService: privateBlobStorageService.Object, - dataImportService: dataImportService.Object, dataArchiveValidationService: dataArchiveValidationService.Object, - fileUploadsValidatorService: fileUploadsValidatorService.Object + fileUploadsValidatorService: fileUploadsValidatorService.Object, + privateBlobStorageService: privateBlobStorageService.Object ); - var result = (await service.UploadAsBulkZip( - releaseVersionId: releaseVersion.Id, - bulkZipFormFile: zipFormFile)) + var result = (await service + .ValidateAndUploadBulkZip(releaseVersion.Id, archive, default)) .AssertRight(); - MockUtils.VerifyAllMocks(privateBlobStorageService, + MockUtils.VerifyAllMocks( dataArchiveValidationService, fileUploadsValidatorService, - dataImportService); + privateBlobStorageService); Assert.Equal(2, result.Count); - } - - await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) - { - var files = contentDbContext.Files.ToList(); - Assert.Equal(9, files.Count); - - var dbReleaseFiles = contentDbContext.ReleaseFiles - .Where(rf => rf.File.Type == FileType.Data) - .OrderBy(rf => rf.Order) - .ToList(); - - Assert.Equal(5, dbReleaseFiles.Count); - - Assert.Equal(0, dbReleaseFiles[0].Order); - Assert.Equal("Original one", dbReleaseFiles[0].Name); - - Assert.Equal(1, dbReleaseFiles[1].Order); - Assert.Equal("Original two", dbReleaseFiles[1].Name); - // NOTE: The Orders of the original ReleaseFiles were 0,1,3 - no releaseFile had an order of 2 + Assert.Equal("One", result[0].Title); + Assert.Equal("one.csv", result[0].DataFilename); + Assert.Equal("one.meta.csv", result[0].MetaFilename); - Assert.Equal(3, dbReleaseFiles[2].Order); - Assert.Equal("Original three", dbReleaseFiles[2].Name); - - Assert.Equal(4, dbReleaseFiles[3].Order); - Assert.Equal("One", dbReleaseFiles[3].Name); - - Assert.Equal(5, dbReleaseFiles[4].Order); - Assert.Equal("Two", dbReleaseFiles[4].Name); + Assert.Equal("Two", result[1].Title); + Assert.Equal("two.csv", result[1].DataFilename); + Assert.Equal("two.meta.csv", result[1].MetaFilename); } } + + private ReleaseDataFileService SetupReleaseDataFileService( ContentDbContext contentDbContext, StatisticsDbContext? statisticsDbContext = null, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseFileServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseFileServiceTests.cs index e70650a2a9f..56eaef34916 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseFileServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseFileServiceTests.cs @@ -2491,7 +2491,7 @@ public async Task UploadAncillary() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), formFile )).Returns(Task.CompletedTask); @@ -2525,7 +2525,7 @@ public async Task UploadAncillary() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), formFile ), Times.Once); @@ -2605,7 +2605,7 @@ public async Task UpdateAncillary() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), newFormFile )).Returns(Task.CompletedTask); @@ -2639,7 +2639,7 @@ public async Task UpdateAncillary() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), newFormFile ), Times.Once); @@ -2729,7 +2729,7 @@ public async Task UpdateAncillary_DoNotRemoveFileAttachedToOtherRelease() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), newFormFile )).Returns(Task.CompletedTask); @@ -2763,7 +2763,7 @@ public async Task UpdateAncillary_DoNotRemoveFileAttachedToOtherRelease() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), newFormFile ), Times.Once); @@ -2886,7 +2886,7 @@ public async Task UploadChart() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Chart))), + path.Contains(FilesPath(releaseVersion.Id, Chart, null))), formFile )).Returns(Task.CompletedTask); @@ -2912,7 +2912,7 @@ public async Task UploadChart() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Chart))), + path.Contains(FilesPath(releaseVersion.Id, Chart, null))), formFile ), Times.Once); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseImageServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseImageServiceTests.cs index cea675a8087..5ee616de33a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseImageServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseImageServiceTests.cs @@ -1,6 +1,4 @@ #nullable enable -using System; -using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; @@ -17,6 +15,8 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using Microsoft.EntityFrameworkCore; using Moq; +using System; +using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.DbUtils; using static GovUk.Education.ExploreEducationStatistics.Common.BlobContainers; using static GovUk.Education.ExploreEducationStatistics.Common.Model.FileType; @@ -208,7 +208,7 @@ public async Task Upload() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Image))), + path.Contains(FilesPath(releaseVersion.Id, Image, null))), formFile )).Returns(Task.CompletedTask); @@ -234,7 +234,7 @@ public async Task Upload() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Image))), + path.Contains(FilesPath(releaseVersion.Id, Image, null))), formFile ), Times.Once); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Services/BlobStorageServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Services/BlobStorageServiceTests.cs index 3099fde10b1..8a3d7196f82 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Services/BlobStorageServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Services/BlobStorageServiceTests.cs @@ -1,12 +1,4 @@ -#nullable enable -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.IO; -using System.Linq; -using System.Net.Mime; -using System.Text.RegularExpressions; -using System.Threading.Tasks; +#nullable enable using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; @@ -19,6 +11,14 @@ using Microsoft.Extensions.Logging; using Moq; using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Net.Mime; +using System.Text.RegularExpressions; +using System.Threading.Tasks; using Xunit; using static Azure.Storage.Blobs.Models.BlobsModelFactory; using static GovUk.Education.ExploreEducationStatistics.Common.BlobContainers; @@ -494,7 +494,7 @@ await service.DeleteBlobs( } [Fact] - private async Task DeleteBlobs_FilterPrioritisesExcludeRegex() + public async Task DeleteBlobs_FilterPrioritisesExcludeRegex() { var blobContainerClient = MockBlobContainerClient(PublicReleaseFiles.Name); var blobServiceClient = MockBlobServiceClient(blobContainerClient); @@ -634,6 +634,62 @@ await service.DeleteBlobs( Assert.Equal("publications/item-4", deletedBlobs[2]); } + [Fact] + public async Task MoveBlob_SourceBlobNotFound_ReturnsFalse() + { + // Arrange + const string sourcePath = "path/to/test.pdf"; + const string destinationPath = "new/path/to/test.pdf"; + + var sourceBlobClient = MockBlobClient(name: sourcePath, exists: false); + + var sourceBlobContainerClient = MockBlobContainerClient(PrivateReleaseTempFiles.Name, sourceBlobClient); + + sourceBlobContainerClient + .Setup(client => client.GetBlobClient(sourcePath)) + .Returns(sourceBlobClient.Object); + + var blobServiceClient = MockBlobServiceClient(sourceBlobContainerClient); + + var service = SetupTestBlobStorageService(blobServiceClient.Object); + + // Act + var result = await service.MoveBlob(PrivateReleaseTempFiles, sourcePath, destinationPath); + + // Assert + Assert.False(result); + } + + [Fact] + public async Task MoveBlob_DestinationBlobAlreadyExists_ReturnsFalse() + { + // Arrange + const string sourcePath = "path/to/test.pdf"; + const string destinationPath = "new/path/to/test.pdf"; + + var sourceBlobClient = MockBlobClient(name: sourcePath, exists: true); + var destinationBlobClient = MockBlobClient(name: destinationPath, exists: true); + + var blobContainerClient = MockBlobContainerClient(PrivateReleaseTempFiles.Name, sourceBlobClient, destinationBlobClient); + + blobContainerClient + .Setup(client => client.GetBlobClient(sourcePath)) + .Returns(sourceBlobClient.Object); + + blobContainerClient + .Setup(client => client.GetBlobClient(destinationPath)) + .Returns(destinationBlobClient.Object); + + var blobServiceClient = MockBlobServiceClient(blobContainerClient); + + var service = SetupTestBlobStorageService(blobServiceClient.Object); + + // Act + var result = await service.MoveBlob(PrivateReleaseTempFiles, sourcePath, destinationPath); + + // Assert + Assert.False(result); + } private static Mock MockBlobServiceClient( params Mock[] blobContainerClients) @@ -719,7 +775,7 @@ public TestBlobStorageService( ILogger logger, IStorageInstanceCreationUtil storageInstanceCreationUtil) : base(connectionString, client, logger, storageInstanceCreationUtil) - {} + { } } private IEnumerable> CreatePages(params List[] pages) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Utils/MockFormTestUtils.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Utils/MockFormTestUtils.cs index 22a12280c5f..da5af913a61 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Utils/MockFormTestUtils.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Utils/MockFormTestUtils.cs @@ -1,6 +1,8 @@ -#nullable enable +#nullable enable using Microsoft.AspNetCore.Http; using Moq; +using System.IO; +using System.Reflection; namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils { @@ -34,5 +36,27 @@ public static Mock CreateFormFileMock(string filename, string content return formFile; } + + public static IFormFile CreateFormFileFromResource(string fileName, string? newFileName = null) + { + var filePath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, + "Resources" + Path.DirectorySeparatorChar + fileName); + + return CreateFormFileFromResourceWithPath(filePath, newFileName ?? fileName); + } + + public static IFormFile CreateFormFileFromResourceWithPath(string filePath, string fileName) + { + var formFile = new Mock(); + formFile + .Setup(f => f.OpenReadStream()) + .Returns(() => System.IO.File.OpenRead(filePath)); + + formFile + .Setup(f => f.FileName) + .Returns(() => fileName); + + return formFile.Object; + } } } diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx index 8238ffb61de..85043e0f208 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx @@ -59,7 +59,7 @@ const ReleaseDataUploadsSection = ({ const [bulkUploadPlan, setBulkUploadPlan] = useState(); const { - data: initialDataFiles = [], + data: initialDataFiles, isLoading, refetch: refetchDataFiles, } = useQuery(releaseDataFileQueries.list(releaseId)); @@ -67,8 +67,8 @@ const ReleaseDataUploadsSection = ({ // Store the data files on state so we can reliably update them // when the permissions/status change. useEffect(() => { - setDataFiles(initialDataFiles); - }, [initialDataFiles, isLoading, setDataFiles]); + setDataFiles(initialDataFiles ?? []); + }, [initialDataFiles]); useEffect(() => { onDataFilesChange?.(dataFiles); diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/DataFileUploadForm.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/DataFileUploadForm.test.tsx index f3baa08ecc1..2664e4cfe5c 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/DataFileUploadForm.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/DataFileUploadForm.test.tsx @@ -149,6 +149,25 @@ describe('DataFileUploadForm', () => { }); }); + test('form submits without error when suitable file is used', async () => { + const onSubmit = jest.fn(); + const { user } = render(); + + const file = new File(['hello, world!'], 'test.zip', { + type: 'application/zip', + }); + + await user.click(screen.getByLabelText('Bulk ZIP upload')); + await user.upload(screen.getByLabelText('Upload bulk ZIP file'), file); + await user.click( + screen.getByRole('button', { + name: 'Upload data files', + }), + ); + + expect(onSubmit).toHaveBeenCalledTimes(1); + }); + test('shows validation message when bulk ZIP file is empty', async () => { const { user } = render(); diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReleaseDataUploadsSection.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReleaseDataUploadsSection.test.tsx index 1ff7d876220..fcf8607f330 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReleaseDataUploadsSection.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReleaseDataUploadsSection.test.tsx @@ -4,6 +4,7 @@ import _releaseDataFileService, { DataFile, UploadDataFilesRequest, UploadZipDataFileRequest, + ArchiveDataSetFile, } from '@admin/services/releaseDataFileService'; import { screen, waitFor, within } from '@testing-library/react'; import React from 'react'; @@ -816,7 +817,7 @@ describe('ReleaseDataUploadsSection', () => { }); }); - test('successful submit with CSV files renders with uploaded data file appended to list', async () => { + test('successful submit with CSV files refetches data files', async () => { releaseDataFileService.uploadDataFiles.mockResolvedValue( testUploadedDataFile, ); @@ -863,59 +864,13 @@ describe('ReleaseDataUploadsSection', () => { } as UploadDataFilesRequest, ); - expect(screen.getAllByTestId('accordionSection')).toHaveLength(3); + expect(releaseDataFileService.getDataFiles).toHaveBeenCalledWith( + 'release-1', + ); }); - - const sections = screen.getAllByTestId('accordionSection'); - - const section1 = within(sections[0]); - - expect( - section1.getByRole('button', { name: /Test data 1/ }), - ).toBeInTheDocument(); - - const section2 = within(sections[1]); - - expect( - section2.getByRole('button', { name: /Test data 2/ }), - ).toBeInTheDocument(); - - expect(section2.getByTestId('Subject title')).toHaveTextContent( - 'Test data 2', - ); - - const section3 = within(sections[2]); - - expect( - section3.getByRole('button', { name: /Test title/ }), - ).toBeInTheDocument(); - - expect(section3.getByTestId('Subject title')).toHaveTextContent( - 'Test title', - ); - - expect(section3.getByTestId('Data file')).toHaveTextContent( - 'test-data.csv', - ); - expect(section3.getByTestId('Metadata file')).toHaveTextContent( - 'test-data.meta.csv', - ); - expect(section3.getByTestId('Data file size')).toHaveTextContent( - '150 Kb', - ); - expect(section3.getByTestId('Number of rows')).toHaveTextContent( - 'Unknown', - ); - expect(section3.getByTestId('Status')).toHaveTextContent('Queued'); - expect(section3.getByTestId('Uploaded by')).toHaveTextContent( - 'user1@test.com', - ); - expect(section3.getByTestId('Date uploaded')).toHaveTextContent( - '18 August 2020 12:00', - ); }); - test('successful submit with ZIP file renders with uploaded data file appended to list', async () => { + test('successful submit with zip file refetches data files', async () => { releaseDataFileService.uploadZipDataFile.mockResolvedValue({ ...testUploadedZipFile, }); @@ -956,68 +911,50 @@ describe('ReleaseDataUploadsSection', () => { zipFile, } as UploadZipDataFileRequest, ); - }); - - const sections = screen.getAllByTestId('accordionSection'); - const section1 = within(sections[0]); - const section2 = within(sections[1]); - const section3 = within(sections[2]); - - expect(sections).toHaveLength(3); - expect( - section1.getByRole('button', { name: /Test data 1/ }), - ).toBeInTheDocument(); - - expect( - section2.getByRole('button', { name: /Test data 2/ }), - ).toBeInTheDocument(); - - expect(section2.getByTestId('Subject title')).toHaveTextContent( - 'Test data 2', - ); - - expect( - section3.getByRole('button', { name: /Test zip title/ }), - ).toBeInTheDocument(); - - expect(section3.getByTestId('Subject title')).toHaveTextContent( - 'Test zip title', - ); - - expect(section3.getByTestId('Data file')).toHaveTextContent( - 'test-data.zip', - ); - expect(section3.getByTestId('Metadata file')).toHaveTextContent( - 'test-data.meta.zip', - ); - expect(section3.getByTestId('Data file size')).toHaveTextContent( - '150 Kb', - ); - expect(section3.getByTestId('Number of rows')).toHaveTextContent( - 'Unknown', - ); - expect(section3.getByTestId('Status')).toHaveTextContent('Queued'); - expect(section3.getByTestId('Uploaded by')).toHaveTextContent( - 'user1@test.com', - ); - expect(section3.getByTestId('Date uploaded')).toHaveTextContent( - '18 August 2020 12:00', - ); + expect(releaseDataFileService.getDataFiles).toHaveBeenCalledWith( + 'release-1', + ); + }); }); - test('successful submit with ZIP file renders with uploaded data file appended to list', async () => { - releaseDataFileService.uploadBulkZipDataFile.mockResolvedValue([ - testUploadedZipFile, - { - ...testUploadedZipFile, - id: 'zip-file-2', - title: 'Test zip title 2', - fileName: 'test-data-2.zip', - metaFileId: 'file-2-meta', - metaFileName: 'test-data-2.meta.zip', + test('successful submit with bulk zip file refetches data files', async () => { + const data: ArchiveDataSetFile = { + dataFileId: 'data-file-1', + dataFilename: 'test.csv', + dataFileSize: 1024, + metaFileId: 'meta-file-1', + metaFilename: 'test.meta.csv', + metaFileSize: 128, + title: 'Data set 1', + }; + + releaseDataFileService.getUploadBulkZipDataFilePlan.mockResolvedValue([ + data, + ]); + + const testDataFile: DataFile = { + id: 'file-1', + rows: 100, + fileName: 'data.csv', + fileSize: { + size: 200, + unit: 'B', + }, + userName: 'test@test.com', + title: 'Test data', + metaFileId: 'file-meta-1', + metaFileName: 'meta.csv', + status: 'COMPLETE', + permissions: { + canCancelImport: false, }, + }; + + releaseDataFileService.importBulkZipDataFile.mockResolvedValue([ + testDataFile, ]); + releaseDataFileService.getDataFileImportStatus.mockResolvedValue( testQueuedImportStatus, ); @@ -1044,80 +981,17 @@ describe('ReleaseDataUploadsSection', () => { name: 'Upload data files', }), ); + await user.click( + screen.getByRole('button', { + name: 'Confirm', + }), + ); await waitFor(() => { - expect( - releaseDataFileService.uploadBulkZipDataFile, - ).toHaveBeenCalledWith('release-1', zipFile); + expect(releaseDataFileService.getDataFiles).toHaveBeenCalledWith( + 'release-1', + ); }); - - const sections = screen.getAllByTestId('accordionSection'); - const section1 = within(sections[0]); - const section2 = within(sections[1]); - const section3 = within(sections[2]); - const section4 = within(sections[3]); - - expect(sections).toHaveLength(4); - - expect( - section1.getByRole('button', { name: /Test data 1/ }), - ).toBeInTheDocument(); - - expect( - section2.getByRole('button', { name: /Test data 2/ }), - ).toBeInTheDocument(); - - expect( - section3.getByRole('button', { name: /Test zip title/ }), - ).toBeInTheDocument(); - expect(section3.getByTestId('Subject title')).toHaveTextContent( - 'Test zip title', - ); - expect(section3.getByTestId('Data file')).toHaveTextContent( - 'test-data.zip', - ); - expect(section3.getByTestId('Metadata file')).toHaveTextContent( - 'test-data.meta.zip', - ); - expect(section3.getByTestId('Data file size')).toHaveTextContent( - '150 Kb', - ); - expect(section3.getByTestId('Number of rows')).toHaveTextContent( - 'Unknown', - ); - expect(section3.getByTestId('Status')).toHaveTextContent('Queued'); - expect(section3.getByTestId('Uploaded by')).toHaveTextContent( - 'user1@test.com', - ); - expect(section3.getByTestId('Date uploaded')).toHaveTextContent( - '18 August 2020 12:00', - ); - - expect( - section4.getByRole('button', { name: /Test zip title 2/ }), - ).toBeInTheDocument(); - expect(section4.getByTestId('Subject title')).toHaveTextContent( - 'Test zip title 2', - ); - expect(section4.getByTestId('Data file')).toHaveTextContent( - 'test-data-2.zip', - ); - expect(section4.getByTestId('Metadata file')).toHaveTextContent( - 'test-data-2.meta.zip', - ); - expect(section4.getByTestId('Data file size')).toHaveTextContent( - '150 Kb', - ); - expect(section4.getByTestId('Number of rows')).toHaveTextContent( - 'Unknown', - ); - expect(section4.getByTestId('Status')).toHaveTextContent('Queued'); - expect(section4.getByTestId('Uploaded by')).toHaveTextContent( - 'user1@test.com', - ); - expect(section4.getByTestId('Date uploaded')).toHaveTextContent( - '18 August 2020 12:00', - ); }); test('updates the number of rows after uploading CSV file when status changes', async () => { @@ -1128,6 +1002,16 @@ describe('ReleaseDataUploadsSection', () => { .mockResolvedValue(testQueuedImportStatus) .mockResolvedValueOnce(testImportingImportStatus); + const testUploadedDataFile2 = { + ...testUploadedDataFile, + title: 'Test title 2', + }; + + releaseDataFileService.getDataFiles.mockResolvedValue([ + ...testDataFiles, + testUploadedDataFile2, + ]); + permissionService.getDataFilePermissions.mockResolvedValue( {} as DataFilePermissions, ); @@ -1171,6 +1055,10 @@ describe('ReleaseDataUploadsSection', () => { metadataFile, } as UploadDataFilesRequest, ); + + expect(releaseDataFileService.getDataFiles).toHaveBeenCalledWith( + 'release-1', + ); }); const sections = screen.getAllByTestId('accordionSection'); @@ -1179,7 +1067,7 @@ describe('ReleaseDataUploadsSection', () => { await waitFor(() => expect( releaseDataFileService.getDataFileImportStatus, - ).toHaveBeenCalledWith('release-1', testUploadedDataFile), + ).toHaveBeenCalledWith('release-1', testUploadedDataFile2), ); await waitFor(() => { expect(section3.getByTestId('Number of rows')).toHaveTextContent('100'); @@ -1195,6 +1083,16 @@ describe('ReleaseDataUploadsSection', () => { .mockResolvedValue(testQueuedImportStatus) .mockResolvedValueOnce(testImportingImportStatus); + const testUploadedDataFile2 = { + ...testUploadedDataFile, + title: 'Test title 2', + }; + + releaseDataFileService.getDataFiles.mockResolvedValue([ + ...testDataFiles, + testUploadedDataFile2, + ]); + permissionService.getDataFilePermissions.mockResolvedValue( {} as DataFilePermissions, ); @@ -1240,7 +1138,7 @@ describe('ReleaseDataUploadsSection', () => { await waitFor(() => expect( releaseDataFileService.getDataFileImportStatus, - ).toHaveBeenCalledWith('release-1', testUploadedZipFile), + ).toHaveBeenCalledWith('release-1', testUploadedDataFile2), ); await waitFor(() => { expect(section3.getByTestId('Number of rows')).toHaveTextContent('100'); From dc0e17f124970e3ab86f8ea9a263d47a9149c270 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Fri, 6 Dec 2024 15:16:07 +0000 Subject: [PATCH 065/144] EES-5583: Use alternative path builder method, and revert optional parameter change. --- .../DataArchiveValidationServiceTests.cs | 4 +--- .../MethodologyImageServiceTests.cs | 4 ++-- .../ReleaseDataFileServicePermissionTests.cs | 4 ++-- .../Services/ReleaseDataFileServiceTests.cs | 18 +++++++++--------- .../Services/ReleaseFileServiceTests.cs | 16 ++++++++-------- .../Services/ReleaseImageServiceTests.cs | 4 ++-- .../Services/ReleaseDataFileService.cs | 12 ++++++------ .../Services/FileStoragePathUtils.cs | 4 ++-- 8 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs index 9faafa5ccdd..27fca5a4bca 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs @@ -387,8 +387,7 @@ public async Task ValidateBulkDataArchiveFiles_IndexFileMissingReferenceToArchiv public async Task IsValidZipFile_NoFileProvided_ReturnsValidationError() { // Arrange - var contentDbContextId = Guid.NewGuid().ToString(); - await using var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId); + await using var contentDbContext = DbUtils.InMemoryApplicationDbContext(); var service = SetupDataArchiveValidationService(contentDbContext: contentDbContext); @@ -410,7 +409,6 @@ public async Task IsValidZipFile_InvalidFileNameAndType_ReturnsValidationErrors( var contentDbContextId = Guid.NewGuid().ToString(); await using var contentDbContext = DbUtils.InMemoryApplicationDbContext(contentDbContextId); - var longFilename = "loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongfilename.csv"; var archive = CreateFormFileFromResource("test-data.csv", longFilename); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyImageServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyImageServiceTests.cs index be12dcb553e..f1a4e5ee689 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyImageServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyImageServiceTests.cs @@ -714,7 +714,7 @@ public async Task Upload() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateMethodologyFiles, It.Is(path => - path.Contains(FilesPath(methodologyVersion.Id, Image, null))), + path.Contains(FilesPath(methodologyVersion.Id, Image))), formFile )).Returns(Task.CompletedTask); @@ -740,7 +740,7 @@ public async Task Upload() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateMethodologyFiles, It.Is(path => - path.Contains(FilesPath(methodologyVersion.Id, Image, null))), + path.Contains(FilesPath(methodologyVersion.Id, Image))), formFile ), Times.Once); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServicePermissionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServicePermissionTests.cs index 54c6dddf040..d73cbe0b940 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServicePermissionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServicePermissionTests.cs @@ -202,7 +202,7 @@ await PolicyCheckBuilder() return service.ValidateAndUploadBulkZip( releaseVersionId: _releaseVersion.Id, zipFile: new Mock().Object, - default); + cancellationToken: default); } ); } @@ -219,7 +219,7 @@ await PolicyCheckBuilder() return service.SaveDataSetsFromTemporaryBlobStorage( releaseVersionId: _releaseVersion.Id, archiveDataSetFiles: new Mock>().Object, - default); + cancellationToken: default); } ); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServiceTests.cs index be5c4778261..93e8f001fe5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseDataFileServiceTests.cs @@ -1842,14 +1842,14 @@ public async Task Upload() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), dataFormFile )).Returns(Task.CompletedTask); privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), metaFormFile )).Returns(Task.CompletedTask); @@ -2014,14 +2014,14 @@ public async Task Upload_Replacing() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), dataFormFile )).Returns(Task.CompletedTask); privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), metaFormFile )).Returns(Task.CompletedTask); @@ -2224,14 +2224,14 @@ public async Task Upload_Order() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), dataFormFile )).Returns(Task.CompletedTask); privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, FileType.Data, null))), + path.Contains(FilesPath(releaseVersion.Id, FileType.Data))), metaFormFile )).Returns(Task.CompletedTask); @@ -2358,7 +2358,7 @@ public async Task UploadAsZip() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, DataZip, null))), + path.Contains(FilesPath(releaseVersion.Id, DataZip))), zipFormFile )).Returns(Task.CompletedTask); @@ -2523,7 +2523,7 @@ public async Task UploadAsZip_Order() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, DataZip, null))), + path.Contains(FilesPath(releaseVersion.Id, DataZip))), zipFormFile )).Returns(Task.CompletedTask); @@ -2676,7 +2676,7 @@ public async Task UploadAsZip_Replacing() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, DataZip, null))), + path.Contains(FilesPath(releaseVersion.Id, DataZip))), zipFormFile )).Returns(Task.CompletedTask); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseFileServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseFileServiceTests.cs index 56eaef34916..e70650a2a9f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseFileServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseFileServiceTests.cs @@ -2491,7 +2491,7 @@ public async Task UploadAncillary() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary))), formFile )).Returns(Task.CompletedTask); @@ -2525,7 +2525,7 @@ public async Task UploadAncillary() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary))), formFile ), Times.Once); @@ -2605,7 +2605,7 @@ public async Task UpdateAncillary() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary))), newFormFile )).Returns(Task.CompletedTask); @@ -2639,7 +2639,7 @@ public async Task UpdateAncillary() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary))), newFormFile ), Times.Once); @@ -2729,7 +2729,7 @@ public async Task UpdateAncillary_DoNotRemoveFileAttachedToOtherRelease() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary))), newFormFile )).Returns(Task.CompletedTask); @@ -2763,7 +2763,7 @@ public async Task UpdateAncillary_DoNotRemoveFileAttachedToOtherRelease() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Ancillary, null))), + path.Contains(FilesPath(releaseVersion.Id, Ancillary))), newFormFile ), Times.Once); @@ -2886,7 +2886,7 @@ public async Task UploadChart() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Chart, null))), + path.Contains(FilesPath(releaseVersion.Id, Chart))), formFile )).Returns(Task.CompletedTask); @@ -2912,7 +2912,7 @@ public async Task UploadChart() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Chart, null))), + path.Contains(FilesPath(releaseVersion.Id, Chart))), formFile ), Times.Once); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseImageServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseImageServiceTests.cs index 5ee616de33a..967e65bda7e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseImageServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ReleaseImageServiceTests.cs @@ -208,7 +208,7 @@ public async Task Upload() privateBlobStorageService.Setup(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Image, null))), + path.Contains(FilesPath(releaseVersion.Id, Image))), formFile )).Returns(Task.CompletedTask); @@ -234,7 +234,7 @@ public async Task Upload() privateBlobStorageService.Verify(mock => mock.UploadFile(PrivateReleaseFiles, It.Is(path => - path.Contains(FilesPath(releaseVersion.Id, Image, null))), + path.Contains(FilesPath(releaseVersion.Id, Image))), formFile ), Times.Once); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs index 8da1a64fc67..04f050d63a7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseDataFileService.cs @@ -516,8 +516,8 @@ public async Task>> SaveDataSetsFromTemp replacingDataFile: replacingFile, order: releaseDataFileOrder); - var sourceDataFilePath = FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Data, archiveDataSetFile.DataFileId); - var destinationDataFilePath = FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Data, dataFile.Id); // Same path, but a new ID has been generated by the creation step above + var sourceDataFilePath = FileExtensions.Path(releaseVersionId, FileType.Data, archiveDataSetFile.DataFileId); + var destinationDataFilePath = FileExtensions.Path(releaseVersionId, FileType.Data, dataFile.Id); // Same path, but a new ID has been generated by the creation step above var dataReleaseFile = await _contentDbContext.ReleaseFiles .Include(rf => rf.File) @@ -535,8 +535,8 @@ public async Task>> SaveDataSetsFromTemp type: FileType.Metadata, createdById: _userService.GetUserId()); - var sourceMetaFilePath = FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Metadata, archiveDataSetFile.MetaFileId); - var destinationMetaFilePath = FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Metadata, metaFile.Id); // Same path, but a new ID has been generated by the creation step above + var sourceMetaFilePath = FileExtensions.Path(releaseVersionId, FileType.Metadata, archiveDataSetFile.MetaFileId); + var destinationMetaFilePath = FileExtensions.Path(releaseVersionId, FileType.Metadata, metaFile.Id); // Same path, but a new ID has been generated by the creation step above await _dataImportService.Import( subjectId: subjectId, @@ -555,8 +555,8 @@ private async Task> ValidateTempDataSetFileExistence( Guid releaseVersionId, ArchiveDataSetFileViewModel fileImportRequest) { - var dataBlobExists = await _privateBlobStorageService.CheckBlobExists(PrivateReleaseTempFiles, $"{FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Data, fileImportRequest.DataFileId)}"); - var metaBlobExists = await _privateBlobStorageService.CheckBlobExists(PrivateReleaseTempFiles, $"{FileStoragePathUtils.FilesPath(releaseVersionId, FileType.Metadata, fileImportRequest.MetaFileId)}"); + var dataBlobExists = await _privateBlobStorageService.CheckBlobExists(PrivateReleaseTempFiles, $"{FileExtensions.Path(releaseVersionId, FileType.Data, fileImportRequest.DataFileId)}"); + var metaBlobExists = await _privateBlobStorageService.CheckBlobExists(PrivateReleaseTempFiles, $"{FileExtensions.Path(releaseVersionId, FileType.Metadata, fileImportRequest.MetaFileId)}"); if (!dataBlobExists || !metaBlobExists) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/FileStoragePathUtils.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/FileStoragePathUtils.cs index 2c1740207a0..3b1c6bee4a5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Services/FileStoragePathUtils.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Services/FileStoragePathUtils.cs @@ -99,10 +99,10 @@ public static string PublicContentReleaseSubjectsPath(string publicationSlug, st return $"{PublicContentReleaseParentPath(publicationSlug, releaseSlug)}/subjects.json"; } - public static string FilesPath(Guid rootPath, FileType type, Guid? filePath = null) + public static string FilesPath(Guid rootPath, FileType type) { var typeFolder = (type == Metadata ? Data : type).GetEnumLabel(); - return $"{rootPath}/{typeFolder}/{filePath}"; + return $"{rootPath}/{typeFolder}/"; } private static string AppendPathSeparator(string? segment = null) From 07211850f8dfdbc2dce81b553b7d0686c7099bd4 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 15:25:35 +0000 Subject: [PATCH 066/144] EES-XXXX - corrected misuse of parameter instead of variable --- infrastructure/templates/template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index dc3c461e975..acfeb56b701 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -3286,7 +3286,7 @@ "App:NotifierStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-notifications')).secretUriWithVersion, ')')]", "App:PublicStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-public')).secretUriWithVersion, ')')]", "App:PublisherStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-publisher')).secretUriWithVersion, ')')]", - "DataFiles:BasePath": "[parameters('publicDataFileShareMountPath')]", + "DataFiles:BasePath": "[variables('publicDataFileShareMountPath')]", "PublicDataDbExists": "[parameters('publicDataDbExists')]" } }, From 53b3211ad544914ef5c03049316a045aeb14e673 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 15:29:10 +0000 Subject: [PATCH 067/144] EES-5685 - reverting change to timeAggregation as Total was not supported --- .../components/alerts/appGateways/backendPoolHealth.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep b/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep index 3fabc48a219..069e01ba904 100644 --- a/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep +++ b/infrastructure/templates/public-api/components/alerts/appGateways/backendPoolHealth.bicep @@ -20,7 +20,7 @@ module alerts '../staticMetricAlert.bicep' = [for name in resourceNames: { resourceType: 'Microsoft.Network/applicationGateways' query: { metric: 'UnhealthyHostCount' - aggregation: 'Total' + aggregation: 'Average' operator: 'GreaterThan' threshold: 0 } From fd0f44904e591ecc9ecda69cd2836ac4e8118d99 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 15:41:13 +0000 Subject: [PATCH 068/144] EES-5724 - fixed missing storage account access key. Updated mount path for Windows which must reside under a \mounts folder at a single level --- infrastructure/templates/template.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index acfeb56b701..9c45c445bc5 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -1198,7 +1198,7 @@ "[concat('Microsoft.Web/sites/', variables('contentAppName'))]", "[concat('Microsoft.Web/sites/', variables('dataAppName'))]" ], - "publicDataFileShareMountPath": "/data/public-api-data", + "publicDataFileShareMountPathWindows": "\\mounts\\public-api-data", "publicDataFileShareName": "[concat(parameters('subscription'), '-ees-papi-fs-data')]", "publicDataStorageAccountName": "[concat(parameters('subscription'), 'eespapisa')]" }, @@ -3286,7 +3286,7 @@ "App:NotifierStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-notifications')).secretUriWithVersion, ')')]", "App:PublicStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-public')).secretUriWithVersion, ')')]", "App:PublisherStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-publisher')).secretUriWithVersion, ')')]", - "DataFiles:BasePath": "[variables('publicDataFileShareMountPath')]", + "DataFiles:BasePath": "[variables('publicDataFileShareMountPathWindows')]", "PublicDataDbExists": "[parameters('publicDataDbExists')]" } }, @@ -3303,8 +3303,9 @@ "[variables('publicDataFileShareName')]": { "type": "AzureFiles", "accountName": "[variables('publicDataStorageAccountName')]", + "accessKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('publicDataStorageAccountName')), '2018-02-01').keys[0].value]", "shareName": "[variables('publicDataFileShareName')]", - "mountPath": "[variables('publicDataFileShareMountPath')]", + "mountPath": "[variables('publicDataFileShareMountPathWindows')]", "protocol": "Smb" } } From 8777d6d2d9ed9810011a0656272b51858bd310e0 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 6 Dec 2024 16:50:40 +0000 Subject: [PATCH 069/144] EES-5724 - grant the Publisher subnet access to the Public API storage account --- .../application/public-api/publicApiStorage.bicep | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep index 744d1217aa7..f61c7f4efd3 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep @@ -31,6 +31,11 @@ resource containerAppEnvironmentSubnet 'Microsoft.Network/virtualNetworks/subnet parent: vNet } +resource publisherSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = { + name: resourceNames.existingResources.subnets.publisherFunction + parent: vNet +} + // TODO EES-5128 - add private endpoints to allow VNet traffic to go directly to Storage Account over the VNet. // Currently supported by subnet whitelisting and Storage service endpoints being enabled on the whitelisted subnets. module publicApiStorageAccountModule '../../components/storageAccount.bicep' = { @@ -41,6 +46,7 @@ module publicApiStorageAccountModule '../../components/storageAccount.bicep' = { allowedSubnetIds: [ dataProcessorSubnet.id containerAppEnvironmentSubnet.id + publisherSubnet.id ] firewallRules: storageFirewallRules skuStorageResource: 'Standard_LRS' From a7a506c90c273342b07fc089aa4c2f435c5ef545 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Fri, 6 Dec 2024 18:03:16 +0000 Subject: [PATCH 070/144] EES-5656 Improve tests in legacy_releases_reordering.robot --- .../components/ReleaseSeriesTable.tsx | 2 +- .../bau/legacy_releases_reordering.robot | 371 ++++++++++++------ 2 files changed, 259 insertions(+), 114 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/publication/components/ReleaseSeriesTable.tsx b/src/explore-education-statistics-admin/src/pages/publication/components/ReleaseSeriesTable.tsx index bcdc478eea9..7fe5f8a361b 100644 --- a/src/explore-education-statistics-admin/src/pages/publication/components/ReleaseSeriesTable.tsx +++ b/src/explore-education-statistics-admin/src/pages/publication/components/ReleaseSeriesTable.tsx @@ -72,7 +72,7 @@ export default function ReleaseSeriesTable({ } return ( - +
    diff --git a/tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot b/tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot index 636fd0c172f..11e7a1a6737 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot @@ -12,13 +12,16 @@ Test Setup fail test fast if required *** Variables *** -${RELEASE_NAME}= Academic year Q1 2022/23 -${PUBLICATION_NAME}= UI tests - legacy releases-%{RUN_IDENTIFIER} -${PUBLICATION_SLUG}= ui-tests-legacy-releases-%{RUN_IDENTIFIER} -${PUBLIC_PUBLICATION_URL_ENDING}= /find-statistics/${PUBLICATION_SLUG} -${DESCRIPTION}= legacy release description -${UPDATED_DESCRIPTION}= updated legacy release description -${PUBLIC_URL_WITHOUT_AUTH} ${EMPTY} +${PUBLICATION_NAME}= UI tests - legacy releases %{RUN_IDENTIFIER} +${PUBLICATION_SLUG}= ui-tests-legacy-releases-%{RUN_IDENTIFIER} +${RELEASE_1_NAME}= Academic year 2020/21 +${RELEASE_2_NAME}= Academic year Q1 2022/23 +${LEGACY_RELEASE_1_DESCRIPTION}= legacy release 1 +${LEGACY_RELEASE_1_URL}= http://test.url/1 +${LEGACY_RELEASE_1_DESCRIPTION_UPDATED}= legacy release 1 updated +${LEGACY_RELEASE_1_URL_UPDATED}= http://test.url/1/updated +${LEGACY_RELEASE_2_DESCRIPTION}= legacy release 2 +${LEGACY_RELEASE_2_URL}= http://test.url/2 *** Test Cases *** @@ -26,231 +29,373 @@ Create new publication via api ${PUBLICATION_ID}= user creates test publication via api ${PUBLICATION_NAME} set suite variable ${PUBLICATION_ID} -Validate that legacy releases do not exist +Validate publication release order table is empty user navigates to publication page from dashboard ${PUBLICATION_NAME} user clicks link Legacy releases user waits until h2 is visible Legacy releases - user checks page does not contain element css:tbody[data-rfd-droppable-id="droppable"] + user checks page contains No releases for this publication. -Create legacy release +Create first legacy release user navigates to publication page from dashboard ${PUBLICATION_NAME} user clicks link Legacy releases user waits until h2 is visible Legacy releases - user creates legacy release ${DESCRIPTION} http://test.com + user creates legacy release ${LEGACY_RELEASE_1_DESCRIPTION} ${LEGACY_RELEASE_1_URL} -Create new release via api - user navigates to publication page from dashboard ${PUBLICATION_NAME} +Validate publication release order table headings + user checks table column heading contains 1 1 Description testid:release-series + user checks table column heading contains 1 2 URL testid:release-series + user checks table column heading contains 1 3 Status testid:release-series + user checks table column heading contains 1 4 Actions testid:release-series + +Validate first legacy release exists in the publication release order + user checks table body has x rows 1 testid:release-series + + user checks table cell contains 1 1 ${LEGACY_RELEASE_1_DESCRIPTION} + user checks table cell contains 1 2 ${LEGACY_RELEASE_1_URL} + user checks table cell contains 1 3 Legacy release + user checks table cell contains 1 4 Edit + user checks table cell contains 1 4 Delete + +Create first release via api user creates test release via api ${PUBLICATION_ID} AY 2020 -Create 2nd legacy release - user navigates to publication page from dashboard ${PUBLICATION_NAME} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases - user creates legacy release ${DESCRIPTION} http://test.com + ${PUBLIC_URL_WITHOUT_AUTH}= remove auth from url %{PUBLIC_URL} + set suite variable ${PUBLIC_URL_WITHOUT_AUTH} + set suite variable ${PUBLIC_RELEASE_1_URL} + ... ${PUBLIC_URL_WITHOUT_AUTH}/find-statistics/${PUBLICATION_SLUG}/2020-21 -Validate that publication and legacy releases exist in the page +Validate first release exists in the publication release order with status unpublished user navigates to publication page from dashboard ${PUBLICATION_NAME} user clicks link Legacy releases user waits until h2 is visible Legacy releases + user checks table body has x rows 2 testid:release-series - ${PUBLIC_URL_WITHOUT_AUTH}= remove auth from url %{PUBLIC_URL} - set suite variable ${PUBLIC_URL_WITHOUT_AUTH} + user checks table cell contains 1 1 ${RELEASE_1_NAME} + user checks table cell contains 1 2 ${PUBLIC_RELEASE_1_URL} + user checks table cell contains 1 3 Unpublished - user checks table cell contains 1 1 Academic year 2020/21 - user checks table cell contains 1 2 ${PUBLIC_URL_WITHOUT_AUTH}/find-statistics/${PUBLICATION_SLUG}/2020-21 + user checks table cell contains 2 1 ${LEGACY_RELEASE_1_DESCRIPTION} + user checks table cell contains 2 2 ${LEGACY_RELEASE_1_URL} + user checks table cell contains 2 3 Legacy release + user checks table cell contains 2 4 Edit + user checks table cell contains 2 4 Delete + +Create second legacy release + user creates legacy release ${LEGACY_RELEASE_2_DESCRIPTION} ${LEGACY_RELEASE_2_URL} + +Validate second legacy release exists in the publication release order + user checks table body has x rows 3 testid:release-series + + user checks table cell contains 1 1 ${RELEASE_1_NAME} + user checks table cell contains 1 2 ${PUBLIC_RELEASE_1_URL} user checks table cell contains 1 3 Unpublished - user checks table cell contains 2 1 ${DESCRIPTION} - user checks table cell contains 2 2 http://test.com + user checks table cell contains 2 1 ${LEGACY_RELEASE_1_DESCRIPTION} + user checks table cell contains 2 2 ${LEGACY_RELEASE_1_URL} user checks table cell contains 2 3 Legacy release user checks table cell contains 2 4 Edit user checks table cell contains 2 4 Delete - user checks table cell contains 3 1 ${DESCRIPTION} - user checks table cell contains 3 2 http://test.com + user checks table cell contains 3 1 ${LEGACY_RELEASE_2_DESCRIPTION} + user checks table cell contains 3 2 ${LEGACY_RELEASE_2_URL} user checks table cell contains 3 3 Legacy release user checks table cell contains 3 4 Edit user checks table cell contains 3 4 Delete -Navigate to release in admin +Add headline text block to first release content page user navigates to draft release page from dashboard ${PUBLICATION_NAME} - ... Academic year 2020/21 - -Add headline text block to Content page + ... ${RELEASE_1_NAME} user navigates to content page ${PUBLICATION_NAME} user adds headlines text block user adds content to headlines text block Headline text block text -Approve release +Approve first release user clicks link Sign off - sleep 1 user approves original release for immediate publication - user waits until page contains element testid:public-release-url - ${PUBLIC_RELEASE_LINK}= get value xpath://*[@data-testid="public-release-url"] - check that variable is not empty PUBLIC_RELEASE_LINK ${PUBLIC_RELEASE_LINK} - set suite variable ${PUBLIC_RELEASE_LINK} -Check legacy release appears on public frontend - user navigates to public frontend ${PUBLIC_RELEASE_LINK} - user opens details dropdown View releases (2) +Validate first release has latest release status in publication release order + user navigates to publication page from dashboard ${PUBLICATION_NAME} + user clicks link Legacy releases + user waits until h2 is visible Legacy releases + + user checks table body has x rows 3 testid:release-series + + user checks table cell contains 1 1 ${RELEASE_1_NAME} + user checks table cell contains 1 2 ${PUBLIC_RELEASE_1_URL} + user checks table cell contains 1 3 Latest release + + user checks table cell contains 2 1 ${LEGACY_RELEASE_1_DESCRIPTION} + user checks table cell contains 2 2 ${LEGACY_RELEASE_1_URL} + user checks table cell contains 2 3 Legacy release + user checks table cell contains 2 4 Edit + user checks table cell contains 2 4 Delete + + user checks table cell contains 3 1 ${LEGACY_RELEASE_2_DESCRIPTION} + user checks table cell contains 3 2 ${LEGACY_RELEASE_2_URL} + user checks table cell contains 3 3 Legacy release + user checks table cell contains 3 4 Edit + user checks table cell contains 3 4 Delete + +Validate other releases section on public frontend + user navigates to public frontend ${PUBLIC_RELEASE_1_URL} + user checks number of other releases is correct 2 + ${view_releases}= user opens details dropdown View releases (2) - ${other_releases}= user gets details content element View releases (2) + user checks other release is shown in position ${LEGACY_RELEASE_1_DESCRIPTION} 1 + user checks other release is shown in position ${LEGACY_RELEASE_2_DESCRIPTION} 2 - user checks list has x items css:ul 2 ${other_releases} + user checks page contains link with text and url + ... ${LEGACY_RELEASE_1_DESCRIPTION} + ... ${LEGACY_RELEASE_1_URL} + ... ${view_releases} - ${other_release_1}= user gets list item element css:ul 2 ${other_releases} - ${other_release_1_link}= get child element ${other_release_1} link:${DESCRIPTION} - user checks element attribute value should be ${other_release_1_link} href http://test.com/ + user checks page contains link with text and url + ... ${LEGACY_RELEASE_2_DESCRIPTION} + ... ${LEGACY_RELEASE_2_URL} + ... ${view_releases} -Navigate to publication to update legacy releases +Update first legacy release user navigates to publication page from dashboard ${PUBLICATION_NAME} user clicks link Legacy releases user waits until h2 is visible Legacy releases -Update legacy release user clicks element xpath://tr[2]//*[text()="Edit"] ${modal}= user waits until modal is visible Edit legacy release user clicks button OK ${modal} user waits until page contains element id:releaseSeriesLegacyLinkForm-description - user enters text into element id:releaseSeriesLegacyLinkForm-description ${UPDATED_DESCRIPTION} - user enters text into element id:releaseSeriesLegacyLinkForm-url http://test2.com + user enters text into element id:releaseSeriesLegacyLinkForm-description + ... ${LEGACY_RELEASE_1_DESCRIPTION_UPDATED} + user enters text into element id:releaseSeriesLegacyLinkForm-url ${LEGACY_RELEASE_1_URL_UPDATED} user clicks button Save legacy release -Validate the updated legacy release +Validate the first legacy release is updated user waits until h2 is visible Legacy releases - user checks element count is x css:tbody tr 3 - user checks table cell contains 2 1 ${UPDATED_DESCRIPTION} - user checks table cell contains 2 2 http://test2.com + user checks table body has x rows 3 testid:release-series + + user checks table cell contains 1 1 ${RELEASE_1_NAME} + user checks table cell contains 1 2 ${PUBLIC_RELEASE_1_URL} + user checks table cell contains 1 3 Latest release + + user checks table cell contains 2 1 ${LEGACY_RELEASE_1_DESCRIPTION_UPDATED} + user checks table cell contains 2 2 ${LEGACY_RELEASE_1_URL_UPDATED} user checks table cell contains 2 3 Legacy release user checks table cell contains 2 4 Edit user checks table cell contains 2 4 Delete -Reorder the legacy releases + user checks table cell contains 3 1 ${LEGACY_RELEASE_2_DESCRIPTION} + user checks table cell contains 3 2 ${LEGACY_RELEASE_2_URL} + user checks table cell contains 3 3 Legacy release + user checks table cell contains 3 4 Edit + user checks table cell contains 3 4 Delete + +Reorder the publication releases user clicks button Reorder releases - user waits until modal is visible Reorder releases - user clicks button OK - user waits until modal is not visible Reorder legacy releases + ${modal}= user waits until modal is visible Reorder releases + user clicks button OK ${modal} + user waits until modal is not visible Reorder releases user waits until page contains button Confirm order + + click element xpath://div[text()="${RELEASE_1_NAME}"] CTRL user presses keys ${SPACE} user presses keys ARROW_DOWN user presses keys ARROW_DOWN user presses keys ${SPACE} + + click element xpath://div[text()="${LEGACY_RELEASE_2_DESCRIPTION}"] CTRL + user presses keys ${SPACE} + user presses keys ARROW_UP + user presses keys ${SPACE} + user clicks button Confirm order sleep 2 -Validate reordered legacy releases +Validate reordered publication releases user waits until page contains button Reorder releases - user checks element count is x css:tbody tr 3 + user checks table body has x rows 3 testid:release-series - user checks table cell contains 1 1 ${UPDATED_DESCRIPTION} - user checks table cell contains 1 2 http://test2.com + user checks table cell contains 1 1 ${LEGACY_RELEASE_2_DESCRIPTION} + user checks table cell contains 1 2 ${LEGACY_RELEASE_2_URL} user checks table cell contains 1 3 Legacy release user checks table cell contains 1 4 Edit user checks table cell contains 1 4 Delete - user checks table cell contains 2 1 ${DESCRIPTION} - user checks table cell contains 2 2 http://test.com + user checks table cell contains 2 1 ${LEGACY_RELEASE_1_DESCRIPTION_UPDATED} + user checks table cell contains 2 2 ${LEGACY_RELEASE_1_URL_UPDATED} user checks table cell contains 2 3 Legacy release user checks table cell contains 2 4 Edit user checks table cell contains 2 4 Delete - user checks table cell contains 3 1 Academic year 2020/21 - user checks table cell contains 3 2 ${PUBLIC_RELEASE_LINK} + user checks table cell contains 3 1 ${RELEASE_1_NAME} + user checks table cell contains 3 2 ${PUBLIC_RELEASE_1_URL} user checks table cell contains 3 3 Latest release -Create a second draft release via api +Validate other releases section on public frontend includes updated legacy release with expected order + user navigates to public frontend ${PUBLIC_RELEASE_1_URL} + user checks number of other releases is correct 2 + ${view_releases}= user opens details dropdown View releases (2) + + user checks other release is shown in position ${LEGACY_RELEASE_2_DESCRIPTION} 1 + user checks other release is shown in position ${LEGACY_RELEASE_1_DESCRIPTION_UPDATED} 2 + + user checks page contains link with text and url + ... ${LEGACY_RELEASE_1_DESCRIPTION_UPDATED} + ... ${LEGACY_RELEASE_1_URL_UPDATED} + ... ${view_releases} + + user checks page contains link with text and url + ... ${LEGACY_RELEASE_2_DESCRIPTION} + ... ${LEGACY_RELEASE_2_URL} + ... ${view_releases} + +Create second release via api + user creates test release via api ${PUBLICATION_ID} AYQ1 2022 + set suite variable ${PUBLIC_RELEASE_2_URL} + ... ${PUBLIC_URL_WITHOUT_AUTH}/find-statistics/${PUBLICATION_SLUG}/2022-23-q1 + +Validate second release exists in the publication release order with status unpublished user navigates to publication page from dashboard ${PUBLICATION_NAME} - user creates release from publication page ${PUBLICATION_NAME} Academic year Q1 2022 + user clicks link Legacy releases + user waits until h2 is visible Legacy releases + user checks table body has x rows 4 testid:release-series + + user checks table cell contains 1 1 ${RELEASE_2_NAME} + user checks table cell contains 1 2 ${PUBLIC_RELEASE_2_URL} + user checks table cell contains 1 3 Unpublished + + user checks table cell contains 2 1 ${LEGACY_RELEASE_2_DESCRIPTION} + user checks table cell contains 2 2 ${LEGACY_RELEASE_2_URL} + user checks table cell contains 2 3 Legacy release + user checks table cell contains 2 4 Edit + user checks table cell contains 2 4 Delete + + user checks table cell contains 3 1 ${LEGACY_RELEASE_1_DESCRIPTION_UPDATED} + user checks table cell contains 3 2 ${LEGACY_RELEASE_1_URL_UPDATED} + user checks table cell contains 3 3 Legacy release + user checks table cell contains 3 4 Edit + user checks table cell contains 3 4 Delete + + user checks table cell contains 4 1 ${RELEASE_1_NAME} + user checks table cell contains 4 2 ${PUBLIC_RELEASE_1_URL} + user checks table cell contains 4 3 Latest release -Add headline text block to content page(2nd release) +Add headline text block to second release content page + user navigates to draft release page from dashboard ${PUBLICATION_NAME} + ... ${RELEASE_2_NAME} user navigates to content page ${PUBLICATION_NAME} user adds headlines text block user adds content to headlines text block Headline text block text -Approve 2nd release +Approve second release user clicks link Sign off user approves original release for immediate publication -Navigate to publication to verify the legacy releases +Validate second release has latest release status in publication release order user navigates to publication page from dashboard ${PUBLICATION_NAME} user clicks link Legacy releases user waits until h2 is visible Legacy releases -Return to Admin and create first amendment + user checks table body has x rows 4 testid:release-series + + user checks table cell contains 1 1 ${RELEASE_2_NAME} + user checks table cell contains 1 2 ${PUBLIC_RELEASE_2_URL} + user checks table cell contains 1 3 Latest release + + user checks table cell contains 2 1 ${LEGACY_RELEASE_2_DESCRIPTION} + user checks table cell contains 2 2 ${LEGACY_RELEASE_2_URL} + user checks table cell contains 2 3 Legacy release + user checks table cell contains 2 4 Edit + user checks table cell contains 2 4 Delete + + user checks table cell contains 3 1 ${LEGACY_RELEASE_1_DESCRIPTION_UPDATED} + user checks table cell contains 3 2 ${LEGACY_RELEASE_1_URL_UPDATED} + user checks table cell contains 3 3 Legacy release + user checks table cell contains 3 4 Edit + user checks table cell contains 3 4 Delete + + user checks table cell contains 4 1 ${RELEASE_1_NAME} + user checks table cell contains 4 2 ${PUBLIC_RELEASE_1_URL} + user checks table cell does not contain 4 3 Latest release + +Validate other releases section on public frontend includes first release with expected order + user navigates to public frontend ${PUBLIC_RELEASE_2_URL} + user checks number of other releases is correct 3 + ${view_releases}= user opens details dropdown View releases (3) + + user checks other release is shown in position ${LEGACY_RELEASE_2_DESCRIPTION} 1 + user checks other release is shown in position ${LEGACY_RELEASE_1_DESCRIPTION_UPDATED} 2 + user checks other release is shown in position ${RELEASE_1_NAME} 3 + +Create amendment of second release user navigates to admin dashboard Bau1 - user creates amendment for release ${PUBLICATION_NAME} ${RELEASE_NAME} + user creates amendment for release ${PUBLICATION_NAME} ${RELEASE_2_NAME} -Navigate to 'Content' page for amendment +Add release note to amendment of second release user clicks link Content - user waits until h2 is visible ${PUBLICATION_NAME} - user waits until page contains button Add a summary text block + user adds a release note Test release note one -Add release note to first amendment - user clicks button Add note - user enters text into element id:create-release-note-form-reason Test release note one - user clicks button Save note - sleep 2 + ${date}= get london date + user waits until element contains css:#release-notes li:nth-of-type(1) time ${date} + user waits until element contains css:#release-notes li:nth-of-type(1) p Test release note one -Navigate to "Sign off" page - user clicks link Sign off - user waits until h3 is visible Release status history - -Approve release amendment +Approve second release amendment user approves amended release for immediate publication - user waits until page contains element testid:public-release-url - ${PUBLIC_AMENDED_RELEASE_LINK}= get value xpath://*[@data-testid="public-release-url"] - check that variable is not empty PUBLIC_RELEASE_LINK ${PUBLIC_AMENDED_RELEASE_LINK} - set suite variable ${PUBLIC_AMENDED_RELEASE_LINK} -Navigate to publication page and verify the amended release +Validate amended second release exists in publication release order user navigates to publication page from dashboard ${PUBLICATION_NAME} user clicks link Legacy releases user waits until h2 is visible Legacy releases -Validate amended legacy releases - user waits until page contains button Reorder releases + user checks table body has x rows 4 testid:release-series - user checks table cell contains 1 1 Academic year Q1 2022/23 - user checks table cell contains 1 2 ${PUBLIC_AMENDED_RELEASE_LINK} + user checks table cell contains 1 1 ${RELEASE_2_NAME} + user checks table cell contains 1 2 ${PUBLIC_RELEASE_2_URL} user checks table cell contains 1 3 Latest release - user checks table cell contains 2 1 ${UPDATED_DESCRIPTION} - user checks table cell contains 2 2 http://test2.com + user checks table cell contains 2 1 ${LEGACY_RELEASE_2_DESCRIPTION} + user checks table cell contains 2 2 ${LEGACY_RELEASE_2_URL} user checks table cell contains 2 3 Legacy release user checks table cell contains 2 4 Edit user checks table cell contains 2 4 Delete - user checks table cell contains 3 1 ${DESCRIPTION} - user checks table cell contains 3 2 http://test.com + user checks table cell contains 3 1 ${LEGACY_RELEASE_1_DESCRIPTION_UPDATED} + user checks table cell contains 3 2 ${LEGACY_RELEASE_1_URL_UPDATED} user checks table cell contains 3 3 Legacy release user checks table cell contains 3 4 Edit user checks table cell contains 3 4 Delete - user checks table cell contains 4 1 Academic year 2020/21 - user checks table cell contains 4 2 ${PUBLIC_RELEASE_LINK} - user checks table cell does not contain 4 3 Latest - -Delete legacy release - user clicks element xpath://tr[3]//*[text()="Delete"] + user checks table cell contains 4 1 ${RELEASE_1_NAME} + user checks table cell contains 4 2 ${PUBLIC_RELEASE_1_URL} + user checks table cell does not contain 4 3 Latest release +Delete first legacy release + user clicks button in table cell 3 4 Delete testid:release-series ${modal}= user waits until modal is visible Delete legacy release user clicks button Confirm ${modal} - user waits until page does not contain element xpath://tr[4] + user waits until modal is not visible Delete legacy release -Validate that deleted legacy release does not exist +Validate first legacy release is deleted from publication release order user waits until page contains button Reorder releases + user checks table body has x rows 3 testid:release-series - user checks table cell contains 1 1 Academic year Q1 2022/23 - user checks table cell contains 1 2 ${PUBLIC_AMENDED_RELEASE_LINK} + user checks table cell contains 1 1 ${RELEASE_2_NAME} + user checks table cell contains 1 2 ${PUBLIC_RELEASE_2_URL} user checks table cell contains 1 3 Latest release - user checks table cell contains 2 1 ${UPDATED_DESCRIPTION} - user checks table cell contains 2 2 http://test2.com + user checks table cell contains 2 1 ${LEGACY_RELEASE_2_DESCRIPTION} + user checks table cell contains 2 2 ${LEGACY_RELEASE_2_URL} user checks table cell contains 2 3 Legacy release user checks table cell contains 2 4 Edit user checks table cell contains 2 4 Delete - user checks table cell contains 3 1 Academic year 2020/21 - user checks table cell contains 3 2 ${PUBLIC_RELEASE_LINK} - user checks table cell does not contain 3 3 Latest + user checks table cell contains 3 1 ${RELEASE_1_NAME} + user checks table cell contains 3 2 ${PUBLIC_RELEASE_1_URL} + user checks table cell does not contain 3 3 Latest release + +Validate other releases section on public frontend does not include first legacy release + user navigates to public frontend ${PUBLIC_RELEASE_2_URL} + user checks number of other releases is correct 2 + ${view_releases}= user opens details dropdown View releases (2) + + user checks other release is shown in position ${LEGACY_RELEASE_2_DESCRIPTION} 1 + user checks other release is shown in position ${RELEASE_1_NAME} 2 From bd2ec556f335b69e81fcebaf9a69f91a70ef303a Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Tue, 3 Dec 2024 16:02:37 +0000 Subject: [PATCH 071/144] EES-5656 Rename page 'Legacy releases' to 'Release order' --- .../PublicationReleaseSeriesPage.tsx | 17 ++++++++----- ...CreateReleaseSeriesLegacyLinkPage.test.tsx | 2 +- ...onEditReleaseSeriesLegacyLinkPage.test.tsx | 2 +- .../PublicationPageContainer.test.tsx | 2 +- .../PublicationReleaseSeriesPage.test.tsx | 24 +++++++++---------- .../__tests__/ReleaseSeriesTable.test.tsx | 4 ++-- .../src/routes/publicationRoutes.ts | 8 +++---- 7 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/publication/PublicationReleaseSeriesPage.tsx b/src/explore-education-statistics-admin/src/pages/publication/PublicationReleaseSeriesPage.tsx index f6eb3b296b9..6dcef500363 100644 --- a/src/explore-education-statistics-admin/src/pages/publication/PublicationReleaseSeriesPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/publication/PublicationReleaseSeriesPage.tsx @@ -32,14 +32,19 @@ export default function PublicationReleaseSeriesPage() { return ( -

    Legacy releases

    +

    Release order

    -

    Releases will be shown in the order below on the publication.

    - Explore education statistics releases from this publication can also be - reordered, including those in draft status or with a draft amendment, - but cannot be edited or deleted. Only releases with a published version - will be shown on the publication. + Releases will be shown in the order below on the publication and can be + reordered. +

    +

    + Legacy releases which link to extenal pages outside the service can be + created, edited, deleted, and are included in the release order. +

    +

    + Only releases with a published version and legacy releases will appear + in the publication.

    {canManageReleaseSeries && !isReordering && ( diff --git a/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationCreateReleaseSeriesLegacyLinkPage.test.tsx b/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationCreateReleaseSeriesLegacyLinkPage.test.tsx index 0b52c8a3972..2b8d2bb3934 100644 --- a/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationCreateReleaseSeriesLegacyLinkPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationCreateReleaseSeriesLegacyLinkPage.test.tsx @@ -30,7 +30,7 @@ describe('PublicationCreateReleaseSeriesLegacyLinkPage', () => { ).toBeInTheDocument(); expect(screen.getByRole('link', { name: 'Cancel' })).toHaveAttribute( 'href', - '/publication/publication-1/legacy', + '/publication/publication-1/releases/order', ); }); diff --git a/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationEditReleaseSeriesLegacyLinkPage.test.tsx b/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationEditReleaseSeriesLegacyLinkPage.test.tsx index 42604e39d97..9267147916c 100644 --- a/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationEditReleaseSeriesLegacyLinkPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationEditReleaseSeriesLegacyLinkPage.test.tsx @@ -67,7 +67,7 @@ describe('PublicationEditReleaseSeriesLegacyLinkPage', () => { ).toBeInTheDocument(); expect(screen.getByRole('link', { name: 'Cancel' })).toHaveAttribute( 'href', - '/publication/publication-1/legacy', + '/publication/publication-1/releases/order', ); }); diff --git a/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationPageContainer.test.tsx b/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationPageContainer.test.tsx index a733ddd5571..13842b835fc 100644 --- a/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationPageContainer.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationPageContainer.test.tsx @@ -60,7 +60,7 @@ describe('PublicationPageContainer', () => { expect(navLinks[3]).not.toHaveAttribute('aria-current'); expect(navLinks[4]).toHaveTextContent('Team access'); expect(navLinks[4]).not.toHaveAttribute('aria-current'); - expect(navLinks[5]).toHaveTextContent('Legacy releases'); + expect(navLinks[5]).toHaveTextContent('Release order'); expect(navLinks[5]).not.toHaveAttribute('aria-current'); expect(screen.getByText('Manage releases')).toBeInTheDocument(); diff --git a/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationReleaseSeriesPage.test.tsx b/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationReleaseSeriesPage.test.tsx index 7ae3c4e1aa0..4928ec727cb 100644 --- a/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationReleaseSeriesPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationReleaseSeriesPage.test.tsx @@ -26,7 +26,7 @@ describe('PublicationReleaseSeriesPage', () => { publicationService.getReleaseSeries.mockResolvedValue(testReleaseSeries); renderPage(testPublication); - expect(await screen.findByText('Legacy releases')).toBeInTheDocument(); + expect(await screen.findByText('Release order')).toBeInTheDocument(); const table = screen.getByRole('table'); expect(within(table).getAllByRole('row')).toHaveLength(5); @@ -52,7 +52,7 @@ describe('PublicationReleaseSeriesPage', () => { publicationService.getReleaseSeries.mockResolvedValue([]); renderPage(testPublication); - expect(await screen.findByText('Legacy releases')).toBeInTheDocument(); + expect(await screen.findByText('Release order')).toBeInTheDocument(); expect( screen.getByText('No releases for this publication.'), @@ -75,7 +75,7 @@ describe('PublicationReleaseSeriesPage', () => { publicationService.getReleaseSeries.mockResolvedValue(testReleaseSeries); renderPage(testPublication); - expect(await screen.findByText('Legacy releases')).toBeInTheDocument(); + expect(await screen.findByText('Release order')).toBeInTheDocument(); await user.click( screen.getByRole('button', { name: 'Reorder releases' }), @@ -101,7 +101,7 @@ describe('PublicationReleaseSeriesPage', () => { publicationService.getReleaseSeries.mockResolvedValue(testReleaseSeries); renderPage(testPublication); - expect(await screen.findByText('Legacy releases')).toBeInTheDocument(); + expect(await screen.findByText('Release order')).toBeInTheDocument(); await user.click( screen.getByRole('button', { name: 'Reorder releases' }), @@ -130,7 +130,7 @@ describe('PublicationReleaseSeriesPage', () => { ).not.toBeInTheDocument(); }); - test('does not show button to reorder when user does not have permission to manage legacy releases', async () => { + test('does not show button to reorder when user does not have permission to manage the release series', async () => { publicationService.getReleaseSeries.mockResolvedValue(testReleaseSeries); renderPage({ ...testPublication, @@ -140,7 +140,7 @@ describe('PublicationReleaseSeriesPage', () => { }, }); - expect(await screen.findByText('Legacy releases')).toBeInTheDocument(); + expect(await screen.findByText('Release order')).toBeInTheDocument(); expect( screen.queryByRole('button', { name: 'Reorder releases' }), @@ -161,7 +161,7 @@ describe('PublicationReleaseSeriesPage', () => { ]); renderPage(testPublication); - expect(await screen.findByText('Legacy releases')).toBeInTheDocument(); + expect(await screen.findByText('Release order')).toBeInTheDocument(); await user.click( screen.getByRole('button', { name: 'Delete Legacy link 1' }), @@ -207,7 +207,7 @@ describe('PublicationReleaseSeriesPage', () => { ); renderPage(testPublication); - expect(await screen.findByText('Legacy releases')).toBeInTheDocument(); + expect(await screen.findByText('Release order')).toBeInTheDocument(); await user.click( screen.getByRole('button', { name: 'Create legacy release' }), @@ -248,7 +248,7 @@ describe('PublicationReleaseSeriesPage', () => { , ); - expect(await screen.findByText('Legacy releases')).toBeInTheDocument(); + expect(await screen.findByText('Release order')).toBeInTheDocument(); await user.click( screen.getByRole('button', { name: 'Create legacy release' }), @@ -261,12 +261,12 @@ describe('PublicationReleaseSeriesPage', () => { ); await waitFor(() => { expect(history.location.pathname).toBe( - `/publication/${testPublication.id}/legacy/create`, + `/publication/${testPublication.id}/releases/legacy/create`, ); }); }); - test('does not show button to create when user does not have permission to manage legacy releases', async () => { + test('does not show button to create when user does not have permission to manage the release series', async () => { publicationService.getReleaseSeries.mockResolvedValueOnce( testReleaseSeries, ); @@ -278,7 +278,7 @@ describe('PublicationReleaseSeriesPage', () => { }, }); - expect(await screen.findByText('Legacy releases')).toBeInTheDocument(); + expect(await screen.findByText('Release order')).toBeInTheDocument(); expect( screen.queryByRole('button', { name: 'Create legacy release' }), diff --git a/src/explore-education-statistics-admin/src/pages/publication/components/__tests__/ReleaseSeriesTable.test.tsx b/src/explore-education-statistics-admin/src/pages/publication/components/__tests__/ReleaseSeriesTable.test.tsx index 4d3e2a499c5..cc59438f0bd 100644 --- a/src/explore-education-statistics-admin/src/pages/publication/components/__tests__/ReleaseSeriesTable.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/publication/components/__tests__/ReleaseSeriesTable.test.tsx @@ -161,13 +161,13 @@ describe('ReleaseSeriesTable', () => { await waitFor(() => { expect(history.location.pathname).toBe( - `/publication/${testPublicationId}/legacy/${testReleaseSeries[3].id}/edit`, + `/publication/${testPublicationId}/releases/legacy/${testReleaseSeries[3].id}/edit`, ); }); }); }); - test('does not show edit and delete actions when user does not have permission to manage legacy releases', () => { + test('does not show edit and delete actions when user does not have permission to manage the release series', () => { render( Date: Fri, 6 Dec 2024 18:28:44 +0000 Subject: [PATCH 072/144] EES-5656 Rename legacy_releases_reordering.robot to release_reordering.robot following rename of Legacy releases page to Release order --- ...rdering.robot => release_reordering.robot} | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) rename tests/robot-tests/tests/admin_and_public/bau/{legacy_releases_reordering.robot => release_reordering.robot} (95%) diff --git a/tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot b/tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot similarity index 95% rename from tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot rename to tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot index 11e7a1a6737..54fcece6839 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/legacy_releases_reordering.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot @@ -31,14 +31,14 @@ Create new publication via api Validate publication release order table is empty user navigates to publication page from dashboard ${PUBLICATION_NAME} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases + user clicks link Release order + user waits until h2 is visible Release order user checks page contains No releases for this publication. Create first legacy release user navigates to publication page from dashboard ${PUBLICATION_NAME} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases + user clicks link Release order + user waits until h2 is visible Release order user creates legacy release ${LEGACY_RELEASE_1_DESCRIPTION} ${LEGACY_RELEASE_1_URL} Validate publication release order table headings @@ -66,8 +66,8 @@ Create first release via api Validate first release exists in the publication release order with status unpublished user navigates to publication page from dashboard ${PUBLICATION_NAME} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases + user clicks link Release order + user waits until h2 is visible Release order user checks table body has x rows 2 testid:release-series user checks table cell contains 1 1 ${RELEASE_1_NAME} @@ -115,8 +115,8 @@ Approve first release Validate first release has latest release status in publication release order user navigates to publication page from dashboard ${PUBLICATION_NAME} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases + user clicks link Release order + user waits until h2 is visible Release order user checks table body has x rows 3 testid:release-series @@ -156,8 +156,8 @@ Validate other releases section on public frontend Update first legacy release user navigates to publication page from dashboard ${PUBLICATION_NAME} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases + user clicks link Release order + user waits until h2 is visible Release order user clicks element xpath://tr[2]//*[text()="Edit"] ${modal}= user waits until modal is visible Edit legacy release @@ -170,7 +170,7 @@ Update first legacy release user clicks button Save legacy release Validate the first legacy release is updated - user waits until h2 is visible Legacy releases + user waits until h2 is visible Release order user checks table body has x rows 3 testid:release-series user checks table cell contains 1 1 ${RELEASE_1_NAME} @@ -255,8 +255,8 @@ Create second release via api Validate second release exists in the publication release order with status unpublished user navigates to publication page from dashboard ${PUBLICATION_NAME} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases + user clicks link Release order + user waits until h2 is visible Release order user checks table body has x rows 4 testid:release-series user checks table cell contains 1 1 ${RELEASE_2_NAME} @@ -292,8 +292,8 @@ Approve second release Validate second release has latest release status in publication release order user navigates to publication page from dashboard ${PUBLICATION_NAME} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases + user clicks link Release order + user waits until h2 is visible Release order user checks table body has x rows 4 testid:release-series @@ -343,8 +343,8 @@ Approve second release amendment Validate amended second release exists in publication release order user navigates to publication page from dashboard ${PUBLICATION_NAME} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases + user clicks link Release order + user waits until h2 is visible Release order user checks table body has x rows 4 testid:release-series From 7d9c21473585e340ef0e825b9f44b174cb99e9c3 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 9 Dec 2024 09:22:29 +0000 Subject: [PATCH 073/144] EES-5724 - enabling content-over-vnet for Publisher Function App, in order to allow it to see the mounted Public API File Share. Without this enabled, the file share, although mounted, is not visible to the machine. --- infrastructure/templates/template.json | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/templates/template.json b/infrastructure/templates/template.json index 9c45c445bc5..f052f537135 100644 --- a/infrastructure/templates/template.json +++ b/infrastructure/templates/template.json @@ -3275,6 +3275,7 @@ "AzureWebJobsStorage": "[concat('@Microsoft.KeyVault(SecretUri=', reference(variables('ees-storage-publisher')).secretUriWithVersion, ')')]", "AzureWebJobs.StageScheduledReleaseVersionsImmediately.Disabled": "[not(parameters('immediatePublicationOfScheduledReleaseVersionsEnabled'))]", "AzureWebJobs.PublishStagedReleaseVersionContentImmediately.Disabled": "[not(parameters('immediatePublicationOfScheduledReleaseVersionsEnabled'))]", + "WEBSITE_CONTENTOVERVNET": "1", "WEBSITE_TIME_ZONE": "[parameters('timeZone')]", "WEBSITE_RUN_FROM_PACKAGE": "1", "FUNCTIONS_EXTENSION_VERSION": "~4", From f82b7d7e752443059989b8726901b7c04ba08f8a Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 9 Dec 2024 13:56:14 +0000 Subject: [PATCH 074/144] EES-5656 Update permissions tests following rename of Legacy releases page to Release order --- .../analyst/analyst_absence_permissions.robot | 14 +++++++------- .../publication_approver_ui_permissions.robot | 11 +++++++---- .../publication_owner_ui_permissions.robot | 11 +++++++---- .../release_approver_ui_permissions.robot | 11 +++++++---- .../release_contributor_ui_permissions.robot | 11 +++++++---- .../release_viewer_ui_permissions.robot | 11 +++++++---- .../seed_data/generate_seed_data_theme_1.robot | 8 ++++---- 7 files changed, 46 insertions(+), 31 deletions(-) diff --git a/tests/robot-tests/tests/admin/analyst/analyst_absence_permissions.robot b/tests/robot-tests/tests/admin/analyst/analyst_absence_permissions.robot index f7393a5b3b2..4da7adf9c73 100644 --- a/tests/robot-tests/tests/admin/analyst/analyst_absence_permissions.robot +++ b/tests/robot-tests/tests/admin/analyst/analyst_absence_permissions.robot @@ -36,12 +36,12 @@ Navigate to Seed Data Theme 1 Publication 1 page user clicks link ${PUPIL_ABSENCE_PUBLICATION_TITLE} user waits until h1 is visible ${PUPIL_ABSENCE_PUBLICATION_TITLE} -Navigate to legacy releases - user clicks link Legacy releases - user waits until h2 is visible Legacy releases +Navigate to publication release order + user clicks link Release order + user waits until h2 is visible Release order -Validate Analyst1 can see correct legacy releases - user checks element count is x css:tbody tr 7 +Validate Analyst1 can see correct publication release order + user checks table body has x rows 7 testid:release-series user checks table cell contains 1 1 Academic year 2016/17 user checks table cell contains 1 2 @@ -81,8 +81,8 @@ Validate Analyst1 can see correct legacy releases Check Analyst1 cannot create a legacy release user checks page does not contain button Create legacy release -Check Analyst1 cannot reorder legacy releases - user checks page does not contain button Reorder legacy releases +Check Analyst1 cannot reorder releases + user checks page does not contain button Reorder releases Check Analyst1 cannot edit a legacy release user checks page does not contain button Edit diff --git a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/publication_approver_ui_permissions.robot b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/publication_approver_ui_permissions.robot index f8377161088..5386c79cbe2 100644 --- a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/publication_approver_ui_permissions.robot +++ b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/publication_approver_ui_permissions.robot @@ -23,14 +23,17 @@ Validates publication approver publication page is correct user waits until page contains link Releases user waits until page contains link Methodologies user waits until page contains link Team access - user waits until page contains link Legacy releases # remove as part of EES-3794 + user waits until page contains link Release order user checks page does not contain link Details user checks page does not contain link Contact -Check cannot create a legacy release - user clicks link Legacy releases - user waits until h2 is visible Legacy releases +Check cannot reorder releases + user clicks link Release order + user waits until h2 is visible Release order + user checks page does not contain button Reorder releases + +Check cannot create legacy releases user checks page does not contain button Create legacy release Check cannot create a Methodology for a Publication if they don't have Publication Owner role diff --git a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/publication_owner_ui_permissions.robot b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/publication_owner_ui_permissions.robot index 6521fa2e270..b51b5c16a02 100644 --- a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/publication_owner_ui_permissions.robot +++ b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/publication_owner_ui_permissions.robot @@ -25,11 +25,14 @@ Navigate to Publication where analyst has Publication Owner role user waits until page contains link Details user waits until page contains link Contact user waits until page contains link Team access - user waits until page contains link Legacy releases + user waits until page contains link Release order -Check can create a legacy release - user clicks link Legacy releases - user waits until h2 is visible Legacy releases +Check can reorder releases + user clicks link Release order + user waits until h2 is visible Release order + user checks page contains button Reorder releases + +Check can create legacy releases user checks page contains button Create legacy release Check can create a Methodology for the owned Publication diff --git a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_approver_ui_permissions.robot b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_approver_ui_permissions.robot index 7bca5c0bc4a..bb8b8071129 100644 --- a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_approver_ui_permissions.robot +++ b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_approver_ui_permissions.robot @@ -23,14 +23,17 @@ Validates release approver publication page is correct user waits until page contains link Releases user waits until page contains link Methodologies user waits until page contains link Team access - user waits until page contains link Legacy releases # remove as part of EES-3794 + user waits until page contains link Release order user checks page does not contain link Details user checks page does not contain link Contact -Check cannot create a legacy release - user clicks link Legacy releases - user waits until h2 is visible Legacy releases +Check cannot reorder releases + user clicks link Release order + user waits until h2 is visible Release order + user checks page does not contain button Reorder releases + +Check cannot create legacy releases user checks page does not contain button Create legacy release Check cannot create a Methodology for a Publication if they don't have Publication Owner role diff --git a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_contributor_ui_permissions.robot b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_contributor_ui_permissions.robot index 86559cd42aa..1eca7e626fb 100644 --- a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_contributor_ui_permissions.robot +++ b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_contributor_ui_permissions.robot @@ -24,14 +24,17 @@ Validates release contributor publication page is correct user waits until page contains link Releases user waits until page contains link Methodologies user waits until page contains link Team access - user waits until page contains link Legacy releases # remove as part of EES-3794 + user waits until page contains link Release order user checks page does not contain link Details user checks page does not contain link Contact -Check cannot create a legacy release - user clicks link Legacy releases - user waits until h2 is visible Legacy releases +Check cannot reorder releases + user clicks link Release order + user waits until h2 is visible Release order + user checks page does not contain button Reorder releases + +Check cannot create legacy releases user checks page does not contain button Create legacy release Check cannot create a Methodology for a Publication if they don't have Publication Owner role diff --git a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_viewer_ui_permissions.robot b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_viewer_ui_permissions.robot index a05ee90c01f..88e0d12bb69 100644 --- a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_viewer_ui_permissions.robot +++ b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_viewer_ui_permissions.robot @@ -24,14 +24,17 @@ Validates release viewer publication page is correct user waits until page contains link Releases user waits until page contains link Methodologies user waits until page contains link Team access - user waits until page contains link Legacy releases # remove as part of EES-3794 + user waits until page contains link Release order user checks page does not contain link Details user checks page does not contain link Contact -Check cannot create a legacy release - user clicks link Legacy releases - user waits until h2 is visible Legacy releases +Check cannot reorder releases + user clicks link Release order + user waits until h2 is visible Release order + user checks page does not contain button Reorder releases + +Check cannot create legacy releases user checks page does not contain button Create legacy release Check cannot create a Methodology for a Publication if they don't have Publication Owner role diff --git a/tests/robot-tests/tests/seed_data/generate_seed_data_theme_1.robot b/tests/robot-tests/tests/seed_data/generate_seed_data_theme_1.robot index 6ce37b2c3ba..fe07971899f 100644 --- a/tests/robot-tests/tests/seed_data/generate_seed_data_theme_1.robot +++ b/tests/robot-tests/tests/seed_data/generate_seed_data_theme_1.robot @@ -48,8 +48,8 @@ Create ${PUPIL_ABSENCE_PUBLICATION_TITLE} ... ${PUPILS_AND_SCHOOLS_THEME_TITLE} Add legacy releases to ${PUPIL_ABSENCE_PUBLICATION_TITLE} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases + user clicks link Release order + user waits until h2 is visible Release order user creates legacy release Academic year 2009/10 ... https://www.gov.uk/government/statistics/pupil-absence-in-schools-in-england-including-pupil-characteristics-academic-year-2009-to-2010 user creates legacy release Academic year 2010/11 @@ -513,8 +513,8 @@ Create ${EXCLUSIONS_PUBLICATION_TITLE} ... ${PUPILS_AND_SCHOOLS_THEME_TITLE} Add legacy releases to ${EXCLUSIONS_PUBLICATION_TITLE} - user clicks link Legacy releases - user waits until h2 is visible Legacy releases + user clicks link Release order + user waits until h2 is visible Release order user creates legacy release Academic year 2008/09 ... https://www.gov.uk/government/statistics/permanent-and-fixed-period-exclusions-in-england-academic-year-2008-to-2009 user creates legacy release Academic year 2009/10 From a46aef532b7d90abd18c44430d1b0d43d4441621 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 2 Dec 2024 09:31:53 +0000 Subject: [PATCH 075/144] EES-5656 Release series tidy up --- .../Services/PublicationService.cs | 35 ++++++++----------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs index 6215b3edc51..f3484a7d780 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs @@ -22,11 +22,10 @@ using System.IO; using System.Linq; using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationErrorMessages; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationUtils; using ExternalMethodologyViewModel = GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.ExternalMethodologyViewModel; -using IPublicationRepository = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IPublicationRepository; -using IPublicationService = GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.IPublicationService; using IReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces.IReleaseVersionRepository; using PublicationViewModel = GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.PublicationViewModel; using ReleaseSummaryViewModel = GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.ReleaseSummaryViewModel; @@ -435,11 +434,12 @@ public async Task>> ListLates }); } - public async Task>> GetReleaseSeries(Guid publicationId) + public async Task>> GetReleaseSeries( + Guid publicationId) { - return await _persistenceHelper - .CheckEntityExists(publicationId) - .OnSuccess(publication => _userService.CheckCanViewPublication(publication)) + return await _context.Publications + .FirstOrNotFoundAsync(p => p.Id == publicationId) + .OnSuccess(_userService.CheckCanViewPublication) .OnSuccess(async publication => { var result = new List(); @@ -512,13 +512,11 @@ public async Task>> LegacyLinkUrl = newLegacyLink.Url, }); - _context.Publications.Update(publication); await _context.SaveChangesAsync(); await _publicationCacheService.UpdatePublication(publication.Slug); - return await GetReleaseSeries(publication.Id) - .OnSuccess(releaseSeries => releaseSeries); + return await GetReleaseSeries(publication.Id); }); } @@ -527,7 +525,6 @@ public async Task>> List updatedReleaseSeriesItems) { return await _context.Publications - .Include(p => p.ReleaseVersions) .FirstOrNotFoundAsync(p => p.Id == publicationId) .OnSuccess(_userService.CheckCanManageReleaseSeries) .OnSuccess(async publication => @@ -551,13 +548,13 @@ public async Task>> } // Check all publication releases are included in updatedReleaseSeriesItems - var publicationReleaseIds = publication.ReleaseVersions - .Select(rv => rv.ReleaseId) - .Distinct() - .ToList(); + var publicationReleaseIds = await _context.Releases + .Where(r => r.PublicationId == publicationId) + .Select(r => r.Id) + .ToListAsync(); var updatedSeriesReleaseIds = updatedReleaseSeriesItems - .Where(rsi => rsi.ReleaseId != null) + .Where(rsi => rsi.ReleaseId.HasValue) .Select(rsi => rsi.ReleaseId!.Value) .ToList(); @@ -570,7 +567,7 @@ public async Task>> // NOTE: A malicious user could change the release series items' Ids, but we don't care - var newReleaseSeries = updatedReleaseSeriesItems + publication.ReleaseSeries = updatedReleaseSeriesItems .Select(request => new ReleaseSeriesItem { Id = request.Id, @@ -579,15 +576,11 @@ public async Task>> LegacyLinkUrl = request.LegacyLinkUrl, }).ToList(); - publication.ReleaseSeries = newReleaseSeries; - _context.Publications.Update(publication); - await _context.SaveChangesAsync(); await _publicationCacheService.UpdatePublication(publication.Slug); - return await GetReleaseSeries(publication.Id) - .OnSuccess(releaseSeries => releaseSeries); + return await GetReleaseSeries(publication.Id); }); } From 249bb0aaf013dd55e5cb7297152d5dbf0f608b94 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 2 Dec 2024 10:23:32 +0000 Subject: [PATCH 076/144] EES-5656 Release series unit test tidy up --- .../Services/PublicationServiceTests.cs | 284 +++++++++--------- 1 file changed, 138 insertions(+), 146 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs index 9c493f09b50..8bfcfd2c321 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs @@ -1326,7 +1326,7 @@ public async Task UpdatePublication_TitleChangesPublicationAndMethodologySlug() } [Fact] - public async void UpdatePublication_NoTitleOrSupersededByChange() + public async Task UpdatePublication_NoTitleOrSupersededByChange() { var theme = new Theme { @@ -2648,26 +2648,19 @@ public async Task ListLatestReleaseVersionsPaginated() [Fact] public async Task GetReleaseSeries() { - var legacyLinks = new List - { - new() - { - Id = Guid.NewGuid(), - LegacyLinkDescription = "legacy link 1", - LegacyLinkUrl = "https://test.com/1", - }, - }; + ReleaseSeriesItem legacyLink = _dataFixture.DefaultLegacyReleaseSeriesItem(); Publication publication = _dataFixture .DefaultPublication() - .WithReleases(ListOf( + .WithReleases([ _dataFixture .DefaultRelease(publishedVersions: 1, year: 2020), _dataFixture .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022))) - .WithLegacyLinks(legacyLinks) + .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022) + ]) + .WithLegacyLinks([legacyLink]) .WithTheme(_dataFixture.DefaultTheme()); var contentDbContextId = Guid.NewGuid().ToString(); @@ -2686,7 +2679,8 @@ public async Task GetReleaseSeries() Assert.Equal(4, viewModels.Count); - var expectedReleaseVersion1 = publication.ReleaseVersions.Single(rv => rv is { Year: 2022, Version: 1 }); + var expectedReleaseVersion1 = + publication.ReleaseVersions.Single(rv => rv is { Year: 2022, Version: 1 }); Assert.Equal(publication.ReleaseSeries[0].Id, viewModels[0].Id); Assert.False(viewModels[0].IsLegacyLink); Assert.Equal(expectedReleaseVersion1.Title, viewModels[0].Description); @@ -2696,7 +2690,8 @@ public async Task GetReleaseSeries() Assert.True(viewModels[0].IsPublished); Assert.Null(viewModels[0].LegacyLinkUrl); - var expectedReleaseVersion2 = publication.ReleaseVersions.Single(rv => rv is { Year: 2021, Version: 0 }); + var expectedReleaseVersion2 = + publication.ReleaseVersions.Single(rv => rv is { Year: 2021, Version: 0 }); Assert.Equal(publication.ReleaseSeries[1].Id, viewModels[1].Id); Assert.False(viewModels[1].IsLegacyLink); Assert.Equal(expectedReleaseVersion2.Title, viewModels[1].Description); @@ -2706,7 +2701,8 @@ public async Task GetReleaseSeries() Assert.False(viewModels[1].IsPublished); Assert.Null(viewModels[1].LegacyLinkUrl); - var expectedReleaseVersion3 = publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); + var expectedReleaseVersion3 = + publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); Assert.Equal(publication.ReleaseSeries[2].Id, viewModels[2].Id); Assert.False(viewModels[2].IsLegacyLink); Assert.Equal(expectedReleaseVersion3.Title, viewModels[2].Description); @@ -2718,12 +2714,12 @@ public async Task GetReleaseSeries() Assert.Equal(publication.ReleaseSeries[3].Id, viewModels[3].Id); Assert.True(viewModels[3].IsLegacyLink); - Assert.Equal(legacyLinks[0].LegacyLinkDescription, viewModels[3].Description); + Assert.Equal(legacyLink.LegacyLinkDescription, viewModels[3].Description); Assert.Null(viewModels[3].ReleaseId); Assert.Null(viewModels[3].ReleaseSlug); Assert.Null(viewModels[3].IsLatest); Assert.Null(viewModels[3].IsPublished); - Assert.Equal(legacyLinks[0].LegacyLinkUrl, viewModels[3].LegacyLinkUrl); + Assert.Equal(legacyLink.LegacyLinkUrl, viewModels[3].LegacyLinkUrl); } } @@ -2732,13 +2728,14 @@ public async Task GetReleaseSeries_NoLegacyLinks() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases(ListOf( + .WithReleases([ _dataFixture .DefaultRelease(publishedVersions: 1, year: 2020), _dataFixture .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022))) + .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022) + ]) .WithTheme(_dataFixture.DefaultTheme()); var contentDbContextId = Guid.NewGuid().ToString(); @@ -2757,7 +2754,8 @@ public async Task GetReleaseSeries_NoLegacyLinks() Assert.Equal(3, viewModels.Count); - var expectedReleaseVersion1 = publication.ReleaseVersions.Single(rv => rv is { Year: 2022, Version: 1 }); + var expectedReleaseVersion1 = + publication.ReleaseVersions.Single(rv => rv is { Year: 2022, Version: 1 }); Assert.Equal(publication.ReleaseSeries[0].Id, viewModels[0].Id); Assert.False(viewModels[0].IsLegacyLink); Assert.Equal(expectedReleaseVersion1.Title, viewModels[0].Description); @@ -2767,7 +2765,8 @@ public async Task GetReleaseSeries_NoLegacyLinks() Assert.True(viewModels[0].IsPublished); Assert.Null(viewModels[0].LegacyLinkUrl); - var expectedReleaseVersion2 = publication.ReleaseVersions.Single(rv => rv is { Year: 2021, Version: 0 }); + var expectedReleaseVersion2 = + publication.ReleaseVersions.Single(rv => rv is { Year: 2021, Version: 0 }); Assert.Equal(publication.ReleaseSeries[1].Id, viewModels[1].Id); Assert.False(viewModels[1].IsLegacyLink); Assert.Equal(expectedReleaseVersion2.Title, viewModels[1].Description); @@ -2777,7 +2776,8 @@ public async Task GetReleaseSeries_NoLegacyLinks() Assert.False(viewModels[1].IsPublished); Assert.Null(viewModels[1].LegacyLinkUrl); - var expectedReleaseVersion3 = publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); + var expectedReleaseVersion3 = + publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); Assert.Equal(publication.ReleaseSeries[2].Id, viewModels[2].Id); Assert.False(viewModels[2].IsLegacyLink); Assert.Equal(expectedReleaseVersion3.Title, viewModels[2].Description); @@ -2792,21 +2792,8 @@ public async Task GetReleaseSeries_NoLegacyLinks() [Fact] public async Task GetReleaseSeries_NoReleases() { - var legacyLinks = new List - { - new() - { - Id = Guid.NewGuid(), - LegacyLinkDescription = "legacy link 1", - LegacyLinkUrl = "https://test.com/1", - }, - new() - { - Id = Guid.NewGuid(), - LegacyLinkDescription = "legacy link 2", - LegacyLinkUrl = "https://test.com/2", - }, - }; + var legacyLinks = _dataFixture.DefaultLegacyReleaseSeriesItem() + .GenerateList(2); Publication publication = _dataFixture .DefaultPublication() @@ -2877,22 +2864,15 @@ public async Task GetReleaseSeries_Empty() [Fact] public async Task AddReleaseSeriesLegacyLink() { - var legacyLinks = new List - { - new() - { - Id = Guid.NewGuid(), - LegacyLinkDescription = "legacy link 1", - LegacyLinkUrl = "https://test.com/1", - }, - }; + ReleaseSeriesItem legacyLink = _dataFixture.DefaultLegacyReleaseSeriesItem(); Publication publication = _dataFixture .DefaultPublication() - .WithReleases(ListOf( + .WithReleases([ _dataFixture - .DefaultRelease(publishedVersions: 1, year: 2020))) - .WithLegacyLinks(legacyLinks) + .DefaultRelease(publishedVersions: 1, year: 2020) + ]) + .WithLegacyLinks([legacyLink]) .WithTheme(_dataFixture.DefaultTheme()); var contentDbContextId = Guid.NewGuid().ToString(); @@ -2920,11 +2900,14 @@ public async Task AddReleaseSeriesLegacyLink() Description = "New legacy link", Url = "https://test.com/new" }); + var viewModels = result.AssertRight(); + VerifyAllMocks(publicationCacheService); Assert.Equal(3, viewModels.Count); - var expectedReleaseVersion1 = publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); + var expectedReleaseVersion1 = + publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); Assert.Equal(publication.ReleaseSeries[0].Id, viewModels[0].Id); Assert.False(viewModels[0].IsLegacyLink); Assert.Equal(expectedReleaseVersion1.Title, viewModels[0].Description); @@ -2936,12 +2919,12 @@ public async Task AddReleaseSeriesLegacyLink() Assert.Equal(publication.ReleaseSeries[1].Id, viewModels[1].Id); Assert.True(viewModels[1].IsLegacyLink); - Assert.Equal(legacyLinks[0].LegacyLinkDescription, viewModels[1].Description); + Assert.Equal(legacyLink.LegacyLinkDescription, viewModels[1].Description); Assert.Null(viewModels[1].ReleaseId); Assert.Null(viewModels[1].ReleaseSlug); Assert.Null(viewModels[1].IsLatest); Assert.Null(viewModels[1].IsPublished); - Assert.Equal(legacyLinks[0].LegacyLinkUrl, viewModels[1].LegacyLinkUrl); + Assert.Equal(legacyLink.LegacyLinkUrl, viewModels[1].LegacyLinkUrl); Assert.True(viewModels[2].IsLegacyLink); Assert.Equal("New legacy link", viewModels[2].Description); @@ -2995,6 +2978,7 @@ public async Task AddReleaseSeriesLegacyLink_AddToEmptySeries() Description = "New legacy link", Url = "https://test.com/new" }); + VerifyAllMocks(publicationCacheService); var viewModels = result.AssertRight(); var newSeriesItem = Assert.Single(viewModels); @@ -3019,26 +3003,17 @@ public async Task AddReleaseSeriesLegacyLink_AddToEmptySeries() [Fact] public async Task UpdateReleaseSeries() { - var legacyLinks = new List - { - new() - { - Id = Guid.NewGuid(), - LegacyLinkDescription = "legacy link 1", - LegacyLinkUrl = "https://test.com/1", - }, - }; - Publication publication = _dataFixture .DefaultPublication() - .WithReleases(ListOf( + .WithReleases([ _dataFixture .DefaultRelease(publishedVersions: 1, year: 2020), _dataFixture .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022))) - .WithLegacyLinks(legacyLinks) + .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022) + ]) + .WithLegacyLinks([_dataFixture.DefaultLegacyReleaseSeriesItem()]) .WithTheme(_dataFixture.DefaultTheme()); var contentDbContextId = Guid.NewGuid().ToString(); @@ -3059,36 +3034,45 @@ public async Task UpdateReleaseSeries() contentDbContext, publicationCacheService: publicationCacheService.Object); - var expectedReleaseVersion2022 = publication.ReleaseVersions.Single(rv => rv is { Year: 2022, Version: 1 }); - var expectedReleaseVersion2021 = publication.ReleaseVersions.Single(rv => rv is { Year: 2021, Version: 0 }); - var expectedReleaseVersion2020 = publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); + var expectedReleaseVersion2022 = + publication.ReleaseVersions.Single(rv => rv is { Year: 2022, Version: 1 }); + var expectedReleaseVersion2021 = + publication.ReleaseVersions.Single(rv => rv is { Year: 2021, Version: 0 }); + var expectedReleaseVersion2020 = + publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); var result = await publicationService.UpdateReleaseSeries( publication.Id, - new List - { - new() + updatedReleaseSeriesItems: + [ + new ReleaseSeriesItemUpdateRequest { Id = Guid.NewGuid(), LegacyLinkDescription = "Legacy link new", LegacyLinkUrl = "https://test.com/new", }, - new() + + new ReleaseSeriesItemUpdateRequest { Id = Guid.NewGuid(), ReleaseId = expectedReleaseVersion2021.ReleaseId, }, - new() + + new ReleaseSeriesItemUpdateRequest { Id = Guid.NewGuid(), ReleaseId = expectedReleaseVersion2020.ReleaseId, }, - new() + + new ReleaseSeriesItemUpdateRequest { Id = Guid.NewGuid(), ReleaseId = expectedReleaseVersion2022.ReleaseId, - }, - }); + } + ]); + + VerifyAllMocks(publicationCacheService); + var viewModels = result.AssertRight(); Assert.Equal(4, viewModels.Count); @@ -3130,19 +3114,9 @@ public async Task UpdateReleaseSeries() [Fact] public async Task UpdateReleaseSeries_SetEmpty() { - var legacyLinks = new List - { - new() - { - Id = Guid.NewGuid(), - LegacyLinkDescription = "legacy link 1", - LegacyLinkUrl = "https://test.com/1", - }, - }; - Publication publication = _dataFixture .DefaultPublication() - .WithLegacyLinks(legacyLinks) + .WithLegacyLinks([_dataFixture.DefaultLegacyReleaseSeriesItem()]) .WithTheme(_dataFixture.DefaultTheme()); var contentDbContextId = Guid.NewGuid().ToString(); @@ -3165,7 +3139,10 @@ public async Task UpdateReleaseSeries_SetEmpty() var result = await publicationService.UpdateReleaseSeries( publication.Id, - new List()); + updatedReleaseSeriesItems: []); + + VerifyAllMocks(publicationCacheService); + var viewModels = result.AssertRight(); Assert.Empty(viewModels); @@ -3177,9 +3154,7 @@ public async Task UpdateReleaseSeries_UnsetRelease() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases(ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 1, year: 2020))) + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2020)]) .WithTheme(_dataFixture.DefaultTheme()); var contentDbContextId = Guid.NewGuid().ToString(); @@ -3200,10 +3175,16 @@ public async Task UpdateReleaseSeries_UnsetRelease() contentDbContext, publicationCacheService: publicationCacheService.Object); - var exception = await Assert.ThrowsAsync(() => publicationService.UpdateReleaseSeries( - publication.Id, - new List())); - Assert.Equal("Missing or duplicate release in new release series. Expected ReleaseIds: " + publication.ReleaseVersions[0].ReleaseId, exception.Message); + var exception = await Assert.ThrowsAsync(() => + publicationService.UpdateReleaseSeries( + publication.Id, + updatedReleaseSeriesItems: [])); + + VerifyAllMocks(publicationCacheService); + + Assert.Equal("Missing or duplicate release in new release series. Expected ReleaseIds: " + + publication.ReleaseVersions[0].ReleaseId, + exception.Message); } } @@ -3212,9 +3193,7 @@ public async Task UpdateReleaseSeries_SetDuplicateRelease() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases(ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 1, year: 2020))) + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2020)]) .WithTheme(_dataFixture.DefaultTheme()); var contentDbContextId = Guid.NewGuid().ToString(); @@ -3235,22 +3214,29 @@ public async Task UpdateReleaseSeries_SetDuplicateRelease() contentDbContext, publicationCacheService: publicationCacheService.Object); - var exception = await Assert.ThrowsAsync(() => publicationService.UpdateReleaseSeries( - publication.Id, - new List - { - new() - { - Id = Guid.NewGuid(), - ReleaseId = publication.ReleaseVersions[0].ReleaseId, - }, - new() - { - Id = Guid.NewGuid(), - ReleaseId = publication.ReleaseVersions[0].ReleaseId, - }, - })); - Assert.Equal("Missing or duplicate release in new release series. Expected ReleaseIds: " + publication.ReleaseVersions[0].ReleaseId, exception.Message); + var exception = await Assert.ThrowsAsync(() => + publicationService.UpdateReleaseSeries( + publication.Id, + updatedReleaseSeriesItems: + [ + new ReleaseSeriesItemUpdateRequest + { + Id = Guid.NewGuid(), + ReleaseId = publication.ReleaseVersions[0].ReleaseId, + }, + + new ReleaseSeriesItemUpdateRequest + { + Id = Guid.NewGuid(), + ReleaseId = publication.ReleaseVersions[0].ReleaseId, + } + ])); + + VerifyAllMocks(publicationCacheService); + + Assert.Equal("Missing or duplicate release in new release series. Expected ReleaseIds: " + + publication.ReleaseVersions[0].ReleaseId, + exception.Message); } } @@ -3259,9 +3245,7 @@ public async Task UpdateReleaseSeries_InvalidSeriesItem1() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases(ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 1, year: 2020))) + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2020)]) .WithTheme(_dataFixture.DefaultTheme()); var contentDbContextId = Guid.NewGuid().ToString(); @@ -3282,20 +3266,24 @@ public async Task UpdateReleaseSeries_InvalidSeriesItem1() contentDbContext, publicationCacheService: publicationCacheService.Object); - var seriesItemId = Guid.NewGuid(); - var exception = await Assert.ThrowsAsync(() => publicationService.UpdateReleaseSeries( - publication.Id, - new List - { - new() - { - Id = seriesItemId, - ReleaseId = publication.ReleaseVersions[0].ReleaseId, - LegacyLinkDescription = "this should be null", - LegacyLinkUrl = "https://should.be/null", - }, - })); - Assert.Equal($"LegacyLink details shouldn't be set if ReleaseId is set. ReleaseSeriesItem: {seriesItemId}", exception.Message); + var exception = await Assert.ThrowsAsync(() => + publicationService.UpdateReleaseSeries( + publication.Id, + [ + new ReleaseSeriesItemUpdateRequest + { + Id = Guid.NewGuid(), + ReleaseId = publication.ReleaseVersions[0].ReleaseId, + LegacyLinkDescription = "this should be null", + LegacyLinkUrl = "https://should.be/null", + } + ])); + + VerifyAllMocks(publicationCacheService); + + Assert.Equal( + $"LegacyLink details shouldn't be set if ReleaseId is set. ReleaseSeriesItem: {Guid.NewGuid()}", + exception.Message); } } @@ -3324,20 +3312,24 @@ public async Task UpdateReleaseSeries_InvalidSeriesItem2() contentDbContext, publicationCacheService: publicationCacheService.Object); - var seriesItemId = Guid.NewGuid(); - var exception = await Assert.ThrowsAsync(() => publicationService.UpdateReleaseSeries( - publication.Id, - new List - { - new() - { - Id = seriesItemId, - ReleaseId = null, - LegacyLinkDescription = null, - LegacyLinkUrl = null, - }, - })); - Assert.Equal($"LegacyLink details should be set if ReleaseId is null. ReleaseSeriesItem: {seriesItemId}", exception.Message); + var exception = await Assert.ThrowsAsync(() => + publicationService.UpdateReleaseSeries( + publication.Id, + [ + new ReleaseSeriesItemUpdateRequest + { + Id = Guid.NewGuid(), + ReleaseId = null, + LegacyLinkDescription = null, + LegacyLinkUrl = null, + } + ])); + + VerifyAllMocks(publicationCacheService); + + Assert.Equal( + $"LegacyLink details should be set if ReleaseId is null. ReleaseSeriesItem: {Guid.NewGuid()}", + exception.Message); } } From b6e1c332cfe5c5f2ef2c85e87f7bab1acd7a3541 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Tue, 3 Dec 2024 17:15:39 +0000 Subject: [PATCH 077/144] EES-5656 Further release series tidy up --- .../PublicationServicePermissionTests.cs | 7 +- .../Services/PublicationServiceTests.cs | 304 ++++++------------ .../Requests/ReleaseSeriesItemRequests.cs | 21 +- .../ManageContent/ManageContentPageService.cs | 5 - .../Services/PublicationService.cs | 52 +-- .../Services/ReleaseService.cs | 2 +- .../ReleaseSeriesTableEntryViewModel.cs | 18 +- .../ViewModels/ReleaseSeriesItemViewModel.cs | 12 +- .../ReleaseSeriesItem.cs | 3 + .../Cache/PublicationCacheServiceTests.cs | 10 +- .../PublicationServiceTests.cs | 1 - .../PublicationService.cs | 5 - ...icationEditReleaseSeriesLegacyLinkPage.tsx | 1 - ...onEditReleaseSeriesLegacyLinkPage.test.tsx | 20 +- .../content/components/ReleaseContent.tsx | 21 +- .../page-view/PrototypeReleaseContent.tsx | 21 +- .../src/services/publicationService.ts | 2 +- .../generators/releaseContentGenerators.ts | 1 - .../src/services/publicationService.ts | 1 - .../PublicationReleasePage.tsx | 21 +- .../__tests__/__data__/testReleaseData.ts | 5 - 21 files changed, 208 insertions(+), 325 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServicePermissionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServicePermissionTests.cs index 215d0cc265c..75477c2195e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServicePermissionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServicePermissionTests.cs @@ -591,7 +591,12 @@ await PermissionTestUtils.PolicyCheckBuilder() var service = BuildPublicationService( context: contentDbContext, userService: userService.Object); - return await service.AddReleaseSeriesLegacyLink(publication.Id, new ReleaseSeriesLegacyLinkAddRequest()); + return await service.AddReleaseSeriesLegacyLink(publication.Id, + new ReleaseSeriesLegacyLinkAddRequest + { + Description = "Test description", + Url = "https://test.url" + }); }); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs index 8bfcfd2c321..4841cb8ce96 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs @@ -2663,6 +2663,10 @@ public async Task GetReleaseSeries() .WithLegacyLinks([legacyLink]) .WithTheme(_dataFixture.DefaultTheme()); + var release2020 = publication.Releases.Single(r => r.Year == 2020); + var release2021 = publication.Releases.Single(r => r.Year == 2021); + var release2022 = publication.Releases.Single(r => r.Year == 2022); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { @@ -2679,40 +2683,30 @@ public async Task GetReleaseSeries() Assert.Equal(4, viewModels.Count); - var expectedReleaseVersion1 = - publication.ReleaseVersions.Single(rv => rv is { Year: 2022, Version: 1 }); - Assert.Equal(publication.ReleaseSeries[0].Id, viewModels[0].Id); Assert.False(viewModels[0].IsLegacyLink); - Assert.Equal(expectedReleaseVersion1.Title, viewModels[0].Description); - Assert.Equal(expectedReleaseVersion1.ReleaseId, viewModels[0].ReleaseId); - Assert.Equal(expectedReleaseVersion1.Slug, viewModels[0].ReleaseSlug); + Assert.Equal(release2022.Title, viewModels[0].Description); + Assert.Equal(release2022.Id, viewModels[0].ReleaseId); + Assert.Equal(release2022.Slug, viewModels[0].ReleaseSlug); Assert.True(viewModels[0].IsLatest); Assert.True(viewModels[0].IsPublished); Assert.Null(viewModels[0].LegacyLinkUrl); - var expectedReleaseVersion2 = - publication.ReleaseVersions.Single(rv => rv is { Year: 2021, Version: 0 }); - Assert.Equal(publication.ReleaseSeries[1].Id, viewModels[1].Id); Assert.False(viewModels[1].IsLegacyLink); - Assert.Equal(expectedReleaseVersion2.Title, viewModels[1].Description); - Assert.Equal(expectedReleaseVersion2.ReleaseId, viewModels[1].ReleaseId); - Assert.Equal(expectedReleaseVersion2.Slug, viewModels[1].ReleaseSlug); + Assert.Equal(release2021.Title, viewModels[1].Description); + Assert.Equal(release2021.Id, viewModels[1].ReleaseId); + Assert.Equal(release2021.Slug, viewModels[1].ReleaseSlug); Assert.False(viewModels[1].IsLatest); Assert.False(viewModels[1].IsPublished); Assert.Null(viewModels[1].LegacyLinkUrl); - var expectedReleaseVersion3 = - publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); - Assert.Equal(publication.ReleaseSeries[2].Id, viewModels[2].Id); Assert.False(viewModels[2].IsLegacyLink); - Assert.Equal(expectedReleaseVersion3.Title, viewModels[2].Description); - Assert.Equal(expectedReleaseVersion3.ReleaseId, viewModels[2].ReleaseId); - Assert.Equal(expectedReleaseVersion3.Slug, viewModels[2].ReleaseSlug); + Assert.Equal(release2020.Title, viewModels[2].Description); + Assert.Equal(release2020.Id, viewModels[2].ReleaseId); + Assert.Equal(release2020.Slug, viewModels[2].ReleaseSlug); Assert.False(viewModels[2].IsLatest); Assert.True(viewModels[2].IsPublished); Assert.Null(viewModels[2].LegacyLinkUrl); - Assert.Equal(publication.ReleaseSeries[3].Id, viewModels[3].Id); Assert.True(viewModels[3].IsLegacyLink); Assert.Equal(legacyLink.LegacyLinkDescription, viewModels[3].Description); Assert.Null(viewModels[3].ReleaseId); @@ -2723,72 +2717,6 @@ public async Task GetReleaseSeries() } } - [Fact] - public async Task GetReleaseSeries_NoLegacyLinks() - { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases([ - _dataFixture - .DefaultRelease(publishedVersions: 1, year: 2020), - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022) - ]) - .WithTheme(_dataFixture.DefaultTheme()); - - var contentDbContextId = Guid.NewGuid().ToString(); - await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) - { - contentDbContext.Publications.Add(publication); - await contentDbContext.SaveChangesAsync(); - } - - await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) - { - var publicationService = BuildPublicationService(contentDbContext); - - var result = await publicationService.GetReleaseSeries(publication.Id); - var viewModels = result.AssertRight(); - - Assert.Equal(3, viewModels.Count); - - var expectedReleaseVersion1 = - publication.ReleaseVersions.Single(rv => rv is { Year: 2022, Version: 1 }); - Assert.Equal(publication.ReleaseSeries[0].Id, viewModels[0].Id); - Assert.False(viewModels[0].IsLegacyLink); - Assert.Equal(expectedReleaseVersion1.Title, viewModels[0].Description); - Assert.Equal(expectedReleaseVersion1.ReleaseId, viewModels[0].ReleaseId); - Assert.Equal(expectedReleaseVersion1.Slug, viewModels[0].ReleaseSlug); - Assert.True(viewModels[0].IsLatest); - Assert.True(viewModels[0].IsPublished); - Assert.Null(viewModels[0].LegacyLinkUrl); - - var expectedReleaseVersion2 = - publication.ReleaseVersions.Single(rv => rv is { Year: 2021, Version: 0 }); - Assert.Equal(publication.ReleaseSeries[1].Id, viewModels[1].Id); - Assert.False(viewModels[1].IsLegacyLink); - Assert.Equal(expectedReleaseVersion2.Title, viewModels[1].Description); - Assert.Equal(expectedReleaseVersion2.ReleaseId, viewModels[1].ReleaseId); - Assert.Equal(expectedReleaseVersion2.Slug, viewModels[1].ReleaseSlug); - Assert.False(viewModels[1].IsLatest); - Assert.False(viewModels[1].IsPublished); - Assert.Null(viewModels[1].LegacyLinkUrl); - - var expectedReleaseVersion3 = - publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); - Assert.Equal(publication.ReleaseSeries[2].Id, viewModels[2].Id); - Assert.False(viewModels[2].IsLegacyLink); - Assert.Equal(expectedReleaseVersion3.Title, viewModels[2].Description); - Assert.Equal(expectedReleaseVersion3.ReleaseId, viewModels[2].ReleaseId); - Assert.Equal(expectedReleaseVersion3.Slug, viewModels[2].ReleaseSlug); - Assert.False(viewModels[2].IsLatest); - Assert.True(viewModels[2].IsPublished); - Assert.Null(viewModels[2].LegacyLinkUrl); - } - } - [Fact] public async Task GetReleaseSeries_NoReleases() { @@ -2816,7 +2744,6 @@ public async Task GetReleaseSeries_NoReleases() Assert.Equal(2, viewModels.Count); - Assert.Equal(publication.ReleaseSeries[0].Id, viewModels[0].Id); Assert.True(viewModels[0].IsLegacyLink); Assert.Equal(legacyLinks[0].LegacyLinkDescription, viewModels[0].Description); Assert.Null(viewModels[0].ReleaseId); @@ -2825,7 +2752,6 @@ public async Task GetReleaseSeries_NoReleases() Assert.Null(viewModels[0].IsPublished); Assert.Equal(legacyLinks[0].LegacyLinkUrl, viewModels[0].LegacyLinkUrl); - Assert.Equal(publication.ReleaseSeries[1].Id, viewModels[1].Id); Assert.True(viewModels[1].IsLegacyLink); Assert.Equal(legacyLinks[1].LegacyLinkDescription, viewModels[1].Description); Assert.Null(viewModels[1].ReleaseId); @@ -2868,13 +2794,12 @@ public async Task AddReleaseSeriesLegacyLink() Publication publication = _dataFixture .DefaultPublication() - .WithReleases([ - _dataFixture - .DefaultRelease(publishedVersions: 1, year: 2020) - ]) + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]) .WithLegacyLinks([legacyLink]) .WithTheme(_dataFixture.DefaultTheme()); + var release = publication.Releases.Single(); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { @@ -2906,18 +2831,14 @@ public async Task AddReleaseSeriesLegacyLink() Assert.Equal(3, viewModels.Count); - var expectedReleaseVersion1 = - publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); - Assert.Equal(publication.ReleaseSeries[0].Id, viewModels[0].Id); Assert.False(viewModels[0].IsLegacyLink); - Assert.Equal(expectedReleaseVersion1.Title, viewModels[0].Description); - Assert.Equal(expectedReleaseVersion1.ReleaseId, viewModels[0].ReleaseId); - Assert.Equal(expectedReleaseVersion1.Slug, viewModels[0].ReleaseSlug); + Assert.Equal(release.Title, viewModels[0].Description); + Assert.Equal(release.Id, viewModels[0].ReleaseId); + Assert.Equal(release.Slug, viewModels[0].ReleaseSlug); Assert.True(viewModels[0].IsLatest); Assert.True(viewModels[0].IsPublished); Assert.Null(viewModels[0].LegacyLinkUrl); - Assert.Equal(publication.ReleaseSeries[1].Id, viewModels[1].Id); Assert.True(viewModels[1].IsLegacyLink); Assert.Equal(legacyLink.LegacyLinkDescription, viewModels[1].Description); Assert.Null(viewModels[1].ReleaseId); @@ -2933,16 +2854,23 @@ public async Task AddReleaseSeriesLegacyLink() Assert.Null(viewModels[2].IsLatest); Assert.Null(viewModels[2].IsPublished); Assert.Equal("https://test.com/new", viewModels[2].LegacyLinkUrl); + } - var dbReleaseSeries = contentDbContext.Publications - .Where(p => p.Id == publication.Id) - .Select(p => p.ReleaseSeries) - .Single(); + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + var actualPublication = await contentDbContext.Publications + .SingleAsync(p => p.Id == publication.Id); - Assert.Equal(3, dbReleaseSeries.Count); - Assert.Equal(viewModels[0].Id, dbReleaseSeries[0].Id); - Assert.Equal(viewModels[1].Id, dbReleaseSeries[1].Id); - Assert.Equal(viewModels[2].Id, dbReleaseSeries[2].Id); + var actualReleaseSeries = actualPublication.ReleaseSeries; + Assert.Equal(3, actualReleaseSeries.Count); + + Assert.Equal(release.Id, actualReleaseSeries[0].ReleaseId); + + Assert.Equal(legacyLink.LegacyLinkDescription, actualReleaseSeries[1].LegacyLinkDescription); + Assert.Equal(legacyLink.LegacyLinkUrl, actualReleaseSeries[1].LegacyLinkUrl); + + Assert.Equal("New legacy link", actualReleaseSeries[2].LegacyLinkDescription); + Assert.Equal("https://test.com/new", actualReleaseSeries[2].LegacyLinkUrl); } } @@ -2978,7 +2906,9 @@ public async Task AddReleaseSeriesLegacyLink_AddToEmptySeries() Description = "New legacy link", Url = "https://test.com/new" }); + VerifyAllMocks(publicationCacheService); + var viewModels = result.AssertRight(); var newSeriesItem = Assert.Single(viewModels); @@ -2989,14 +2919,17 @@ public async Task AddReleaseSeriesLegacyLink_AddToEmptySeries() Assert.Null(newSeriesItem.IsLatest); Assert.Null(newSeriesItem.IsPublished); Assert.Equal("https://test.com/new", newSeriesItem.LegacyLinkUrl); + } + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + var actualPublication = await contentDbContext.Publications + .SingleAsync(p => p.Id == publication.Id); - var dbReleaseSeries = contentDbContext.Publications - .Where(p => p.Id == publication.Id) - .Select(p => p.ReleaseSeries) - .Single(); + var actualReleaseSeriesItem = Assert.Single(actualPublication.ReleaseSeries); - var dbSeriesItem = Assert.Single(dbReleaseSeries); - Assert.Equal(viewModels[0].Id, dbSeriesItem.Id); + Assert.Equal("New legacy link", actualReleaseSeriesItem.LegacyLinkDescription); + Assert.Equal("https://test.com/new", actualReleaseSeriesItem.LegacyLinkUrl); } } @@ -3016,6 +2949,10 @@ public async Task UpdateReleaseSeries() .WithLegacyLinks([_dataFixture.DefaultLegacyReleaseSeriesItem()]) .WithTheme(_dataFixture.DefaultTheme()); + var release2020 = publication.Releases.Single(r => r.Year == 2020); + var release2021 = publication.Releases.Single(r => r.Year == 2021); + var release2022 = publication.Releases.Single(r => r.Year == 2022); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { @@ -3034,40 +2971,26 @@ public async Task UpdateReleaseSeries() contentDbContext, publicationCacheService: publicationCacheService.Object); - var expectedReleaseVersion2022 = - publication.ReleaseVersions.Single(rv => rv is { Year: 2022, Version: 1 }); - var expectedReleaseVersion2021 = - publication.ReleaseVersions.Single(rv => rv is { Year: 2021, Version: 0 }); - var expectedReleaseVersion2020 = - publication.ReleaseVersions.Single(rv => rv is { Year: 2020, Version: 0 }); - var result = await publicationService.UpdateReleaseSeries( publication.Id, updatedReleaseSeriesItems: [ new ReleaseSeriesItemUpdateRequest { - Id = Guid.NewGuid(), LegacyLinkDescription = "Legacy link new", LegacyLinkUrl = "https://test.com/new", }, - new ReleaseSeriesItemUpdateRequest { - Id = Guid.NewGuid(), - ReleaseId = expectedReleaseVersion2021.ReleaseId, + ReleaseId = release2021.Id }, - new ReleaseSeriesItemUpdateRequest { - Id = Guid.NewGuid(), - ReleaseId = expectedReleaseVersion2020.ReleaseId, + ReleaseId = release2020.Id }, - new ReleaseSeriesItemUpdateRequest { - Id = Guid.NewGuid(), - ReleaseId = expectedReleaseVersion2022.ReleaseId, + ReleaseId = release2022.Id } ]); @@ -3086,29 +3009,45 @@ public async Task UpdateReleaseSeries() Assert.Equal("https://test.com/new", viewModels[0].LegacyLinkUrl); Assert.False(viewModels[1].IsLegacyLink); - Assert.Equal(expectedReleaseVersion2021.Title, viewModels[1].Description); - Assert.Equal(expectedReleaseVersion2021.ReleaseId, viewModels[1].ReleaseId); - Assert.Equal(expectedReleaseVersion2021.Slug, viewModels[1].ReleaseSlug); + Assert.Equal(release2021.Title, viewModels[1].Description); + Assert.Equal(release2021.Id, viewModels[1].ReleaseId); + Assert.Equal(release2021.Slug, viewModels[1].ReleaseSlug); Assert.False(viewModels[1].IsLatest); Assert.False(viewModels[1].IsPublished); Assert.Null(viewModels[1].LegacyLinkUrl); Assert.False(viewModels[2].IsLegacyLink); - Assert.Equal(expectedReleaseVersion2020.Title, viewModels[2].Description); - Assert.Equal(expectedReleaseVersion2020.ReleaseId, viewModels[2].ReleaseId); - Assert.Equal(expectedReleaseVersion2020.Slug, viewModels[2].ReleaseSlug); + Assert.Equal(release2020.Title, viewModels[2].Description); + Assert.Equal(release2020.Id, viewModels[2].ReleaseId); + Assert.Equal(release2020.Slug, viewModels[2].ReleaseSlug); Assert.False(viewModels[2].IsLatest); Assert.True(viewModels[2].IsPublished); Assert.Null(viewModels[2].LegacyLinkUrl); Assert.False(viewModels[3].IsLegacyLink); - Assert.Equal(expectedReleaseVersion2022.Title, viewModels[3].Description); - Assert.Equal(expectedReleaseVersion2022.ReleaseId, viewModels[3].ReleaseId); - Assert.Equal(expectedReleaseVersion2022.Slug, viewModels[3].ReleaseSlug); + Assert.Equal(release2022.Title, viewModels[3].Description); + Assert.Equal(release2022.Id, viewModels[3].ReleaseId); + Assert.Equal(release2022.Slug, viewModels[3].ReleaseSlug); Assert.True(viewModels[3].IsLatest); Assert.True(viewModels[3].IsPublished); Assert.Null(viewModels[3].LegacyLinkUrl); } + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + var actualPublication = await contentDbContext.Publications + .SingleAsync(p => p.Id == publication.Id); + + var actualReleaseSeries = actualPublication.ReleaseSeries; + Assert.Equal(4, actualReleaseSeries.Count); + + Assert.Equal("Legacy link new", actualReleaseSeries[0].LegacyLinkDescription); + Assert.Equal("https://test.com/new", actualReleaseSeries[0].LegacyLinkUrl); + + Assert.Equal(release2021.Id, actualReleaseSeries[1].ReleaseId); + Assert.Equal(release2020.Id, actualReleaseSeries[2].ReleaseId); + Assert.Equal(release2022.Id, actualReleaseSeries[3].ReleaseId); + } } [Fact] @@ -3147,6 +3086,14 @@ public async Task UpdateReleaseSeries_SetEmpty() Assert.Empty(viewModels); } + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + var actualPublication = await contentDbContext.Publications + .SingleAsync(p => p.Id == publication.Id); + + Assert.Empty(actualPublication.ReleaseSeries); + } } [Fact] @@ -3154,9 +3101,11 @@ public async Task UpdateReleaseSeries_UnsetRelease() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2020)]) + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]) .WithTheme(_dataFixture.DefaultTheme()); + var release = publication.Releases.Single(); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { @@ -3166,24 +3115,14 @@ public async Task UpdateReleaseSeries_UnsetRelease() await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - var publicationCacheService = new Mock(Strict); - publicationCacheService.Setup(mock => - mock.UpdatePublication(publication.Slug)) - .ReturnsAsync(new PublicationCacheViewModel()); - - var publicationService = BuildPublicationService( - contentDbContext, - publicationCacheService: publicationCacheService.Object); + var publicationService = BuildPublicationService(contentDbContext); var exception = await Assert.ThrowsAsync(() => publicationService.UpdateReleaseSeries( publication.Id, updatedReleaseSeriesItems: [])); - VerifyAllMocks(publicationCacheService); - - Assert.Equal("Missing or duplicate release in new release series. Expected ReleaseIds: " + - publication.ReleaseVersions[0].ReleaseId, + Assert.Equal($"Missing or duplicate release in new release series. Expected ReleaseIds: {release.Id}", exception.Message); } } @@ -3193,9 +3132,11 @@ public async Task UpdateReleaseSeries_SetDuplicateRelease() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2020)]) + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]) .WithTheme(_dataFixture.DefaultTheme()); + var release = publication.Releases.Single(); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { @@ -3205,14 +3146,7 @@ public async Task UpdateReleaseSeries_SetDuplicateRelease() await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - var publicationCacheService = new Mock(Strict); - publicationCacheService.Setup(mock => - mock.UpdatePublication(publication.Slug)) - .ReturnsAsync(new PublicationCacheViewModel()); - - var publicationService = BuildPublicationService( - contentDbContext, - publicationCacheService: publicationCacheService.Object); + var publicationService = BuildPublicationService(contentDbContext); var exception = await Assert.ThrowsAsync(() => publicationService.UpdateReleaseSeries( @@ -3221,21 +3155,15 @@ public async Task UpdateReleaseSeries_SetDuplicateRelease() [ new ReleaseSeriesItemUpdateRequest { - Id = Guid.NewGuid(), - ReleaseId = publication.ReleaseVersions[0].ReleaseId, + ReleaseId = release.Id }, - new ReleaseSeriesItemUpdateRequest { - Id = Guid.NewGuid(), - ReleaseId = publication.ReleaseVersions[0].ReleaseId, + ReleaseId = release.Id } ])); - VerifyAllMocks(publicationCacheService); - - Assert.Equal("Missing or duplicate release in new release series. Expected ReleaseIds: " + - publication.ReleaseVersions[0].ReleaseId, + Assert.Equal($"Missing or duplicate release in new release series. Expected ReleaseIds: {release.Id}", exception.Message); } } @@ -3245,9 +3173,11 @@ public async Task UpdateReleaseSeries_InvalidSeriesItem1() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2020)]) + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]) .WithTheme(_dataFixture.DefaultTheme()); + var release = publication.Releases.Single(); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { @@ -3257,14 +3187,7 @@ public async Task UpdateReleaseSeries_InvalidSeriesItem1() await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - var publicationCacheService = new Mock(Strict); - publicationCacheService.Setup(mock => - mock.UpdatePublication(publication.Slug)) - .ReturnsAsync(new PublicationCacheViewModel()); - - var publicationService = BuildPublicationService( - contentDbContext, - publicationCacheService: publicationCacheService.Object); + var publicationService = BuildPublicationService(contentDbContext); var exception = await Assert.ThrowsAsync(() => publicationService.UpdateReleaseSeries( @@ -3272,18 +3195,13 @@ public async Task UpdateReleaseSeries_InvalidSeriesItem1() [ new ReleaseSeriesItemUpdateRequest { - Id = Guid.NewGuid(), - ReleaseId = publication.ReleaseVersions[0].ReleaseId, + ReleaseId = release.Id, LegacyLinkDescription = "this should be null", LegacyLinkUrl = "https://should.be/null", } ])); - VerifyAllMocks(publicationCacheService); - - Assert.Equal( - $"LegacyLink details shouldn't be set if ReleaseId is set. ReleaseSeriesItem: {Guid.NewGuid()}", - exception.Message); + Assert.Equal("LegacyLink details shouldn't be set if ReleaseId is set.", exception.Message); } } @@ -3303,14 +3221,7 @@ public async Task UpdateReleaseSeries_InvalidSeriesItem2() await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { - var publicationCacheService = new Mock(Strict); - publicationCacheService.Setup(mock => - mock.UpdatePublication(publication.Slug)) - .ReturnsAsync(new PublicationCacheViewModel()); - - var publicationService = BuildPublicationService( - contentDbContext, - publicationCacheService: publicationCacheService.Object); + var publicationService = BuildPublicationService(contentDbContext); var exception = await Assert.ThrowsAsync(() => publicationService.UpdateReleaseSeries( @@ -3318,18 +3229,13 @@ public async Task UpdateReleaseSeries_InvalidSeriesItem2() [ new ReleaseSeriesItemUpdateRequest { - Id = Guid.NewGuid(), ReleaseId = null, LegacyLinkDescription = null, LegacyLinkUrl = null, } ])); - VerifyAllMocks(publicationCacheService); - - Assert.Equal( - $"LegacyLink details should be set if ReleaseId is null. ReleaseSeriesItem: {Guid.NewGuid()}", - exception.Message); + Assert.Equal("LegacyLink details should be set if ReleaseId is null.", exception.Message); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseSeriesItemRequests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseSeriesItemRequests.cs index ffb39b58cbe..cd020aeaada 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseSeriesItemRequests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseSeriesItemRequests.cs @@ -1,21 +1,22 @@ #nullable enable - using System; +using System.ComponentModel.DataAnnotations; namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests; -public class ReleaseSeriesLegacyLinkAddRequest +public record ReleaseSeriesLegacyLinkAddRequest { - public string Description { get; set; } = string.Empty; - public string Url { get; set; } = string.Empty; + [Required] + public required string Description { get; init; } + + [Required] + public required string Url { get; init; } } -public class ReleaseSeriesItemUpdateRequest +public record ReleaseSeriesItemUpdateRequest { - public Guid Id { get; set; } - public Guid? ReleaseId { get; set; } + public Guid? ReleaseId { get; init; } - public string? LegacyLinkDescription { get; set; } - public string? LegacyLinkUrl { get; set; } + public string? LegacyLinkDescription { get; init; } + public string? LegacyLinkUrl { get; init; } } - diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ManageContent/ManageContentPageService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ManageContent/ManageContentPageService.cs index 56a0515af77..f6c6c038a9c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ManageContent/ManageContentPageService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ManageContent/ManageContentPageService.cs @@ -118,8 +118,6 @@ await _releaseVersionRepository { return new ReleaseSeriesItemViewModel { - Id = rsi.Id, - IsLegacyLink = rsi.IsLegacyLink, Description = rsi.LegacyLinkDescription!, LegacyLinkUrl = rsi.LegacyLinkUrl, }; @@ -130,10 +128,7 @@ await _releaseVersionRepository return new ReleaseSeriesItemViewModel { - Id = rsi.Id, - IsLegacyLink = rsi.IsLegacyLink, Description = latestReleaseVersion.Title, - ReleaseId = latestReleaseVersion.ReleaseId, ReleaseSlug = latestReleaseVersion.Slug, }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs index f3484a7d780..1341e018e56 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs @@ -19,7 +19,6 @@ using Microsoft.EntityFrameworkCore; using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; @@ -450,43 +449,28 @@ public async Task>> result.Add(new ReleaseSeriesTableEntryViewModel { Id = seriesItem.Id, - IsLegacyLink = true, Description = seriesItem.LegacyLinkDescription!, LegacyLinkUrl = seriesItem.LegacyLinkUrl, }); } else { - // prefer getting the latest published version over an unpublished amendment - var latestVersion = await _context.ReleaseVersions - .LatestReleaseVersion(seriesItem.ReleaseId!.Value, publishedOnly: true) - .SingleOrDefaultAsync(); + var release = await _context.Releases + .SingleAsync(r => r.Id == seriesItem.ReleaseId); - if (latestVersion == null) - { - // if the release has no published version, then use its original unpublished version - latestVersion = await _context.ReleaseVersions - .LatestReleaseVersion(seriesItem.ReleaseId!.Value) - .SingleOrDefaultAsync(); - - if (latestVersion == null) - { - throw new InvalidDataException( - "ReleaseSeriesItem with ReleaseId set should have an associated " + - $"ReleaseVersion. Release: {seriesItem.ReleaseId} " + - $"ReleaseSeriesItem: {seriesItem.Id}"); - } - } + var latestPublishedReleaseVersion = await _context.ReleaseVersions + .LatestReleaseVersion(releaseId: seriesItem.ReleaseId!.Value, publishedOnly: true) + .SingleOrDefaultAsync(); result.Add(new ReleaseSeriesTableEntryViewModel { Id = seriesItem.Id, - IsLegacyLink = false, - Description = latestVersion.Title, - ReleaseId = latestVersion.ReleaseId, - ReleaseSlug = latestVersion.Slug, - IsLatest = latestVersion.Id == publication.LatestPublishedReleaseVersionId, - IsPublished = latestVersion.Live, + ReleaseId = release.Id, + Description = release.Title, + ReleaseSlug = release.Slug, + IsLatest = publication.LatestPublishedReleaseVersionId != null && + latestPublishedReleaseVersion?.Id == publication.LatestPublishedReleaseVersionId, + IsPublished = latestPublishedReleaseVersion != null }); } } @@ -507,11 +491,11 @@ public async Task>> publication.ReleaseSeries.Add(new ReleaseSeriesItem { Id = Guid.NewGuid(), - ReleaseId = null, LegacyLinkDescription = newLegacyLink.Description, LegacyLinkUrl = newLegacyLink.Url, }); + _context.Publications.Update(publication); await _context.SaveChangesAsync(); await _publicationCacheService.UpdatePublication(publication.Slug); @@ -535,15 +519,13 @@ public async Task>> if (seriesItem.ReleaseId != null && ( seriesItem.LegacyLinkDescription != null || seriesItem.LegacyLinkUrl != null)) { - throw new ArgumentException( - $"LegacyLink details shouldn't be set if ReleaseId is set. ReleaseSeriesItem: {seriesItem.Id}"); + throw new ArgumentException("LegacyLink details shouldn't be set if ReleaseId is set."); } if (seriesItem.ReleaseId == null && ( seriesItem.LegacyLinkDescription == null || seriesItem.LegacyLinkUrl == null)) { - throw new ArgumentException( - $"LegacyLink details should be set if ReleaseId is null. ReleaseSeriesItem: {seriesItem.Id}"); + throw new ArgumentException("LegacyLink details should be set if ReleaseId is null."); } } @@ -562,15 +544,13 @@ public async Task>> { throw new ArgumentException( "Missing or duplicate release in new release series. Expected ReleaseIds: " + - publicationReleaseIds.Select(id => id.ToString()).JoinToString(",")); + publicationReleaseIds.JoinToString(",")); } - // NOTE: A malicious user could change the release series items' Ids, but we don't care - publication.ReleaseSeries = updatedReleaseSeriesItems .Select(request => new ReleaseSeriesItem { - Id = request.Id, + Id = Guid.NewGuid(), ReleaseId = request.ReleaseId, LegacyLinkDescription = request.LegacyLinkDescription, LegacyLinkUrl = request.LegacyLinkUrl, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs index abad7c72c1b..d969dbfac2c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ReleaseService.cs @@ -197,7 +197,7 @@ await CreateGenericContentFromTemplate(releaseCreate.TemplateReleaseId.Value, publication.ReleaseSeries.Insert(0, new ReleaseSeriesItem { Id = Guid.NewGuid(), - ReleaseId = newReleaseVersion.ReleaseId, + ReleaseId = release.Id }); _context.Publications.Update(publication); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ReleaseSeriesTableEntryViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ReleaseSeriesTableEntryViewModel.cs index 83eb3c842b2..7140ad73187 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ReleaseSeriesTableEntryViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ReleaseSeriesTableEntryViewModel.cs @@ -5,17 +5,17 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; public record ReleaseSeriesTableEntryViewModel { - public Guid Id { get; set; } - public bool IsLegacyLink { get; set; } - public string Description { get; set; } = string.Empty; + public required Guid Id { get; init; } + public required string Description { get; init; } // used by EES release series item - public Guid? ReleaseId { get; set; } - public string? ReleaseSlug { get; set; } - public bool? IsLatest { get; set; } - public bool? IsPublished { get; set; } + public Guid? ReleaseId { get; init; } + public string? ReleaseSlug { get; init; } + public bool? IsLatest { get; init; } + public bool? IsPublished { get; init; } // used by legacy link series item - public string? LegacyLinkUrl { get; set; } -} + public string? LegacyLinkUrl { get; init; } + public bool IsLegacyLink => ReleaseId == null; +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/ViewModels/ReleaseSeriesItemViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/ViewModels/ReleaseSeriesItemViewModel.cs index 2da3516d111..1d32d83477c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/ViewModels/ReleaseSeriesItemViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/ViewModels/ReleaseSeriesItemViewModel.cs @@ -5,14 +5,14 @@ namespace GovUk.Education.ExploreEducationStatistics.Common.ViewModels; public record ReleaseSeriesItemViewModel { - public Guid Id { get; set; } - public bool IsLegacyLink { get; set; } - public string Description { get; set; } = string.Empty; + public required string Description { get; init; } // used by EES release series item - public Guid? ReleaseId { get; set; } - public string? ReleaseSlug { get; set; } + public Guid? ReleaseId { get; init; } + public string? ReleaseSlug { get; init; } // used by legacy link series item - public string? LegacyLinkUrl { get; set; } + public string? LegacyLinkUrl { get; init; } + + public bool IsLegacyLink => ReleaseId == null; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseSeriesItem.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseSeriesItem.cs index e31cfc68dcf..75ca3662e94 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseSeriesItem.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseSeriesItem.cs @@ -6,6 +6,9 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Model; public record ReleaseSeriesItem { + /// + /// Unique identifier for the ReleaseSeriesItem which exists to allow safely managing legacy links in the UI. + /// public Guid Id { get; set; } public Guid? ReleaseId { get; set; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/Cache/PublicationCacheServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/Cache/PublicationCacheServiceTests.cs index 3cc4a6ee302..55bd9df9411 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/Cache/PublicationCacheServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/Cache/PublicationCacheServiceTests.cs @@ -52,16 +52,14 @@ public class PublicationCacheServiceTests : CacheServiceTestFixture Title = "" } }, - ReleaseSeries = new() - { - new() + ReleaseSeries = + [ + new ReleaseSeriesItemViewModel { - Id = Guid.NewGuid(), - IsLegacyLink = true, Description = "legacy link description", LegacyLinkUrl = "http://test.com/", } - }, + ], Theme = new ThemeViewModel( Guid.NewGuid(), Slug: "", diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/PublicationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/PublicationServiceTests.cs index bc28e6d61ed..6821c31bde4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/PublicationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/PublicationServiceTests.cs @@ -205,7 +205,6 @@ public async Task Success() Assert.Null(releaseSeriesItem2.LegacyLinkUrl); var releaseSeriesItem3 = publicationViewModel.ReleaseSeries[2]; - Assert.Equal(_legacyLinks[0].Id, releaseSeriesItem3.Id); Assert.True(releaseSeriesItem3.IsLegacyLink); Assert.Null(releaseSeriesItem3.ReleaseId); Assert.Equal(_legacyLinks[0].LegacyLinkDescription, releaseSeriesItem3.Description); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs index 5e298c79739..7792a5f2b93 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs @@ -100,8 +100,6 @@ public async Task> Get(string pu { return new ReleaseSeriesItemViewModel { - Id = rsi.Id, - IsLegacyLink = rsi.IsLegacyLink, Description = rsi.LegacyLinkDescription!, LegacyLinkUrl = rsi.LegacyLinkUrl, }; @@ -112,10 +110,7 @@ public async Task> Get(string pu return new ReleaseSeriesItemViewModel { - Id = rsi.Id, - IsLegacyLink = rsi.IsLegacyLink, Description = latestReleaseVersion.Title, - ReleaseId = latestReleaseVersion.ReleaseId, ReleaseSlug = latestReleaseVersion.Slug, }; diff --git a/src/explore-education-statistics-admin/src/pages/publication/PublicationEditReleaseSeriesLegacyLinkPage.tsx b/src/explore-education-statistics-admin/src/pages/publication/PublicationEditReleaseSeriesLegacyLinkPage.tsx index 1c9e437e492..31d990847c0 100644 --- a/src/explore-education-statistics-admin/src/pages/publication/PublicationEditReleaseSeriesLegacyLinkPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/publication/PublicationEditReleaseSeriesLegacyLinkPage.tsx @@ -19,7 +19,6 @@ export const mapToReleaseSeriesItemUpdateRequest = ( releaseSeries: ReleaseSeriesTableEntry[], ): ReleaseSeriesItemUpdateRequest[] => { return releaseSeries.map(seriesItem => ({ - id: seriesItem.id, releaseId: seriesItem.releaseId, legacyLinkDescription: seriesItem.isLegacyLink ? seriesItem.description diff --git a/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationEditReleaseSeriesLegacyLinkPage.test.tsx b/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationEditReleaseSeriesLegacyLinkPage.test.tsx index 9267147916c..5be827048c1 100644 --- a/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationEditReleaseSeriesLegacyLinkPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/publication/__tests__/PublicationEditReleaseSeriesLegacyLinkPage.test.tsx @@ -25,24 +25,24 @@ const publicationService = _publicationService as jest.Mocked< describe('PublicationEditReleaseSeriesLegacyLinkPage', () => { const releaseSeries: ReleaseSeriesTableEntry[] = [ { - id: 'legacy-release-1', + id: 'release-series-item-1', isLegacyLink: true, description: 'Legacy link 1', legacyLinkUrl: 'https://gov.uk/1', }, { - id: 'release-1', + id: 'release-series-item-2', isLegacyLink: false, description: 'Academic Year 2000/01', - releaseId: 'release-parent-1', + releaseId: 'release-1', releaseSlug: 'release-slug', isLatest: true, isPublished: true, }, { - id: 'legacy-release-2', + id: 'release-series-item-3', isLegacyLink: true, description: 'Legacy link 2', @@ -55,7 +55,7 @@ describe('PublicationEditReleaseSeriesLegacyLinkPage', () => { }); test('renders the edit legacy release page', async () => { - renderPage(testPublication, 'legacy-release-1'); + renderPage(testPublication, 'release-series-item-1'); await waitFor(() => { expect(screen.getByText('Edit legacy release')).toBeInTheDocument(); @@ -72,7 +72,7 @@ describe('PublicationEditReleaseSeriesLegacyLinkPage', () => { }); test('handles successfully submitting the form', async () => { - renderPage(testPublication, 'legacy-release-1'); + renderPage(testPublication, 'release-series-item-1'); await waitFor(() => { expect(screen.getByText('Edit legacy release')).toBeInTheDocument(); }); @@ -90,19 +90,13 @@ describe('PublicationEditReleaseSeriesLegacyLinkPage', () => { 'publication-1', [ { - id: 'legacy-release-1', - legacyLinkDescription: 'Legacy link 1 edited', legacyLinkUrl: 'https://gov.uk/1/edit', }, { - id: 'release-1', - - releaseId: 'release-parent-1', + releaseId: 'release-1', }, { - id: 'legacy-release-2', - legacyLinkDescription: 'Legacy link 2', legacyLinkUrl: 'https://gov.uk/2', }, diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/ReleaseContent.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/ReleaseContent.tsx index 7d766922ba8..945c68dfc14 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/ReleaseContent.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/ReleaseContent.tsx @@ -346,14 +346,19 @@ const ReleaseContent = ({
      {[ ...releaseSeries.map( - ({ - id, - isLegacyLink, - description, - legacyLinkUrl, - releaseSlug, - }) => ( -
    • + ( + { + isLegacyLink, + description, + legacyLinkUrl, + releaseSlug, + }, + index, + ) => ( +
    • {isLegacyLink ? ( {description} ) : ( diff --git a/src/explore-education-statistics-admin/src/prototypes/page-view/PrototypeReleaseContent.tsx b/src/explore-education-statistics-admin/src/prototypes/page-view/PrototypeReleaseContent.tsx index 13cd6458699..aea585bcebb 100644 --- a/src/explore-education-statistics-admin/src/prototypes/page-view/PrototypeReleaseContent.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/page-view/PrototypeReleaseContent.tsx @@ -308,14 +308,19 @@ const PrototypeReleaseContent = ({
        {[ ...releaseSeries.map( - ({ - id, - isLegacyLink, - description, - legacyLinkUrl, - releaseSlug, - }) => ( -
      • + ( + { + isLegacyLink, + description, + legacyLinkUrl, + releaseSlug, + }, + index, + ) => ( +
      • {isLegacyLink ? ( {description} ) : ( diff --git a/src/explore-education-statistics-admin/src/services/publicationService.ts b/src/explore-education-statistics-admin/src/services/publicationService.ts index f20174ef199..71465d3190c 100644 --- a/src/explore-education-statistics-admin/src/services/publicationService.ts +++ b/src/explore-education-statistics-admin/src/services/publicationService.ts @@ -83,7 +83,6 @@ export interface ReleaseSeriesLegacyLinkAddRequest { } export interface ReleaseSeriesItemUpdateRequest { - id: string; releaseId?: string; legacyLinkDescription?: string; legacyLinkUrl?: string; @@ -97,6 +96,7 @@ export interface ListReleasesParams { } export interface ReleaseSeriesTableEntry extends ReleaseSeriesItem { + id: string; isLatest?: boolean; isPublished?: boolean; } diff --git a/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts b/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts index a8dc2067805..ab2c4eb9faa 100644 --- a/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts +++ b/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts @@ -95,7 +95,6 @@ const defaultPublication: Publication = { releases: [], releaseSeries: [ { - id: 'legacylink-id', isLegacyLink: true, description: 'legacy link 1', legacyLinkUrl: 'https://test.com/1', diff --git a/src/explore-education-statistics-common/src/services/publicationService.ts b/src/explore-education-statistics-common/src/services/publicationService.ts index a512cb26639..d32c4c8898d 100644 --- a/src/explore-education-statistics-common/src/services/publicationService.ts +++ b/src/explore-education-statistics-common/src/services/publicationService.ts @@ -39,7 +39,6 @@ export interface Publication { } export interface ReleaseSeriesItem { - id: string; isLegacyLink: boolean; description: string; releaseId?: string; diff --git a/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx b/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx index 5f6784870ff..ac3a4d19f6b 100644 --- a/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx +++ b/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx @@ -341,14 +341,19 @@ const PublicationReleasePage: NextPage = ({ release }) => {
          {[ ...releaseSeries.map( - ({ - id, - isLegacyLink, - description, - legacyLinkUrl, - releaseSlug, - }) => ( -
        • + ( + { + isLegacyLink, + description, + legacyLinkUrl, + releaseSlug, + }, + index, + ) => ( +
        • {isLegacyLink ? ( {description} ) : ( diff --git a/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/__data__/testReleaseData.ts b/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/__data__/testReleaseData.ts index b750540f349..c7c697c1db3 100644 --- a/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/__data__/testReleaseData.ts +++ b/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/__data__/testReleaseData.ts @@ -18,33 +18,28 @@ export const testPublication: Publication = { ], releaseSeries: [ { - id: 'release-2', isLegacyLink: false, description: 'Academic year 2018/19', releaseSlug: '2018-19', }, { - id: 'legacy-release-3', isLegacyLink: true, description: 'Academic year 2014/15', legacyLinkUrl: 'https://www.gov.uk/government/statistics/pupil-absence-in-schools-in-england-2014-to-2015', }, { - id: 'release-1', isLegacyLink: false, description: 'Academic year 2017/18', releaseSlug: '2017-18', }, { - id: 'legacy-release-2', isLegacyLink: true, description: 'Academic year 2013/14', legacyLinkUrl: 'https://www.gov.uk/government/statistics/pupil-absence-in-schools-in-england-2013-to-2014', }, { - id: 'legacy-release-1', isLegacyLink: true, description: 'Academic year 2012/13', legacyLinkUrl: From 504166c925bf221d0cb78c86c2c58b868dd087f6 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 9 Dec 2024 18:18:42 +0000 Subject: [PATCH 078/144] EES-5656 No need to use `RequiredAttribute` with nullable context enabled --- .../Requests/ReleaseSeriesItemRequests.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseSeriesItemRequests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseSeriesItemRequests.cs index cd020aeaada..33ca1bdb6e2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseSeriesItemRequests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseSeriesItemRequests.cs @@ -1,15 +1,12 @@ #nullable enable using System; -using System.ComponentModel.DataAnnotations; namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests; public record ReleaseSeriesLegacyLinkAddRequest { - [Required] public required string Description { get; init; } - [Required] public required string Url { get; init; } } From 2afd48d10ad012d8fca9c2de83825bfdfb582c04 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 22 Nov 2024 17:06:26 +0000 Subject: [PATCH 079/144] EES-5446 - removing unhealthy check for Data Processor staging slot to reduce noise during deployments. Prior to this we would see lots of FileNotFoundExceptions in the staging slot only during deploys. --- .../public-api/publicApiDataProcessor.bicep | 38 ++----------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 29a9a450634..59741bcaf46 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -6,6 +6,9 @@ param resourceNames ResourceNames @description('Specifies the location for all resources.') param location string +@description('Alert metric name prefix') +param metricsNamePrefix string + @description('The Application Insights key that is associated with this resource') param applicationInsightsKey string @@ -113,41 +116,6 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { } } -module functionAppHealthAlert '../../components/alerts/sites/healthAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.dataProcessor}HealthDeploy' - params: { - resourceNames: [resourceNames.publicApi.dataProcessor] - alertsGroupName: resourceNames.existingResources.alertsGroup - tagValues: tagValues - } -} - -module storageAccountAvailabilityAlerts '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.dataProcessor}StorageAvailabilityDeploy' - params: { - resourceNames: [ - dataProcessorFunctionAppModule.outputs.managementStorageAccountName - dataProcessorFunctionAppModule.outputs.slot1StorageAccountName - dataProcessorFunctionAppModule.outputs.slot2StorageAccountName - ] - alertsGroupName: resourceNames.existingResources.alertsGroup - tagValues: tagValues - } -} - -module fileServiceAvailabilityAlerts '../../components/alerts/fileServices/availabilityAlert.bicep' = if (deployAlerts) { - name: '${resourceNames.publicApi.dataProcessor}FsAvailabilityDeploy' - params: { - resourceNames: [ - dataProcessorFunctionAppModule.outputs.managementStorageAccountName - dataProcessorFunctionAppModule.outputs.slot1StorageAccountName - dataProcessorFunctionAppModule.outputs.slot2StorageAccountName - ] - alertsGroupName: resourceNames.existingResources.alertsGroup - tagValues: tagValues - } -} - output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath From af64f987e09016f0392ac8c9bbb603f76577214a Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 25 Nov 2024 16:12:39 +0000 Subject: [PATCH 080/144] EES-5446 - added StatusCheck endpoint to provide count of active orchestrations via HTTP call. Added script to Data Processor deploy tasks to poll until active orchestrations are all complete prior to initiating slot swap --- .../public-api/publicApiDataProcessor.bicep | 1 + .../ci/jobs/deploy-data-processor.yml | 32 ++++++++++++++++++ .../public-api/components/functionApp.bicep | 2 +- .../templates/public-api/main.bicep | 5 +-- .../ProcessorFunctionsIntegrationTest.cs | 1 + .../Functions/StatusCheckFunction.cs | 33 +++++++++++++++++++ 6 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 59741bcaf46..ef734a30278 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -119,3 +119,4 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath +output url string = dataProcessorFunctionAppModule.outputs.url diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index c55178694b8..1e7a08daca8 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -135,6 +135,38 @@ jobs: publicNetworkAccess=Disabled \ siteConfig.publicNetworkAccess=Disabled + - task: AzureCLI@2 + displayName: Wait for active orchestrations to complete + retryCountOnTaskFailure: 1 + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + + pollingTimeSeconds=5 + maxAttempts=24 + attempts=1 + + while [ "$attempts" -le "$maxAttempts" ]; do + + echo "Attempt number $attempts to check if the Data Processor is ready for deployment" + + activeOrchestrations=`curl -s $dataProcessorFunctionAppUrl/api/StatusCheck | jq -r '.activeOrchestrations != 0'` + + if [[ "$activeOrchestrations" == "false2" ]]; then + echo "No active orchestrations running on the Data Processor - slot swapping can proceed" + exit 0 + fi + + attempts=$((attempts + 1)) + + done + + echo "Timed out waiting for active Data processor orchestrations to complete." + exit 1 + - task: AzureCLI@2 displayName: Swap slots retryCountOnTaskFailure: 1 diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 4e274ad4cdf..a2ccd99f2da 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -363,6 +363,6 @@ module privateEndpointModule 'privateEndpoint.bicep' = if (privateEndpointSubnet } output functionAppName string = functionApp.name -output managementStorageAccountName string = sharedStorageAccountName +output url string = 'https://${functionApp.properties.defaultHostName}'output managementStorageAccountName string = sharedStorageAccountName output slot1StorageAccountName string = slot1StorageAccountName output slot2StorageAccountName string = slot2StorageAccountName diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 92c0cafd7e0..9137450fb3b 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -73,7 +73,7 @@ param deployContainerApp bool = true // TODO EES-5128 - Note that this has been added temporarily to avoid 10+ minute deploys where it appears that PSQL // will redeploy even if no changes exist in this deploy from the previous one. @description('Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys.') -param updatePsqlFlexibleServer bool = false +param deployPsqlFlexibleServer bool = false param deployAlerts bool = false @@ -208,7 +208,7 @@ module logAnalyticsWorkspaceModule 'application/shared/logAnalyticsWorkspace.bic } } -module postgreSqlServerModule 'application/shared/postgreSqlFlexibleServer.bicep' = if (updatePsqlFlexibleServer) { +module postgreSqlServerModule 'application/shared/postgreSqlFlexibleServer.bicep' = if (deployPsqlFlexibleServer) { name: 'postgreSqlFlexibleServerApplicationModuleDeploy' params: { location: location @@ -387,6 +387,7 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' output dataProcessorContentDbConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-contentdb' output dataProcessorPsqlConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-publicdatadb' output dataProcessorFunctionAppManagedIdentityClientId string = dataProcessorModule.outputs.managedIdentityClientId +output dataProcessorFunctionAppUrl string = dataProcessorModule.outputs.url output coreStorageConnectionStringSecretKey string = coreStorage.outputs.coreStorageConnectionStringSecretKey output keyVaultName string = resourceNames.existingResources.keyVault diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs index b15821ee4c5..b79e4c82673 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs @@ -301,6 +301,7 @@ protected override IEnumerable GetFunctionTypes() typeof(HandleProcessingFailureFunction), typeof(HealthCheckFunctions), typeof(BulkDeleteDataSetVersionsFunction), + typeof(StatusCheckFunction), ]; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs new file mode 100644 index 00000000000..1878f9defda --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs @@ -0,0 +1,33 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask.Client; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class StatusCheckFunction +{ + private static readonly OrchestrationQuery ActiveOrchestrationsQuery = new() + { + Statuses = new List + { + OrchestrationRuntimeStatus.Pending, + OrchestrationRuntimeStatus.Running + } + }; + + [Function("StatusCheck")] + [Produces("application/json")] + public static async Task StatusCheck( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")] +#pragma warning disable IDE0060 + HttpRequestMessage request, +#pragma warning restore IDE0060 + [DurableClient] DurableTaskClient client) + { + var activeOrchestrations = await client + .GetAllInstancesAsync(filter: ActiveOrchestrationsQuery) + .ToListAsync(); + + return new OkObjectResult(new { ActiveOrchestrations = activeOrchestrations.Count }); + } +} From 457d63a27cc79477266917fc18d458b77dc155f5 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 26 Nov 2024 10:19:24 +0000 Subject: [PATCH 081/144] EES-5446 - adding in support for allowing specific IP address ranges access to the Data Processor app. Temporarily allowing anonymous access and public internet access while trialling active orchestrations endpoint --- .../public-api/publicApiDataProcessor.bicep | 10 ++- .../public-api/ci/azure-pipelines.yml | 22 +++++-- .../ci/jobs/deploy-data-processor.yml | 66 +++++++++---------- .../templates/public-api/ci/stages/deploy.yml | 1 + .../public-api/components/functionApp.bicep | 23 +++++++ .../templates/public-api/main.bicep | 24 +++++-- .../templates/public-api/types.bicep | 1 + 7 files changed, 99 insertions(+), 48 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index ef734a30278..efd3861fa45 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -18,8 +18,11 @@ param dataProcessorFunctionAppExists bool @description('Specifies the Application (Client) Id of a pre-existing App Registration used to represent the Data Processor Function App.') param dataProcessorAppRegistrationClientId string -@description('Public API Storage : Firewall rules.') -param storageFirewallRules FirewallRule[] = [] +@description('The IP address ranges that can access the Data Processor storage accounts.') +param storageFirewallRules FirewallRule[] + +@description('The IP address ranges that can access the Data Processor Function App endpoints.') +param functionAppFirewallRules FirewallRule[] = [] @description('Whether to create or update Azure Monitor alerts during this deploy') param deployAlerts bool @@ -75,7 +78,8 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { applicationInsightsKey: applicationInsightsKey subnetId: outboundVnetSubnet.id privateEndpointSubnetId: inboundVnetSubnet.id - publicNetworkAccessEnabled: false + publicNetworkAccessEnabled: true + functionAppEndpointFirewallRules: functionAppFirewallRules entraIdAuthentication: { appRegistrationClientId: dataProcessorAppRegistrationClientId allowedClientIds: [ diff --git a/infrastructure/templates/public-api/ci/azure-pipelines.yml b/infrastructure/templates/public-api/ci/azure-pipelines.yml index f3d3a984229..09421669437 100644 --- a/infrastructure/templates/public-api/ci/azure-pipelines.yml +++ b/infrastructure/templates/public-api/ci/azure-pipelines.yml @@ -1,12 +1,18 @@ trigger: none parameters: - - name: deployContainerApp - displayName: Can we deploy the Container App yet? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added. - default: true - - name: updatePsqlFlexibleServer + - name: deploySharedPrivateDnsZones + displayName: Do the shared Private DNS Zones need creating or updating? + default: false + - name: deployPsqlFlexibleServer displayName: Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys. default: false + - name: deployContainerApp + displayName: Does the Public API Container App need creating or updating? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added. + default: true + - name: deployDataProcessor + displayName: Does the Data Processor need creating or updating? + default: true - name: deployAlerts displayName: Whether to create or update Azure Monitor alerts during this deploy. default: false @@ -41,10 +47,14 @@ variables: value: $[eq(variables['Build.SourceBranch'], 'refs/heads/master')] - name: vmImageName value: ubuntu-latest + - name: deploySharedPrivateDnsZones + value: ${{ parameters.deploySharedPrivateDnsZones }} + - name: deployPsqlFlexibleServer + value: ${{ parameters.deployPsqlFlexibleServer }} - name: deployContainerApp value: ${{ parameters.deployContainerApp }} - - name: updatePsqlFlexibleServer - value: ${{ parameters.updatePsqlFlexibleServer }} + - name: deployDataProcessor + value: ${{ parameters.deployDataProcessor }} - name: deployAlerts value: ${{ parameters.deployAlerts }} diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 1e7a08daca8..d75023e53b6 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -67,23 +67,23 @@ jobs: # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow # DevOps to deploy the Data Processor Function App without having to temporarily # make it publicly accessible. - - task: AzureCLI@2 - displayName: Temporarily enable public network access before deploy - retryCountOnTaskFailure: 1 - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e + # - task: AzureCLI@2 + # displayName: Temporarily enable public network access before deploy + # retryCountOnTaskFailure: 1 + # inputs: + # azureSubscription: ${{ parameters.serviceConnection }} + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # set -e - az functionapp update \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --slot staging \ - --set \ - publicNetworkAccess=Enabled \ - siteConfig.publicNetworkAccess=Enabled + # az functionapp update \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --slot staging \ + # --set \ + # publicNetworkAccess=Enabled \ + # siteConfig.publicNetworkAccess=Enabled # TODO EES-5128 # Retry deploying the Function App in order to allow the staging slot the time to @@ -116,24 +116,24 @@ jobs: # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow # DevOps to deploy the Data Processor Function App without having to temporarily # make it publicly accessible. - - task: AzureCLI@2 - displayName: Disable public network access after deploy - retryCountOnTaskFailure: 1 - condition: always() - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e + # - task: AzureCLI@2 + # displayName: Disable public network access after deploy + # retryCountOnTaskFailure: 1 + # condition: always() + # inputs: + # azureSubscription: ${{ parameters.serviceConnection }} + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # set -e - az functionapp update \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --slot staging \ - --set \ - publicNetworkAccess=Disabled \ - siteConfig.publicNetworkAccess=Disabled + # az functionapp update \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --slot staging \ + # --set \ + # publicNetworkAccess=Disabled \ + # siteConfig.publicNetworkAccess=Disabled - task: AzureCLI@2 displayName: Wait for active orchestrations to complete diff --git a/infrastructure/templates/public-api/ci/stages/deploy.yml b/infrastructure/templates/public-api/ci/stages/deploy.yml index 62adedfe7cc..b3f2f4a7b9c 100644 --- a/infrastructure/templates/public-api/ci/stages/deploy.yml +++ b/infrastructure/templates/public-api/ci/stages/deploy.yml @@ -60,4 +60,5 @@ stages: parameters: serviceConnection: ${{ parameters.serviceConnection }} environment: ${{ parameters.environment }} + condition: eq(variables.deployDataProcessor, true) dependsOn: DeployPublicApiInfrastructure diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index a2ccd99f2da..6363cff5017 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -36,6 +36,9 @@ param privateEndpointSubnetId string? @description('Specifies whether this Function App is accessible from the public internet') param publicNetworkAccessEnabled bool = false +@description('IP address ranges that are allowed to access the Function App endpoints. Dependent on "publicNetworkAccessEnabled" being true.') +param functionAppEndpointFirewallRules FirewallRule[] = [] + @description('An existing Managed Identity\'s Resource Id with which to associate this Function App') param userAssignedManagedIdentityParams { id: string @@ -172,6 +175,15 @@ resource slot2FileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2 ] } +var firewallRules = [for firewallRule in functionAppEndpointFirewallRules: { + ipAddress: firewallRule.cidr + action: 'Allow' + tag: 'Default' + priority: 100 + name: firewallRule.name + description: firewallRule.description +}] + var commonSiteProperties = { enabled: true httpsOnly: true @@ -192,6 +204,17 @@ var commonSiteProperties = { } keyVaultReferenceIdentity: keyVaultReferenceIdentity publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' + // ipSecurityRestrictions: firewallRules + ipSecurityRestrictionsDefaultAction: 'Deny' + scmIpSecurityRestrictions: [ + { + ipAddress: 'Any' + action: 'Allow' + priority: 2147483647 + name: 'Allow all' + description: 'Allow all access' + } + ] } // Create the main production deploy slot. diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 9137450fb3b..96a4820e5e3 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -67,6 +67,9 @@ param dateProvisioned string = utcNow('u') @description('The tags of the Docker images to deploy.') param dockerImagesTag string = '' +@description('Do the shared Private DNS Zones need creating or updating?') +param deploySharedPrivateDnsZones bool = false + @description('Can we deploy the Container App yet? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added.') param deployContainerApp bool = true @@ -170,7 +173,8 @@ module coreStorage 'application/shared/coreStorage.bicep' = { } } -module privateDnsZonesModule 'application/shared/privateDnsZones.bicep' = { +module privateDnsZonesModule 'application/shared/privateDnsZones.bicep' = + if (deploySharedPrivateDnsZones || deployPsqlFlexibleServer || deployDataProcessor) { name: 'privateDnsZonesApplicationModuleDeploy' params: { resourceNames: resourceNames @@ -366,11 +370,12 @@ module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContai } } -module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' = { +module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' = if (deployDataProcessor) { name: 'publicApiDataProcessorApplicationModuleDeploy' params: { location: location resourceNames: resourceNames + metricsNamePrefix: '${subscription}PublicDataProcessor' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId storageFirewallRules: storageFirewallRules @@ -386,12 +391,19 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' output dataProcessorContentDbConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-contentdb' output dataProcessorPsqlConnectionStringSecretKey string = 'ees-publicapi-data-processor-connectionstring-publicdatadb' -output dataProcessorFunctionAppManagedIdentityClientId string = dataProcessorModule.outputs.managedIdentityClientId -output dataProcessorFunctionAppUrl string = dataProcessorModule.outputs.url + +output dataProcessorFunctionAppManagedIdentityClientId string = deployDataProcessor + ? dataProcessorModule.outputs.managedIdentityClientId + : '' +output dataProcessorFunctionAppUrl string = deployDataProcessor + ? dataProcessorModule.outputs.url + : '' + +output dataProcessorPublicApiDataFileShareMountPath string = deployDataProcessor + ? dataProcessorModule.outputs.publicApiDataFileShareMountPath + : '' output coreStorageConnectionStringSecretKey string = coreStorage.outputs.coreStorageConnectionStringSecretKey output keyVaultName string = resourceNames.existingResources.keyVault -output dataProcessorPublicApiDataFileShareMountPath string = dataProcessorModule.outputs.publicApiDataFileShareMountPath - output enableThemeDeletion bool = enableThemeDeletion diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index fb5199aafa5..8dc60f8b9a6 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -43,6 +43,7 @@ type ResourceNames = { @export() type FirewallRule = { name: string + description: string? cidr: string } From c4b07e2cc0f126c879d777145c6f06edb853d3a5 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 26 Nov 2024 11:08:18 +0000 Subject: [PATCH 082/144] EES-5446 - added dummy long-running orchestration with which we can test the deploy delay of the Data Processor. --- .../public-api/publicApiDataProcessor.bicep | 2 +- .../ci/jobs/deploy-data-processor.yml | 114 +++++++++--------- .../ci/jobs/deploy-infrastructure.yml | 6 +- .../templates/public-api/ci/stages/deploy.yml | 1 - .../public-api/ci/tasks/deploy-bicep.yml | 12 +- .../public-api/components/functionApp.bicep | 28 +++-- .../components/siteAzureAuthentication.bicep | 2 +- .../templates/public-api/main.bicep | 4 +- .../Extensions/HttpRequestExtensions.cs | 9 ++ .../Functions/LogRunningTriggerFunction.cs | 49 ++++++++ .../Functions/LongRunningOrchestration.cs | 55 +++++++++ .../Model/LongRunningOrchestrationContext.cs | 6 + 12 files changed, 212 insertions(+), 76 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/LongRunningOrchestrationContext.cs diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index efd3861fa45..08fad8fa149 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -123,4 +123,4 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath -output url string = dataProcessorFunctionAppModule.outputs.url +output stagingUrl string = dataProcessorFunctionAppModule.outputs.stagingUrl diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index d75023e53b6..40f856fd682 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -10,15 +10,16 @@ parameters: jobs: - deployment: DeployPublicDataProcessor displayName: Deploy Public Data Processor + condition: and(succeeded(), eq(variables.deployDataProcessor, true)) dependsOn: ${{ parameters.dependsOn }} environment: ${{ parameters.environment }} strategy: runOnce: deploy: steps: - - download: MainBuild - displayName: Download Public API Data Processor artifact - artifact: public-api-data-processor + # - download: MainBuild + # displayName: Download Public API Data Processor artifact + # artifact: public-api-data-processor - template: ../tasks/bicep-output-variables.yml parameters: @@ -27,41 +28,41 @@ jobs: # We do config updates out of Bicep template so we can implement slot swapping. # Changes are first deployed to the staging slot and combined with a fresh # code deploy prior to being swapped with the production slot. - - task: AzureCLI@2 - displayName: Update staging slot app settings - retryCountOnTaskFailure: 1 - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e + # - task: AzureCLI@2 + # displayName: Update staging slot app settings + # retryCountOnTaskFailure: 1 + # inputs: + # azureSubscription: ${{ parameters.serviceConnection }} + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # set -e - az functionapp config appsettings set \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --slot staging \ - --settings \ - "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ - "App__EnableThemeDeletion=$(enableThemeDeletion)" \ - "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ - "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)" + # az functionapp config appsettings set \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --slot staging \ + # --settings \ + # "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ + # "App__EnableThemeDeletion=$(enableThemeDeletion)" \ + # "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ + # "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)" - az webapp config connection-string set \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --connection-string-type SQLAzure \ - --slot staging \ - --settings \ - "ContentDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorContentDbConnectionStringSecretKey))" + # az webapp config connection-string set \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --connection-string-type SQLAzure \ + # --slot staging \ + # --settings \ + # "ContentDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorContentDbConnectionStringSecretKey))" - az webapp config connection-string set \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --connection-string-type PostgreSQL \ - --slot staging \ - --settings \ - "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" + # az webapp config connection-string set \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --connection-string-type PostgreSQL \ + # --slot staging \ + # --settings \ + # "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" # TODO EES-5128 # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow @@ -97,20 +98,20 @@ jobs: # Client IDs / Identities that can access the Function App. The Service Principal # that is performing the deploy can be accessed by using the `addSpnToEnvironment` # config option in the task definition and using the $(servicePrincipalId) variable. - - task: AzureCLI@2 - displayName: Deploy to staging slot - retryCountOnTaskFailure: 10 - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -e - az functionapp deployment source config-zip \ - --src '$(Pipeline.Workspace)/MainBuild/public-api-data-processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.zip' \ - --name $(dataProcessorFunctionAppName) \ - --resource-group $(resourceGroupName) \ - --slot staging + # - task: AzureCLI@2 + # displayName: Deploy to staging slot + # retryCountOnTaskFailure: 10 + # inputs: + # azureSubscription: ${{ parameters.serviceConnection }} + # scriptType: bash + # scriptLocation: inlineScript + # inlineScript: | + # set -e + # az functionapp deployment source config-zip \ + # --src '$(Pipeline.Workspace)/MainBuild/public-api-data-processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.zip' \ + # --name $(dataProcessorFunctionAppName) \ + # --resource-group $(resourceGroupName) \ + # --slot staging # TODO EES-5128 # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow @@ -137,30 +138,35 @@ jobs: - task: AzureCLI@2 displayName: Wait for active orchestrations to complete - retryCountOnTaskFailure: 1 inputs: azureSubscription: ${{ parameters.serviceConnection }} scriptType: bash scriptLocation: inlineScript inlineScript: | set -e + set -x pollingTimeSeconds=5 - maxAttempts=24 + maxAttempts=50 attempts=1 while [ "$attempts" -le "$maxAttempts" ]; do echo "Attempt number $attempts to check if the Data Processor is ready for deployment" + echo "Calling $(dataProcessorFunctionAppStagingUrl)/api/StatusCheck to check for active orchestrations" - activeOrchestrations=`curl -s $dataProcessorFunctionAppUrl/api/StatusCheck | jq -r '.activeOrchestrations != 0'` - - if [[ "$activeOrchestrations" == "false2" ]]; then + activeOrchestrations=`curl -s $(dataProcessorFunctionAppStagingUrl)/api/StatusCheck | jq -r '.activeOrchestrations != 0'` + + echo "Orchestration results are $activeOrchestrations" + + if [[ "$activeOrchestrations" == "false" ]]; then echo "No active orchestrations running on the Data Processor - slot swapping can proceed" exit 0 fi attempts=$((attempts + 1)) + + sleep $pollingTimeSeconds done diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml b/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml index 4deeb28a247..541651a2296 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml @@ -35,8 +35,9 @@ jobs: action: validate serviceConnection: ${{ parameters.serviceConnection }} parameterFile: $(paramFile) + deploySharedPrivateDnsZones: false + deployPsqlFlexibleServer: false deployContainerApp: true - updatePsqlFlexibleServer: false deployAlerts: false dataProcessorExists: true @@ -62,8 +63,9 @@ jobs: action: create serviceConnection: ${{ parameters.serviceConnection }} parameterFile: $(paramFile) + deploySharedPrivateDnsZones: $(deploySharedPrivateDnsZones) + deployPsqlFlexibleServer: $(deployPsqlFlexibleServer) deployContainerApp: $(deployContainerApp) - updatePsqlFlexibleServer: $(updatePsqlFlexibleServer) deployAlerts: $(deployAlerts) dataProcessorExists: $(dataProcessorExists) diff --git a/infrastructure/templates/public-api/ci/stages/deploy.yml b/infrastructure/templates/public-api/ci/stages/deploy.yml index b3f2f4a7b9c..62adedfe7cc 100644 --- a/infrastructure/templates/public-api/ci/stages/deploy.yml +++ b/infrastructure/templates/public-api/ci/stages/deploy.yml @@ -60,5 +60,4 @@ stages: parameters: serviceConnection: ${{ parameters.serviceConnection }} environment: ${{ parameters.environment }} - condition: eq(variables.deployDataProcessor, true) dependsOn: DeployPublicApiInfrastructure diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index fa5ceeaa331..a4b5102cce2 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -11,14 +11,20 @@ parameters: type: string - name: parameterFile type: string + - name: deploySharedPrivateDnsZones + type: string + - name: deployPsqlFlexibleServer + type: string - name: deployContainerApp type: string - - name: updatePsqlFlexibleServer + default: true + - name: deployDataProcessor type: string - name: deployAlerts type: string - name: dataProcessorExists type: string + default: true steps: - task: AzureCLI@2 @@ -45,8 +51,10 @@ steps: storageFirewallRules='$(maintenanceFirewallRules)' \ acrResourceGroupName='$(acrResourceGroupName)' \ dockerImagesTag='$(resources.pipeline.MainBuild.runName)' \ + deploySharedPrivateDnsZones=${{ parameters.deploySharedPrivateDnsZones }} \ + deployPsqlFlexibleServer=${{ parameters.deployPsqlFlexibleServer }} \ deployContainerApp=${{ parameters.deployContainerApp }} \ - updatePsqlFlexibleServer=${{ parameters.updatePsqlFlexibleServer }} \ + deployDataProcessor=${{ parameters.deployDataProcessor }} \ deployAlerts=${{ parameters.deployAlerts }} \ dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \ dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \ diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 6363cff5017..af3cc7c910a 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -201,20 +201,20 @@ var commonSiteProperties = { netFrameworkVersion: '8.0' linuxFxVersion: appServicePlanOS == 'Linux' ? 'DOTNET-ISOLATED|8.0' : null keyVaultReferenceIdentity: keyVaultReferenceIdentity + publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' + // ipSecurityRestrictions: firewallRules + ipSecurityRestrictionsDefaultAction: 'Allow' // TODO Deny! + scmIpSecurityRestrictions: [ + { + ipAddress: 'Any' + action: 'Allow' + priority: 2147483647 + name: 'Allow all' + description: 'Allow all access' + } + ] } keyVaultReferenceIdentity: keyVaultReferenceIdentity - publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' - // ipSecurityRestrictions: firewallRules - ipSecurityRestrictionsDefaultAction: 'Deny' - scmIpSecurityRestrictions: [ - { - ipAddress: 'Any' - action: 'Allow' - priority: 2147483647 - name: 'Allow all' - description: 'Allow all access' - } - ] } // Create the main production deploy slot. @@ -386,6 +386,8 @@ module privateEndpointModule 'privateEndpoint.bicep' = if (privateEndpointSubnet } output functionAppName string = functionApp.name -output url string = 'https://${functionApp.properties.defaultHostName}'output managementStorageAccountName string = sharedStorageAccountName +output url string = 'https://${functionApp.properties.defaultHostName}' +output stagingUrl string = 'https://${functionApp.name}-staging.azurewebsites.net' +output managementStorageAccountName string = sharedStorageAccountName output slot1StorageAccountName string = slot1StorageAccountName output slot2StorageAccountName string = slot2StorageAccountName diff --git a/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep b/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep index 0bbce6d3f9a..6b1864737bf 100644 --- a/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep +++ b/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep @@ -19,7 +19,7 @@ param requireAuthentication bool = true var properties = { globalValidation: { requireAuthentication: requireAuthentication - unauthenticatedClientAction: requireAuthentication ? 'Return401' : null + unauthenticatedClientAction: requireAuthentication ? 'Return401' : 'AllowAnonymous' } httpSettings: { requireHttps: true diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 96a4820e5e3..b65b17d44ed 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -395,8 +395,8 @@ output dataProcessorPsqlConnectionStringSecretKey string = 'ees-publicapi-data-p output dataProcessorFunctionAppManagedIdentityClientId string = deployDataProcessor ? dataProcessorModule.outputs.managedIdentityClientId : '' -output dataProcessorFunctionAppUrl string = deployDataProcessor - ? dataProcessorModule.outputs.url +output dataProcessorFunctionAppStagingUrl string = deployDataProcessor + ? dataProcessorModule.outputs.stagingUrl : '' output dataProcessorPublicApiDataFileShareMountPath string = deployDataProcessor diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs index 97709252ed5..3d95e325919 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs @@ -114,4 +114,13 @@ public static bool GetRequestParamBool( var paramValue = GetRequestParam(httpRequest, paramName, defaultValue + ""); return bool.Parse(paramValue); } + + public static int GetRequestParamInt( + this HttpRequest httpRequest, + string paramName, + int defaultValue) + { + var paramValue = GetRequestParam(httpRequest, paramName, defaultValue + ""); + return int.Parse(paramValue); + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs new file mode 100644 index 00000000000..916f801a561 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs @@ -0,0 +1,49 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class LogRunningTriggerFunction( + ILogger logger) +{ + [Function(nameof(TriggerLongRunningOrchestration))] + public async Task TriggerLongRunningOrchestration( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = nameof(TriggerLongRunningOrchestration))] + HttpRequest httpRequest, + [DurableClient] DurableTaskClient client, + CancellationToken cancellationToken) + { + var instanceId = Guid.NewGuid(); + + var durationSeconds = + httpRequest.GetRequestParamInt(paramName: "durationSeconds", 60); + + const string orchestratorName = + nameof(LogRunningOrchestration.ProcessLongRunningOrchestration); + + var options = new StartOrchestrationOptions { InstanceId = instanceId.ToString() }; + + logger.LogInformation( + "Scheduling '{OrchestratorName}' (InstanceId={InstanceId}, DurationSeconds={DurationSeconds}))", + orchestratorName, + instanceId, + durationSeconds); + + await client.ScheduleNewOrchestrationInstanceAsync( + orchestratorName, + new LongRunningOrchestrationContext + { + DurationSeconds = durationSeconds + }, + options, + cancellationToken); + + return new OkResult(); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs new file mode 100644 index 00000000000..0e789ff8bf2 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs @@ -0,0 +1,55 @@ +using System.Diagnostics; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class LogRunningOrchestration(ILogger logger) +{ + [Function(nameof(ProcessLongRunningOrchestration))] + public static async Task ProcessLongRunningOrchestration( + [OrchestrationTrigger] TaskOrchestrationContext context, + LongRunningOrchestrationContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessLongRunningOrchestration)); + + logger.LogInformation( + "Processing long-running orchestration (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", + context.InstanceId, + input.DurationSeconds); + + try + { + await context.CallActivity(nameof(LongRunningActivity), logger, context.InstanceId); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", + context.InstanceId, + input.DurationSeconds); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } + + [Function(nameof(LongRunningActivity))] + public async Task LongRunningActivity( + [ActivityTrigger] Guid instanceId, + int durationSeconds, + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + + while (stopwatch.Elapsed.Seconds < durationSeconds) + { + await Task.Delay(10000, cancellationToken); + + logger.LogInformation($"Long-running orchestration running for {stopwatch.Elapsed.Seconds} " + + $"out of {durationSeconds} seconds"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/LongRunningOrchestrationContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/LongRunningOrchestrationContext.cs new file mode 100644 index 00000000000..40eb0ff82dc --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/LongRunningOrchestrationContext.cs @@ -0,0 +1,6 @@ +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; + +public record LongRunningOrchestrationContext +{ + public required int DurationSeconds { get; init; } +} From d2c5cc8e85c79d7d63d8bc9043d0261a7262d2eb Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 26 Nov 2024 15:58:59 +0000 Subject: [PATCH 083/144] EES-5446 - grant Azure DevOps SPN access to Data Processor via EasyAuth and granting of an access token --- .../public-api/publicApiDataProcessor.bicep | 11 +- .../ci/jobs/deploy-data-processor.yml | 160 +++++++++++------- .../templates/public-api/ci/stages/deploy.yml | 10 +- .../public-api/ci/tasks/deploy-bicep.yml | 6 +- .../public-api/components/functionApp.bicep | 2 +- .../components/siteAzureAuthentication.bicep | 11 +- .../templates/public-api/main.bicep | 9 + 7 files changed, 130 insertions(+), 79 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 08fad8fa149..a7218c96c6b 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -18,6 +18,10 @@ param dataProcessorFunctionAppExists bool @description('Specifies the Application (Client) Id of a pre-existing App Registration used to represent the Data Processor Function App.') param dataProcessorAppRegistrationClientId string +@description('Specifies the principal id of the Azure DevOps SPN.') +@secure() +param devopsServicePrincipalId string + @description('The IP address ranges that can access the Data Processor storage accounts.') param storageFirewallRules FirewallRule[] @@ -42,7 +46,6 @@ resource adminAppServiceIdentity 'Microsoft.ManagedIdentity/identities@2023-01-3 } var adminAppClientId = adminAppServiceIdentity.properties.clientId -var adminAppPrincipalId = adminAppServiceIdentity.properties.principalId resource publicApiStorageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' existing = { name: resourceNames.publicApi.publicApiStorageAccount @@ -84,10 +87,9 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { appRegistrationClientId: dataProcessorAppRegistrationClientId allowedClientIds: [ adminAppClientId + devopsServicePrincipalId ] - allowedPrincipalIds: [ - adminAppPrincipalId - ] + allowedPrincipalIds: [] requireAuthentication: true } userAssignedManagedIdentityParams: { @@ -123,4 +125,5 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath +output url string = dataProcessorFunctionAppModule.outputs.url output stagingUrl string = dataProcessorFunctionAppModule.outputs.stagingUrl diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 40f856fd682..66841918584 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -17,9 +17,9 @@ jobs: runOnce: deploy: steps: - # - download: MainBuild - # displayName: Download Public API Data Processor artifact - # artifact: public-api-data-processor + - download: MainBuild + displayName: Download Public API Data Processor artifact + artifact: public-api-data-processor - template: ../tasks/bicep-output-variables.yml parameters: @@ -28,41 +28,41 @@ jobs: # We do config updates out of Bicep template so we can implement slot swapping. # Changes are first deployed to the staging slot and combined with a fresh # code deploy prior to being swapped with the production slot. - # - task: AzureCLI@2 - # displayName: Update staging slot app settings - # retryCountOnTaskFailure: 1 - # inputs: - # azureSubscription: ${{ parameters.serviceConnection }} - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # set -e + - task: AzureCLI@2 + displayName: Update staging slot app settings + retryCountOnTaskFailure: 1 + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e - # az functionapp config appsettings set \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --slot staging \ - # --settings \ - # "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ - # "App__EnableThemeDeletion=$(enableThemeDeletion)" \ - # "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ - # "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)" + az functionapp config appsettings set \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --slot staging \ + --settings \ + "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ + "App__EnableThemeDeletion=$(enableThemeDeletion)" \ + "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ + "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)" - # az webapp config connection-string set \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --connection-string-type SQLAzure \ - # --slot staging \ - # --settings \ - # "ContentDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorContentDbConnectionStringSecretKey))" + az webapp config connection-string set \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --connection-string-type SQLAzure \ + --slot staging \ + --settings \ + "ContentDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorContentDbConnectionStringSecretKey))" - # az webapp config connection-string set \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --connection-string-type PostgreSQL \ - # --slot staging \ - # --settings \ - # "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" + az webapp config connection-string set \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --connection-string-type PostgreSQL \ + --slot staging \ + --settings \ + "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" # TODO EES-5128 # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow @@ -98,20 +98,20 @@ jobs: # Client IDs / Identities that can access the Function App. The Service Principal # that is performing the deploy can be accessed by using the `addSpnToEnvironment` # config option in the task definition and using the $(servicePrincipalId) variable. - # - task: AzureCLI@2 - # displayName: Deploy to staging slot - # retryCountOnTaskFailure: 10 - # inputs: - # azureSubscription: ${{ parameters.serviceConnection }} - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # set -e - # az functionapp deployment source config-zip \ - # --src '$(Pipeline.Workspace)/MainBuild/public-api-data-processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.zip' \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --slot staging + - task: AzureCLI@2 + displayName: Deploy to staging slot + retryCountOnTaskFailure: 10 + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + az functionapp deployment source config-zip \ + --src '$(Pipeline.Workspace)/MainBuild/public-api-data-processor/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.zip' \ + --name $(dataProcessorFunctionAppName) \ + --resource-group $(resourceGroupName) \ + --slot staging # TODO EES-5128 # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow @@ -136,6 +136,43 @@ jobs: # publicNetworkAccess=Disabled \ # siteConfig.publicNetworkAccess=Disabled + - task: AzureCLI@2 + displayName: Wait for Data Processor to start up + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + + accessToken=`az account get-access-token \ + --resource $(dataProcessorAppRegistrationClientId) \ + --query "accessToken" \ + -o tsv` + + pollingTimeSeconds=5 + maxAttempts=50 + attempt=1 + + while [ "$attempt" -le "$maxAttempts" ]; do + + httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck` + + echo "Attempt number $attempt - calling $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck to check for Data Processor staging slot to be healthy - got HTTP Status Code $httpStatusCode." + + if [[ "$httpStatusCode" == "200" ]]; then + echo "Data Processor staging slot has started successfully and is healthy - deployment can continue." + exit 0 + fi + + attempt=$((attempt + 1)) + sleep $pollingTimeSeconds + + done + + echo "Timed out waiting for Data processor to start up." + exit 1 + - task: AzureCLI@2 displayName: Wait for active orchestrations to complete inputs: @@ -144,33 +181,33 @@ jobs: scriptLocation: inlineScript inlineScript: | set -e - set -x + + accessToken=`az account get-access-token \ + --resource $(dataProcessorAppRegistrationClientId) \ + --query "accessToken" \ + -o tsv` pollingTimeSeconds=5 maxAttempts=50 - attempts=1 + attempt=1 - while [ "$attempts" -le "$maxAttempts" ]; do + while [ "$attempt" -le "$maxAttempts" ]; do - echo "Attempt number $attempts to check if the Data Processor is ready for deployment" - echo "Calling $(dataProcessorFunctionAppStagingUrl)/api/StatusCheck to check for active orchestrations" - - activeOrchestrations=`curl -s $(dataProcessorFunctionAppStagingUrl)/api/StatusCheck | jq -r '.activeOrchestrations != 0'` + activeOrchestrations=`curl -H "Authorization: Bearer $accessToken" -s $(dataProcessorFunctionAppUrl)/api/StatusCheck | jq -r '.activeOrchestrations != 0'` - echo "Orchestration results are $activeOrchestrations" + echo "Attempt number $attempt - called $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations - currently $activeOrchestrations active orchestrations running." if [[ "$activeOrchestrations" == "false" ]]; then - echo "No active orchestrations running on the Data Processor - slot swapping can proceed" + echo "No active orchestrations running on the Data Processor - slot swapping can proceed." exit 0 fi - attempts=$((attempts + 1)) - + attempt=$((attempt + 1)) sleep $pollingTimeSeconds done - echo "Timed out waiting for active Data processor orchestrations to complete." + echo "Timed out waiting for active Data Processor orchestrations to complete." exit 1 - task: AzureCLI@2 @@ -182,6 +219,7 @@ jobs: scriptLocation: inlineScript inlineScript: | set -e + az functionapp deployment slot swap \ --name $(dataProcessorFunctionAppName) \ --resource-group $(resourceGroupName) \ diff --git a/infrastructure/templates/public-api/ci/stages/deploy.yml b/infrastructure/templates/public-api/ci/stages/deploy.yml index 62adedfe7cc..aad36c1949c 100644 --- a/infrastructure/templates/public-api/ci/stages/deploy.yml +++ b/infrastructure/templates/public-api/ci/stages/deploy.yml @@ -50,11 +50,11 @@ stages: environment: ${{ parameters.environment }} bicepParamFile: ${{ parameters.bicepParamFile }} - - template: ../jobs/deploy-api-docs.yml - parameters: - serviceConnection: ${{ parameters.serviceConnection }} - environment: ${{ parameters.environment }} - dependsOn: DeployPublicApiInfrastructure + # - template: ../jobs/deploy-api-docs.yml + # parameters: + # serviceConnection: ${{ parameters.serviceConnection }} + # environment: ${{ parameters.environment }} + # dependsOn: DeployPublicApiInfrastructure - template: ../jobs/deploy-data-processor.yml parameters: diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index a4b5102cce2..17efac2dd53 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -33,9 +33,10 @@ steps: azureSubscription: ${{ parameters.serviceConnection }} scriptType: bash scriptLocation: inlineScript + addSpnToEnvironment: true inlineScript: | set -e - + az deployment group ${{ parameters.action }} \ --name $(infraDeployName) \ --resource-group $(resourceGroupName) \ @@ -58,4 +59,5 @@ steps: deployAlerts=${{ parameters.deployAlerts }} \ dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \ dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \ - apiAppRegistrationClientId='$(apiAppRegistrationClientId)' + apiAppRegistrationClientId='$(apiAppRegistrationClientId)' \ + devopsServicePrincipalId="$servicePrincipalId" \ No newline at end of file diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index af3cc7c910a..c212fc814c5 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -386,7 +386,7 @@ module privateEndpointModule 'privateEndpoint.bicep' = if (privateEndpointSubnet } output functionAppName string = functionApp.name -output url string = 'https://${functionApp.properties.defaultHostName}' +output url string = 'https://${functionApp.name}.azurewebsites.net' output stagingUrl string = 'https://${functionApp.name}-staging.azurewebsites.net' output managementStorageAccountName string = sharedStorageAccountName output slot1StorageAccountName string = slot1StorageAccountName diff --git a/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep b/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep index 6b1864737bf..e0f525dbcde 100644 --- a/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep +++ b/infrastructure/templates/public-api/components/siteAzureAuthentication.bicep @@ -11,7 +11,7 @@ param stagingSlotName string = 'none' param allowedClientIds string[] = [] @description('Specifies an optional set of Principal Ids of Managed Identities that are allowed to access this resource') -param allowedPrincipalIds string[] = [] +param allowedPrincipalIds string[] @description('Specifies whether all calls to this resource should be authenticated or not. Defaults to true') param requireAuthentication bool = true @@ -35,15 +35,14 @@ var properties = { allowedAudiences: [ 'api://${clientId}' ] - defaultAuthorizationPolicy: { + defaultAuthorizationPolicy: union({ allowedApplications: union( [clientId], allowedClientIds ) - allowedPrincipals: { - identities: allowedPrincipalIds - } - } + }, length(allowedPrincipalIds) > 0 ? { + identities: allowedPrincipalIds + } : {}) } } } diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index b65b17d44ed..30834a98c57 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -96,6 +96,10 @@ param dataProcessorAppRegistrationClientId string = '' @description('Specifies the Application (Client) Id of a pre-existing App Registration used to represent the API Container App.') param apiAppRegistrationClientId string = '' +@description('Specifies the principal id of the Azure DevOps SPN.') +@secure() +param devopsServicePrincipalId string = '' + @description('Specifies whether or not test Themes can be deleted in the environment.') param enableThemeDeletion bool = false @@ -378,6 +382,7 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' metricsNamePrefix: '${subscription}PublicDataProcessor' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId + devopsServicePrincipalId: devopsServicePrincipalId storageFirewallRules: storageFirewallRules dataProcessorFunctionAppExists: dataProcessorFunctionAppExists deployAlerts: deployAlerts @@ -395,6 +400,10 @@ output dataProcessorPsqlConnectionStringSecretKey string = 'ees-publicapi-data-p output dataProcessorFunctionAppManagedIdentityClientId string = deployDataProcessor ? dataProcessorModule.outputs.managedIdentityClientId : '' + +output dataProcessorFunctionAppUrl string = deployDataProcessor + ? dataProcessorModule.outputs.url + : '' output dataProcessorFunctionAppStagingUrl string = deployDataProcessor ? dataProcessorModule.outputs.stagingUrl : '' From 94ac0901a90f79f1acd3d87695362c3e6e7dda92 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 27 Nov 2024 20:40:00 +0000 Subject: [PATCH 084/144] EES-5446 - adding network restrictions to lock down Data Processor down to Admin, Azure Pipeline runners (temporarily disabled in favour of whitelisting AzureCloud service tag) and general maintenance firewall rules. --- azure-pipelines-main.yml | 625 +++++++++--------- .../application/shared/virtualNetwork.bicep | 3 + .../ci/jobs/deploy-data-processor.yml | 51 +- .../public-api/ci/tasks/deploy-bicep.yml | 6 +- .../public-api/components/functionApp.bicep | 13 +- .../templates/public-api/main.bicep | 44 +- .../templates/public-api/types.bicep | 3 +- ...ction.cs => LongRunningTriggerFunction.cs} | 4 +- .../Functions/StatusCheckFunction.cs | 6 +- 9 files changed, 410 insertions(+), 345 deletions(-) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/{LogRunningTriggerFunction.cs => LongRunningTriggerFunction.cs} (95%) diff --git a/azure-pipelines-main.yml b/azure-pipelines-main.yml index d5fe21618fc..3ede71f80fe 100644 --- a/azure-pipelines-main.yml +++ b/azure-pipelines-main.yml @@ -6,6 +6,7 @@ parameters: - master - dev - test + - EES-5446-prevent-function-app-slot-swap-until-orchestrations-complete variables: BuildConfiguration: Release @@ -59,64 +60,64 @@ jobs: # custom: format whitespace src/GovUk.Education.ExploreEducationStatistics.sln --verify-no-changes --severity error # arguments: --verify-no-changes --verbosity diagnostic - - task: DotNetCoreCLI@2 - displayName: Verify Formatting and Style - inputs: - command: custom - custom: format - ## TODO: Remove "--severity error" once style formatter has been run across project - arguments: style --verify-no-changes --verbosity diagnostic --severity error - projects: src/GovUk.Education.ExploreEducationStatistics.sln - - - task: DotNetCoreCLI@2 - displayName: Verify Formatting and Style - inputs: - command: custom - custom: format - ## TODO: Remove "--severity error" once work has been done to resolve build warnings (https://dfedigital.atlassian.net/browse/EES-4594). - arguments: analyzers --verify-no-changes --verbosity diagnostic --severity error - projects: src/GovUk.Education.ExploreEducationStatistics.sln +# - task: DotNetCoreCLI@2 +# displayName: Verify Formatting and Style +# inputs: +# command: custom +# custom: format +# ## TODO: Remove "--severity error" once style formatter has been run across project +# arguments: style --verify-no-changes --verbosity diagnostic --severity error +# projects: src/GovUk.Education.ExploreEducationStatistics.sln +# +# - task: DotNetCoreCLI@2 +# displayName: Verify Formatting and Style +# inputs: +# command: custom +# custom: format +# ## TODO: Remove "--severity error" once work has been done to resolve build warnings (https://dfedigital.atlassian.net/browse/EES-4594). +# arguments: analyzers --verify-no-changes --verbosity diagnostic --severity error +# projects: src/GovUk.Education.ExploreEducationStatistics.sln # TODO: Wrap these ^ three tasks up into a single `dotnet format` task once all 3 above TODOs are TO-DONE ;) - - task: DotNetCoreCLI@2 - displayName: Test - inputs: - command: test - projects: | - **/GovUk.*[Tt]ests/*.csproj - !**/GovUk.Education.ExploreEducationStatistics.Admin.Tests/*csproj - arguments: --configuration $(BuildConfiguration) - - - task: DotNetCoreCLI@2 - displayName: Package Data API - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Api.csproj' - arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/data-api - zipAfterPublish: True - - - task: PublishPipelineArtifact@0 - displayName: Publish Data API artifact - inputs: - artifactName: data-api - targetPath: $(Build.ArtifactStagingDirectory)/data-api - - - task: DotNetCoreCLI@2 - displayName: Package Content API - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Content.Api.csproj' - arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/content-api - zipAfterPublish: True - - - task: PublishPipelineArtifact@0 - displayName: Publish Content API artifact - inputs: - artifactName: content-api - targetPath: $(Build.ArtifactStagingDirectory)/content-api +# - task: DotNetCoreCLI@2 +# displayName: Test +# inputs: +# command: test +# projects: | +# **/GovUk.*[Tt]ests/*.csproj +# !**/GovUk.Education.ExploreEducationStatistics.Admin.Tests/*csproj +# arguments: --configuration $(BuildConfiguration) +# +# - task: DotNetCoreCLI@2 +# displayName: Package Data API +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Api.csproj' +# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/data-api +# zipAfterPublish: True +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Data API artifact +# inputs: +# artifactName: data-api +# targetPath: $(Build.ArtifactStagingDirectory)/data-api +# +# - task: DotNetCoreCLI@2 +# displayName: Package Content API +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Content.Api.csproj' +# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/content-api +# zipAfterPublish: True +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Content API artifact +# inputs: +# artifactName: content-api +# targetPath: $(Build.ArtifactStagingDirectory)/content-api - task: DotNetCoreCLI@2 displayName: Package Public API @@ -166,260 +167,260 @@ jobs: artifactName: public-api-data-processor targetPath: $(Build.ArtifactStagingDirectory)/public-api-data-processor - - task: DotNetCoreCLI@2 - displayName: Package Notifier Function - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Notifier.csproj' - arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/notifier - zipAfterPublish: True - - - task: PublishPipelineArtifact@0 - displayName: Publish Notifier artifact - inputs: - artifactName: notifier - targetPath: $(Build.ArtifactStagingDirectory)/notifier - - - task: DotNetCoreCLI@2 - displayName: Package Publisher Function - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Publisher.csproj' - arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/publisher - zipAfterPublish: True - - - task: PublishPipelineArtifact@0 - displayName: Publish Publisher artifact - inputs: - artifactName: publisher - targetPath: $(Build.ArtifactStagingDirectory)/publisher - - - task: DotNetCoreCLI@2 - displayName: Package Processor Function - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Processor.csproj' - arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/processor - zipAfterPublish: True - - - task: PublishPipelineArtifact@0 - displayName: Publish Processor artifact - inputs: - artifactName: processor - targetPath: $(Build.ArtifactStagingDirectory)/processor - - - job: Admin - pool: ees-ubuntu2204-xlarge - workspace: - clean: all - steps: - - task: UseNode@1 - displayName: Install Node.js $(NodeVersion) - inputs: - version: $(NodeVersion) - - - task: Bash@3 - displayName: corepack enable - inputs: - workingDir: . - targetType: inline - script: corepack enable - - - task: UseDotNet@2 - displayName: Install .NET 8.0 SDK - inputs: - version: 8.0.x - performMultiLevelLookup: true - - - task: DotNetCoreCLI@2 - displayName: Build - inputs: - projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' - arguments: --configuration $(BuildConfiguration) - - - task: DotNetCoreCLI@2 - displayName: Test - inputs: - command: test - projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.Tests.csproj' - arguments: --configuration $(BuildConfiguration) --collect "Code coverage" - - - task: Bash@3 - displayName: pnpm i - inputs: - targetType: inline - script: pnpm i - - - task: Bash@3 - displayName: pnpm run build - inputs: - targetType: inline - script: pnpm --filter=explore-education-statistics-admin run build - - - task: CopyFiles@2 - displayName: Copy files to wwwroot - inputs: - SourceFolder: src/explore-education-statistics-admin/build - TargetFolder: src/GovUk.Education.ExploreEducationStatistics.Admin/wwwroot - - - task: DotNetCoreCLI@2 - displayName: Package Admin app - inputs: - command: publish - publishWebProjects: false - projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' - arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: Publish Admin artifact - inputs: - artifactName: admin - targetPath: $(Build.ArtifactStagingDirectory) - - - job: Frontend - pool: ees-ubuntu2204-xlarge - workspace: - clean: all - steps: - - task: UseNode@1 - displayName: Install Node.js $(NodeVersion) - inputs: - version: $(NodeVersion) - - - task: Bash@3 - displayName: corepack enable - inputs: - workingDir: . - targetType: inline - script: corepack enable - - - task: Bash@3 - displayName: pnpm i - inputs: - workingDir: . - targetType: inline - script: pnpm i - - - task: Bash@3 - displayName: pnpm tsc - inputs: - workingDir: . - targetType: inline - script: pnpm tsc - - - task: Bash@3 - displayName: pnpm lint - inputs: - workingDir: . - targetType: inline - script: pnpm lint - - - task: Bash@3 - displayName: pnpm format:check - inputs: - workingDir: . - targetType: inline - script: pnpm format:check - - - task: Bash@3 - displayName: pnpm test:ci - inputs: - workingDir: . - targetType: inline - script: pnpm test:ci - - - task: PublishTestResults@2 - displayName: Publish frontend test results - inputs: - testResultsFormat: JUnit - testResultsFiles: explore-education-statistics-*/junit-*.xml - searchFolder: ./src - testRunTitle: Release Jest tests - mergeTestResults: true - - - task: Bash@3 - displayName: pnpm run build - inputs: - targetType: inline - script: pnpm --filter=explore-education-statistics-frontend run build - - - task: Docker@2 - displayName: Build Public frontend Docker image - condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) - inputs: - containerRegistry: $(AcrServiceConnection) - repository: ees-public-frontend - command: build - Dockerfile: docker/public-frontend/Dockerfile - buildContext: $(System.DefaultWorkingDirectory) - tags: $(Build.BuildNumber) - arguments: --build-arg BUILD_BUILDNUMBER=$(Build.BuildNumber) - env: - DOCKER_BUILDKIT: 1 - - - task: Docker@2 - displayName: Push Public frontend Docker image - condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) - inputs: - containerRegistry: $(AcrServiceConnection) - repository: ees-public-frontend - command: push - tags: $(Build.BuildNumber) - - - job: ApiDocs - pool: - vmImage: ubuntu-22.04 - workspace: - clean: all - variables: - WorkingDirectory: src/explore-education-statistics-api-docs - steps: - - task: UseNode@1 - displayName: Install Node.js $(NodeVersion) - inputs: - version: $(NodeVersion) - - - task: UseRubyVersion@0 - displayName: Install Ruby $(RubyVersion) - inputs: - versionSpec: '>= $(RubyVersion)' - - - task: Bash@3 - displayName: Build - env: - TECH_DOCS_API_URL: https://dev.statistics.api.education.gov.uk - inputs: - workingDirectory: $(WorkingDirectory) - targetType: inline - script: | - bundle install - bundle exec middleman build - - - task: PublishPipelineArtifact@1 - displayName: Publish artifact - inputs: - artifactName: public-api-docs - targetPath: $(WorkingDirectory)/build - - - job: MiscellaneousArtifacts - pool: - vmImage: ubuntu-22.04 - workspace: - clean: all - steps: - - task: CopyFiles@2 - displayName: Copy Pipfiles to tests - inputs: - Contents: | - Pipfile - Pipfile.lock - TargetFolder: tests - - - task: PublishPipelineArtifact@0 - displayName: Publish test files - inputs: - artifactName: tests - targetPath: tests +# - task: DotNetCoreCLI@2 +# displayName: Package Notifier Function +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Notifier.csproj' +# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/notifier +# zipAfterPublish: True +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Notifier artifact +# inputs: +# artifactName: notifier +# targetPath: $(Build.ArtifactStagingDirectory)/notifier +# +# - task: DotNetCoreCLI@2 +# displayName: Package Publisher Function +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Publisher.csproj' +# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/publisher +# zipAfterPublish: True +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Publisher artifact +# inputs: +# artifactName: publisher +# targetPath: $(Build.ArtifactStagingDirectory)/publisher +# +# - task: DotNetCoreCLI@2 +# displayName: Package Processor Function +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Processor.csproj' +# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/processor +# zipAfterPublish: True +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Processor artifact +# inputs: +# artifactName: processor +# targetPath: $(Build.ArtifactStagingDirectory)/processor + +# - job: Admin +# pool: ees-ubuntu2204-xlarge +# workspace: +# clean: all +# steps: +# - task: UseNode@1 +# displayName: Install Node.js $(NodeVersion) +# inputs: +# version: $(NodeVersion) +# +# - task: Bash@3 +# displayName: corepack enable +# inputs: +# workingDir: . +# targetType: inline +# script: corepack enable +# +# - task: UseDotNet@2 +# displayName: Install .NET 8.0 SDK +# inputs: +# version: 8.0.x +# performMultiLevelLookup: true +# +# - task: DotNetCoreCLI@2 +# displayName: Build +# inputs: +# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' +# arguments: --configuration $(BuildConfiguration) +# +# - task: DotNetCoreCLI@2 +# displayName: Test +# inputs: +# command: test +# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.Tests.csproj' +# arguments: --configuration $(BuildConfiguration) --collect "Code coverage" +# +# - task: Bash@3 +# displayName: pnpm i +# inputs: +# targetType: inline +# script: pnpm i +# +# - task: Bash@3 +# displayName: pnpm run build +# inputs: +# targetType: inline +# script: pnpm --filter=explore-education-statistics-admin run build +# +# - task: CopyFiles@2 +# displayName: Copy files to wwwroot +# inputs: +# SourceFolder: src/explore-education-statistics-admin/build +# TargetFolder: src/GovUk.Education.ExploreEducationStatistics.Admin/wwwroot +# +# - task: DotNetCoreCLI@2 +# displayName: Package Admin app +# inputs: +# command: publish +# publishWebProjects: false +# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' +# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory) +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish Admin artifact +# inputs: +# artifactName: admin +# targetPath: $(Build.ArtifactStagingDirectory) +# +# - job: Frontend +# pool: ees-ubuntu2204-xlarge +# workspace: +# clean: all +# steps: +# - task: UseNode@1 +# displayName: Install Node.js $(NodeVersion) +# inputs: +# version: $(NodeVersion) +# +# - task: Bash@3 +# displayName: corepack enable +# inputs: +# workingDir: . +# targetType: inline +# script: corepack enable +# +# - task: Bash@3 +# displayName: pnpm i +# inputs: +# workingDir: . +# targetType: inline +# script: pnpm i +# +# - task: Bash@3 +# displayName: pnpm tsc +# inputs: +# workingDir: . +# targetType: inline +# script: pnpm tsc +# +# - task: Bash@3 +# displayName: pnpm lint +# inputs: +# workingDir: . +# targetType: inline +# script: pnpm lint +# +# - task: Bash@3 +# displayName: pnpm format:check +# inputs: +# workingDir: . +# targetType: inline +# script: pnpm format:check +# +# - task: Bash@3 +# displayName: pnpm test:ci +# inputs: +# workingDir: . +# targetType: inline +# script: pnpm test:ci +# +# - task: PublishTestResults@2 +# displayName: Publish frontend test results +# inputs: +# testResultsFormat: JUnit +# testResultsFiles: explore-education-statistics-*/junit-*.xml +# searchFolder: ./src +# testRunTitle: Release Jest tests +# mergeTestResults: true +# +# - task: Bash@3 +# displayName: pnpm run build +# inputs: +# targetType: inline +# script: pnpm --filter=explore-education-statistics-frontend run build +# +# - task: Docker@2 +# displayName: Build Public frontend Docker image +# condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) +# inputs: +# containerRegistry: $(AcrServiceConnection) +# repository: ees-public-frontend +# command: build +# Dockerfile: docker/public-frontend/Dockerfile +# buildContext: $(System.DefaultWorkingDirectory) +# tags: $(Build.BuildNumber) +# arguments: --build-arg BUILD_BUILDNUMBER=$(Build.BuildNumber) +# env: +# DOCKER_BUILDKIT: 1 +# +# - task: Docker@2 +# displayName: Push Public frontend Docker image +# condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) +# inputs: +# containerRegistry: $(AcrServiceConnection) +# repository: ees-public-frontend +# command: push +# tags: $(Build.BuildNumber) + +# - job: ApiDocs +# pool: +# vmImage: ubuntu-22.04 +# workspace: +# clean: all +# variables: +# WorkingDirectory: src/explore-education-statistics-api-docs +# steps: +# - task: UseNode@1 +# displayName: Install Node.js $(NodeVersion) +# inputs: +# version: $(NodeVersion) +# +# - task: UseRubyVersion@0 +# displayName: Install Ruby $(RubyVersion) +# inputs: +# versionSpec: '>= $(RubyVersion)' +# +# - task: Bash@3 +# displayName: Build +# env: +# TECH_DOCS_API_URL: https://dev.statistics.api.education.gov.uk +# inputs: +# workingDirectory: $(WorkingDirectory) +# targetType: inline +# script: | +# bundle install +# bundle exec middleman build +# +# - task: PublishPipelineArtifact@1 +# displayName: Publish artifact +# inputs: +# artifactName: public-api-docs +# targetPath: $(WorkingDirectory)/build +# +# - job: MiscellaneousArtifacts +# pool: +# vmImage: ubuntu-22.04 +# workspace: +# clean: all +# steps: +# - task: CopyFiles@2 +# displayName: Copy Pipfiles to tests +# inputs: +# Contents: | +# Pipfile +# Pipfile.lock +# TargetFolder: tests +# +# - task: PublishPipelineArtifact@0 +# displayName: Publish test files +# inputs: +# artifactName: tests +# targetPath: tests diff --git a/infrastructure/templates/public-api/application/shared/virtualNetwork.bicep b/infrastructure/templates/public-api/application/shared/virtualNetwork.bicep index ad5f11225b7..04e86e6a381 100644 --- a/infrastructure/templates/public-api/application/shared/virtualNetwork.bicep +++ b/infrastructure/templates/public-api/application/shared/virtualNetwork.bicep @@ -83,6 +83,9 @@ output adminAppServiceSubnetStartIpAddress string = parseCidr(adminSubnet.proper @description('The last usable IP address for the Admin App Service Subnet.') output adminAppServiceSubnetEndIpAddress string = parseCidr(adminSubnet.properties.addressPrefix).lastUsable +@description('The IP address range for the Admin App Service Subnet.') +output adminAppServiceSubnetCidr string = adminSubnet.properties.addressPrefix + @description('The first usable IP address for the Publisher Function App Subnet.') output publisherFunctionAppSubnetStartIpAddress string = parseCidr(publisherSubnet.properties.addressPrefix).firstUsable diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 66841918584..351f4a29352 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -137,14 +137,13 @@ jobs: # siteConfig.publicNetworkAccess=Disabled - task: AzureCLI@2 - displayName: Wait for Data Processor to start up + displayName: Wait for Data Processor staging slot to start up inputs: azureSubscription: ${{ parameters.serviceConnection }} scriptType: bash scriptLocation: inlineScript inlineScript: | - set -e - + accessToken=`az account get-access-token \ --resource $(dataProcessorAppRegistrationClientId) \ --query "accessToken" \ @@ -180,7 +179,6 @@ jobs: scriptType: bash scriptLocation: inlineScript inlineScript: | - set -e accessToken=`az account get-access-token \ --resource $(dataProcessorAppRegistrationClientId) \ @@ -193,9 +191,13 @@ jobs: while [ "$attempt" -le "$maxAttempts" ]; do - activeOrchestrations=`curl -H "Authorization: Bearer $accessToken" -s $(dataProcessorFunctionAppUrl)/api/StatusCheck | jq -r '.activeOrchestrations != 0'` + activeOrchestrationResults=`curl -H "Authorization: Bearer $accessToken" -s $(dataProcessorFunctionAppUrl)/api/StatusCheck` + activeOrchestrationCount=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations'` + activeOrchestrations=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations != 0'` - echo "Attempt number $attempt - called $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations - currently $activeOrchestrations active orchestrations running." + echo $activeOrchestrationResults + + echo "Attempt number $attempt - called $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations - currently $activeOrchestrationCount active orchestrations running." if [[ "$activeOrchestrations" == "false" ]]; then echo "No active orchestrations running on the Data Processor - slot swapping can proceed." @@ -225,3 +227,40 @@ jobs: --resource-group $(resourceGroupName) \ --slot staging \ --target-slot production + + - task: AzureCLI@2 + displayName: Check that Data Processor is ready to serve new requests + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -x + + accessToken=`az account get-access-token \ + --resource $(dataProcessorAppRegistrationClientId) \ + --query "accessToken" \ + -o tsv` + + pollingTimeSeconds=5 + maxAttempts=50 + attempt=1 + + while [ "$attempt" -le "$maxAttempts" ]; do + + httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppUrl)/api/HealthCheck` + + echo "Attempt number $attempt - calling $(dataProcessorFunctionAppUrl)/api/HealthCheck to check for Data Processor to be healthy - got HTTP Status Code $httpStatusCode." + + if [[ "$httpStatusCode" == "200" ]]; then + echo "Data Processor has started successfully and is healthy - ready to serve new requests." + exit 0 + fi + + attempt=$((attempt + 1)) + sleep $pollingTimeSeconds + + done + + echo "Timed out waiting for Data processor to start up." + exit 1 diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index 17efac2dd53..c18b37ca570 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -47,9 +47,8 @@ steps: resourceTags='$(resourceTags)' \ postgreSqlAdminName='$(postgreSqlAdminName)' \ postgreSqlAdminPassword='$(postgreSqlAdminPassword)' \ - postgreSqlFirewallRules='$(maintenanceFirewallRules)' \ postgreSqlEntraIdAdminPrincipals='$(postgreSqlEntraIdAdminPrincipals)' \ - storageFirewallRules='$(maintenanceFirewallRules)' \ + maintenanceFirewallRules='$(maintenanceFirewallRules)' \ acrResourceGroupName='$(acrResourceGroupName)' \ dockerImagesTag='$(resources.pipeline.MainBuild.runName)' \ deploySharedPrivateDnsZones=${{ parameters.deploySharedPrivateDnsZones }} \ @@ -60,4 +59,5 @@ steps: dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \ dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \ apiAppRegistrationClientId='$(apiAppRegistrationClientId)' \ - devopsServicePrincipalId="$servicePrincipalId" \ No newline at end of file + devopsServicePrincipalId="$servicePrincipalId" \ + pipelineRunnerCidr="$(pipelineRunnerCidr)" \ No newline at end of file diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index c212fc814c5..0126d4c0504 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -175,13 +175,12 @@ resource slot2FileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2 ] } -var firewallRules = [for firewallRule in functionAppEndpointFirewallRules: { +var firewallRules = [for (firewallRule, index) in functionAppEndpointFirewallRules: { + name: firewallRule.name ipAddress: firewallRule.cidr action: 'Allow' - tag: 'Default' - priority: 100 - name: firewallRule.name - description: firewallRule.description + tag: firewallRule.tag ?? 'Default' + priority: firewallRule.priority ?? (100 + index) }] var commonSiteProperties = { @@ -202,8 +201,8 @@ var commonSiteProperties = { linuxFxVersion: appServicePlanOS == 'Linux' ? 'DOTNET-ISOLATED|8.0' : null keyVaultReferenceIdentity: keyVaultReferenceIdentity publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' - // ipSecurityRestrictions: firewallRules - ipSecurityRestrictionsDefaultAction: 'Allow' // TODO Deny! + ipSecurityRestrictions: publicNetworkAccessEnabled && length(firewallRules) > 0 ? firewallRules : null + ipSecurityRestrictionsDefaultAction: publicNetworkAccessEnabled && length(firewallRules) > 0 ? 'Deny' : 'Allow' scmIpSecurityRestrictions: [ { ipAddress: 'Any' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 30834a98c57..b3824c9c907 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -10,8 +10,8 @@ param location string = resourceGroup().location @description('Public API Storage : Size of the file share in GB.') param publicApiDataFileShareQuota int = 1 -@description('Public API Storage : Firewall rules.') -param storageFirewallRules FirewallRule[] = [] +@description('Firewall rules for maintenance of the service by allowing key IP ranges access to resources.') +param maintenanceFirewallRules FirewallRule[] = [] @description('Database : administrator login name.') @minLength(0) @@ -36,9 +36,6 @@ param postgreSqlStorageSizeGB int = 32 @description('Database : Azure Database for PostgreSQL Autogrow setting.') param postgreSqlAutoGrowStatus string = 'Disabled' -@description('Database : Firewall rules.') -param postgreSqlFirewallRules FirewallRule[] = [] - @description('Database : Entra ID admin principal names for this resource') param postgreSqlEntraIdAdminPrincipals PrincipalNameAndId[] = [] @@ -70,14 +67,17 @@ param dockerImagesTag string = '' @description('Do the shared Private DNS Zones need creating or updating?') param deploySharedPrivateDnsZones bool = false -@description('Can we deploy the Container App yet? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added.') -param deployContainerApp bool = true - // TODO EES-5128 - Note that this has been added temporarily to avoid 10+ minute deploys where it appears that PSQL // will redeploy even if no changes exist in this deploy from the previous one. @description('Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys.') param deployPsqlFlexibleServer bool = false +@description('Does the Public API Container App need creating or updating? This is dependent on the PostgreSQL Flexible Server being set up and having users manually added.') +param deployContainerApp bool = true + +@description('Does the Data Processor need creating or updating?') +param deployDataProcessor bool = true + param deployAlerts bool = false @description('Public URLs of other components in the service.') @@ -100,6 +100,9 @@ param apiAppRegistrationClientId string = '' @secure() param devopsServicePrincipalId string = '' +@description('Specifies the IP address range of the pipeline runners.') +param pipelineRunnerCidr string = '' + @description('Specifies whether or not test Themes can be deleted in the environment.') param enableThemeDeletion bool = false @@ -192,7 +195,7 @@ module publicApiStorageModule 'application/public-api/publicApiStorage.bicep' = location: location resourceNames: resourceNames publicApiDataFileShareQuota: publicApiDataFileShareQuota - storageFirewallRules: storageFirewallRules + storageFirewallRules: maintenanceFirewallRules deployAlerts: deployAlerts tagValues: tagValues } @@ -226,7 +229,7 @@ module postgreSqlServerModule 'application/shared/postgreSqlFlexibleServer.bicep entraIdAdminPrincipals: postgreSqlEntraIdAdminPrincipals privateEndpointSubnetId: vNetModule.outputs.psqlFlexibleServerSubnetRef autoGrowStatus: postgreSqlAutoGrowStatus - firewallRules: postgreSqlFirewallRules + firewallRules: maintenanceFirewallRules sku: postgreSqlSkuName storageSizeGB: postgreSqlStorageSizeGB deployAlerts: deployAlerts @@ -374,6 +377,11 @@ module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContai } } +var adminSubnetFirewallRule = { + name: 'Admin App Service subnet range' + cidr: vNetModule.outputs.adminAppServiceSubnetCidr +} + module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' = if (deployDataProcessor) { name: 'publicApiDataProcessorApplicationModuleDeploy' params: { @@ -383,7 +391,21 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId devopsServicePrincipalId: devopsServicePrincipalId - storageFirewallRules: storageFirewallRules + storageFirewallRules: maintenanceFirewallRules + functionAppFirewallRules: union([ + adminSubnetFirewallRule + // TODO EES-5446 - add in when static IP range available for runner scale sets + // { + // name: 'Pipeline runner IP address range' + // cidr: pipelineRunnerCidr + // } + { + cidr: 'AzureCloud' + tag: 'ServiceTag' + priority: 100 + name: 'AzureCloud' + } + ], maintenanceFirewallRules) dataProcessorFunctionAppExists: dataProcessorFunctionAppExists deployAlerts: deployAlerts tagValues: tagValues diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index 8dc60f8b9a6..eee0b62a22c 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -43,8 +43,9 @@ type ResourceNames = { @export() type FirewallRule = { name: string - description: string? cidr: string + priority: int? + tag: string? } @export() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs similarity index 95% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs index 916f801a561..c37e5a8c6dc 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LogRunningTriggerFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs @@ -9,8 +9,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class LogRunningTriggerFunction( - ILogger logger) +public class LongRunningTriggerFunction( + ILogger logger) { [Function(nameof(TriggerLongRunningOrchestration))] public async Task TriggerLongRunningOrchestration( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs index 1878f9defda..877e663232f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.Functions.Worker; using Microsoft.DurableTask.Client; @@ -14,20 +15,19 @@ public class StatusCheckFunction OrchestrationRuntimeStatus.Running } }; - + [Function("StatusCheck")] [Produces("application/json")] public static async Task StatusCheck( [HttpTrigger(AuthorizationLevel.Anonymous, "get")] #pragma warning disable IDE0060 - HttpRequestMessage request, + HttpRequest request, #pragma warning restore IDE0060 [DurableClient] DurableTaskClient client) { var activeOrchestrations = await client .GetAllInstancesAsync(filter: ActiveOrchestrationsQuery) .ToListAsync(); - return new OkObjectResult(new { ActiveOrchestrations = activeOrchestrations.Count }); } } From e61c935c8f13b6ed3898697dc71d379971928d73 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 28 Nov 2024 15:23:55 +0000 Subject: [PATCH 085/144] EES-5446 - corrected long-running test orchestration. --- .../public-api/publicApiDataProcessor.bicep | 4 +-- .../public-api/publicApiStorage.bicep | 4 +-- .../shared/postgreSqlFlexibleServer.bicep | 4 +-- .../ci/jobs/deploy-data-processor.yml | 18 +++++++------ .../public-api/ci/tasks/deploy-bicep.yml | 2 +- .../public-api/components/functionApp.bicep | 8 +++--- .../components/postgresqlDatabase.bicep | 4 +-- .../components/storageAccount.bicep | 4 +-- .../templates/public-api/main.bicep | 21 +++++++++++----- .../templates/public-api/types.bicep | 9 +++++-- .../Functions/LongRunningFunctions.cs | 25 +++++++++++++++++++ .../Functions/LongRunningOrchestration.cs | 22 ++-------------- 12 files changed, 75 insertions(+), 50 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index a7218c96c6b..5891ab45d52 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -1,4 +1,4 @@ -import { ResourceNames, FirewallRule } from '../../types.bicep' +import { ResourceNames, FirewallRule, IpRange } from '../../types.bicep' @description('Specifies common resource naming variables.') param resourceNames ResourceNames @@ -23,7 +23,7 @@ param dataProcessorAppRegistrationClientId string param devopsServicePrincipalId string @description('The IP address ranges that can access the Data Processor storage accounts.') -param storageFirewallRules FirewallRule[] +param storageFirewallRules IpRange[] @description('The IP address ranges that can access the Data Processor Function App endpoints.') param functionAppFirewallRules FirewallRule[] = [] diff --git a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep index f61c7f4efd3..1207e1a971a 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiStorage.bicep @@ -1,4 +1,4 @@ -import { ResourceNames, FirewallRule } from '../../types.bicep' +import { ResourceNames, IpRange } from '../../types.bicep' param resourceNames ResourceNames @@ -9,7 +9,7 @@ param location string param publicApiDataFileShareQuota int @description('Public API Storage : Firewall rules.') -param storageFirewallRules FirewallRule[] +param storageFirewallRules IpRange[] @description('Specifies a set of tags with which to tag the resource in Azure.') param tagValues object diff --git a/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep b/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep index 2f9469be5d8..a5a96609131 100644 --- a/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep +++ b/infrastructure/templates/public-api/application/shared/postgreSqlFlexibleServer.bicep @@ -1,4 +1,4 @@ -import { ResourceNames, FirewallRule, PrincipalNameAndId } from '../../types.bicep' +import { ResourceNames, IpRange, PrincipalNameAndId } from '../../types.bicep' @description('Specifies common resource naming variables.') param resourceNames ResourceNames @@ -23,7 +23,7 @@ param storageSizeGB int = 32 param autoGrowStatus string = 'Disabled' @description('Firewall rules.') -param firewallRules FirewallRule[] = [] +param firewallRules IpRange[] = [] @description('Specifies the subnet id that the PostgreSQL private endpoint will be attached to.') param privateEndpointSubnetId string diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 351f4a29352..794677c2929 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -143,7 +143,7 @@ jobs: scriptType: bash scriptLocation: inlineScript inlineScript: | - + accessToken=`az account get-access-token \ --resource $(dataProcessorAppRegistrationClientId) \ --query "accessToken" \ @@ -155,9 +155,11 @@ jobs: while [ "$attempt" -le "$maxAttempts" ]; do + echo "Attempt number $attempt of $maxAttempts- calling $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck to check for Data Processor staging being healthy." + httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck` - echo "Attempt number $attempt - calling $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck to check for Data Processor staging slot to be healthy - got HTTP Status Code $httpStatusCode." + echo "Got HTTP status code $httpStatusCode." if [[ "$httpStatusCode" == "200" ]]; then echo "Data Processor staging slot has started successfully and is healthy - deployment can continue." @@ -188,17 +190,17 @@ jobs: pollingTimeSeconds=5 maxAttempts=50 attempt=1 - + while [ "$attempt" -le "$maxAttempts" ]; do + echo "Attempt number $attempt of $maxAttempts - callinf $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations." + activeOrchestrationResults=`curl -H "Authorization: Bearer $accessToken" -s $(dataProcessorFunctionAppUrl)/api/StatusCheck` activeOrchestrationCount=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations'` activeOrchestrations=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations != 0'` - echo $activeOrchestrationResults + echo "Currently $activeOrchestrationCount active orchestrations running." - echo "Attempt number $attempt - called $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations - currently $activeOrchestrationCount active orchestrations running." - if [[ "$activeOrchestrations" == "false" ]]; then echo "No active orchestrations running on the Data Processor - slot swapping can proceed." exit 0 @@ -248,9 +250,11 @@ jobs: while [ "$attempt" -le "$maxAttempts" ]; do + echo "Attempt number $attempt of $maxAttempts - calling $(dataProcessorFunctionAppUrl)/api/HealthCheck to check for Data Processor being healthy." + httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppUrl)/api/HealthCheck` - echo "Attempt number $attempt - calling $(dataProcessorFunctionAppUrl)/api/HealthCheck to check for Data Processor to be healthy - got HTTP Status Code $httpStatusCode." + echo "Got HTTP status code $httpStatusCode." if [[ "$httpStatusCode" == "200" ]]; then echo "Data Processor has started successfully and is healthy - ready to serve new requests." diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index c18b37ca570..1903174be77 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -48,7 +48,7 @@ steps: postgreSqlAdminName='$(postgreSqlAdminName)' \ postgreSqlAdminPassword='$(postgreSqlAdminPassword)' \ postgreSqlEntraIdAdminPrincipals='$(postgreSqlEntraIdAdminPrincipals)' \ - maintenanceFirewallRules='$(maintenanceFirewallRules)' \ + maintenanceIpRanges='$(maintenanceFirewallRules)' \ acrResourceGroupName='$(acrResourceGroupName)' \ dockerImagesTag='$(resources.pipeline.MainBuild.runName)' \ deploySharedPrivateDnsZones=${{ parameters.deploySharedPrivateDnsZones }} \ diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index 0126d4c0504..f77f7808822 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -1,4 +1,4 @@ -import { FirewallRule, AzureFileShareMount, EntraIdAuthentication } from '../types.bicep' +import { FirewallRule, IpRange, AzureFileshareMount, EntraIdAuthentication } from '../types.bicep' @description('Specifies the location for all resources.') param location string @@ -71,7 +71,7 @@ param healthCheckPath string? param azureFileShares AzureFileShareMount[] = [] @description('Specifies firewall rules for the various storage accounts in use by the Function App') -param storageFirewallRules FirewallRule[] = [] +param storageFirewallRules IpRange[] = [] var reserved = appServicePlanOS == 'Linux' @@ -179,8 +179,8 @@ var firewallRules = [for (firewallRule, index) in functionAppEndpointFirewallRul name: firewallRule.name ipAddress: firewallRule.cidr action: 'Allow' - tag: firewallRule.tag ?? 'Default' - priority: firewallRule.priority ?? (100 + index) + tag: firewallRule.tag != null ? firewallRule.tag : 'Default' + priority: firewallRule.priority != null ? firewallRule.priority : 100 + index }] var commonSiteProperties = { diff --git a/infrastructure/templates/public-api/components/postgresqlDatabase.bicep b/infrastructure/templates/public-api/components/postgresqlDatabase.bicep index 273e148d322..006c67627b4 100644 --- a/infrastructure/templates/public-api/components/postgresqlDatabase.bicep +++ b/infrastructure/templates/public-api/components/postgresqlDatabase.bicep @@ -1,4 +1,4 @@ -import { FirewallRule, PrincipalNameAndId } from '../types.bicep' +import { IpRange, PrincipalNameAndId } from '../types.bicep' @description('Specifies the location for all resources.') param location string @@ -40,7 +40,7 @@ param geoRedundantBackup string = 'Disabled' param databaseNames string[] @description('An array of firewall rules containing IP address ranges') -param firewallRules FirewallRule[] = [] +param firewallRules IpRange[] = [] @description('An array of Entra ID admin principal names for this resource') param entraIdAdminPrincipals PrincipalNameAndId[] = [] diff --git a/infrastructure/templates/public-api/components/storageAccount.bicep b/infrastructure/templates/public-api/components/storageAccount.bicep index 9d5576d2312..a3e865d0b13 100644 --- a/infrastructure/templates/public-api/components/storageAccount.bicep +++ b/infrastructure/templates/public-api/components/storageAccount.bicep @@ -1,4 +1,4 @@ -import { FirewallRule } from '../types.bicep' +import { IpRange } from '../types.bicep' @description('Specifies the location for all resources.') param location string @@ -10,7 +10,7 @@ param storageAccountName string param allowedSubnetIds string[] = [] @description('Storage Account Network Firewall Rules') -param firewallRules FirewallRule[] = [] +param firewallRules IpRange[] = [] @description('Storage Account SKU') param skuStorageResource 'Standard_LRS' | 'Standard_GRS' | 'Standard_RAGRS' | 'Standard_ZRS' | 'Premium_LRS' | 'Premium_ZRS' | 'Standard_GZRS' | 'Standard_RAGZRS' = 'Standard_LRS' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index b3824c9c907..b985dd102ff 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -1,5 +1,5 @@ import { abbreviations } from 'abbreviations.bicep' -import { FirewallRule, PrincipalNameAndId, StaticWebAppSku } from 'types.bicep' +import { IpRange, PrincipalNameAndId, StaticWebAppSku } from 'types.bicep' @description('Environment : Subscription name e.g. s101d01. Used as a prefix for created resources.') param subscription string = '' @@ -11,7 +11,7 @@ param location string = resourceGroup().location param publicApiDataFileShareQuota int = 1 @description('Firewall rules for maintenance of the service by allowing key IP ranges access to resources.') -param maintenanceFirewallRules FirewallRule[] = [] +param maintenanceIpRanges IpRange[] = [] @description('Database : administrator login name.') @minLength(0) @@ -166,6 +166,13 @@ var resourceNames = { } } +var maintenanceFirewallRules = [for maintenanceIpRange in maintenanceIpRanges: { + name: maintenanceIpRange.name + cidr: maintenanceIpRange.cidr + tag: 'Default' + priority: 100 +}] + module vNetModule 'application/shared/virtualNetwork.bicep' = { name: 'virtualNetworkApplicationModuleDeploy' params: { @@ -195,7 +202,7 @@ module publicApiStorageModule 'application/public-api/publicApiStorage.bicep' = location: location resourceNames: resourceNames publicApiDataFileShareQuota: publicApiDataFileShareQuota - storageFirewallRules: maintenanceFirewallRules + storageFirewallRules: maintenanceIpRanges deployAlerts: deployAlerts tagValues: tagValues } @@ -229,7 +236,7 @@ module postgreSqlServerModule 'application/shared/postgreSqlFlexibleServer.bicep entraIdAdminPrincipals: postgreSqlEntraIdAdminPrincipals privateEndpointSubnetId: vNetModule.outputs.psqlFlexibleServerSubnetRef autoGrowStatus: postgreSqlAutoGrowStatus - firewallRules: maintenanceFirewallRules + firewallRules: maintenanceIpRanges sku: postgreSqlSkuName storageSizeGB: postgreSqlStorageSizeGB deployAlerts: deployAlerts @@ -380,6 +387,8 @@ module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContai var adminSubnetFirewallRule = { name: 'Admin App Service subnet range' cidr: vNetModule.outputs.adminAppServiceSubnetCidr + tag: 'Default' + priority: 100 } module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' = if (deployDataProcessor) { @@ -391,7 +400,7 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId devopsServicePrincipalId: devopsServicePrincipalId - storageFirewallRules: maintenanceFirewallRules + storageFirewallRules: maintenanceIpRanges functionAppFirewallRules: union([ adminSubnetFirewallRule // TODO EES-5446 - add in when static IP range available for runner scale sets @@ -402,7 +411,7 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' { cidr: 'AzureCloud' tag: 'ServiceTag' - priority: 100 + priority: 101 name: 'AzureCloud' } ], maintenanceFirewallRules) diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index eee0b62a22c..6d73131a9af 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -40,12 +40,17 @@ type ResourceNames = { } } +@export() +type IpRange = { + name: string + cidr: string +} @export() type FirewallRule = { name: string cidr: string - priority: int? - tag: string? + priority: int + tag: string } @export() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs new file mode 100644 index 00000000000..f17bc7a8128 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs @@ -0,0 +1,25 @@ +using System.Diagnostics; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class LongRunningFunctions(ILogger logger) +{ + [Function(nameof(LongRunningActivity))] + public async Task LongRunningActivity( + [ActivityTrigger] LongRunningOrchestrationContext input, + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + + while (stopwatch.Elapsed.Seconds < input.DurationSeconds) + { + await Task.Delay(10000, cancellationToken); + + logger.LogInformation($"Long-running orchestration running for {stopwatch.Elapsed.Seconds} " + + $"out of {input.DurationSeconds} seconds"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs index 0e789ff8bf2..98e3330ec3f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using Microsoft.Azure.Functions.Worker; @@ -7,7 +6,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class LogRunningOrchestration(ILogger logger) +public static class LogRunningOrchestration { [Function(nameof(ProcessLongRunningOrchestration))] public static async Task ProcessLongRunningOrchestration( @@ -23,7 +22,7 @@ public static async Task ProcessLongRunningOrchestration( try { - await context.CallActivity(nameof(LongRunningActivity), logger, context.InstanceId); + await context.CallActivity(nameof(LongRunningFunctions.LongRunningActivity), logger, input); } catch (Exception e) { @@ -35,21 +34,4 @@ public static async Task ProcessLongRunningOrchestration( await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); } } - - [Function(nameof(LongRunningActivity))] - public async Task LongRunningActivity( - [ActivityTrigger] Guid instanceId, - int durationSeconds, - CancellationToken cancellationToken) - { - var stopwatch = Stopwatch.StartNew(); - - while (stopwatch.Elapsed.Seconds < durationSeconds) - { - await Task.Delay(10000, cancellationToken); - - logger.LogInformation($"Long-running orchestration running for {stopwatch.Elapsed.Seconds} " + - $"out of {durationSeconds} seconds"); - } - } } From 22af14968f6397c001b4325a3dbd7eef1243e263 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 29 Nov 2024 13:30:47 +0000 Subject: [PATCH 086/144] EES-5446 - breaking out deploy tasks into template files --- .../public-api/ci/azure-pipelines.yml | 5 + .../ci/jobs/deploy-data-processor.yml | 181 ++---------------- .../public-api/ci/tasks/deploy-bicep.yml | 3 +- .../ci/tasks/wait-for-endpoint-success.yml | 63 ++++++ .../wait-for-orchestrations-to-complete.yml | 74 +++++++ .../templates/public-api/main.bicep | 8 +- .../templates/public-api/types.bicep | 1 + 7 files changed, 169 insertions(+), 166 deletions(-) create mode 100644 infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml create mode 100644 infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml diff --git a/infrastructure/templates/public-api/ci/azure-pipelines.yml b/infrastructure/templates/public-api/ci/azure-pipelines.yml index 09421669437..5412ea98168 100644 --- a/infrastructure/templates/public-api/ci/azure-pipelines.yml +++ b/infrastructure/templates/public-api/ci/azure-pipelines.yml @@ -16,6 +16,9 @@ parameters: - name: deployAlerts displayName: Whether to create or update Azure Monitor alerts during this deploy. default: false + - name: awaitActiveOrchestrations + displayName: Should this deploy wait for active orchestrations in Function Apps to complete prior to deploying? + default: true - name: forceDeployToEnvironment displayName: Set to either dev or test to force a deploy to that environment from the chosen branch. type: string @@ -57,6 +60,8 @@ variables: value: ${{ parameters.deployDataProcessor }} - name: deployAlerts value: ${{ parameters.deployAlerts }} + - name: awaitActiveOrchestrations + value: ${{ parameters.awaitActiveOrchestrations }} pool: vmImage: $(vmImageName) diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 794677c2929..5124c1da6d4 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -64,28 +64,6 @@ jobs: --settings \ "PublicDataDb=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(dataProcessorPsqlConnectionStringSecretKey))" - # TODO EES-5128 - # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow - # DevOps to deploy the Data Processor Function App without having to temporarily - # make it publicly accessible. - # - task: AzureCLI@2 - # displayName: Temporarily enable public network access before deploy - # retryCountOnTaskFailure: 1 - # inputs: - # azureSubscription: ${{ parameters.serviceConnection }} - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # set -e - - # az functionapp update \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --slot staging \ - # --set \ - # publicNetworkAccess=Enabled \ - # siteConfig.publicNetworkAccess=Enabled - # TODO EES-5128 # Retry deploying the Function App in order to allow the staging slot the time to # fully restart after config and network settings have been updated prior to deploy. @@ -113,107 +91,21 @@ jobs: --resource-group $(resourceGroupName) \ --slot staging - # TODO EES-5128 - # Add Private Endpoint to Data Processor Function App into the VMSS VNet to allow - # DevOps to deploy the Data Processor Function App without having to temporarily - # make it publicly accessible. - # - task: AzureCLI@2 - # displayName: Disable public network access after deploy - # retryCountOnTaskFailure: 1 - # condition: always() - # inputs: - # azureSubscription: ${{ parameters.serviceConnection }} - # scriptType: bash - # scriptLocation: inlineScript - # inlineScript: | - # set -e - - # az functionapp update \ - # --name $(dataProcessorFunctionAppName) \ - # --resource-group $(resourceGroupName) \ - # --slot staging \ - # --set \ - # publicNetworkAccess=Disabled \ - # siteConfig.publicNetworkAccess=Disabled - - - task: AzureCLI@2 - displayName: Wait for Data Processor staging slot to start up - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - - accessToken=`az account get-access-token \ - --resource $(dataProcessorAppRegistrationClientId) \ - --query "accessToken" \ - -o tsv` - - pollingTimeSeconds=5 - maxAttempts=50 - attempt=1 - - while [ "$attempt" -le "$maxAttempts" ]; do - - echo "Attempt number $attempt of $maxAttempts- calling $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck to check for Data Processor staging being healthy." - - httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck` - - echo "Got HTTP status code $httpStatusCode." - - if [[ "$httpStatusCode" == "200" ]]; then - echo "Data Processor staging slot has started successfully and is healthy - deployment can continue." - exit 0 - fi - - attempt=$((attempt + 1)) - sleep $pollingTimeSeconds - - done - - echo "Timed out waiting for Data processor to start up." - exit 1 - - - task: AzureCLI@2 - displayName: Wait for active orchestrations to complete - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - - accessToken=`az account get-access-token \ - --resource $(dataProcessorAppRegistrationClientId) \ - --query "accessToken" \ - -o tsv` - - pollingTimeSeconds=5 - maxAttempts=50 - attempt=1 - - while [ "$attempt" -le "$maxAttempts" ]; do - - echo "Attempt number $attempt of $maxAttempts - callinf $(dataProcessorFunctionAppUrl)/api/StatusCheck to check for active orchestrations." - - activeOrchestrationResults=`curl -H "Authorization: Bearer $accessToken" -s $(dataProcessorFunctionAppUrl)/api/StatusCheck` - activeOrchestrationCount=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations'` - activeOrchestrations=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations != 0'` - - echo "Currently $activeOrchestrationCount active orchestrations running." - - if [[ "$activeOrchestrations" == "false" ]]; then - echo "No active orchestrations running on the Data Processor - slot swapping can proceed." - exit 0 - fi - - attempt=$((attempt + 1)) - sleep $pollingTimeSeconds - - done - - echo "Timed out waiting for active Data Processor orchestrations to complete." - exit 1 + - template: ../tasks/wait-for-endpoint-success.yml + parameters: + serviceConnection: ${{ parameters.serviceConnection }} + displayName: Waiting for staging slot to start successfully + accessTokenScope: $(dataProcessorAppRegistrationClientId) + endpoint: $(dataProcessorFunctionAppStagingUrl)/api/HealthCheck + - template: ../tasks/wait-for-orchestrations-to-complete.yml + parameters: + serviceConnection: ${{ parameters.serviceConnection }} + displayName: Waiting for active orchestrations in the production slot to complete + accessTokenScope: $(dataProcessorAppRegistrationClientId) + endpoint: $(dataProcessorFunctionAppUrl)/api/StatusCheck + condition: eq(variables.awaitActiveOrchestrations, true) + - task: AzureCLI@2 displayName: Swap slots retryCountOnTaskFailure: 1 @@ -223,48 +115,15 @@ jobs: scriptLocation: inlineScript inlineScript: | set -e - az functionapp deployment slot swap \ --name $(dataProcessorFunctionAppName) \ --resource-group $(resourceGroupName) \ --slot staging \ --target-slot production - - task: AzureCLI@2 - displayName: Check that Data Processor is ready to serve new requests - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -x - - accessToken=`az account get-access-token \ - --resource $(dataProcessorAppRegistrationClientId) \ - --query "accessToken" \ - -o tsv` - - pollingTimeSeconds=5 - maxAttempts=50 - attempt=1 - - while [ "$attempt" -le "$maxAttempts" ]; do - - echo "Attempt number $attempt of $maxAttempts - calling $(dataProcessorFunctionAppUrl)/api/HealthCheck to check for Data Processor being healthy." - - httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null $(dataProcessorFunctionAppUrl)/api/HealthCheck` - - echo "Got HTTP status code $httpStatusCode." - - if [[ "$httpStatusCode" == "200" ]]; then - echo "Data Processor has started successfully and is healthy - ready to serve new requests." - exit 0 - fi - - attempt=$((attempt + 1)) - sleep $pollingTimeSeconds - - done - - echo "Timed out waiting for Data processor to start up." - exit 1 + - template: ../tasks/wait-for-endpoint-success.yml + parameters: + serviceConnection: ${{ parameters.serviceConnection }} + displayName: Checking that production slot is healthy after slot swap + accessTokenScope: $(dataProcessorAppRegistrationClientId) + endpoint: $(dataProcessorFunctionAppUrl)/api/HealthCheck diff --git a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml index 1903174be77..2e9d5ca3518 100644 --- a/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml +++ b/infrastructure/templates/public-api/ci/tasks/deploy-bicep.yml @@ -59,5 +59,4 @@ steps: dataProcessorFunctionAppExists=${{ parameters.dataProcessorExists }} \ dataProcessorAppRegistrationClientId='$(dataProcessorAppRegistrationClientId)' \ apiAppRegistrationClientId='$(apiAppRegistrationClientId)' \ - devopsServicePrincipalId="$servicePrincipalId" \ - pipelineRunnerCidr="$(pipelineRunnerCidr)" \ No newline at end of file + devopsServicePrincipalId="$servicePrincipalId" \ No newline at end of file diff --git a/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml b/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml new file mode 100644 index 00000000000..77e9454e7d4 --- /dev/null +++ b/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml @@ -0,0 +1,63 @@ +parameters: + + - name: serviceConnection + type: string + + - name: displayName + type: string + default: Waiting for a successful response from endpoint + + - name: accessTokenScope + type: string + default: null + + - name: pollingDelaySeconds + type: number + default: 5 + + - name: maxAttempts + type: number + default: 50 + + - name: endpoint + type: string + +steps: + - task: AzureCLI@2 + displayName: ${{ parameters.displayName }} + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + + if [ -n "${{ parameters.accessTokenScope }}" ]; then + accessToken=`az account get-access-token \ + --resource ${{ parameters.accessTokenScope }} \ + --query "accessToken" \ + -o tsv` + fi + + for attempt in $(seq 1 ${{ parameters.maxAttempts }}); + do + + echo "Attempt number $attempt of ${{ parameters.maxAttempts }} - calling ${{ parameters.endpoint }} to check for successful response." + + if [ -n "$accessToken" ]; then + httpStatusCode=`curl --write-out '%{http_code}' -H "Authorization: Bearer $accessToken" -s --output /dev/null ${{ parameters.endpoint }}` + else + httpStatusCode=`curl --write-out '%{http_code}' -s --output /dev/null ${{ parameters.endpoint }}` + fi + + if (( $httpStatusCode >= 200 && $httpStatusCode <= 204 )); then + echo "Received successful response with status code $httpStatusCode." + exit 0 + fi + + echo "Received response with status code $httpStatusCode. Retrying in ${{ parameters.pollingDelaySeconds }} seconds." + sleep ${{ parameters.pollingDelaySeconds }} + + done + + echo "Timed out waiting for successful response." + exit 1 \ No newline at end of file diff --git a/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml b/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml new file mode 100644 index 00000000000..cf957664c6d --- /dev/null +++ b/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml @@ -0,0 +1,74 @@ +parameters: + + - name: serviceConnection + type: string + + - name: displayName + type: string + default: Waiting for active orchestrations to complete + + - name: condition + type: string + + - name: accessTokenScope + type: string + default: null + + - name: pollingDelaySeconds + type: number + default: 5 + + - name: maxAttempts + type: number + default: 50 + + - name: endpoint + type: string + + - name: dependsOn + type: object + default: [] + +steps: + - task: AzureCLI@2 + displayName: ${{ parameters.displayName }} + condition: ${{ parameters.condition}} + inputs: + azureSubscription: ${{ parameters.serviceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + + if [ -n "${{ parameters.accessTokenScope }}" ]; then + accessToken=`az account get-access-token \ + --resource ${{ parameters.accessTokenScope }} \ + --query "accessToken" \ + -o tsv` + fi + + for attempt in $(seq 1 ${{ parameters.maxAttempts }}); + do + + echo "Attempt number $attempt of ${{ parameters.maxAttempts }} - calling ${{ parameters.endpoint }} to check for active orchestrations." + + if [ -n "$accessToken" ]; then + activeOrchestrationResults=`curl -H "Authorization: Bearer $accessToken" -s ${{ parameters.endpoint }}` + else + activeOrchestrationResults=`curl -s ${{ parameters.endpoint }}` + fi + + activeOrchestrationCount=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations'` + activeOrchestrations=`echo $activeOrchestrationResults | jq -r '.activeOrchestrations != 0'` + + if [[ "$activeOrchestrations" == "false" ]]; then + echo "No active orchestrations are running." + exit 0 + fi + + echo "$activeOrchestrationCount active orchestrations are still running. Retrying in ${{ parameters.pollingDelaySeconds }} seconds." + sleep ${{ parameters.pollingDelaySeconds }} + + done + + echo "Timed out waiting for active orchestrations to complete." + exit 1 \ No newline at end of file diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index b985dd102ff..4b7f611bced 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -100,8 +100,9 @@ param apiAppRegistrationClientId string = '' @secure() param devopsServicePrincipalId string = '' -@description('Specifies the IP address range of the pipeline runners.') -param pipelineRunnerCidr string = '' +// TODO EES-5446 - reinstate pipelineRunnerCidr when the DevOps runners have a static IP range available. +// @description('Specifies the IP address range of the pipeline runners.') +// param pipelineRunnerCidr string = '' @description('Specifies whether or not test Themes can be deleted in the environment.') param enableThemeDeletion bool = false @@ -403,11 +404,12 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' storageFirewallRules: maintenanceIpRanges functionAppFirewallRules: union([ adminSubnetFirewallRule - // TODO EES-5446 - add in when static IP range available for runner scale sets + // TODO EES-5446 - reinstate when static IP range available for runner scale sets // { // name: 'Pipeline runner IP address range' // cidr: pipelineRunnerCidr // } + // TODO EES-5446 - remove service tag whitelisting when runner scale set IP range reinstated { cidr: 'AzureCloud' tag: 'ServiceTag' diff --git a/infrastructure/templates/public-api/types.bicep b/infrastructure/templates/public-api/types.bicep index 6d73131a9af..d21d724fad2 100644 --- a/infrastructure/templates/public-api/types.bicep +++ b/infrastructure/templates/public-api/types.bicep @@ -45,6 +45,7 @@ type IpRange = { name: string cidr: string } + @export() type FirewallRule = { name: string From 1251ecbe510154f0949ddfaa50aa70d97cee077d Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 2 Dec 2024 09:16:45 +0000 Subject: [PATCH 087/144] EES-5446 - reverting temporary pipeline changes for speeding up test deploys --- azure-pipelines-main.yml | 625 +++++++++--------- .../templates/public-api/ci/stages/deploy.yml | 12 +- 2 files changed, 318 insertions(+), 319 deletions(-) diff --git a/azure-pipelines-main.yml b/azure-pipelines-main.yml index 3ede71f80fe..d5fe21618fc 100644 --- a/azure-pipelines-main.yml +++ b/azure-pipelines-main.yml @@ -6,7 +6,6 @@ parameters: - master - dev - test - - EES-5446-prevent-function-app-slot-swap-until-orchestrations-complete variables: BuildConfiguration: Release @@ -60,64 +59,64 @@ jobs: # custom: format whitespace src/GovUk.Education.ExploreEducationStatistics.sln --verify-no-changes --severity error # arguments: --verify-no-changes --verbosity diagnostic -# - task: DotNetCoreCLI@2 -# displayName: Verify Formatting and Style -# inputs: -# command: custom -# custom: format -# ## TODO: Remove "--severity error" once style formatter has been run across project -# arguments: style --verify-no-changes --verbosity diagnostic --severity error -# projects: src/GovUk.Education.ExploreEducationStatistics.sln -# -# - task: DotNetCoreCLI@2 -# displayName: Verify Formatting and Style -# inputs: -# command: custom -# custom: format -# ## TODO: Remove "--severity error" once work has been done to resolve build warnings (https://dfedigital.atlassian.net/browse/EES-4594). -# arguments: analyzers --verify-no-changes --verbosity diagnostic --severity error -# projects: src/GovUk.Education.ExploreEducationStatistics.sln + - task: DotNetCoreCLI@2 + displayName: Verify Formatting and Style + inputs: + command: custom + custom: format + ## TODO: Remove "--severity error" once style formatter has been run across project + arguments: style --verify-no-changes --verbosity diagnostic --severity error + projects: src/GovUk.Education.ExploreEducationStatistics.sln + + - task: DotNetCoreCLI@2 + displayName: Verify Formatting and Style + inputs: + command: custom + custom: format + ## TODO: Remove "--severity error" once work has been done to resolve build warnings (https://dfedigital.atlassian.net/browse/EES-4594). + arguments: analyzers --verify-no-changes --verbosity diagnostic --severity error + projects: src/GovUk.Education.ExploreEducationStatistics.sln # TODO: Wrap these ^ three tasks up into a single `dotnet format` task once all 3 above TODOs are TO-DONE ;) -# - task: DotNetCoreCLI@2 -# displayName: Test -# inputs: -# command: test -# projects: | -# **/GovUk.*[Tt]ests/*.csproj -# !**/GovUk.Education.ExploreEducationStatistics.Admin.Tests/*csproj -# arguments: --configuration $(BuildConfiguration) -# -# - task: DotNetCoreCLI@2 -# displayName: Package Data API -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Api.csproj' -# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/data-api -# zipAfterPublish: True -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Data API artifact -# inputs: -# artifactName: data-api -# targetPath: $(Build.ArtifactStagingDirectory)/data-api -# -# - task: DotNetCoreCLI@2 -# displayName: Package Content API -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Content.Api.csproj' -# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/content-api -# zipAfterPublish: True -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Content API artifact -# inputs: -# artifactName: content-api -# targetPath: $(Build.ArtifactStagingDirectory)/content-api + - task: DotNetCoreCLI@2 + displayName: Test + inputs: + command: test + projects: | + **/GovUk.*[Tt]ests/*.csproj + !**/GovUk.Education.ExploreEducationStatistics.Admin.Tests/*csproj + arguments: --configuration $(BuildConfiguration) + + - task: DotNetCoreCLI@2 + displayName: Package Data API + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Api.csproj' + arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/data-api + zipAfterPublish: True + + - task: PublishPipelineArtifact@0 + displayName: Publish Data API artifact + inputs: + artifactName: data-api + targetPath: $(Build.ArtifactStagingDirectory)/data-api + + - task: DotNetCoreCLI@2 + displayName: Package Content API + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Content.Api.csproj' + arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/content-api + zipAfterPublish: True + + - task: PublishPipelineArtifact@0 + displayName: Publish Content API artifact + inputs: + artifactName: content-api + targetPath: $(Build.ArtifactStagingDirectory)/content-api - task: DotNetCoreCLI@2 displayName: Package Public API @@ -167,260 +166,260 @@ jobs: artifactName: public-api-data-processor targetPath: $(Build.ArtifactStagingDirectory)/public-api-data-processor -# - task: DotNetCoreCLI@2 -# displayName: Package Notifier Function -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Notifier.csproj' -# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/notifier -# zipAfterPublish: True -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Notifier artifact -# inputs: -# artifactName: notifier -# targetPath: $(Build.ArtifactStagingDirectory)/notifier -# -# - task: DotNetCoreCLI@2 -# displayName: Package Publisher Function -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Publisher.csproj' -# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/publisher -# zipAfterPublish: True -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Publisher artifact -# inputs: -# artifactName: publisher -# targetPath: $(Build.ArtifactStagingDirectory)/publisher -# -# - task: DotNetCoreCLI@2 -# displayName: Package Processor Function -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Processor.csproj' -# arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/processor -# zipAfterPublish: True -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Processor artifact -# inputs: -# artifactName: processor -# targetPath: $(Build.ArtifactStagingDirectory)/processor - -# - job: Admin -# pool: ees-ubuntu2204-xlarge -# workspace: -# clean: all -# steps: -# - task: UseNode@1 -# displayName: Install Node.js $(NodeVersion) -# inputs: -# version: $(NodeVersion) -# -# - task: Bash@3 -# displayName: corepack enable -# inputs: -# workingDir: . -# targetType: inline -# script: corepack enable -# -# - task: UseDotNet@2 -# displayName: Install .NET 8.0 SDK -# inputs: -# version: 8.0.x -# performMultiLevelLookup: true -# -# - task: DotNetCoreCLI@2 -# displayName: Build -# inputs: -# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' -# arguments: --configuration $(BuildConfiguration) -# -# - task: DotNetCoreCLI@2 -# displayName: Test -# inputs: -# command: test -# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.Tests.csproj' -# arguments: --configuration $(BuildConfiguration) --collect "Code coverage" -# -# - task: Bash@3 -# displayName: pnpm i -# inputs: -# targetType: inline -# script: pnpm i -# -# - task: Bash@3 -# displayName: pnpm run build -# inputs: -# targetType: inline -# script: pnpm --filter=explore-education-statistics-admin run build -# -# - task: CopyFiles@2 -# displayName: Copy files to wwwroot -# inputs: -# SourceFolder: src/explore-education-statistics-admin/build -# TargetFolder: src/GovUk.Education.ExploreEducationStatistics.Admin/wwwroot -# -# - task: DotNetCoreCLI@2 -# displayName: Package Admin app -# inputs: -# command: publish -# publishWebProjects: false -# projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' -# arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory) -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish Admin artifact -# inputs: -# artifactName: admin -# targetPath: $(Build.ArtifactStagingDirectory) -# -# - job: Frontend -# pool: ees-ubuntu2204-xlarge -# workspace: -# clean: all -# steps: -# - task: UseNode@1 -# displayName: Install Node.js $(NodeVersion) -# inputs: -# version: $(NodeVersion) -# -# - task: Bash@3 -# displayName: corepack enable -# inputs: -# workingDir: . -# targetType: inline -# script: corepack enable -# -# - task: Bash@3 -# displayName: pnpm i -# inputs: -# workingDir: . -# targetType: inline -# script: pnpm i -# -# - task: Bash@3 -# displayName: pnpm tsc -# inputs: -# workingDir: . -# targetType: inline -# script: pnpm tsc -# -# - task: Bash@3 -# displayName: pnpm lint -# inputs: -# workingDir: . -# targetType: inline -# script: pnpm lint -# -# - task: Bash@3 -# displayName: pnpm format:check -# inputs: -# workingDir: . -# targetType: inline -# script: pnpm format:check -# -# - task: Bash@3 -# displayName: pnpm test:ci -# inputs: -# workingDir: . -# targetType: inline -# script: pnpm test:ci -# -# - task: PublishTestResults@2 -# displayName: Publish frontend test results -# inputs: -# testResultsFormat: JUnit -# testResultsFiles: explore-education-statistics-*/junit-*.xml -# searchFolder: ./src -# testRunTitle: Release Jest tests -# mergeTestResults: true -# -# - task: Bash@3 -# displayName: pnpm run build -# inputs: -# targetType: inline -# script: pnpm --filter=explore-education-statistics-frontend run build -# -# - task: Docker@2 -# displayName: Build Public frontend Docker image -# condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) -# inputs: -# containerRegistry: $(AcrServiceConnection) -# repository: ees-public-frontend -# command: build -# Dockerfile: docker/public-frontend/Dockerfile -# buildContext: $(System.DefaultWorkingDirectory) -# tags: $(Build.BuildNumber) -# arguments: --build-arg BUILD_BUILDNUMBER=$(Build.BuildNumber) -# env: -# DOCKER_BUILDKIT: 1 -# -# - task: Docker@2 -# displayName: Push Public frontend Docker image -# condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) -# inputs: -# containerRegistry: $(AcrServiceConnection) -# repository: ees-public-frontend -# command: push -# tags: $(Build.BuildNumber) - -# - job: ApiDocs -# pool: -# vmImage: ubuntu-22.04 -# workspace: -# clean: all -# variables: -# WorkingDirectory: src/explore-education-statistics-api-docs -# steps: -# - task: UseNode@1 -# displayName: Install Node.js $(NodeVersion) -# inputs: -# version: $(NodeVersion) -# -# - task: UseRubyVersion@0 -# displayName: Install Ruby $(RubyVersion) -# inputs: -# versionSpec: '>= $(RubyVersion)' -# -# - task: Bash@3 -# displayName: Build -# env: -# TECH_DOCS_API_URL: https://dev.statistics.api.education.gov.uk -# inputs: -# workingDirectory: $(WorkingDirectory) -# targetType: inline -# script: | -# bundle install -# bundle exec middleman build -# -# - task: PublishPipelineArtifact@1 -# displayName: Publish artifact -# inputs: -# artifactName: public-api-docs -# targetPath: $(WorkingDirectory)/build -# -# - job: MiscellaneousArtifacts -# pool: -# vmImage: ubuntu-22.04 -# workspace: -# clean: all -# steps: -# - task: CopyFiles@2 -# displayName: Copy Pipfiles to tests -# inputs: -# Contents: | -# Pipfile -# Pipfile.lock -# TargetFolder: tests -# -# - task: PublishPipelineArtifact@0 -# displayName: Publish test files -# inputs: -# artifactName: tests -# targetPath: tests + - task: DotNetCoreCLI@2 + displayName: Package Notifier Function + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Notifier.csproj' + arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/notifier + zipAfterPublish: True + + - task: PublishPipelineArtifact@0 + displayName: Publish Notifier artifact + inputs: + artifactName: notifier + targetPath: $(Build.ArtifactStagingDirectory)/notifier + + - task: DotNetCoreCLI@2 + displayName: Package Publisher Function + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Publisher.csproj' + arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/publisher + zipAfterPublish: True + + - task: PublishPipelineArtifact@0 + displayName: Publish Publisher artifact + inputs: + artifactName: publisher + targetPath: $(Build.ArtifactStagingDirectory)/publisher + + - task: DotNetCoreCLI@2 + displayName: Package Processor Function + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Data.Processor.csproj' + arguments: --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)/processor + zipAfterPublish: True + + - task: PublishPipelineArtifact@0 + displayName: Publish Processor artifact + inputs: + artifactName: processor + targetPath: $(Build.ArtifactStagingDirectory)/processor + + - job: Admin + pool: ees-ubuntu2204-xlarge + workspace: + clean: all + steps: + - task: UseNode@1 + displayName: Install Node.js $(NodeVersion) + inputs: + version: $(NodeVersion) + + - task: Bash@3 + displayName: corepack enable + inputs: + workingDir: . + targetType: inline + script: corepack enable + + - task: UseDotNet@2 + displayName: Install .NET 8.0 SDK + inputs: + version: 8.0.x + performMultiLevelLookup: true + + - task: DotNetCoreCLI@2 + displayName: Build + inputs: + projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' + arguments: --configuration $(BuildConfiguration) + + - task: DotNetCoreCLI@2 + displayName: Test + inputs: + command: test + projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.Tests.csproj' + arguments: --configuration $(BuildConfiguration) --collect "Code coverage" + + - task: Bash@3 + displayName: pnpm i + inputs: + targetType: inline + script: pnpm i + + - task: Bash@3 + displayName: pnpm run build + inputs: + targetType: inline + script: pnpm --filter=explore-education-statistics-admin run build + + - task: CopyFiles@2 + displayName: Copy files to wwwroot + inputs: + SourceFolder: src/explore-education-statistics-admin/build + TargetFolder: src/GovUk.Education.ExploreEducationStatistics.Admin/wwwroot + + - task: DotNetCoreCLI@2 + displayName: Package Admin app + inputs: + command: publish + publishWebProjects: false + projects: '**/GovUk.Education.ExploreEducationStatistics.Admin.csproj' + arguments: --self-contained true -r win-x64 --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: Publish Admin artifact + inputs: + artifactName: admin + targetPath: $(Build.ArtifactStagingDirectory) + + - job: Frontend + pool: ees-ubuntu2204-xlarge + workspace: + clean: all + steps: + - task: UseNode@1 + displayName: Install Node.js $(NodeVersion) + inputs: + version: $(NodeVersion) + + - task: Bash@3 + displayName: corepack enable + inputs: + workingDir: . + targetType: inline + script: corepack enable + + - task: Bash@3 + displayName: pnpm i + inputs: + workingDir: . + targetType: inline + script: pnpm i + + - task: Bash@3 + displayName: pnpm tsc + inputs: + workingDir: . + targetType: inline + script: pnpm tsc + + - task: Bash@3 + displayName: pnpm lint + inputs: + workingDir: . + targetType: inline + script: pnpm lint + + - task: Bash@3 + displayName: pnpm format:check + inputs: + workingDir: . + targetType: inline + script: pnpm format:check + + - task: Bash@3 + displayName: pnpm test:ci + inputs: + workingDir: . + targetType: inline + script: pnpm test:ci + + - task: PublishTestResults@2 + displayName: Publish frontend test results + inputs: + testResultsFormat: JUnit + testResultsFiles: explore-education-statistics-*/junit-*.xml + searchFolder: ./src + testRunTitle: Release Jest tests + mergeTestResults: true + + - task: Bash@3 + displayName: pnpm run build + inputs: + targetType: inline + script: pnpm --filter=explore-education-statistics-frontend run build + + - task: Docker@2 + displayName: Build Public frontend Docker image + condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) + inputs: + containerRegistry: $(AcrServiceConnection) + repository: ees-public-frontend + command: build + Dockerfile: docker/public-frontend/Dockerfile + buildContext: $(System.DefaultWorkingDirectory) + tags: $(Build.BuildNumber) + arguments: --build-arg BUILD_BUILDNUMBER=$(Build.BuildNumber) + env: + DOCKER_BUILDKIT: 1 + + - task: Docker@2 + displayName: Push Public frontend Docker image + condition: and(succeeded(), eq(variables.IsBranchDeployable, true)) + inputs: + containerRegistry: $(AcrServiceConnection) + repository: ees-public-frontend + command: push + tags: $(Build.BuildNumber) + + - job: ApiDocs + pool: + vmImage: ubuntu-22.04 + workspace: + clean: all + variables: + WorkingDirectory: src/explore-education-statistics-api-docs + steps: + - task: UseNode@1 + displayName: Install Node.js $(NodeVersion) + inputs: + version: $(NodeVersion) + + - task: UseRubyVersion@0 + displayName: Install Ruby $(RubyVersion) + inputs: + versionSpec: '>= $(RubyVersion)' + + - task: Bash@3 + displayName: Build + env: + TECH_DOCS_API_URL: https://dev.statistics.api.education.gov.uk + inputs: + workingDirectory: $(WorkingDirectory) + targetType: inline + script: | + bundle install + bundle exec middleman build + + - task: PublishPipelineArtifact@1 + displayName: Publish artifact + inputs: + artifactName: public-api-docs + targetPath: $(WorkingDirectory)/build + + - job: MiscellaneousArtifacts + pool: + vmImage: ubuntu-22.04 + workspace: + clean: all + steps: + - task: CopyFiles@2 + displayName: Copy Pipfiles to tests + inputs: + Contents: | + Pipfile + Pipfile.lock + TargetFolder: tests + + - task: PublishPipelineArtifact@0 + displayName: Publish test files + inputs: + artifactName: tests + targetPath: tests diff --git a/infrastructure/templates/public-api/ci/stages/deploy.yml b/infrastructure/templates/public-api/ci/stages/deploy.yml index aad36c1949c..956272d6ccb 100644 --- a/infrastructure/templates/public-api/ci/stages/deploy.yml +++ b/infrastructure/templates/public-api/ci/stages/deploy.yml @@ -16,7 +16,7 @@ parameters: type: string - name: dependsOn type: object - default: [] + default: [ ] - name: trigger type: string default: automatic @@ -50,11 +50,11 @@ stages: environment: ${{ parameters.environment }} bicepParamFile: ${{ parameters.bicepParamFile }} - # - template: ../jobs/deploy-api-docs.yml - # parameters: - # serviceConnection: ${{ parameters.serviceConnection }} - # environment: ${{ parameters.environment }} - # dependsOn: DeployPublicApiInfrastructure + - template: ../jobs/deploy-api-docs.yml + parameters: + serviceConnection: ${{ parameters.serviceConnection }} + environment: ${{ parameters.environment }} + dependsOn: DeployPublicApiInfrastructure - template: ../jobs/deploy-data-processor.yml parameters: From 782043fa7e734484ce68762ac6120908b5a49985 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 2 Dec 2024 11:26:40 +0000 Subject: [PATCH 088/144] EES-5446 - making data processor and PSQL deploys independent of private DNS zone deployment to save lots of time during a standard deploy --- .../templates/public-api/ci/stages/deploy.yml | 2 +- .../public-api/ci/tasks/wait-for-endpoint-success.yml | 8 +------- .../ci/tasks/wait-for-orchestrations-to-complete.yml | 10 +--------- infrastructure/templates/public-api/main.bicep | 4 ++-- .../Functions/LongRunningOrchestration.cs | 2 +- .../Functions/LongRunningTriggerFunction.cs | 2 +- 6 files changed, 7 insertions(+), 21 deletions(-) diff --git a/infrastructure/templates/public-api/ci/stages/deploy.yml b/infrastructure/templates/public-api/ci/stages/deploy.yml index 956272d6ccb..62adedfe7cc 100644 --- a/infrastructure/templates/public-api/ci/stages/deploy.yml +++ b/infrastructure/templates/public-api/ci/stages/deploy.yml @@ -16,7 +16,7 @@ parameters: type: string - name: dependsOn type: object - default: [ ] + default: [] - name: trigger type: string default: automatic diff --git a/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml b/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml index 77e9454e7d4..9b79efecbcc 100644 --- a/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml +++ b/infrastructure/templates/public-api/ci/tasks/wait-for-endpoint-success.yml @@ -1,24 +1,18 @@ parameters: - - name: serviceConnection type: string - - name: displayName type: string default: Waiting for a successful response from endpoint - - name: accessTokenScope type: string default: null - - name: pollingDelaySeconds type: number default: 5 - - name: maxAttempts type: number default: 50 - - name: endpoint type: string @@ -60,4 +54,4 @@ steps: done echo "Timed out waiting for successful response." - exit 1 \ No newline at end of file + exit 1 diff --git a/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml b/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml index cf957664c6d..435dc897c9e 100644 --- a/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml +++ b/infrastructure/templates/public-api/ci/tasks/wait-for-orchestrations-to-complete.yml @@ -1,30 +1,22 @@ parameters: - - name: serviceConnection type: string - - name: displayName type: string default: Waiting for active orchestrations to complete - - name: condition type: string - - name: accessTokenScope type: string default: null - - name: pollingDelaySeconds type: number default: 5 - - name: maxAttempts type: number default: 50 - - name: endpoint type: string - - name: dependsOn type: object default: [] @@ -71,4 +63,4 @@ steps: done echo "Timed out waiting for active orchestrations to complete." - exit 1 \ No newline at end of file + exit 1 diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 4b7f611bced..499e5b6cfb7 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -10,7 +10,7 @@ param location string = resourceGroup().location @description('Public API Storage : Size of the file share in GB.') param publicApiDataFileShareQuota int = 1 -@description('Firewall rules for maintenance of the service by allowing key IP ranges access to resources.') +@description('Provides access to resources for specific IP address ranges used for service maintenance.') param maintenanceIpRanges IpRange[] = [] @description('Database : administrator login name.') @@ -189,7 +189,7 @@ module coreStorage 'application/shared/coreStorage.bicep' = { } module privateDnsZonesModule 'application/shared/privateDnsZones.bicep' = - if (deploySharedPrivateDnsZones || deployPsqlFlexibleServer || deployDataProcessor) { + if (deploySharedPrivateDnsZones) { name: 'privateDnsZonesApplicationModuleDeploy' params: { resourceNames: resourceNames diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs index 98e3330ec3f..a2e55a20dee 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs @@ -6,7 +6,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public static class LogRunningOrchestration +public static class LongRunningOrchestration { [Function(nameof(ProcessLongRunningOrchestration))] public static async Task ProcessLongRunningOrchestration( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs index c37e5a8c6dc..b57a6108b70 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs @@ -25,7 +25,7 @@ public async Task TriggerLongRunningOrchestration( httpRequest.GetRequestParamInt(paramName: "durationSeconds", 60); const string orchestratorName = - nameof(LogRunningOrchestration.ProcessLongRunningOrchestration); + nameof(LongRunningOrchestration.ProcessLongRunningOrchestration); var options = new StartOrchestrationOptions { InstanceId = instanceId.ToString() }; From 05a108580f7a357496da5a39bb2e1f3cbb6a4264 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 10 Dec 2024 16:41:41 +0000 Subject: [PATCH 089/144] EES-5446 - responded to various PR comments. Disabling long-running orchestration trigger by default. Code improvements and variable renaming. Tweaks to Bicep templates. --- .../public-api/publicApiDataProcessor.bicep | 43 ++++++++++++++++--- .../ci/jobs/deploy-data-processor.yml | 4 +- .../public-api/components/functionApp.bicep | 11 +++-- .../templates/public-api/main.bicep | 25 +++++------ .../Extensions/HttpRequestExtensions.cs | 4 +- .../Functions/LongRunningTriggerFunction.cs | 7 +-- .../Functions/StatusCheckFunction.cs | 13 +++--- 7 files changed, 68 insertions(+), 39 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 5891ab45d52..3abc099f17e 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -6,9 +6,6 @@ param resourceNames ResourceNames @description('Specifies the location for all resources.') param location string -@description('Alert metric name prefix') -param metricsNamePrefix string - @description('The Application Insights key that is associated with this resource') param applicationInsightsKey string @@ -26,7 +23,7 @@ param devopsServicePrincipalId string param storageFirewallRules IpRange[] @description('The IP address ranges that can access the Data Processor Function App endpoints.') -param functionAppFirewallRules FirewallRule[] = [] +param functionAppFirewallRules FirewallRule[] @description('Whether to create or update Azure Monitor alerts during this deploy') param deployAlerts bool @@ -107,9 +104,6 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { } preWarmedInstanceCount: 1 healthCheckPath: '/api/HealthCheck' - appSettings: { - App__MetaInsertBatchSize: 1000 - } azureFileShares: [{ storageName: resourceNames.publicApi.publicApiFileShare storageAccountKey: publicApiStorageAccount.listKeys().keys[0].value @@ -122,6 +116,41 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { } } +module functionAppHealthAlert '../../components/alerts/sites/healthAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}HealthDeploy' + params: { + resourceNames: [resourceNames.publicApi.dataProcessor] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} + +module storageAccountAvailabilityAlerts '../../components/alerts/storageAccounts/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}StorageAvailabilityDeploy' + params: { + resourceNames: [ + dataProcessorFunctionAppModule.outputs.managementStorageAccountName + dataProcessorFunctionAppModule.outputs.slot1StorageAccountName + dataProcessorFunctionAppModule.outputs.slot2StorageAccountName + ] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} + +module fileServiceAvailabilityAlerts '../../components/alerts/fileServices/availabilityAlert.bicep' = if (deployAlerts) { + name: '${resourceNames.publicApi.dataProcessor}FsAvailabilityDeploy' + params: { + resourceNames: [ + dataProcessorFunctionAppModule.outputs.managementStorageAccountName + dataProcessorFunctionAppModule.outputs.slot1StorageAccountName + dataProcessorFunctionAppModule.outputs.slot2StorageAccountName + ] + alertsGroupName: resourceNames.existingResources.alertsGroup + tagValues: tagValues + } +} + output managedIdentityName string = dataProcessorFunctionAppManagedIdentity.name output managedIdentityClientId string = dataProcessorFunctionAppManagedIdentity.properties.clientId output publicApiDataFileShareMountPath string = publicApiDataFileShareMountPath diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml index 5124c1da6d4..1ec275354e8 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-data-processor.yml @@ -43,9 +43,11 @@ jobs: --resource-group $(resourceGroupName) \ --slot staging \ --settings \ - "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ + "App__MetaInsertBatchSize=1000" \ "App__EnableThemeDeletion=$(enableThemeDeletion)" \ + "App__PrivateStorageConnectionString=@Microsoft.KeyVault(VaultName=$(keyVaultName); SecretName=$(coreStorageConnectionStringSecretKey))" \ "AZURE_CLIENT_ID=$(dataProcessorFunctionAppManagedIdentityClientId)" \ + "AzureWebJobs.TriggerLongRunningOrchestration.Disabled=true" \ "DataFiles__BasePath=$(dataProcessorPublicApiDataFileShareMountPath)" az webapp config connection-string set \ diff --git a/infrastructure/templates/public-api/components/functionApp.bicep b/infrastructure/templates/public-api/components/functionApp.bicep index f77f7808822..12d55ea2dc9 100644 --- a/infrastructure/templates/public-api/components/functionApp.bicep +++ b/infrastructure/templates/public-api/components/functionApp.bicep @@ -1,4 +1,4 @@ -import { FirewallRule, IpRange, AzureFileshareMount, EntraIdAuthentication } from '../types.bicep' +import { FirewallRule, IpRange, AzureFileShareMount, EntraIdAuthentication } from '../types.bicep' @description('Specifies the location for all resources.') param location string @@ -37,7 +37,7 @@ param privateEndpointSubnetId string? param publicNetworkAccessEnabled bool = false @description('IP address ranges that are allowed to access the Function App endpoints. Dependent on "publicNetworkAccessEnabled" being true.') -param functionAppEndpointFirewallRules FirewallRule[] = [] +param functionAppFirewallRules FirewallRule[] = [] @description('An existing Managed Identity\'s Resource Id with which to associate this Function App') param userAssignedManagedIdentityParams { @@ -175,7 +175,7 @@ resource slot2FileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2 ] } -var firewallRules = [for (firewallRule, index) in functionAppEndpointFirewallRules: { +var firewallRules = [for (firewallRule, index) in functionAppFirewallRules: { name: firewallRule.name ipAddress: firewallRule.cidr action: 'Allow' @@ -202,7 +202,10 @@ var commonSiteProperties = { keyVaultReferenceIdentity: keyVaultReferenceIdentity publicNetworkAccess: publicNetworkAccessEnabled ? 'Enabled' : 'Disabled' ipSecurityRestrictions: publicNetworkAccessEnabled && length(firewallRules) > 0 ? firewallRules : null - ipSecurityRestrictionsDefaultAction: publicNetworkAccessEnabled && length(firewallRules) > 0 ? 'Deny' : 'Allow' + ipSecurityRestrictionsDefaultAction: 'Deny' + // TODO EES-5446 - this setting controls access to the deploy site for the Function App. + // This is currently the default value, but ideally we would lock this down to only be accessible + // by our runners and certain other whitelisted IP address ranges (e.g. trusted VPNs). scmIpSecurityRestrictions: [ { ipAddress: 'Any' diff --git a/infrastructure/templates/public-api/main.bicep b/infrastructure/templates/public-api/main.bicep index 499e5b6cfb7..4426a92c6ac 100644 --- a/infrastructure/templates/public-api/main.bicep +++ b/infrastructure/templates/public-api/main.bicep @@ -385,30 +385,22 @@ module appGatewayModule 'application/shared/appGateway.bicep' = if (deployContai } } -var adminSubnetFirewallRule = { - name: 'Admin App Service subnet range' - cidr: vNetModule.outputs.adminAppServiceSubnetCidr - tag: 'Default' - priority: 100 -} - module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' = if (deployDataProcessor) { name: 'publicApiDataProcessorApplicationModuleDeploy' params: { location: location resourceNames: resourceNames - metricsNamePrefix: '${subscription}PublicDataProcessor' applicationInsightsKey: appInsightsModule.outputs.appInsightsKey dataProcessorAppRegistrationClientId: dataProcessorAppRegistrationClientId devopsServicePrincipalId: devopsServicePrincipalId storageFirewallRules: maintenanceIpRanges functionAppFirewallRules: union([ - adminSubnetFirewallRule - // TODO EES-5446 - reinstate when static IP range available for runner scale sets - // { - // name: 'Pipeline runner IP address range' - // cidr: pipelineRunnerCidr - // } + { + name: 'Admin App Service subnet range' + cidr: vNetModule.outputs.adminAppServiceSubnetCidr + tag: 'Default' + priority: 100 + } // TODO EES-5446 - remove service tag whitelisting when runner scale set IP range reinstated { cidr: 'AzureCloud' @@ -416,6 +408,11 @@ module dataProcessorModule 'application/public-api/publicApiDataProcessor.bicep' priority: 101 name: 'AzureCloud' } + // TODO EES-5446 - reinstate when static IP range available for runner scale sets + // { + // name: 'Pipeline runner IP address range' + // cidr: pipelineRunnerCidr + // } ], maintenanceFirewallRules) dataProcessorFunctionAppExists: dataProcessorFunctionAppExists deployAlerts: deployAlerts diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs index 3d95e325919..9e60dc9f13a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/HttpRequestExtensions.cs @@ -111,7 +111,7 @@ public static bool GetRequestParamBool( string paramName, bool defaultValue) { - var paramValue = GetRequestParam(httpRequest, paramName, defaultValue + ""); + var paramValue = GetRequestParam(httpRequest, paramName, defaultValue.ToString()); return bool.Parse(paramValue); } @@ -120,7 +120,7 @@ public static int GetRequestParamInt( string paramName, int defaultValue) { - var paramValue = GetRequestParam(httpRequest, paramName, defaultValue + ""); + var paramValue = GetRequestParam(httpRequest, paramName, defaultValue.ToString()); return int.Parse(paramValue); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs index b57a6108b70..5012da598e7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs @@ -22,7 +22,7 @@ public async Task TriggerLongRunningOrchestration( var instanceId = Guid.NewGuid(); var durationSeconds = - httpRequest.GetRequestParamInt(paramName: "durationSeconds", 60); + httpRequest.GetRequestParamInt("durationSeconds", defaultValue: 60); const string orchestratorName = nameof(LongRunningOrchestration.ProcessLongRunningOrchestration); @@ -37,10 +37,7 @@ public async Task TriggerLongRunningOrchestration( await client.ScheduleNewOrchestrationInstanceAsync( orchestratorName, - new LongRunningOrchestrationContext - { - DurationSeconds = durationSeconds - }, + new LongRunningOrchestrationContext { DurationSeconds = durationSeconds }, options, cancellationToken); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs index 877e663232f..7a59add4431 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/StatusCheckFunction.cs @@ -9,14 +9,14 @@ public class StatusCheckFunction { private static readonly OrchestrationQuery ActiveOrchestrationsQuery = new() { - Statuses = new List - { + Statuses = + [ OrchestrationRuntimeStatus.Pending, OrchestrationRuntimeStatus.Running - } + ] }; - [Function("StatusCheck")] + [Function(nameof(StatusCheck))] [Produces("application/json")] public static async Task StatusCheck( [HttpTrigger(AuthorizationLevel.Anonymous, "get")] @@ -27,7 +27,8 @@ public static async Task StatusCheck( { var activeOrchestrations = await client .GetAllInstancesAsync(filter: ActiveOrchestrationsQuery) - .ToListAsync(); - return new OkObjectResult(new { ActiveOrchestrations = activeOrchestrations.Count }); + .CountAsync(); + + return new OkObjectResult(new { ActiveOrchestrations = activeOrchestrations }); } } From a6a783207e0b8ab25a65504bfda8452b97952b8d Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 10 Dec 2024 16:52:44 +0000 Subject: [PATCH 090/144] EES-5446 - grouped long-running test functions into single class --- .../Functions/LongRunningFunctions.cs | 65 +++++++++++++++++++ .../Functions/LongRunningOrchestration.cs | 37 ----------- .../Functions/LongRunningTriggerFunction.cs | 46 ------------- 3 files changed, 65 insertions(+), 83 deletions(-) delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs index f17bc7a8128..4835da3a1c5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningFunctions.cs @@ -1,12 +1,77 @@ using System.Diagnostics; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; public class LongRunningFunctions(ILogger logger) { + [Function(nameof(TriggerLongRunningOrchestration))] + public async Task TriggerLongRunningOrchestration( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = nameof(TriggerLongRunningOrchestration))] + HttpRequest httpRequest, + [DurableClient] DurableTaskClient client, + CancellationToken cancellationToken) + { + var instanceId = Guid.NewGuid(); + + var durationSeconds = + httpRequest.GetRequestParamInt("durationSeconds", defaultValue: 60); + + const string orchestratorName = + nameof(ProcessLongRunningOrchestration); + + var options = new StartOrchestrationOptions { InstanceId = instanceId.ToString() }; + + logger.LogInformation( + "Scheduling '{OrchestratorName}' (InstanceId={InstanceId}, DurationSeconds={DurationSeconds}))", + orchestratorName, + instanceId, + durationSeconds); + + await client.ScheduleNewOrchestrationInstanceAsync( + orchestratorName, + new LongRunningOrchestrationContext { DurationSeconds = durationSeconds }, + options, + cancellationToken); + + return new OkResult(); + } + + [Function(nameof(ProcessLongRunningOrchestration))] + public static async Task ProcessLongRunningOrchestration( + [OrchestrationTrigger] TaskOrchestrationContext context, + LongRunningOrchestrationContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessLongRunningOrchestration)); + + logger.LogInformation( + "Processing long-running orchestration (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", + context.InstanceId, + input.DurationSeconds); + + try + { + await context.CallActivity(nameof(LongRunningActivity), logger, input); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", + context.InstanceId, + input.DurationSeconds); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } + [Function(nameof(LongRunningActivity))] public async Task LongRunningActivity( [ActivityTrigger] LongRunningOrchestrationContext input, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs deleted file mode 100644 index a2e55a20dee..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningOrchestration.cs +++ /dev/null @@ -1,37 +0,0 @@ -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -using Microsoft.Azure.Functions.Worker; -using Microsoft.DurableTask; -using Microsoft.Extensions.Logging; - -namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; - -public static class LongRunningOrchestration -{ - [Function(nameof(ProcessLongRunningOrchestration))] - public static async Task ProcessLongRunningOrchestration( - [OrchestrationTrigger] TaskOrchestrationContext context, - LongRunningOrchestrationContext input) - { - var logger = context.CreateReplaySafeLogger(nameof(ProcessLongRunningOrchestration)); - - logger.LogInformation( - "Processing long-running orchestration (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", - context.InstanceId, - input.DurationSeconds); - - try - { - await context.CallActivity(nameof(LongRunningFunctions.LongRunningActivity), logger, input); - } - catch (Exception e) - { - logger.LogError(e, - "Activity failed with an exception (InstanceId={InstanceId}, DurationSeconds={DurationSeconds})", - context.InstanceId, - input.DurationSeconds); - - await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); - } - } -} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs deleted file mode 100644 index 5012da598e7..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/LongRunningTriggerFunction.cs +++ /dev/null @@ -1,46 +0,0 @@ -using GovUk.Education.ExploreEducationStatistics.Common.Extensions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Azure.Functions.Worker; -using Microsoft.DurableTask; -using Microsoft.DurableTask.Client; -using Microsoft.Extensions.Logging; - -namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; - -public class LongRunningTriggerFunction( - ILogger logger) -{ - [Function(nameof(TriggerLongRunningOrchestration))] - public async Task TriggerLongRunningOrchestration( - [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = nameof(TriggerLongRunningOrchestration))] - HttpRequest httpRequest, - [DurableClient] DurableTaskClient client, - CancellationToken cancellationToken) - { - var instanceId = Guid.NewGuid(); - - var durationSeconds = - httpRequest.GetRequestParamInt("durationSeconds", defaultValue: 60); - - const string orchestratorName = - nameof(LongRunningOrchestration.ProcessLongRunningOrchestration); - - var options = new StartOrchestrationOptions { InstanceId = instanceId.ToString() }; - - logger.LogInformation( - "Scheduling '{OrchestratorName}' (InstanceId={InstanceId}, DurationSeconds={DurationSeconds}))", - orchestratorName, - instanceId, - durationSeconds); - - await client.ScheduleNewOrchestrationInstanceAsync( - orchestratorName, - new LongRunningOrchestrationContext { DurationSeconds = durationSeconds }, - options, - cancellationToken); - - return new OkResult(); - } -} From ea94d10251d7f57084a14d2d51b438a319ab2463 Mon Sep 17 00:00:00 2001 From: Nusrath Moh Date: Tue, 10 Dec 2024 17:34:24 +0000 Subject: [PATCH 091/144] EES-5730 Fix UI test failure in general_public suite --- ...table_tool_absence_by_characteristic.robot | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot b/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot index 92c9a07b598..2e1d53e5e41 100644 --- a/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot +++ b/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot @@ -192,16 +192,17 @@ Click Update and view reordered table button user clicks button Update and view reordered table Validate results table column headings after reordering - user checks table column heading contains 1 1 Gender male - user checks table column heading contains 1 2 Gender female - user checks table column heading contains 2 1 2013/14 - user checks table column heading contains 2 2 2014/15 - user checks table column heading contains 2 3 2015/16 - user checks table column heading contains 2 4 2012/13 - user checks table column heading contains 2 5 2013/14 - user checks table column heading contains 2 6 2014/15 - user checks table column heading contains 2 7 2015/16 - user checks table column heading contains 2 8 2012/13 + user checks table column heading contains 1 1 Gender + user checks table column heading contains 2 1 Gender male + user checks table column heading contains 2 2 Gender female + user checks table column heading contains 3 1 2013/14 + user checks table column heading contains 3 2 2014/15 + user checks table column heading contains 3 3 2015/16 + user checks table column heading contains 3 4 2012/13 + user checks table column heading contains 3 5 2013/14 + user checks table column heading contains 3 6 2014/15 + user checks table column heading contains 3 7 2015/16 + user checks table column heading contains 3 8 2012/13 Validate results table row headings after reordering user checks table row heading contains 1 1 Overall absence rate @@ -251,16 +252,17 @@ User validates permanent link works correctly user waits until h1 is visible 'Absence by characteristic' from '${PUPIL_ABSENCE_PUBLICATION_TITLE}' User validates permalink table - user checks table column heading contains 1 1 Gender male - user checks table column heading contains 1 2 Gender female - user checks table column heading contains 2 1 2013/14 - user checks table column heading contains 2 2 2014/15 - user checks table column heading contains 2 3 2015/16 - user checks table column heading contains 2 4 2012/13 - user checks table column heading contains 2 5 2013/14 - user checks table column heading contains 2 6 2014/15 - user checks table column heading contains 2 7 2015/16 - user checks table column heading contains 2 8 2012/13 + user checks table column heading contains 1 1 Gender + user checks table column heading contains 2 1 Gender male + user checks table column heading contains 2 2 Gender female + user checks table column heading contains 3 1 2013/14 + user checks table column heading contains 3 2 2014/15 + user checks table column heading contains 3 3 2015/16 + user checks table column heading contains 3 4 2012/13 + user checks table column heading contains 3 5 2013/14 + user checks table column heading contains 3 6 2014/15 + user checks table column heading contains 3 7 2015/16 + user checks table column heading contains 3 8 2012/13 user checks table row heading contains 1 1 Overall absence rate user checks table row heading contains 2 1 Unauthorised absence rate From 2f657daada55f58d8110cf9bc898c99e71a60aed Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 10 Dec 2024 18:23:41 +0000 Subject: [PATCH 092/144] EES-5705 - in addition to posting the UI test zip file if suites failed, also post if all tests passed but it took more than 1 run to achieve it. This will help us spot flaky tests. --- tests/robot-tests/tests/libs/slack.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/robot-tests/tests/libs/slack.py b/tests/robot-tests/tests/libs/slack.py index 2c8c40f0988..6071874ae9b 100644 --- a/tests/robot-tests/tests/libs/slack.py +++ b/tests/robot-tests/tests/libs/slack.py @@ -130,7 +130,9 @@ def send_test_report(self, env: str, suites_ran: str, suites_failed: [], number_ response = self.client.chat_postMessage(channel=self.slack_channel, text="All results", blocks=attachments) - if suites_failed: + # Post the test report if there were test failures or if more than 1 run attempt was necessary in + # order to get all tests to pass. The latter case will help to identify which of our tests are flaky. + if suites_failed or number_of_test_runs > 1: date = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S") report_name = f"UI-test-report-{suites_ran.replace('tests/', '')}-{env}-{date}.zip" From 6ee55af0faaf2d9998296006755d33922bc281a9 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 11 Dec 2024 15:06:39 +0000 Subject: [PATCH 093/144] EES-5446 - minor tweaks to fix pipeline after mere conflicts resolved --- .../application/public-api/publicApiDataProcessor.bicep | 2 +- .../templates/public-api/ci/jobs/deploy-infrastructure.yml | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep index 3abc099f17e..9e7719ab209 100644 --- a/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep +++ b/infrastructure/templates/public-api/application/public-api/publicApiDataProcessor.bicep @@ -79,7 +79,7 @@ module dataProcessorFunctionAppModule '../../components/functionApp.bicep' = { subnetId: outboundVnetSubnet.id privateEndpointSubnetId: inboundVnetSubnet.id publicNetworkAccessEnabled: true - functionAppEndpointFirewallRules: functionAppFirewallRules + functionAppFirewallRules: functionAppFirewallRules entraIdAuthentication: { appRegistrationClientId: dataProcessorAppRegistrationClientId allowedClientIds: [ diff --git a/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml b/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml index 541651a2296..af0c39481de 100644 --- a/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml +++ b/infrastructure/templates/public-api/ci/jobs/deploy-infrastructure.yml @@ -37,9 +37,10 @@ jobs: parameterFile: $(paramFile) deploySharedPrivateDnsZones: false deployPsqlFlexibleServer: false - deployContainerApp: true + deployContainerApp: false + deployDataProcessor: false deployAlerts: false - dataProcessorExists: true + dataProcessorExists: false - task: AzureCLI@2 displayName: Check if Data Processor Function App exists @@ -66,6 +67,7 @@ jobs: deploySharedPrivateDnsZones: $(deploySharedPrivateDnsZones) deployPsqlFlexibleServer: $(deployPsqlFlexibleServer) deployContainerApp: $(deployContainerApp) + deployDataProcessor: $(deployDataProcessor) deployAlerts: $(deployAlerts) dataProcessorExists: $(dataProcessorExists) From fe2cf609b474342353214cf23738a3456b143ae7 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Wed, 11 Dec 2024 17:04:12 +0000 Subject: [PATCH 094/144] EES-5543: Correct BoundaryLevel property in ChartDataSetConfig. --- .../Model/Chart/ChartDataSetConfig.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Chart/ChartDataSetConfig.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Chart/ChartDataSetConfig.cs index 5172c43f308..0569a6a9f64 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Chart/ChartDataSetConfig.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Model/Chart/ChartDataSetConfig.cs @@ -1,8 +1,7 @@ -#nullable enable -using System.Collections.Generic; -using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +#nullable enable using Newtonsoft.Json; using Newtonsoft.Json.Converters; +using System.Collections.Generic; namespace GovUk.Education.ExploreEducationStatistics.Common.Model.Chart { @@ -10,7 +9,7 @@ public class ChartDataSetConfig { public ChartBaseDataSet DataSet; public ChartDataGrouping DataGrouping; - public BoundaryLevel BoundaryLevels; + public long BoundaryLevel; } [JsonConverter(typeof(StringEnumConverter))] From 735211604930f7983a1e7c3c4d9c78873f2bcf22 Mon Sep 17 00:00:00 2001 From: dfe-sdt Date: Thu, 12 Dec 2024 09:45:06 +0000 Subject: [PATCH 095/144] chore(tests): update test snapshots --- .../tests/snapshots/data_catalogue_snapshot.json | 2 +- .../tests/snapshots/find_statistics_snapshot.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json index 3308500e031..eeeed40583d 100644 --- a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json +++ b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json @@ -1,5 +1,5 @@ { - "num_datasets": "924 data sets", + "num_datasets": "926 data sets", "themes": [ { "publications": [ diff --git a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json index 7705e194c2f..6986b264be7 100644 --- a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json +++ b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json @@ -65,7 +65,7 @@ { "publication_summary": "Statistics on the characteristics of early years providers in England.", "publication_title": "Childcare and early years provider survey", - "published": "14 Dec 2023", + "published": "12 Dec 2024", "release_type": "Official statistics", "theme": "Early years" }, @@ -100,7 +100,7 @@ { "publication_summary": "This publication provides data reported by local authorities on children missing education (CME) in England.", "publication_title": "Children missing education", - "published": "29 Feb 2024", + "published": "12 Dec 2024", "release_type": "Official statistics in development", "theme": "Pupils and schools" }, @@ -205,7 +205,7 @@ { "publication_summary": "This publication provides data reported by local authorities on children in\u202felective home education (EHE) in England.", "publication_title": "Elective home education", - "published": "29 Feb 2024", + "published": "12 Dec 2024", "release_type": "Official statistics in development", "theme": "Pupils and schools" }, @@ -331,7 +331,7 @@ { "publication_summary": "Local authority and school spending on education, children's services and social care across financial years. Also includes income of LA maintained schools.", "publication_title": "LA and school expenditure", - "published": "25 Jan 2024", + "published": "12 Dec 2024", "release_type": "Official statistics", "theme": "Finance and funding" }, From d357fe56c63aa4945ea836c256e3249b75dd1ad8 Mon Sep 17 00:00:00 2001 From: jack-hive <148866614+jack-hive@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:47:54 +0000 Subject: [PATCH 096/144] EES-5627 Adding `Label` to the `Release` (#5457) * EES-5627 Adding `Label` to the `Release` * EES-5627 Fixing unique index to treat a `NULL` `Label` as unique * EES-5627 Adding support to change the `Label` in the `ReleaseGeneratorExtensions` --- ...8_EES5627_AddingLabelToRelease.Designer.cs | 2237 +++++++++++++++++ ...1212113038_EES5627_AddingLabelToRelease.cs | 69 + .../ContentDbContextModelSnapshot.cs | 10 +- .../Fixtures/ReleaseGeneratorExtensions.cs | 10 + .../Release.cs | 13 +- 5 files changed, 2333 insertions(+), 6 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241212113038_EES5627_AddingLabelToRelease.Designer.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241212113038_EES5627_AddingLabelToRelease.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241212113038_EES5627_AddingLabelToRelease.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241212113038_EES5627_AddingLabelToRelease.Designer.cs new file mode 100644 index 00000000000..699d1745362 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241212113038_EES5627_AddingLabelToRelease.Designer.cs @@ -0,0 +1,2237 @@ +// +using System; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + [DbContext(typeof(ContentDbContext))] + [Migration("20241212113038_EES5627_AddingLabelToRelease")] + partial class EES5627_AddingLabelToRelease + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContactTelNo") + .HasColumnType("nvarchar(max)"); + + b.Property("TeamEmail") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TeamName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Contacts"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.FreeTextRank", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Rank") + .HasColumnType("int"); + + b.ToTable((string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("LegacyCreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("Resolved") + .HasColumnType("datetime2"); + + b.Property("ResolvedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ResolvedById"); + + b.ToTable("Comment"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentSectionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Locked") + .HasColumnType("datetime2"); + + b.Property("LockedById") + .IsConcurrencyToken() + .HasColumnType("uniqueidentifier"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentSectionId"); + + b.HasIndex("LockedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentBlock", (string)null); + + b.HasDiscriminator("Type").HasValue("ContentBlock"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Caption") + .HasColumnType("nvarchar(max)"); + + b.Property("Heading") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentSections"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestDraftVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("LatestDraftVersionId") + .IsUnique() + .HasFilter("[LatestDraftVersionId] IS NOT NULL"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("DataBlocks", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier") + .HasColumnName("DataBlockId"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("DataBlockVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("ExpectedImportedRows") + .HasColumnType("int"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("GeographicLevels") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ImportedRows") + .HasColumnType("int"); + + b.Property("LastProcessedRowIndex") + .HasColumnType("int"); + + b.Property("MetaFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("StagePercentageComplete") + .HasColumnType("int"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("TotalRows") + .HasColumnType("int"); + + b.Property("ZipFileId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId") + .IsUnique(); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("FileId"), new[] { "Status" }); + + b.HasIndex("MetaFileId") + .IsUnique(); + + b.HasIndex("ZipFileId"); + + b.ToTable("DataImports"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataImportId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("DataImportId"); + + b.ToTable("DataImportErrors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("EmbedBlocks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DataBlockId") + .IsUnique(); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("FeaturedTables"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentLength") + .HasColumnType("bigint"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileMeta") + .HasColumnType("nvarchar(max)"); + + b.Property("DataSetFileVersion") + .HasColumnType("int"); + + b.Property("Filename") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReplacedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReplacingId") + .HasColumnType("uniqueidentifier"); + + b.Property("RootPath") + .HasColumnType("uniqueidentifier"); + + b.Property("SourceId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReplacedById") + .IsUnique() + .HasFilter("[ReplacedById] IS NOT NULL"); + + b.HasIndex("ReplacingId") + .IsUnique() + .HasFilter("[ReplacingId] IS NOT NULL"); + + b.HasIndex("SourceId"); + + b.HasIndex("Type"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.ToTable("GlossaryEntries"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("GuidanceText") + .HasColumnType("nvarchar(max)"); + + b.Property("GuidanceTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Trend") + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("KeyStatistics"); + + b.UseTptMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("OwningPublicationSlug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OwningPublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("Methodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DisplayDate") + .HasColumnType("datetime2"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("MethodologyNotes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("MethodologyVersionId", "Slug"); + + b.ToTable("MethodologyRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("AlternativeSlug") + .HasColumnType("nvarchar(max)"); + + b.Property("AlternativeTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("PublishingStrategy") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ScheduledWithReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyId"); + + b.HasIndex("PreviousVersionId"); + + b.HasIndex("ScheduledWithReleaseVersionId"); + + b.ToTable("MethodologyVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Annexes") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("MethodologyVersionId"); + + b.ToTable("MethodologyVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Permalink", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataSetTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MigratedFromLegacy") + .HasColumnType("bit"); + + b.Property("PublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("SubjectId"); + + b.ToTable("Permalinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseSeries") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .IsRequired() + .HasMaxLength(160) + .HasColumnType("nvarchar(160)"); + + b.Property("SupersededById") + .HasColumnType("uniqueidentifier"); + + b.Property("ThemeId") + .HasColumnType("uniqueidentifier"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContactId"); + + b.HasIndex("LatestPublishedReleaseVersionId") + .IsUnique() + .HasFilter("[LatestPublishedReleaseVersionId] IS NOT NULL"); + + b.HasIndex("SupersededById"); + + b.HasIndex("ThemeId"); + + b.ToTable("Publications"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("Owner") + .HasColumnType("bit"); + + b.HasKey("PublicationId", "MethodologyId"); + + b.HasIndex("MethodologyId"); + + b.ToTable("PublicationMethodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("PublicationId", "Slug"); + + b.ToTable("PublicationRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Label") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(81) + .HasColumnType("nvarchar(81)"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("nvarchar(5)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Year") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("PublicationId", "Year", "TimePeriodCoverage", "Label") + .IsUnique(); + + b.ToTable("Releases"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("FilterSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("IndicatorSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("PublicApiDataSetId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicApiDataSetVersion") + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("ReleaseVersionId", "FileId") + .IsUnique(); + + b.HasIndex("ReleaseVersionId", "PublicApiDataSetId", "PublicApiDataSetVersion") + .IsUnique() + .HasFilter("[PublicApiDataSetId] IS NOT NULL AND [PublicApiDataSetVersion] IS NOT NULL"); + + b.ToTable("ReleaseFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseRedirect", b => + { + b.Property("ReleaseId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("ReleaseId", "Slug"); + + b.ToTable("ReleaseRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("ReleaseStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataGuidance") + .HasColumnType("nvarchar(max)"); + + b.Property("NextReleaseDate") + .HasColumnType("nvarchar(max)"); + + b.Property("NotifiedOn") + .HasColumnType("datetime2"); + + b.Property("NotifySubscribers") + .HasColumnType("bit"); + + b.Property("PreReleaseAccessList") + .HasColumnType("nvarchar(max)"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublishScheduled") + .HasColumnType("datetime2"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("RelatedInformation") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseName") + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(6) + .HasColumnType("nvarchar(6)"); + + b.Property("Type") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("UpdatePublishedDate") + .HasColumnType("bit"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("ReleaseId"); + + b.HasIndex("Type"); + + b.HasIndex("PreviousVersionId", "Version"); + + b.ToTable("ReleaseVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Themes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("On") + .HasColumnType("datetime2"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("Update"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstName") + .HasColumnType("nvarchar(max)"); + + b.Property("LastName") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("DeletedById"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.ToTable("UserPublicationInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("UserId"); + + b.ToTable("UserPublicationRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EmailSent") + .HasColumnType("bit"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("UserReleaseInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UserId"); + + b.ToTable("UserReleaseRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Charts") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Charts"); + + b.Property("Heading") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Heading"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Query") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Query"); + + b.Property("Source") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Table") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Table"); + + b.HasDiscriminator().HasValue("DataBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("EmbedBlockId") + .HasColumnType("uniqueidentifier") + .HasColumnName("EmbedBlockId"); + + b.HasIndex("EmbedBlockId") + .IsUnique() + .HasFilter("[EmbedBlockId] IS NOT NULL"); + + b.HasDiscriminator().HasValue("EmbedBlockLink"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.HtmlBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("HtmlBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MarkDownBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("MarkDownBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.HasIndex("DataBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.ToTable("KeyStatisticsDataBlock", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("Statistic") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.ToTable("KeyStatisticsText", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", "ContentBlock") + .WithMany("Comments") + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "ResolvedBy") + .WithMany() + .HasForeignKey("ResolvedById"); + + b.Navigation("ContentBlock"); + + b.Navigation("CreatedBy"); + + b.Navigation("ResolvedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", "ContentSection") + .WithMany("Content") + .HasForeignKey("ContentSectionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "LockedBy") + .WithMany() + .HasForeignKey("LockedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentSection"); + + b.Navigation("LockedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Content") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestDraftVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestDraftVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestPublishedVersionId"); + + b.Navigation("LatestDraftVersion"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "ContentBlock") + .WithMany() + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("DataBlockVersions") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "FileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "MetaFile") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "MetaFileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ZipFile") + .WithMany() + .HasForeignKey("ZipFileId") + .OnDelete(DeleteBehavior.Restrict); + + b.Navigation("File"); + + b.Navigation("MetaFile"); + + b.Navigation("ZipFile"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "DataImport") + .WithMany("Errors") + .HasForeignKey("DataImportId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataImport"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", "DataBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("FeaturedTables") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedBy") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Replacing") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacingId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Source") + .WithMany() + .HasForeignKey("SourceId"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReplacedBy"); + + b.Navigation("Replacing"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("CreatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("KeyStatistics") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "LatestPublishedVersionId"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany("Notes") + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Versions") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ScheduledWithReleaseVersion") + .WithMany() + .HasForeignKey("ScheduledWithReleaseVersionId"); + + b.Navigation("CreatedBy"); + + b.Navigation("Methodology"); + + b.Navigation("PreviousVersion"); + + b.Navigation("ScheduledWithReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", null) + .WithOne("MethodologyContent") + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", "MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", "Contact") + .WithMany() + .HasForeignKey("ContactId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "LatestPublishedReleaseVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "LatestPublishedReleaseVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "SupersededBy") + .WithMany() + .HasForeignKey("SupersededById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", "Theme") + .WithMany("Publications") + .HasForeignKey("ThemeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ExternalMethodology", "ExternalMethodology", b1 => + { + b1.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b1.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.HasKey("PublicationId"); + + b1.ToTable("ExternalMethodology", (string)null); + + b1.WithOwner() + .HasForeignKey("PublicationId"); + }); + + b.Navigation("Contact"); + + b.Navigation("ExternalMethodology"); + + b.Navigation("LatestPublishedReleaseVersion"); + + b.Navigation("SupersededBy"); + + b.Navigation("Theme"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Publications") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Methodologies") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Methodology"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Releases") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") + .WithMany() + .HasForeignKey("ReleaseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Release"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("ReleaseStatuses") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("ReleaseVersions") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") + .WithMany("Versions") + .HasForeignKey("ReleaseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("PreviousVersion"); + + b.Navigation("Publication"); + + b.Navigation("Release"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Updates") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.User", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.Navigation("DeletedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("Publication"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", "EmbedBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", "EmbedBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("EmbedBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithMany() + .HasForeignKey("DataBlockId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Navigation("Comments"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Navigation("Content"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Navigation("Errors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Navigation("Publications"); + + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Navigation("MethodologyContent") + .IsRequired(); + + b.Navigation("Notes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Navigation("Methodologies"); + + b.Navigation("ReleaseVersions"); + + b.Navigation("Releases"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Navigation("Content"); + + b.Navigation("DataBlockVersions"); + + b.Navigation("FeaturedTables"); + + b.Navigation("KeyStatistics"); + + b.Navigation("ReleaseStatuses"); + + b.Navigation("Updates"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Navigation("Publications"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241212113038_EES5627_AddingLabelToRelease.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241212113038_EES5627_AddingLabelToRelease.cs new file mode 100644 index 00000000000..48c14588423 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241212113038_EES5627_AddingLabelToRelease.cs @@ -0,0 +1,69 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + /// + public partial class EES5627_AddingLabelToRelease : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_Releases_PublicationId_Year_TimePeriodCoverage", + table: "Releases"); + + migrationBuilder.AlterColumn( + name: "Slug", + table: "Releases", + type: "nvarchar(81)", + maxLength: 81, + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(30)", + oldMaxLength: 30); + + migrationBuilder.AddColumn( + name: "Label", + table: "Releases", + type: "nvarchar(50)", + maxLength: 50, + nullable: true); + + migrationBuilder.CreateIndex( + name: "IX_Releases_PublicationId_Year_TimePeriodCoverage_Label", + table: "Releases", + columns: new[] { "PublicationId", "Year", "TimePeriodCoverage", "Label" }, + unique: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_Releases_PublicationId_Year_TimePeriodCoverage_Label", + table: "Releases"); + + migrationBuilder.DropColumn( + name: "Label", + table: "Releases"); + + migrationBuilder.AlterColumn( + name: "Slug", + table: "Releases", + type: "nvarchar(30)", + maxLength: 30, + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(81)", + oldMaxLength: 81); + + migrationBuilder.CreateIndex( + name: "IX_Releases_PublicationId_Year_TimePeriodCoverage", + table: "Releases", + columns: new[] { "PublicationId", "Year", "TimePeriodCoverage" }, + unique: true); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs index 67bff7569b4..830ba7175ab 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs @@ -894,13 +894,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Created") .HasColumnType("datetime2"); + b.Property("Label") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + b.Property("PublicationId") .HasColumnType("uniqueidentifier"); b.Property("Slug") .IsRequired() - .HasMaxLength(30) - .HasColumnType("nvarchar(30)"); + .HasMaxLength(81) + .HasColumnType("nvarchar(81)"); b.Property("TimePeriodCoverage") .IsRequired() @@ -915,7 +919,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Id"); - b.HasIndex("PublicationId", "Year", "TimePeriodCoverage") + b.HasIndex("PublicationId", "Year", "TimePeriodCoverage", "Label") .IsUnique(); b.ToTable("Releases"); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs index 35f6c2675b0..5ebea8621a9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs @@ -97,6 +97,11 @@ public static Generator WithYear( int year) => generator.ForInstance(s => s.SetYear(year)); + public static Generator WithLabel( + this Generator generator, + string? label) + => generator.ForInstance(s => s.SetLabel(label)); + public static InstanceSetters SetDefaults( this InstanceSetters setters, int? year = null) @@ -234,4 +239,9 @@ public static InstanceSetters SetYear( this InstanceSetters setters, int year) => setters.Set(p => p.Year, year); + + public static InstanceSetters SetLabel( + this InstanceSetters setters, + string? label) + => setters.Set(p => p.Label, label); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs index 0e34b8b65a2..699e6998cb5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs @@ -24,6 +24,8 @@ public class Release : ICreatedUpdatedTimestamps public required int Year { get; set; } + public string? Label { get; set; } + public List Versions { get; set; } = []; public DateTime Created { get; set; } @@ -40,7 +42,10 @@ internal class Config : IEntityTypeConfiguration public void Configure(EntityTypeBuilder builder) { builder.Property(m => m.Slug) - .HasMaxLength(30); + .HasMaxLength(81); + + builder.Property(m => m.Label) + .HasMaxLength(50); builder.Property(m => m.TimePeriodCoverage) .HasConversion(new EnumToEnumValueConverter()) @@ -50,9 +55,11 @@ public void Configure(EntityTypeBuilder builder) { dsv.PublicationId, dsv.Year, - dsv.TimePeriodCoverage + dsv.TimePeriodCoverage, + dsv.Label, }) - .IsUnique(); + .IsUnique() + .HasFilter(null); } } } From 6d53130cbe6b1d7ef700fcb45fb4defdd6baf37f Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 12 Dec 2024 15:11:03 +0000 Subject: [PATCH 097/144] EES-5732 prevent ui test failures after data replacement --- .../release/data/components/ReleaseDataUploadsSection.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx index 85043e0f208..b2b954f0d1d 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataUploadsSection.tsx @@ -111,6 +111,14 @@ const ReleaseDataUploadsSection = ({ dataFile: DataFile, { totalRows, status }: DataFileImportStatus, ) => { + // EES-5732 UI tests related to data replacement sometimes fail + // because of a permission call for the replaced file being called, + // probably caused by the speed of the tests. + // This prevents this happening. + if (status === 'NOT_FOUND') { + return; + } + const permissions = await permissionService.getDataFilePermissions( releaseId, dataFile.id, From 7e51403a74d6270cd8eb5645126f80d5b974879b Mon Sep 17 00:00:00 2001 From: jack-hive <148866614+jack-hive@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:49:05 +0000 Subject: [PATCH 098/144] EES-5632 Distinguishing release redirects across different publications (#5446) * EES-5632 [WIP] Refactoring BE to return release redirects grouped by publication * EES-5632 [WIP] Integration tests * EES-5632 Enabling a different Azurite TestContainer instance to be used across each test so that the stored data doesn't interfere with other tests * EES-5632 FE - Changing the view-model field names to match the new BE field names * EES-5632 Some requested changes as per PR review * EES-5632 removing some duplicated tests * EES-5632 Reformatting tests * EES-5632 remove unused import --- .../MethodologyApprovalServiceTests.cs | 6 +- .../Methodologies/MethodologyServiceTests.cs | 42 +- .../Services/PublicationServiceTests.cs | 24 +- .../DataSetFilesControllerTests.cs | 27 +- .../Controllers/RedirectsControllerTests.cs | 522 ++++++++++++------ .../Fixtures/IntegrationTestFixture.cs | 77 ++- .../Fixtures/TestApplicationFactory.cs | 76 +-- .../MethodologyGeneratorExtensions.cs | 24 +- .../MethodologyVersionGeneratorExtensions.cs | 34 ++ .../PublicationGeneratorExtensions.cs | 43 ++ .../Fixtures/ReleaseGeneratorExtensions.cs | 43 ++ .../MethodologyVersion.cs | 2 + .../Publication.cs | 2 + .../PublicationRedirect.cs | 8 +- .../Release.cs | 2 + .../ReleaseRedirect.cs | 6 +- .../RedirectsServiceTests.cs | 142 +---- .../RedirectsService.cs | 62 ++- .../RedirectViewModels.cs | 6 +- .../pages/__tests__/redirectPages.test.ts | 4 +- .../src/middleware/pages/redirectPages.ts | 4 +- .../src/services/redirectService.ts | 4 +- 22 files changed, 680 insertions(+), 480 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs index 0105556c036..351828f7342 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyApprovalServiceTests.cs @@ -681,9 +681,9 @@ public async Task UpdateApprovalStatus_ApprovingUsingImmediateStrategy() redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); await using (var context = InMemoryApplicationDbContext(contentDbContextId)) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs index 17499e9130b..8c4afcd5a6e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/Methodologies/MethodologyServiceTests.cs @@ -3600,9 +3600,9 @@ public async Task PublicationTitleOrSlugChanged() var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -3662,9 +3662,9 @@ public async Task PublicationTitleOrSlugChanged_NoMethodologyRedirectAsMethodolo var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -3745,9 +3745,9 @@ await contentDbContext.PublicationMethodologies.AddRangeAsync( var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -3847,9 +3847,9 @@ public async Task PublicationTitleOrSlugChanged_MethodologySlugIsAlternativeSlug var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -3917,9 +3917,9 @@ public async Task PublicationTitleOrSlugChanged_MethodologyIsLive() var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -4004,9 +4004,9 @@ public async Task var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); @@ -4110,9 +4110,9 @@ public async Task var redirectsCacheService = new Mock(MockBehavior.Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var service = SetupMethodologyService(contentDbContext, redirectsCacheService: redirectsCacheService.Object); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs index 4841cb8ce96..39fb7872c3f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs @@ -1127,9 +1127,9 @@ public async Task UpdatePublication_AlreadyPublished() redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var publicationService = BuildPublicationService(context, methodologyService: methodologyService.Object, @@ -1272,9 +1272,9 @@ public async Task UpdatePublication_TitleChangesPublicationAndMethodologySlug() redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var publicationService = BuildPublicationService(context, methodologyService: methodologyService.Object, @@ -1579,9 +1579,9 @@ public async Task UpdatePublication_CreateRedirectIfLiveSlugChanged() var redirectsCacheService = new Mock(Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var publicationService = BuildPublicationService(context, methodologyService: methodologyService.Object, @@ -1678,9 +1678,9 @@ public async Task UpdatePublication_ChangeBackToPreviousLiveSlug() var redirectsCacheService = new Mock(Strict); redirectsCacheService.Setup(mock => mock.UpdateRedirects()) .ReturnsAsync(new RedirectsViewModel( - Publications: [], - Methodologies: [], - Releases: [])); + PublicationRedirects: [], + MethodologyRedirects: [], + ReleaseRedirectsByPublicationSlug: [])); var publicationService = BuildPublicationService(context, methodologyService: methodologyService.Object, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index 1de823466ab..91a6b434769 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -1759,7 +1759,7 @@ private static Mock ContentDbContextMock( public class DownloadDataSetFileTests(TestApplicationFactory testApp) : DataSetFilesControllerTests(testApp) { - public override async Task InitializeAsync() => await TestApp.StartAzurite(); + public override async Task InitializeAsync() => await StartAzurite(); [Fact] public async Task DownloadDataSetFile() @@ -1844,7 +1844,7 @@ public async Task DownloadDataSetFile_NotPublished() .WithReleaseVersion(publication.ReleaseVersions[0]) .WithFile(_fixture.DefaultFile(FileType.Data)); - await TestApp.StartAzurite(); + await StartAzurite(); var testApp = BuildApp(enableAzurite: true); var publicBlobStorageService = testApp.Services.GetRequiredService(); @@ -1871,7 +1871,7 @@ await TestApp.AddTestData(context => public class ListSitemapItemsTests(TestApplicationFactory testApp) : DataSetFilesControllerTests(testApp) { - public override async Task InitializeAsync() => await TestApp.StartAzurite(); + public override async Task InitializeAsync() => await StartAzurite(); private async Task InvokeListSitemapItems( WebApplicationFactory? app = null) @@ -1904,7 +1904,7 @@ public async Task ListSitemapItems() )) ); - await TestApp.StartAzurite(); + await StartAzurite(); var testApp = BuildApp(enableAzurite: true); var publicBlobStorageService = testApp.Services.GetRequiredService(); @@ -1944,7 +1944,7 @@ public class GetDataSetFileTests(TestApplicationFactory testApp) : DataSetFilesC { public override async Task InitializeAsync() { - await TestApp.StartAzurite(); + await StartAzurite(); } [Fact] @@ -1975,7 +1975,7 @@ await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile); }); - await TestApp.StartAzurite(); + await StartAzurite(); var testApp = BuildApp(enableAzurite: true); @@ -2098,7 +2098,7 @@ await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile); }); - await TestApp.StartAzurite(); + await StartAzurite(); var testApp = BuildApp(enableAzurite: true); @@ -2163,7 +2163,7 @@ await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile); }); - await TestApp.StartAzurite(); + await StartAzurite(); var testApp = BuildApp(enableAzurite: true); @@ -2224,7 +2224,7 @@ await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile); }); - await TestApp.StartAzurite(); + await StartAzurite(); var testApp = BuildApp(enableAzurite: true); @@ -2281,7 +2281,7 @@ await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile); }); - await TestApp.StartAzurite(); + await StartAzurite(); var testApp = BuildApp(enableAzurite: true); @@ -2362,7 +2362,7 @@ await TestApp.AddTestData(context => context.ReleaseFootnote.AddRange(releaseFootnote1, releaseFootnote2); }); - await TestApp.StartAzurite(); + await StartAzurite(); var testApp = BuildApp(enableAzurite: true); @@ -2466,7 +2466,7 @@ await TestApp.AddTestData(context => context.ReleaseFiles.AddRange(releaseFile0, releaseFile1, releaseFile2); }); - await TestApp.StartAzurite(); + await StartAzurite(); var testApp = BuildApp(enableAzurite: true); @@ -2532,8 +2532,7 @@ private WebApplicationFactory BuildApp( StatisticsDbContext? statisticsDbContext = null, bool enableAzurite = false) { - return TestApp - .WithAzurite(enabled: enableAzurite) + return WithAzurite(enabled: enableAzurite) .ConfigureServices(services => { services.ReplaceService(MemoryCacheService); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/RedirectsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/RedirectsControllerTests.cs index 7577fb5075c..f2336a62773 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/RedirectsControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/RedirectsControllerTests.cs @@ -20,189 +20,388 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.Controlle public abstract class RedirectsControllerTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { - public class ListTests(TestApplicationFactory testApp) : RedirectsControllerTests(testApp) + public abstract class ListTests(TestApplicationFactory testApp) : RedirectsControllerTests(testApp) { - public override async Task InitializeAsync() => await TestApp.StartAzurite(); + public override async Task InitializeAsync() => await StartAzurite(); - [Fact] - public async Task RedirectsExist_Returns200WithRedirects() + public class GeneralTests(TestApplicationFactory testApp) : ListTests(testApp) { - var publicationRedirects = DataFixture.DefaultPublicationRedirect() - .WithPublication(DataFixture.DefaultPublication()) - .GenerateList(3); - - var releaseRedirects = DataFixture.DefaultReleaseRedirect() - .WithRelease(DataFixture.DefaultRelease()) - .GenerateList(3); - - var methodologyVersions = DataFixture.DefaultMethodologyVersion() - .ForIndex(3, s => s.SetPublished(DateTime.UtcNow)) - .GenerateList(4); + [Fact] + public async Task RedirectsExistInCache_Returns200WithRedirects() + { + var app = BuildApp(enableAzurite: true); + var client = app.CreateClient(); + + var blobCacheService = app.Services.GetRequiredService(); + + List cachedMethodologyRedirects = + [ + new(FromSlug: "original-methodology-slug-1", ToSlug: "updated-methodology-slug-1"), + new(FromSlug: "original-methodology-slug-2", ToSlug: "updated-methodology-slug-2"), + ]; + + List cachedPublicationRedirects = + [ + new(FromSlug: "original-publication-slug-1", ToSlug: "updated-publication-slug-1"), + new(FromSlug: "original-publication-slug-2", ToSlug: "updated-publication-slug-2"), + ]; + + var cachedReleaseRedirectsByPublicationSlug = new Dictionary> + { + { + "updated-publication-slug-1", + new List() + { + new(FromSlug: "original-release-slug-1", ToSlug: "updated-release-slug-1"), + new(FromSlug: "original-release-slug-2", ToSlug: "updated-release-slug-2"), + } + } + }; + + var cachedViewModel = new RedirectsViewModel( + PublicationRedirects: cachedPublicationRedirects, + ReleaseRedirectsByPublicationSlug: cachedReleaseRedirectsByPublicationSlug, + MethodologyRedirects: cachedMethodologyRedirects); + + await blobCacheService.SetItemAsync(new RedirectsCacheKey(), cachedViewModel); + + var response = await ListRedirects(client); + + var viewModel = response.AssertOk(); + + Assert.Equal(cachedMethodologyRedirects.Count, viewModel.MethodologyRedirects.Count); + Assert.All( + cachedMethodologyRedirects, + cmr => Assert.Contains( + viewModel.MethodologyRedirects, + rvm => cmr.FromSlug == rvm.FromSlug && cmr.ToSlug == rvm.ToSlug)); + + Assert.Equal(cachedPublicationRedirects.Count, viewModel.PublicationRedirects.Count); + Assert.All( + cachedPublicationRedirects, + cpr => Assert.Contains( + viewModel.PublicationRedirects, + rvm => cpr.FromSlug == rvm.FromSlug && cpr.ToSlug == rvm.ToSlug)); + + var cachedReleaseRedirectsForPublication = Assert.Single(cachedReleaseRedirectsByPublicationSlug); + Assert.Equal(cachedReleaseRedirectsForPublication.Value.Count, + viewModel.ReleaseRedirectsByPublicationSlug["updated-publication-slug-1"].Count); + Assert.Equal("updated-publication-slug-1", cachedReleaseRedirectsForPublication.Key); + Assert.All( + cachedReleaseRedirectsForPublication.Value, + crr => Assert.Contains( + viewModel.ReleaseRedirectsByPublicationSlug["updated-publication-slug-1"], + rvm => crr.FromSlug == rvm.FromSlug && crr.ToSlug == rvm.ToSlug)); + } + + [Fact] + public async Task NoRedirectsExist_Returns200WithNoRedirects() + { + var client = BuildApp(enableAzurite: true).CreateClient(); + var response = await ListRedirects(client); - Methodology methodology = DataFixture.DefaultMethodology() - .WithMethodologyVersions(methodologyVersions) - .WithLatestPublishedVersion(methodologyVersions.Last()); + var viewModel = response.AssertOk(); - var methodologyRedirects = DataFixture.DefaultMethodologyRedirect() - .ForIndex(0, s => s.SetMethodologyVersion(methodologyVersions[0])) - .ForIndex(1, s => s.SetMethodologyVersion(methodologyVersions[1])) - .ForIndex(2, s => s.SetMethodologyVersion(methodologyVersions[2])) - .GenerateList(3); + Assert.Empty(viewModel.PublicationRedirects); + Assert.Empty(viewModel.ReleaseRedirectsByPublicationSlug); + Assert.Empty(viewModel.MethodologyRedirects); + } + } - await TestApp.AddTestData(context => + public class MethodologyRedirectsTests(TestApplicationFactory testApp) : ListTests(testApp) + { + [Fact] + public async Task RedirectsExist_Returns200WithRedirects() { - context.PublicationRedirects.AddRange(publicationRedirects); - context.ReleaseRedirects.AddRange(releaseRedirects); - context.MethodologyRedirects.AddRange(methodologyRedirects); - }); - - var response = await ListRedirects(); - - var viewModel = response.AssertOk(); - - Assert.All( - publicationRedirects, - pr => Assert.Contains( - viewModel.Publications, - rvm => pr.Slug == rvm.FromSlug && pr.Publication.Slug == rvm.ToSlug)); - Assert.All( - releaseRedirects, - rr => Assert.Contains( - viewModel.Releases, - rvm => rr.Slug == rvm.FromSlug && rr.Release.Slug == rvm.ToSlug)); - Assert.All( - methodologyRedirects, - mr => Assert.Contains( - viewModel.Methodologies, - rvm => mr.Slug == rvm.FromSlug && mr.MethodologyVersion.Methodology.OwningPublicationSlug == rvm.ToSlug)); + Methodology methodology = DataFixture.DefaultMethodology() + .WithMethodologyVersions( + DataFixture.DefaultMethodologyVersion() + .WithRedirects([DataFixture.DefaultMethodologyRedirect()]) + .ForIndex(0, s => s.SetRedirects([DataFixture.DefaultMethodologyRedirect()])) + .ForIndex(1, + s => s + .SetRedirects([DataFixture.DefaultMethodologyRedirect()]) + .SetPublished(DateTime.UtcNow)) + .GenerateList(2)); + + await TestApp.AddTestData(context => context.Methodologies.Add(methodology)); + + var client = BuildApp(enableAzurite: true).CreateClient(); + var response = await ListRedirects(client); + + var viewModel = response.AssertOk(); + + var redirects = methodology.Versions + .SelectMany(mv => mv.MethodologyRedirects) + .ToList(); + + Assert.Equal(redirects.Count, viewModel.MethodologyRedirects.Count); + Assert.All( + redirects, + mr => Assert.Contains( + viewModel.MethodologyRedirects, + rvm => + mr.Slug == rvm.FromSlug && + mr.MethodologyVersion.Methodology.OwningPublicationSlug == rvm.ToSlug)); + } + + [Fact] + public async Task RedirectsExist_RedirectsAreCached() + { + Methodology methodology = DataFixture.DefaultMethodology() + .WithMethodologyVersions( + DataFixture.DefaultMethodologyVersion() + .WithRedirects([DataFixture.DefaultMethodologyRedirect()]) + .ForIndex(0, s => s.SetRedirects([DataFixture.DefaultMethodologyRedirect()])) + .ForIndex(1, + s => s + .SetRedirects([DataFixture.DefaultMethodologyRedirect()]) + .SetPublished(DateTime.UtcNow)) + .GenerateList(2)); + + await TestApp.AddTestData(context => context.Methodologies.Add(methodology)); + + var app = BuildApp(enableAzurite: true); + var client = app.CreateClient(); + + await ListRedirects(client); + + var blobCacheService = app.Services.GetRequiredService(); + + var cachedValue = + await blobCacheService.GetItemAsync(new RedirectsCacheKey(), typeof(RedirectsViewModel)); + var cachedRedirectsViewModel = Assert.IsType(cachedValue); + + var redirects = methodology.Versions + .SelectMany(mv => mv.MethodologyRedirects) + .ToList(); + + Assert.Empty(cachedRedirectsViewModel.PublicationRedirects); + Assert.Empty(cachedRedirectsViewModel.ReleaseRedirectsByPublicationSlug); + + Assert.Equal(redirects.Count, cachedRedirectsViewModel.MethodologyRedirects.Count); + Assert.All( + redirects, + mr => Assert.Contains( + cachedRedirectsViewModel.MethodologyRedirects, + rvm => + mr.Slug == rvm.FromSlug && + mr.MethodologyVersion.Methodology.OwningPublicationSlug == rvm.ToSlug)); + } } - [Fact] - public async Task RedirectsExist_RedirectsAreCached() + public class PublicationRedirectsTests(TestApplicationFactory testApp) : ListTests(testApp) { - var publicationRedirects = DataFixture.DefaultPublicationRedirect() - .WithPublication(DataFixture.DefaultPublication()) - .GenerateList(3); + [Fact] + public async Task RedirectsExist_Returns200WithRedirects() + { + var publicationRedirects = DataFixture.DefaultPublicationRedirect() + .WithPublication(DataFixture.DefaultPublication()) + .GenerateList(2); + + await TestApp.AddTestData(context => + context.PublicationRedirects.AddRange(publicationRedirects)); + + var client = BuildApp(enableAzurite: true).CreateClient(); + var response = await ListRedirects(client); + + var viewModel = response.AssertOk(); + + Assert.Equal(publicationRedirects.Count, viewModel.PublicationRedirects.Count); + Assert.All( + publicationRedirects, + pr => Assert.Contains( + viewModel.PublicationRedirects, + rvm => + pr.Slug == rvm.FromSlug && + pr.Publication.Slug == rvm.ToSlug)); + } + + [Fact] + public async Task RedirectsExist_RedirectsAreCached() + { + var publicationRedirects = DataFixture.DefaultPublicationRedirect() + .WithPublication(DataFixture.DefaultPublication()) + .GenerateList(2); - var releaseRedirects = DataFixture.DefaultReleaseRedirect() - .WithRelease(DataFixture.DefaultRelease()) - .GenerateList(3); + await TestApp.AddTestData(context => + context.PublicationRedirects.AddRange(publicationRedirects)); - var methodologyVersions = DataFixture.DefaultMethodologyVersion() - .ForIndex(3, s => s.SetPublished(DateTime.UtcNow)) - .GenerateList(4); + await StartAzurite(); - Methodology methodology = DataFixture.DefaultMethodology() - .WithMethodologyVersions(methodologyVersions) - .WithLatestPublishedVersion(methodologyVersions.Last()); + var app = BuildApp(enableAzurite: true); + var client = app.CreateClient(); - var methodologyRedirects = DataFixture.DefaultMethodologyRedirect() - .ForIndex(0, s => s.SetMethodologyVersion(methodologyVersions[0])) - .ForIndex(1, s => s.SetMethodologyVersion(methodologyVersions[1])) - .ForIndex(2, s => s.SetMethodologyVersion(methodologyVersions[2])) - .GenerateList(3); + await ListRedirects(client); - await TestApp.AddTestData(context => - { - context.PublicationRedirects.AddRange(publicationRedirects); - context.ReleaseRedirects.AddRange(releaseRedirects); - context.MethodologyRedirects.AddRange(methodologyRedirects); - }); - - var app = BuildApp(); - var client = app.CreateClient(); - - await ListRedirects(client); - - var blobCacheService = app.Services.GetRequiredService(); - - var cachedValue = await blobCacheService.GetItemAsync(new RedirectsCacheKey(), typeof(RedirectsViewModel)); - var cachedRedirectsViewModel = Assert.IsType(cachedValue); - - Assert.All( - publicationRedirects, - pr => Assert.Contains( - cachedRedirectsViewModel.Publications, - rvm => pr.Slug == rvm.FromSlug && pr.Publication.Slug == rvm.ToSlug)); - Assert.All( - releaseRedirects, - rr => Assert.Contains( - cachedRedirectsViewModel.Releases, - rvm => rr.Slug == rvm.FromSlug && rr.Release.Slug == rvm.ToSlug)); - Assert.All( - methodologyRedirects, - mr => Assert.Contains( - cachedRedirectsViewModel.Methodologies, - rvm => mr.Slug == rvm.FromSlug && mr.MethodologyVersion.Methodology.OwningPublicationSlug == rvm.ToSlug)); - } + var blobCacheService = app.Services.GetRequiredService(); - [Fact] - public async Task RedirectsExistInCache_Returns200WithRedirects() - { - var app = BuildApp(); - var client = app.CreateClient(); + var cachedValue = + await blobCacheService.GetItemAsync(new RedirectsCacheKey(), typeof(RedirectsViewModel)); + var cachedRedirectsViewModel = Assert.IsType(cachedValue); - var blobCacheService = app.Services.GetRequiredService(); + Assert.Empty(cachedRedirectsViewModel.MethodologyRedirects); + Assert.Empty(cachedRedirectsViewModel.ReleaseRedirectsByPublicationSlug); - var cachedPublicationRedirects = new List() - { - new(FromSlug: "publication_fromSlug_1", ToSlug: "publication_toSlug_1"), - new(FromSlug: "publication_fromSlug_2", ToSlug: "publication_toSlug_2"), - new(FromSlug: "publication_fromSlug_3", ToSlug: "publication_toSlug_3") - }; + Assert.Equal(publicationRedirects.Count, cachedRedirectsViewModel.PublicationRedirects.Count); + Assert.All( + publicationRedirects, + pr => Assert.Contains( + cachedRedirectsViewModel.PublicationRedirects, + rvm => + pr.Slug == rvm.FromSlug && + pr.Publication.Slug == rvm.ToSlug)); + } + } - var cachedReleaseRedirects = new List() + public class ReleaseRedirectsTests(TestApplicationFactory testApp) : ListTests(testApp) + { + [Fact] + public async Task ReleaseRedirectDoesNotExistForPublicationWithRedirect_Returns200WithRedirects() { - new(FromSlug: "release_fromSlug_1", ToSlug: "release_toSlug_1"), - new(FromSlug: "release_fromSlug_2", ToSlug: "release_toSlug_2"), - new(FromSlug: "release_fromSlug_3", ToSlug: "release_toSlug_3") - }; + PublicationRedirect publicationRedirect = DataFixture.DefaultPublicationRedirect() + .WithPublication(DataFixture.DefaultPublication()); - var cachedMethodologyRedirects = new List() - { - new(FromSlug: "methodology_fromSlug_1", ToSlug: "methodology_toSlug_1"), - new(FromSlug: "methodology_fromSlug_2", ToSlug: "methodology_toSlug_2"), - new(FromSlug: "methodology_fromSlug_3", ToSlug: "methodology_toSlug_3") - }; - - var cachedViewModel = new RedirectsViewModel( - Publications: cachedPublicationRedirects, - Releases: cachedReleaseRedirects, - Methodologies: cachedMethodologyRedirects); - - await blobCacheService.SetItemAsync(new RedirectsCacheKey(), cachedViewModel); - - var response = await ListRedirects(client); - - var viewModel = response.AssertOk(); - - Assert.All( - cachedPublicationRedirects, - cpr => Assert.Contains( - viewModel.Publications, - rvm => cpr.FromSlug == rvm.FromSlug && cpr.ToSlug == rvm.ToSlug)); - Assert.All( - cachedReleaseRedirects, - crr => Assert.Contains( - viewModel.Releases, - rvm => crr.FromSlug == rvm.FromSlug && crr.ToSlug == rvm.ToSlug)); - Assert.All( - cachedMethodologyRedirects, - cmr => Assert.Contains( - viewModel.Methodologies, - rvm => cmr.FromSlug == rvm.FromSlug && cmr.ToSlug == rvm.ToSlug)); - } + await TestApp.AddTestData(context => + context.PublicationRedirects.Add(publicationRedirect)); - [Fact] - public async Task NoRedirectsExist_Returns200WithNoRedirects() - { - var response = await ListRedirects(); + var client = BuildApp(enableAzurite: true).CreateClient(); + var response = await ListRedirects(client); - var viewModel = response.AssertOk(); + var viewModel = response.AssertOk(); - Assert.Empty(viewModel.Publications); - Assert.Empty(viewModel.Releases); - Assert.Empty(viewModel.Methodologies); + var publicationRedirectViewModel = Assert.Single(viewModel.PublicationRedirects); + Assert.Equal(publicationRedirect.Slug, publicationRedirectViewModel.FromSlug); + Assert.Equal(publicationRedirect.Publication.Slug, publicationRedirectViewModel.ToSlug); + + Assert.Empty(viewModel.ReleaseRedirectsByPublicationSlug); + } + + [Fact] + public async Task ReleaseRedirectExistsForPublicationWithRedirect_Returns200WithRedirects() + { + Publication publication = DataFixture.DefaultPublication() + .WithReleases( + [ + DataFixture + .DefaultRelease(publishedVersions: 1) + .WithSlug("updated-release-slug-1") + .WithRedirects( + DataFixture.DefaultReleaseRedirect() + .ForIndex(0, s => s.SetSlug("first-release-slug-1")) + .ForIndex(1, s => s.SetSlug("second-release-slug-1")) + .GenerateList(2) + ) + ]) + .WithSlug("updated-publication-slug-1") + .WithRedirects( + [ + DataFixture.DefaultPublicationRedirect() + .WithSlug("original-publication-slug-1") + ]); + + await TestApp.AddTestData(context => context.Publications.Add(publication)); + + var client = BuildApp(enableAzurite: true).CreateClient(); + var response = await ListRedirects(client); + + var viewModel = response.AssertOk(); + + var publicationRedirectViewModel = Assert.Single(viewModel.PublicationRedirects); + Assert.Equal("original-publication-slug-1", publicationRedirectViewModel.FromSlug); + Assert.Equal("updated-publication-slug-1", publicationRedirectViewModel.ToSlug); + + var releaseRedirectsForPublication = Assert.Single(viewModel.ReleaseRedirectsByPublicationSlug); + Assert.Equal("updated-publication-slug-1", releaseRedirectsForPublication.Key); + + Assert.Equal(2, releaseRedirectsForPublication.Value.Count); + Assert.Contains(releaseRedirectsForPublication.Value, + r => r.FromSlug == "first-release-slug-1" && r.ToSlug == "updated-release-slug-1"); + Assert.Contains(releaseRedirectsForPublication.Value, + r => r.FromSlug == "second-release-slug-1" && r.ToSlug == "updated-release-slug-1"); + } + + [Fact] + public async Task ReleaseRedirectExistsForPublicationWithoutRedirect_Returns200WithRedirects() + { + Publication publication = DataFixture.DefaultPublication() + .WithReleases( + [ + DataFixture + .DefaultRelease(publishedVersions: 1) + .WithSlug("updated-release-slug-1") + .WithRedirects( + DataFixture.DefaultReleaseRedirect() + .ForIndex(0, s => s.SetSlug("first-release-slug-1")) + .ForIndex(1, s => s.SetSlug("second-release-slug-1")) + .GenerateList(2) + ) + ]) + .WithSlug("original-publication-slug-1"); + + await TestApp.AddTestData(context => context.Publications.Add(publication)); + + var client = BuildApp(enableAzurite: true).CreateClient(); + var response = await ListRedirects(client); + + var viewModel = response.AssertOk(); + + Assert.Empty(viewModel.PublicationRedirects); + + var releaseRedirectsForPublication = Assert.Single(viewModel.ReleaseRedirectsByPublicationSlug); + Assert.Equal("original-publication-slug-1", releaseRedirectsForPublication.Key); + + Assert.Equal(2, releaseRedirectsForPublication.Value.Count); + Assert.Contains(releaseRedirectsForPublication.Value, + r => r.FromSlug == "first-release-slug-1" && r.ToSlug == "updated-release-slug-1"); + Assert.Contains(releaseRedirectsForPublication.Value, + r => r.FromSlug == "second-release-slug-1" && r.ToSlug == "updated-release-slug-1"); + } + + [Fact] + public async Task RedirectsExist_RedirectsAreCached() + { + Publication publication = DataFixture.DefaultPublication() + .WithReleases( + [ + DataFixture + .DefaultRelease(publishedVersions: 1) + .WithSlug("updated-release-slug-1") + .WithRedirects( + DataFixture.DefaultReleaseRedirect() + .ForIndex(0, s => s.SetSlug("first-release-slug-1")) + .ForIndex(1, s => s.SetSlug("second-release-slug-1")) + .GenerateList(2) + ) + ]) + .WithSlug("original-publication-slug-1"); + + await TestApp.AddTestData(context => context.Publications.Add(publication)); + + var app = BuildApp(enableAzurite: true); + var client = app.CreateClient(); + + await ListRedirects(client); + + var blobCacheService = app.Services.GetRequiredService(); + + var cachedValue = + await blobCacheService.GetItemAsync(new RedirectsCacheKey(), typeof(RedirectsViewModel)); + var cachedRedirectsViewModel = Assert.IsType(cachedValue); + + Assert.Empty(cachedRedirectsViewModel.MethodologyRedirects); + Assert.Empty(cachedRedirectsViewModel.PublicationRedirects); + + var releaseRedirectsForPublication = + Assert.Single(cachedRedirectsViewModel.ReleaseRedirectsByPublicationSlug); + Assert.Equal("original-publication-slug-1", releaseRedirectsForPublication.Key); + + Assert.Equal(2, releaseRedirectsForPublication.Value.Count); + Assert.Contains(releaseRedirectsForPublication.Value, + r => r.FromSlug == "first-release-slug-1" && r.ToSlug == "updated-release-slug-1"); + Assert.Contains(releaseRedirectsForPublication.Value, + r => r.FromSlug == "second-release-slug-1" && r.ToSlug == "updated-release-slug-1"); + } } private async Task ListRedirects( @@ -214,9 +413,8 @@ private async Task ListRedirects( } } - private WebApplicationFactory BuildApp() + private WebApplicationFactory BuildApp(bool enableAzurite = false) { - return TestApp - .WithAzurite(enabled: true); + return WithAzurite(enabled: enableAzurite); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Fixtures/IntegrationTestFixture.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Fixtures/IntegrationTestFixture.cs index 8590f56d116..ad9485636ad 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Fixtures/IntegrationTestFixture.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Fixtures/IntegrationTestFixture.cs @@ -1,25 +1,50 @@ #nullable enable +using System; +using System.Collections.Generic; using System.Threading.Tasks; +using DotNet.Testcontainers.Containers; +using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; -using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; -using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; +using Testcontainers.Azurite; using Xunit; +using GovUk.Education.ExploreEducationStatistics.Common.Services; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.Fixtures; [Collection(CacheTestFixture.CollectionName)] -public abstract class IntegrationTestFixture : +public abstract class IntegrationTestFixture(TestApplicationFactory testApp) : CacheServiceTestFixture, IClassFixture, IAsyncLifetime { + private readonly AzuriteContainer _azuriteContainer = new AzuriteBuilder() + .WithImage("mcr.microsoft.com/azure-storage/azurite:3.31.0") + .Build(); + protected readonly DataFixture DataFixture = new(); - protected readonly TestApplicationFactory TestApp; + protected readonly TestApplicationFactory TestApp = testApp; - internal IntegrationTestFixture(TestApplicationFactory testApp) + /// + /// Start the Azurite container. Once started, the test app must also + /// be configured with to use it. + /// + /// + /// We don't start the Azurite container in a class fixture as there currently + /// isn't a good way to clear it after each test. The current approach is to + /// restart the container for each test case (which is quite slow). + /// See: https://github.com/Azure/Azurite/issues/588. + /// For now, we should manually control the Azurite container's lifecycle by + /// calling this on a case-by-case basis. + /// + public async Task StartAzurite() { - TestApp = testApp; + await _azuriteContainer.StartAsync(); } public virtual async Task InitializeAsync() @@ -29,8 +54,42 @@ public virtual async Task InitializeAsync() public virtual async Task DisposeAsync() { - await TestApp.EnsureDatabaseDeleted(); - await TestApp.EnsureDatabaseDeleted(); - await TestApp.StopAzurite(); + await TestApp.ClearAllTestData(); + await _azuriteContainer.DisposeAsync(); + } + + public WebApplicationFactory WithAzurite(bool enabled = true) + { + if (!enabled) + { + return TestApp; + } + + if (_azuriteContainer.State != TestcontainersStates.Running) + { + throw new InvalidOperationException( + $"Azurite container must be started via '{nameof(StartAzurite)}' method first"); + } + + return TestApp.WithWebHostBuilder(builder => + { + builder + .ConfigureAppConfiguration((_, config) => + { + config.AddInMemoryCollection( + [ + new KeyValuePair("PublicStorage", _azuriteContainer.GetConnectionString()) + ]); + }) + .ConfigureServices(services => + { + services.ReplaceService(sp => + new PublicBlobStorageService( + _azuriteContainer.GetConnectionString(), + sp.GetRequiredService>() + ) + ); + }); + }); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Fixtures/TestApplicationFactory.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Fixtures/TestApplicationFactory.cs index 2b7a6016445..8e430dc34c6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Fixtures/TestApplicationFactory.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Fixtures/TestApplicationFactory.cs @@ -1,91 +1,21 @@ #nullable enable -using System; -using System.Collections.Generic; using System.Threading.Tasks; -using DotNet.Testcontainers.Containers; -using GovUk.Education.ExploreEducationStatistics.Common.Services; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; -using Microsoft.AspNetCore.Mvc.Testing; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; using Moq; -using Testcontainers.Azurite; namespace GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.Fixtures; public sealed class TestApplicationFactory : TestApplicationFactory { - private readonly AzuriteContainer _azuriteContainer = new AzuriteBuilder() - .WithImage("mcr.microsoft.com/azure-storage/azurite:3.31.0") - .WithInMemoryPersistence() - .Build(); - - public override async ValueTask DisposeAsync() - { - await _azuriteContainer.DisposeAsync(); - } - - /// - /// Start the Azurite container. Once started, the test app must also - /// be configured with to use it. - /// - /// - /// We don't start the Azurite container in a class fixture as there currently - /// isn't a good way to clear it after each test. The current approach is to - /// restart the container for each test case (which is quite slow). - /// See: https://github.com/Azure/Azurite/issues/588. - /// For now, we should manually control the Azurite container's lifecycle by - /// calling this on a case-by-case basis. - /// - public async Task StartAzurite() - { - await _azuriteContainer.StartAsync(); - } - - public async Task StopAzurite() - { - await _azuriteContainer.StopAsync(); - } - - public WebApplicationFactory WithAzurite(bool enabled = true) + public async Task ClearAllTestData() { - if (!enabled) - { - return this; - } - - if (_azuriteContainer.State != TestcontainersStates.Running) - { - throw new InvalidOperationException( - $"Azurite container must be started via '{nameof(StartAzurite)}' method first"); - } - - return WithWebHostBuilder(builder => - { - builder - .ConfigureAppConfiguration((_, config) => - { - config.AddInMemoryCollection( - [ - new KeyValuePair("PublicStorage", _azuriteContainer.GetConnectionString()) - ]); - }) - .ConfigureServices(services => - { - services.ReplaceService(sp => - new PublicBlobStorageService( - _azuriteContainer.GetConnectionString(), - sp.GetRequiredService>() - ) - ); - }); - }); + await EnsureDatabaseDeleted(); + await EnsureDatabaseDeleted(); } protected override IHostBuilder CreateHostBuilder() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyGeneratorExtensions.cs index 12c6dc1bc79..4ee528301a0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyGeneratorExtensions.cs @@ -58,16 +58,24 @@ private static InstanceSetters SetMethodologyVersions( this InstanceSetters setters, Func> methodologyVersions) => setters.Set( - m => m.Versions, - (_, methodology, context) => - { - var list = methodologyVersions.Invoke(context).ToList(); + m => m.Versions, + (_, methodology, context) => + { + var list = methodologyVersions.Invoke(context).ToList(); - list.ForEach(methodologyVersion => methodologyVersion.Methodology = methodology); + list.ForEach(methodologyVersion => methodologyVersion.Methodology = methodology); - return list; - } - ); + var latestPublishedVersion = list + .Where(mv => mv.Published.HasValue) + .OrderBy(mv => mv.Published!) + .LastOrDefault(); + + methodology.LatestPublishedVersion = latestPublishedVersion; + methodology.LatestPublishedVersionId = latestPublishedVersion?.Id; + + return list; + } + ); private static InstanceSetters SetLatestPublishedVersion( this InstanceSetters setters, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs index 4c909e32735..6ebfe049ac6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/MethodologyVersionGeneratorExtensions.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; @@ -38,6 +39,16 @@ public static Generator WithPublished( DateTime published) => generator.ForInstance(d => d.SetPublished(published)); + public static Generator WithRedirects( + this Generator generator, + IEnumerable methodologyRedirects) + => generator.ForInstance(s => s.SetRedirects(methodologyRedirects)); + + public static Generator WithRedirects( + this Generator generator, + Func> methodologyRedirects) + => generator.ForInstance(s => s.SetRedirects(methodologyRedirects.Invoke)); + public static InstanceSetters SetDefaults(this InstanceSetters setters) => setters .SetDefault(p => p.Id) @@ -60,4 +71,27 @@ public static InstanceSetters SetPublished( this InstanceSetters setters, DateTime published) => setters.Set(mv => mv.Published, published); + + public static InstanceSetters SetRedirects( + this InstanceSetters setters, + IEnumerable methodologyRedirects) + => setters.SetRedirects(_ => methodologyRedirects); + + private static InstanceSetters SetRedirects( + this InstanceSetters setters, + Func> methodologyRedirects) + => setters.Set( + mv => mv.MethodologyRedirects, + (_, methodologyVersion, context) => + { + var list = methodologyRedirects.Invoke(context).ToList(); + + list.ForEach(methodologyRedirect => + { + methodologyRedirect.MethodologyVersion = methodologyVersion; + methodologyRedirect.MethodologyVersionId = methodologyVersion.Id; + }); + + return list; + }); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs index 989976bb1c4..9a22c3c570c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/PublicationGeneratorExtensions.cs @@ -80,6 +80,11 @@ public static Generator WithTheme( Theme theme) => generator.ForInstance(s => s.SetTheme(theme)); + public static Generator WithSlug( + this Generator generator, + string slug) + => generator.ForInstance(s => s.SetSlug(slug)); + public static InstanceSetters SetId( this InstanceSetters setters, Guid id) @@ -106,6 +111,16 @@ public static Generator WithThemes( return generator; } + public static Generator WithRedirects( + this Generator generator, + IEnumerable publicationRedirects) + => generator.ForInstance(s => s.SetRedirects(publicationRedirects)); + + public static Generator WithRedirects( + this Generator generator, + Func> publicationRedirects) + => generator.ForInstance(s => s.SetRedirects(publicationRedirects.Invoke)); + public static InstanceSetters SetReleases( this InstanceSetters setters, IEnumerable releases) @@ -270,4 +285,32 @@ private static InstanceSetters SetTheme( Theme theme) => setters.Set(p => p.Theme, theme) .SetThemeId(theme.Id); + + public static InstanceSetters SetSlug( + this InstanceSetters setters, + string slug) + => setters.Set(p => p.Slug, slug); + + public static InstanceSetters SetRedirects( + this InstanceSetters setters, + IEnumerable publicationRedirects) + => setters.SetRedirects(_ => publicationRedirects); + + private static InstanceSetters SetRedirects( + this InstanceSetters setters, + Func> publicationRedirects) + => setters.Set( + mv => mv.PublicationRedirects, + (_, publication, context) => + { + var list = publicationRedirects.Invoke(context).ToList(); + + list.ForEach(publicationRedirect => + { + publicationRedirect.Publication = publication; + publicationRedirect.PublicationId = publication.Id; + }); + + return list; + }); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs index 5ebea8621a9..2ed14ecaa44 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/ReleaseGeneratorExtensions.cs @@ -97,6 +97,21 @@ public static Generator WithYear( int year) => generator.ForInstance(s => s.SetYear(year)); + public static Generator WithSlug( + this Generator generator, + string slug) + => generator.ForInstance(s => s.SetSlug(slug)); + + public static Generator WithRedirects( + this Generator generator, + IEnumerable releaseRedirects) + => generator.ForInstance(s => s.SetRedirects(releaseRedirects)); + + public static Generator WithRedirects( + this Generator generator, + Func> releaseRedirects) + => generator.ForInstance(s => s.SetRedirects(releaseRedirects.Invoke)); + public static Generator WithLabel( this Generator generator, string? label) @@ -240,6 +255,34 @@ public static InstanceSetters SetYear( int year) => setters.Set(p => p.Year, year); + public static InstanceSetters SetSlug( + this InstanceSetters setters, + string slug) + => setters.Set(p => p.Slug, slug); + + public static InstanceSetters SetRedirects( + this InstanceSetters setters, + IEnumerable releaseRedirects) + => setters.SetRedirects(_ => releaseRedirects); + + private static InstanceSetters SetRedirects( + this InstanceSetters setters, + Func> releaseRedirects) + => setters.Set( + mv => mv.ReleaseRedirects, + (_, release, context) => + { + var list = releaseRedirects.Invoke(context).ToList(); + + list.ForEach(releaseRedirect => + { + releaseRedirect.Release = release; + releaseRedirect.ReleaseId = release.Id; + }); + + return list; + }); + public static InstanceSetters SetLabel( this InstanceSetters setters, string? label) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/MethodologyVersion.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/MethodologyVersion.cs index 1a2dc950440..9fc2ed2083f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/MethodologyVersion.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/MethodologyVersion.cs @@ -42,6 +42,8 @@ public class MethodologyVersion : ICreatedTimestamp public List Notes { get; set; } = new(); + public List MethodologyRedirects { get; set; } = []; + public Methodology Methodology { get; set; } = null!; public Guid MethodologyId { get; set; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Publication.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Publication.cs index 8386148e2c3..5f15ace12eb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Publication.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Publication.cs @@ -21,6 +21,8 @@ public class Publication public List ReleaseVersions { get; set; } = []; + public List PublicationRedirects { get; set; } = []; + public List Methodologies { get; set; } = []; public ExternalMethodology? ExternalMethodology { get; set; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/PublicationRedirect.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/PublicationRedirect.cs index 08881ca4d7a..14da8256319 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/PublicationRedirect.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/PublicationRedirect.cs @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable using System; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -6,11 +6,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Model; public class PublicationRedirect : ICreatedTimestamp { - public string Slug { get; init; } = null!; + public string Slug { get; set; } = null!; - public Guid PublicationId { get; init; } + public Guid PublicationId { get; set; } - public Publication Publication { get; init; } = null!; + public Publication Publication { get; set; } = null!; public DateTime Created { get; set; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs index 699e6998cb5..aa7bc75d42d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Release.cs @@ -28,6 +28,8 @@ public class Release : ICreatedUpdatedTimestamps public List Versions { get; set; } = []; + public List ReleaseRedirects { get; set; } = []; + public DateTime Created { get; set; } public DateTime? Updated { get; set; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseRedirect.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseRedirect.cs index 0b6d93f226f..5efe0ab2466 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseRedirect.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/ReleaseRedirect.cs @@ -6,11 +6,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Model; public class ReleaseRedirect : ICreatedTimestamp { - public string Slug { get; init; } = null!; + public string Slug { get; set; } = null!; - public Guid ReleaseId { get; init; } + public Guid ReleaseId { get; set; } - public Release Release { get; init; } = null!; + public Release Release { get; set; } = null!; public DateTime Created { get; set; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/RedirectsServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/RedirectsServiceTests.cs index 19ab2c5c116..33c729d978f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/RedirectsServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/RedirectsServiceTests.cs @@ -11,136 +11,6 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Services.Tests { public class RedirectsServiceTests { - [Fact] - public async Task List_PublicationRedirect() - { - var publication = new Publication - { - Slug = "redirect-to", - }; - - var publicationRedirects = new List - { - new() - { - Slug = "redirect-from-2", - Publication = publication, - }, - new() - { - Slug = "redirect-from-1", - Publication = publication, - }, - new() - { - Slug = "redirect-to", // should be excluded in results, as same as current publication slug - Publication = publication, - }, - }; - - var contentDbContextId = Guid.NewGuid().ToString(); - await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) - { - await contentDbContext.PublicationRedirects.AddRangeAsync(publicationRedirects); - await contentDbContext.SaveChangesAsync(); - } - - await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) - { - var redirectsService = SetupRedirectsService(contentDbContext); - - var result = await redirectsService.List(); - - var viewModel = result.AssertRight(); - var publicationsRedirectViewModel = viewModel.Publications; - - Assert.Equal(2, publicationsRedirectViewModel.Count); - - Assert.Equal("redirect-from-2", publicationsRedirectViewModel[0].FromSlug); - Assert.Equal("redirect-to", publicationsRedirectViewModel[0].ToSlug); - - Assert.Equal("redirect-from-1", publicationsRedirectViewModel[1].FromSlug); - Assert.Equal("redirect-to", publicationsRedirectViewModel[1].ToSlug); - } - } - - [Fact] - public async Task List_MethodologyRedirect() - { - var latestPublishedVersionId = Guid.NewGuid(); - var methodology = new Methodology - { - LatestPublishedVersionId = latestPublishedVersionId, - OwningPublicationSlug = "no-redirect-to-1", - Versions = new List - { - new() - { - // previous version - AlternativeSlug = "no-redirect-to-2", - Version = 0, - }, - new() - { - Id = latestPublishedVersionId, - AlternativeSlug = "redirect-to", - Version = 1, - }, - new() - { - // latestVersion but unpublished - AlternativeSlug = "no-redirect-to-3", - Version = 2, - }, - } - }; - - var methodologyRedirects = new List - { - new() - { - Slug = "redirect-from-2", - MethodologyVersion = methodology.Versions[0], - }, - new() - { - Slug = "redirect-from-1", - MethodologyVersion = methodology.Versions[1], - }, - new() - { - Slug = "no-redirect-from-1", - MethodologyVersion = methodology.Versions[2], - }, - }; - - var contentDbContextId = Guid.NewGuid().ToString(); - await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) - { - await contentDbContext.Methodologies.AddAsync(methodology); - await contentDbContext.MethodologyRedirects.AddRangeAsync(methodologyRedirects); - await contentDbContext.SaveChangesAsync(); - } - - await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) - { - var redirectsService = SetupRedirectsService(contentDbContext); - - var result = await redirectsService.List(); - - var viewModel = result.AssertRight(); - var methodologyRedirectViewModel = viewModel.Methodologies; - - Assert.Equal(2, methodologyRedirectViewModel.Count); - - Assert.Equal("redirect-from-2", methodologyRedirectViewModel[0].FromSlug); - Assert.Equal("redirect-to", methodologyRedirectViewModel[0].ToSlug); - - Assert.Equal("redirect-from-1", methodologyRedirectViewModel[1].FromSlug); - Assert.Equal("redirect-to", methodologyRedirectViewModel[1].ToSlug); - } - } - [Fact] public async Task List_MethodologyRedirect_MethodologyNotPublished() { @@ -178,7 +48,7 @@ public async Task List_MethodologyRedirect_MethodologyNotPublished() var viewModel = result.AssertRight(); - Assert.Empty(viewModel.Methodologies); + Assert.Empty(viewModel.MethodologyRedirects); } } @@ -231,9 +101,9 @@ public async Task List_MethodologyRedirect_PreviousVersionRedirectOnly() var result = await redirectsService.List(); var viewModel = result.AssertRight(); - var methodologyRedirectViewModel = viewModel.Methodologies; + var methodologyRedirectsViewModel = viewModel.MethodologyRedirects; - var redirect = Assert.Single(methodologyRedirectViewModel); + var redirect = Assert.Single(methodologyRedirectsViewModel); Assert.Equal("redirect-from-1", redirect.FromSlug); Assert.Equal("redirect-to", redirect.ToSlug); @@ -318,7 +188,7 @@ public async Task List_MethodologyRedirect_MultipleMethodologies() var result = await redirectsService.List(); var viewModel = result.AssertRight(); - var methodologyRedirectsViewModel = viewModel.Methodologies; + var methodologyRedirectsViewModel = viewModel.MethodologyRedirects; Assert.Equal(2, methodologyRedirectsViewModel.Count); @@ -379,7 +249,7 @@ public async Task List_MethodologyRedirect_FilterRedirectIfSameAsCurrentSlug() var result = await redirectsService.List(); var viewModel = result.AssertRight(); - Assert.Empty(viewModel.Methodologies); + Assert.Empty(viewModel.MethodologyRedirects); } } @@ -438,7 +308,7 @@ public async Task List_MethodologyRedirect_DuplicateRedirects() var viewModel = result.AssertRight(); - var redirect = Assert.Single(viewModel.Methodologies); + var redirect = Assert.Single(viewModel.MethodologyRedirects); Assert.Equal("duplicated-redirect", redirect.FromSlug); Assert.Equal("redirect-to", redirect.ToSlug); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/RedirectsService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/RedirectsService.cs index 10a392a8be3..6f03b39aa12 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/RedirectsService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/RedirectsService.cs @@ -22,7 +22,7 @@ public RedirectsService(ContentDbContext contentDbContext) public async Task> List() { - var publicationRedirectViewModels = await _contentDbContext.PublicationRedirects + var publicationRedirects = await _contentDbContext.PublicationRedirects .Where(pr => pr.Slug != pr.Publication.Slug) // don't use redirects to the current live slug .Distinct() .Select(pr => new RedirectViewModel( @@ -34,21 +34,21 @@ public async Task> List() // A redirect for a MethodologyVersion that isn't published shouldn't appear in the list. A redirect becomes // active once the associated MethodologyVersion is published. It remains active if subsequent methodology // amendments are published. We establish this by checking against each MethodologyVersion's Version number. - var methodologyRedirects = await _contentDbContext.MethodologyRedirects - .Where(mr => mr.MethodologyVersion.Methodology.LatestPublishedVersion != null - && mr.MethodologyVersion.Methodology.LatestPublishedVersion.Version >= - mr.MethodologyVersion.Version + var methodologyRedirects = ( + await _contentDbContext.MethodologyRedirects + .Where(mr => mr.MethodologyVersion.Methodology.LatestPublishedVersion != null + && mr.MethodologyVersion.Methodology.LatestPublishedVersion.Version >= + mr.MethodologyVersion.Version + ) + .Select(mr => new + { + RedirectSlug = mr.Slug, + // MethodologyVersion.Slug cannot be translated into SQL, so we do this... + LatestPublishedSlug = mr.MethodologyVersion.Methodology.LatestPublishedVersion!.AlternativeSlug + ?? mr.MethodologyVersion.Methodology.OwningPublicationSlug, + }) + .ToListAsync() ) - .Select(mr => new - { - RedirectSlug = mr.Slug, - // MethodologyVersion.Slug cannot be translated into SQL, so we do this... - LatestPublishedSlug = mr.MethodologyVersion.Methodology.LatestPublishedVersion!.AlternativeSlug - ?? mr.MethodologyVersion.Methodology.OwningPublicationSlug, - }) - .ToListAsync(); - - var methodologyRedirectViewModels = methodologyRedirects .Select(mr => { if (mr.RedirectSlug == mr.LatestPublishedSlug) @@ -64,18 +64,28 @@ public async Task> List() .Distinct() .ToList(); - var releaseRedirectViewModels = await _contentDbContext.ReleaseRedirects - .Where(rr => rr.Slug != rr.Release.Slug) // don't use redirects to the current live slug - .Distinct() - .Select(rr => new RedirectViewModel( - rr.Slug, - rr.Release.Slug - )) - .ToListAsync(); + var releaseRedirectsByPublicationSlug = ( + await _contentDbContext.ReleaseRedirects + .Where(rr => rr.Slug != rr.Release.Slug) // don't use redirects to the current live slug + .Distinct() + .Select(rr => new + { + PublicationSlug = rr.Release.Publication.Slug, + FromSlug = rr.Slug, + ToSlug = rr.Release.Slug + }) + .ToListAsync() + ) + .GroupBy(rr => rr.PublicationSlug) + .ToDictionary( + g => g.Key, + g => g + .Select(a => new RedirectViewModel(FromSlug: a.FromSlug, ToSlug: a.ToSlug)) + .ToList()); return new RedirectsViewModel( - publicationRedirectViewModels, - methodologyRedirectViewModels, - releaseRedirectViewModels); + PublicationRedirects: publicationRedirects, + MethodologyRedirects: methodologyRedirects, + ReleaseRedirectsByPublicationSlug: releaseRedirectsByPublicationSlug); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/RedirectViewModels.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/RedirectViewModels.cs index e211a620400..5c68b68e2d2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/RedirectViewModels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/RedirectViewModels.cs @@ -1,9 +1,9 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.ViewModels; public record RedirectsViewModel( - List Publications, - List Methodologies, - List Releases); + List PublicationRedirects, + List MethodologyRedirects, + Dictionary> ReleaseRedirectsByPublicationSlug); public record RedirectViewModel( string FromSlug, diff --git a/src/explore-education-statistics-frontend/src/middleware/pages/__tests__/redirectPages.test.ts b/src/explore-education-statistics-frontend/src/middleware/pages/__tests__/redirectPages.test.ts index 328a848ce0b..772cc444b81 100644 --- a/src/explore-education-statistics-frontend/src/middleware/pages/__tests__/redirectPages.test.ts +++ b/src/explore-education-statistics-frontend/src/middleware/pages/__tests__/redirectPages.test.ts @@ -15,11 +15,11 @@ describe('redirectPages', () => { const nextSpy = jest.spyOn(NextResponse, 'next'); const testRedirects: Redirects = { - methodologies: [ + methodologyRedirects: [ { fromSlug: 'original-slug-1', toSlug: 'updated-slug-1' }, { fromSlug: 'original-slug-2', toSlug: 'updated-slug-2' }, ], - publications: [ + publicationRedirects: [ { fromSlug: 'original-slug-3', toSlug: 'updated-slug-3' }, { fromSlug: 'original-slug-4', toSlug: 'updated-slug-4' }, ], diff --git a/src/explore-education-statistics-frontend/src/middleware/pages/redirectPages.ts b/src/explore-education-statistics-frontend/src/middleware/pages/redirectPages.ts index e9c4e07bbbd..ea5e6a09509 100644 --- a/src/explore-education-statistics-frontend/src/middleware/pages/redirectPages.ts +++ b/src/explore-education-statistics-frontend/src/middleware/pages/redirectPages.ts @@ -15,8 +15,8 @@ const cacheTime = getCacheTime(); let cachedRedirects: CachedRedirects | undefined; const redirectPaths = { - methodologies: '/methodology', - publications: '/find-statistics', + methodologyRedirects: '/methodology', + publicationRedirects: '/find-statistics', }; export default async function redirectPages( diff --git a/src/explore-education-statistics-frontend/src/services/redirectService.ts b/src/explore-education-statistics-frontend/src/services/redirectService.ts index 4daf2724edb..f90347cc773 100644 --- a/src/explore-education-statistics-frontend/src/services/redirectService.ts +++ b/src/explore-education-statistics-frontend/src/services/redirectService.ts @@ -1,6 +1,6 @@ export interface Redirects { - methodologies: Redirect[]; - publications: Redirect[]; + methodologyRedirects: Redirect[]; + publicationRedirects: Redirect[]; } export type RedirectType = keyof Redirects; From 910df94ce77403019f1779e530adbbac95bde037 Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 21 Nov 2024 10:04:05 +0000 Subject: [PATCH 099/144] EES-5587 refactor reordering filters and indicators --- .../src/components/DraggableItem.module.scss | 57 -- .../src/components/DraggableItem.tsx | 76 -- .../src/components/DroppableArea.module.scss | 14 - .../src/components/DroppableArea.tsx | 38 - .../data/components/ReorderFiltersList.tsx | 213 +++--- .../data/components/ReorderIndicatorsList.tsx | 121 ++-- .../data/components/ReorderList.module.scss | 46 -- .../release/data/components/ReorderList.tsx | 210 ------ .../__tests__/ReorderFiltersList.test.tsx | 240 +++---- .../__tests__/ReorderIndicatorsList.test.tsx | 105 ++- .../components/ReorderableItem.module.scss | 3 +- .../src/components/ReorderableItem.tsx | 32 +- .../src/components/ReorderableList.tsx | 7 +- .../ReorderableNestedList.module.scss | 26 + .../src/components/ReorderableNestedList.tsx | 105 +++ .../__tests__/ReorderableItem.test.tsx | 186 +++++ .../__tests__/ReorderableNestedList.test.tsx | 655 ++++++++++++++++++ .../bau/data_reordering.robot | 181 ++--- 18 files changed, 1438 insertions(+), 877 deletions(-) delete mode 100644 src/explore-education-statistics-admin/src/components/DraggableItem.module.scss delete mode 100644 src/explore-education-statistics-admin/src/components/DraggableItem.tsx delete mode 100644 src/explore-education-statistics-admin/src/components/DroppableArea.module.scss delete mode 100644 src/explore-education-statistics-admin/src/components/DroppableArea.tsx delete mode 100644 src/explore-education-statistics-admin/src/pages/release/data/components/ReorderList.module.scss delete mode 100644 src/explore-education-statistics-admin/src/pages/release/data/components/ReorderList.tsx create mode 100644 src/explore-education-statistics-common/src/components/ReorderableNestedList.module.scss create mode 100644 src/explore-education-statistics-common/src/components/ReorderableNestedList.tsx create mode 100644 src/explore-education-statistics-common/src/components/__tests__/ReorderableItem.test.tsx create mode 100644 src/explore-education-statistics-common/src/components/__tests__/ReorderableNestedList.test.tsx diff --git a/src/explore-education-statistics-admin/src/components/DraggableItem.module.scss b/src/explore-education-statistics-admin/src/components/DraggableItem.module.scss deleted file mode 100644 index 3dd029332b3..00000000000 --- a/src/explore-education-statistics-admin/src/components/DraggableItem.module.scss +++ /dev/null @@ -1,57 +0,0 @@ -@import '~govuk-frontend/dist/govuk/base'; - -$vertical-spacing: govuk-spacing(3); - -.draggable { - background: govuk-colour('white'); - border-bottom: 1px solid govuk-colour('dark-grey'); - margin-bottom: 0; - padding: $vertical-spacing govuk-spacing(1) $vertical-spacing 40px; - position: relative; - - &:hover { - background: govuk-colour('light-grey'); - } - - &:focus { - @include govuk-focused-text; - z-index: 1; - } -} - -.isDragging { - @include govuk-focused-text; - z-index: 1; -} - -.isDraggedOutside { - opacity: 0.7; -} - -.dragHandle { - color: govuk-colour('dark-grey'); - font-size: 1.6rem; - left: 4px; - line-height: 1; - position: absolute; - top: $vertical-spacing; -} - -.hideDragHandle { - padding-left: 0; - - &::before { - content: none; - } -} - -/* stylelint-disable selector-no-qualifying-type */ -tr.draggable { - border-bottom: 0; - display: table-row; - - &.isDragging { - display: table; - } -} -/* stylelint-enable */ diff --git a/src/explore-education-statistics-admin/src/components/DraggableItem.tsx b/src/explore-education-statistics-admin/src/components/DraggableItem.tsx deleted file mode 100644 index 683b3aa113f..00000000000 --- a/src/explore-education-statistics-admin/src/components/DraggableItem.tsx +++ /dev/null @@ -1,76 +0,0 @@ -import styles from '@admin/components/DraggableItem.module.scss'; -import classNames from 'classnames'; -import React, { ReactNode } from 'react'; -import { Draggable } from '@hello-pangea/dnd'; - -export const DragHandle = ({ className }: { className?: string }) => ( - - ☰ - -); - -interface Props { - children: ReactNode; - className?: string; - // draggableClassName replaces the default draggable style - draggableClassName?: string; - dragHandle?: ReactNode; - // dragHandleClassName replaces the default dragHandle style - dragHandleClassName?: string; - hideDragHandle?: boolean; - id: string; - index: number; - isDisabled?: boolean; - isReordering: boolean; - tag?: 'div' | 'li' | 'tr'; - testId?: string; -} - -const DraggableItem = ({ - children, - className, - draggableClassName, - dragHandleClassName, - dragHandle: overrideDragHandle, - hideDragHandle = false, - id, - index, - isDisabled = false, - isReordering, - tag: Element = 'div', - testId, -}: Props) => { - return ( - - {(draggableProvided, draggableSnapshot) => ( - - {!hideDragHandle && isReordering && ( - <> - {overrideDragHandle ?? - DragHandle({ className: dragHandleClassName })} - - )} - {children} - - )} - - ); -}; - -export default DraggableItem; diff --git a/src/explore-education-statistics-admin/src/components/DroppableArea.module.scss b/src/explore-education-statistics-admin/src/components/DroppableArea.module.scss deleted file mode 100644 index e77698975ef..00000000000 --- a/src/explore-education-statistics-admin/src/components/DroppableArea.module.scss +++ /dev/null @@ -1,14 +0,0 @@ -@import '~govuk-frontend/dist/govuk/base'; - -/* stylelint-disable selector-no-qualifying-type */ -ol.dropArea, -ul.dropArea { - list-style: none; - padding: 0; -} -/* stylelint-enable */ - -.dropAreaActive { - background: govuk-colour('light-grey'); - outline: $govuk-focus-width solid $govuk-focus-colour; -} diff --git a/src/explore-education-statistics-admin/src/components/DroppableArea.tsx b/src/explore-education-statistics-admin/src/components/DroppableArea.tsx deleted file mode 100644 index 3d1f1a65f2e..00000000000 --- a/src/explore-education-statistics-admin/src/components/DroppableArea.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import styles from '@admin/components/DroppableArea.module.scss'; -import classNames from 'classnames'; -import React, { ReactNode } from 'react'; -import { DroppableProvided, DroppableStateSnapshot } from '@hello-pangea/dnd'; - -interface Props { - children: ReactNode; - className?: string; - droppableProvided: DroppableProvided; - droppableSnapshot: DroppableStateSnapshot; - tag?: 'div' | 'ol' | 'tbody'; - testId?: string; -} - -const DroppableArea = ({ - children, - className, - droppableProvided, - droppableSnapshot, - tag: Element = 'div', - testId, -}: Props) => { - return ( - - {children} - {droppableProvided.placeholder} - - ); -}; - -export default DroppableArea; diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderFiltersList.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderFiltersList.tsx index be437fded69..b7f8823a394 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderFiltersList.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderFiltersList.tsx @@ -1,18 +1,24 @@ -import ReorderList, { - FormattedGroup, - FormattedFilters, - ReorderProps, -} from '@admin/pages/release/data/components/ReorderList'; import Button from '@common/components/Button'; import ButtonGroup from '@common/components/ButtonGroup'; import LoadingSpinner from '@common/components/LoadingSpinner'; +import { + ReorderableListItem, + ReorderResult, +} from '@common/components/ReorderableItem'; +import ReorderableNestedList from '@common/components/ReorderableNestedList'; import useAsyncHandledRetry from '@common/hooks/useAsyncHandledRetry'; import tableBuilderService, { Subject, } from '@common/services/tableBuilderService'; +import reorder from '@common/utils/reorder'; import orderBy from 'lodash/orderBy'; import React, { useEffect, useState } from 'react'; +interface ReorderHandler extends ReorderResult { + expandedItemId?: string; + expandedItemParentId?: string; +} + interface UpdatedFilter { id: string; filterGroups: { id: string; filterItems: string[] }[]; @@ -26,18 +32,18 @@ interface Props { onSave: (subjectId: string, requestFilters: UpdateFiltersRequest) => void; } -const ReorderFiltersList = ({ +export default function ReorderFiltersList({ releaseId, subject, onCancel, onSave, -}: Props) => { +}: Props) { const { value: subjectMeta, isLoading } = useAsyncHandledRetry( () => tableBuilderService.getSubjectMeta(subject.id, releaseId), [subject.id, releaseId], ); - const [filters, setFilters] = useState([]); + const [filters, setFilters] = useState([]); useEffect(() => { if (isLoading || !subjectMeta) { @@ -47,105 +53,136 @@ const ReorderFiltersList = ({ // Transforming the filters to be nested arrays rather than keyed objects. // Order by the order fields. - const formattedFilters = orderBy( - Object.entries(filtersMeta).map(([, group]) => { - return { - id: group.id, - label: group.legend, - order: group.order, - groups: orderBy( - Object.entries(group.options).map(([, item]) => { - return { - id: item.id, - label: item.label, - order: item.order, - items: item.options.map(option => ({ - id: option.value, - label: option.label, - })), - }; - }), - 'order', - ), - }; - }), + const formattedFilters: ReorderableListItem[] = orderBy( + Object.values(filtersMeta), 'order', - ); + ).map(group => { + return { + id: group.id, + label: group.legend, + childOptions: orderBy(Object.values(group.options), 'order').map( + item => ({ + id: item.id, + label: item.label, + parentId: group.id, + childOptions: item.options.map(option => ({ + id: option.value, + label: option.label, + parentId: item.id, + })), + }), + ), + }; + }); + setFilters(formattedFilters); }, [isLoading, subjectMeta]); const handleReorder = ({ - reordered, - parentCategoryId, - parentGroupId, - }: ReorderProps) => { - // reordering filters - if (!parentCategoryId && !parentGroupId) { - setFilters(reordered as FormattedFilters[]); + prevIndex, + nextIndex, + expandedItemId, + expandedItemParentId, + }: ReorderHandler) => { + // Top level + if (!expandedItemId) { + setFilters(reorder(filters, prevIndex, nextIndex)); return; } - - const reorderedFilters = filters.map(filter => { - if (filter.id !== parentCategoryId) { - return filter; + const reordered = filters.map(filter => { + // Second level + if (filter.id === expandedItemId && filter.childOptions?.length) { + // Only one child option when has default group. + // In this case the children of the group are shown instead of the group. + return filter.childOptions.length === 1 + ? { + ...filter, + childOptions: [ + { + ...filter.childOptions[0], + childOptions: filter.childOptions[0].childOptions + ? reorder( + filter.childOptions[0].childOptions, + prevIndex, + nextIndex, + ) + : [], + }, + ], + } + : { + ...filter, + childOptions: reorder(filter.childOptions, prevIndex, nextIndex), + }; } - // Reordering groups - if (!parentGroupId) { - return { ...filter, groups: reordered }; - } - // Reordering items - const updatedFilterGroups = filter.groups.map( - (filterGroup: FormattedGroup) => { - if (filterGroup.id !== parentGroupId) { - return filterGroup; - } - return { - ...filterGroup, - items: reordered, - }; - }, - ); - return { ...filter, groups: updatedFilterGroups }; - }) as FormattedFilters[]; + // Third level + if (expandedItemParentId) { + return filter.id === expandedItemParentId + ? { + ...filter, + childOptions: filter.childOptions?.map(option => + option.id === expandedItemId + ? { + ...option, + childOptions: option.childOptions?.length + ? reorder(option.childOptions, prevIndex, nextIndex) + : [], + } + : option, + ), + } + : filter; + } + return filter; + }); - setFilters(reorderedFilters); + setFilters(reordered); }; const handleSave = () => { - const updateFiltersRequest: UpdateFiltersRequest = filters.map(filter => { - return { - id: filter.id, - filterGroups: filter.groups.map(group => { - return { + const updateFiltersRequest: UpdateFiltersRequest = filters.map(filter => ({ + id: filter.id, + filterGroups: filter.childOptions + ? filter.childOptions?.map(group => ({ id: group.id, - filterItems: group.items.map(item => item.id), - }; - }), - }; - }); + filterItems: group.childOptions + ? group.childOptions?.map(item => item.id) + : [], + })) + : [], + })); onSave(subject.id, updateFiltersRequest); }; return ( - <> -

          {`Reorder filters for ${subject.name}`}

          - - {filters.length === 0 ? ( + + {filters.length === 0 ? ( + <>

          No filters available.

          - ) : ( - - )} - - {filters.length > 0 && ( - - )} - -           - + + ) : ( + <> + + + {filters.length > 0 && ( + + )} + + + + )} +           ); -}; -export default ReorderFiltersList; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderIndicatorsList.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderIndicatorsList.tsx index da5b01e7e46..e0918a01079 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderIndicatorsList.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderIndicatorsList.tsx @@ -1,17 +1,23 @@ -import ReorderList, { - FormattedIndicators, - ReorderProps, -} from '@admin/pages/release/data/components/ReorderList'; import Button from '@common/components/Button'; import ButtonGroup from '@common/components/ButtonGroup'; import LoadingSpinner from '@common/components/LoadingSpinner'; +import ReorderableNestedList from '@common/components/ReorderableNestedList'; +import { + ReorderableListItem, + ReorderResult, +} from '@common/components/ReorderableItem'; import useAsyncHandledRetry from '@common/hooks/useAsyncHandledRetry'; +import reorder from '@common/utils/reorder'; import tableBuilderService, { Subject, } from '@common/services/tableBuilderService'; import orderBy from 'lodash/orderBy'; import React, { useEffect, useState } from 'react'; +interface ReorderHandler extends ReorderResult { + expandedItemId?: string; +} + interface UpdatedIndicator { id: string; indicators: string[]; @@ -28,18 +34,18 @@ interface Props { ) => void; } -const ReorderIndicatorsList = ({ +export default function ReorderIndicatorsList({ releaseId, subject, onCancel, onSave, -}: Props) => { +}: Props) { const { value: subjectMeta, isLoading } = useAsyncHandledRetry( () => tableBuilderService.getSubjectMeta(subject.id, releaseId), [subject.id, releaseId], ); - const [indicators, setIndicators] = useState([]); + const [indicators, setIndicators] = useState([]); useEffect(() => { if (isLoading || !subjectMeta) { @@ -49,38 +55,46 @@ const ReorderIndicatorsList = ({ // Transforming the indicators to be nested arrays rather than keyed objects. // Order by the order field. - const formattedIndicators = orderBy( - Object.entries(indicatorsMeta).map(([, item]) => { - return { - id: item.id, - label: item.label, - order: item.order, - items: item.options.map(option => ({ - id: option.value, - label: option.label, - })), - }; - }), + const formattedIndicators: ReorderableListItem[] = orderBy( + Object.values(indicatorsMeta), 'order', - ); + ).map(item => { + return { + id: item.id, + label: item.label, + childOptions: item.options.map(option => ({ + id: option.value, + label: option.label, + })), + }; + }); setIndicators(formattedIndicators); }, [isLoading, subjectMeta]); - const handleReorder = ({ reordered, parentCategoryId }: ReorderProps) => { - // reordering indicators - if (!parentCategoryId) { - setIndicators(reordered as FormattedIndicators[]); + const handleReorder = ({ + prevIndex, + nextIndex, + expandedItemId, + }: ReorderHandler) => { + // Top level + if (!expandedItemId) { + setIndicators(reorder(indicators, prevIndex, nextIndex)); return; } - // reordering indicator items - const reorderedIndicators = indicators.map(indicator => - indicator.id !== parentCategoryId + // Second level + const reordered = indicators.map(indicator => + indicator.id !== expandedItemId ? indicator - : { ...indicator, items: reordered }, - ) as FormattedIndicators[]; + : { + ...indicator, + childOptions: indicator.childOptions + ? reorder(indicator.childOptions, prevIndex, nextIndex) + : [], + }, + ); - setIndicators(reorderedIndicators); + setIndicators(reordered); }; const handleSave = () => { @@ -88,7 +102,9 @@ const ReorderIndicatorsList = ({ indicator => { return { id: indicator.id, - indicators: indicator.items.map(item => item.id), + indicators: indicator.childOptions + ? indicator.childOptions?.map(item => item.id) + : [], }; }, ); @@ -96,24 +112,33 @@ const ReorderIndicatorsList = ({ }; return ( - <> -

          {`Reorder indicators for ${subject.name}`}

          - - {indicators.length === 0 ? ( + + {indicators.length === 0 ? ( + <>

          No indicators available.

          - ) : ( - - )} - - {indicators.length > 0 && ( - - )} - -           - + + ) : ( + <> + + + {indicators.length > 0 && ( + + )} + + + + )} +           ); -}; -export default ReorderIndicatorsList; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderList.module.scss b/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderList.module.scss deleted file mode 100644 index e47c0da7ed3..00000000000 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderList.module.scss +++ /dev/null @@ -1,46 +0,0 @@ -@import '~govuk-frontend/dist/govuk/base'; - -.isExpandedItem { - background: govuk-colour('light-grey') !important; - - .isExpandedItem { - background: lighten(govuk-colour('light-grey'), 2); - } -} - -.dragHandle { - font-size: 1rem; - left: 12px; - position: absolute; - top: govuk-spacing(3); -} - -.draggableInner { - align-items: center; - display: flex; - flex-grow: 1; - justify-content: space-between; - padding: 0 govuk-spacing(2) 0 0; -} - -.optionLabel { - &.isExpanded { - font-weight: $govuk-font-weight-bold; - } - - &.hideDragHandle { - padding-left: 40px; - } -} - -.inner { - width: 100%; -} - -.dropArea { - margin-bottom: govuk-spacing(6); - - .dropArea & { - margin: govuk-spacing(2) govuk-spacing(1) govuk-spacing(4) 0; - } -} diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderList.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderList.tsx deleted file mode 100644 index 5d4d3cc0613..00000000000 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ReorderList.tsx +++ /dev/null @@ -1,210 +0,0 @@ -import DroppableArea from '@admin/components/DroppableArea'; -import DraggableItem from '@admin/components/DraggableItem'; -import styles from '@admin/pages/release/data/components/ReorderList.module.scss'; -import ButtonText from '@common/components/ButtonText'; -import reorder from '@common/utils/reorder'; -import classNames from 'classnames'; -import React, { useState } from 'react'; -import { DragDropContext, Droppable } from '@hello-pangea/dnd'; - -export interface FormattedOption { - id: string; - label: string; -} -export interface FormattedGroup { - id: string; - label: string; - items: FormattedOption[]; -} -export interface FormattedFilters { - id: string; - label: string; - groups: FormattedGroup[]; -} -export interface FormattedIndicators { - id: string; - label: string; - items: FormattedOption[]; -} -export interface ReorderProps { - reordered: ( - | FormattedIndicators - | FormattedFilters - | FormattedOption - | FormattedGroup - )[]; - parentCategoryId?: string; - parentGroupId?: string; -} - -const getChildItems = ( - option: - | FormattedFilters - | FormattedGroup - | FormattedOption - | FormattedIndicators, -): { - childOptions?: (FormattedOption | FormattedGroup)[]; - parentGroupId?: string; -} => { - if ('groups' in option && option.groups.length > 1) { - return { childOptions: option.groups }; - } - if ('groups' in option && option.groups.length === 1) { - return { - childOptions: option.groups[0].items, - parentGroupId: option.groups[0].id, - }; - } - if ('items' in option && option.items.length > 1) { - return { childOptions: option.items, parentGroupId: option.id }; - } - return { childOptions: undefined }; -}; - -interface ReorderListProps { - listItems: ( - | FormattedIndicators - | FormattedFilters - | FormattedOption - | FormattedGroup - )[]; - categoryId?: string; - groupId?: string; - testId?: string; - onReorder: ({ - reordered, - parentCategoryId, - parentGroupId, - }: ReorderProps) => void; -} -const ReorderList = ({ - listItems, - categoryId, - groupId, - testId, - onReorder, -}: ReorderListProps) => { - const [reorderingGroups, setReorderingGroups] = useState([]); - - // If category has only one group, just show its options not the group. - const options = - listItems.length === 1 && 'items' in listItems[0] - ? listItems[0].items - : listItems; - - const parentCategoryId = - listItems.length === 1 && 'items' in listItems[0] - ? listItems[0].id - : categoryId; - - return ( - { - if (!result.destination) { - return; - } - - const reordered = reorder( - options, - result.source.index, - result.destination.index, - ); - - // Update the order property if it's a category or group, don't if it's items - onReorder({ - reordered: reordered.map((option, index) => { - if ('order' in reordered[0]) { - return { - ...option, - order: index, - }; - } - return option; - }), - parentCategoryId, - parentGroupId: groupId, - }); - }} - > - - {(droppableProvided, droppableSnapshot) => ( - - {options.map((option, index) => { - const key = option.id || `key-${index}`; - const isExpanded = reorderingGroups.includes(key); - const { childOptions, parentGroupId } = getChildItems(option); - return ( - 0 && !isExpanded} - dragHandle={ - isExpanded ? ( - - ▼ - - ) : undefined - } - id={key} - index={index} - isDisabled={reorderingGroups.length !== 0} - isReordering - key={key} - tag="li" - > -
          - - 0 && !isExpanded, - })} - > - {option.label} - - {childOptions && ( - { - setReorderingGroups( - isExpanded - ? reorderingGroups.filter(item => item !== key) - : [...reorderingGroups, key], - ); - }} - > - {isExpanded - ? 'Done' - : 'Reorder options within this group'} - - )} - - {childOptions && isExpanded && ( - - )} -
          -           - ); - })} -           - )} -           -           - ); -}; -export default ReorderList; diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReorderFiltersList.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReorderFiltersList.test.tsx index 372ef854aeb..f73a25cef98 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReorderFiltersList.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReorderFiltersList.test.tsx @@ -1,10 +1,12 @@ -import ReorderFiltersList from '@admin/pages/release/data/components/ReorderFiltersList'; +import ReorderFiltersList, { + UpdateFiltersRequest, +} from '@admin/pages/release/data/components/ReorderFiltersList'; +import render from '@common-test/render'; import _tableBuilderService, { Subject, SubjectMeta, } from '@common/services/tableBuilderService'; -import { render, screen, waitFor, within } from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; +import { screen, waitFor, within } from '@testing-library/react'; import noop from 'lodash/noop'; import React from 'react'; @@ -120,28 +122,23 @@ describe('ReorderFiltersList', () => { ); expect( - screen.getByRole('heading', { name: 'Reorder filters for Subject Name' }), - ); - - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); + await screen.findByText('Reorder filters for Subject Name'), + ).toBeInTheDocument(); - const filters = within(screen.getByTestId('reorder-list')).getAllByRole( - 'button', + const filters = within(screen.getByTestId('reorder-filters')).getAllByRole( + 'listitem', ); - // the listitems are given role 'button' by react-dnd expect(within(filters[0]).getByText('Category 2')); expect( within(filters[0]).getByRole('button', { - name: 'Reorder options within this group', + name: 'Reorder options within Category 2', }), ); - expect(within(filters[2]).getByText('Category 1')); + expect(within(filters[1]).getByText('Category 1')); expect( - within(filters[2]).getByRole('button', { - name: 'Reorder options within this group', + within(filters[1]).getByRole('button', { + name: 'Reorder options within Category 1', }), ); }); @@ -171,13 +168,14 @@ describe('ReorderFiltersList', () => { await waitFor(() => { expect(screen.getByText('No filters available.')); - expect(screen.queryByTestId('reorder-list')).not.toBeInTheDocument(); }); + + expect(screen.queryByTestId('reorder-filters')).not.toBeInTheDocument(); }); test('clicking the `reorder options` button on a filter shows the filter groups for the filter ordered by the order property', async () => { tableBuilderService.getSubjectMeta.mockResolvedValue(testSubjectMeta); - render( + const { user } = render( { onSave={noop} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - const filtersList = screen.getByTestId('reorder-list'); - await userEvent.click( - screen.getAllByRole('button', { - name: 'Reorder options within this group', - })[1], + expect( + await screen.findByText('Reorder filters for Subject Name'), + ).toBeInTheDocument(); + + await user.click( + screen.getByRole('button', { name: 'Reorder options within Category 1' }), ); + expect(await screen.findByText('Category 1 Group 2')); - await waitFor(() => { - expect(screen.getByText('Category 1 Group 2')); - }); - const groupsList = within(filtersList).getByRole('list'); - const groups = within(groupsList).getAllByRole('button'); + const groups = within( + screen.getByTestId('reorder-filters-children'), + ).getAllByRole('listitem'); expect(within(groups[0]).getByText('Category 1 Group 2')); - expect(within(groups[2]).getByText('Category 1 Group 1')); - expect(within(groups[3]).getByText('Category 1 Group 3')); + expect(within(groups[1]).getByText('Category 1 Group 1')); + expect(within(groups[2]).getByText('Category 1 Group 3')); }); test('filter groups have a `reorder options` button if they have more than one filter item', async () => { tableBuilderService.getSubjectMeta.mockResolvedValue(testSubjectMeta); - render( + const { user } = render( { onSave={noop} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - const filtersList = screen.getByTestId('reorder-list'); - await userEvent.click( - screen.getAllByRole('button', { - name: 'Reorder options within this group', - })[1], + expect( + await screen.findByText('Reorder filters for Subject Name'), + ).toBeInTheDocument(); + + await user.click( + screen.getByRole('button', { name: 'Reorder options within Category 1' }), ); + expect(await screen.findByText('Category 1 Group 2')); await waitFor(() => { expect(screen.getByText('Category 1 Group 2')); }); - const groupsList = within(filtersList).getByRole('list'); - const groups = within(groupsList).getAllByRole('button'); - expect(within(groups[0]).getByText('Category 1 Group 2')); expect( - within(groups[0]).getByRole('button', { - name: 'Reorder options within this group', + screen.getByRole('button', { + name: 'Reorder options within Category 1 Group 2', }), ); - expect(within(groups[2]).getByText('Category 1 Group 1')); expect( - within(groups[2]).queryByRole('button', { - name: 'Reorder options within this group', + screen.queryByRole('button', { + name: 'Reorder options within Category 1 Group 1', }), - ).not.toBeInTheDocument(); - - expect(within(groups[3]).getByText('Category 1 Group 3')); + ); expect( - within(groups[3]).getByRole('button', { - name: 'Reorder options within this group', + screen.getByRole('button', { + name: 'Reorder options within Category 1 Group 3', }), ); }); - test('clicking the `done` button on a filter hides the groups for the filter', async () => { + test('clicking the `close` button on a filter hides the groups for the filter', async () => { tableBuilderService.getSubjectMeta.mockResolvedValue(testSubjectMeta); - render( + const { user } = render( { onSave={noop} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - const filtersList = screen.getByTestId('reorder-list'); - await userEvent.click( - screen.getAllByRole('button', { - name: 'Reorder options within this group', - })[1], + expect( + await screen.findByText('Reorder filters for Subject Name'), + ).toBeInTheDocument(); + + await user.click( + screen.getByRole('button', { name: 'Reorder options within Category 1' }), ); + expect(await screen.findByText('Category 1 Group 2')); await waitFor(() => { expect(screen.getByText('Category 1 Group 2')); }); - expect(within(filtersList).getByRole('list')); - await userEvent.click(screen.getByRole('button', { name: 'Done' })); + await user.click(screen.getByRole('button', { name: 'Close Category 1' })); expect(screen.queryByText('Category 1 Group 2')).not.toBeInTheDocument(); - expect(within(filtersList).queryByRole('list')).not.toBeInTheDocument(); }); test('clicking the `reorder options` button on a filter group shows the options for that group', async () => { tableBuilderService.getSubjectMeta.mockResolvedValue(testSubjectMeta); - render( + const { user } = render( { onSave={noop} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - const filtersList = screen.getByTestId('reorder-list'); - await userEvent.click( - screen.getAllByRole('button', { - name: 'Reorder options within this group', - })[1], - ); + expect( + await screen.findByText('Reorder filters for Subject Name'), + ).toBeInTheDocument(); - await waitFor(() => { - expect(screen.getByText('Category 1 Group 2')); - }); - const buttons = within(within(filtersList).getByRole('list')).getAllByRole( - 'button', + await user.click( + screen.getByRole('button', { name: 'Reorder options within Category 1' }), ); - await userEvent.click( - within(buttons[0]).getByRole('button', { - name: 'Reorder options within this group', + expect(await screen.findByText('Category 1 Group 2')); + + await user.click( + screen.getByRole('button', { + name: 'Reorder options within Category 1 Group 2', }), ); + expect(await screen.findByText('Category 1 Group 2 Item 1')); - const options = within(buttons[0]).getAllByRole('button'); + const options = within( + screen.getByTestId('reorder-filters-children'), + ).getAllByRole('listitem'); expect(within(options[1]).getByText('Category 1 Group 2 Item 1')); expect(within(options[2]).getByText('Category 1 Group 2 Item 2')); expect(within(options[3]).getByText('Category 1 Group 2 Item 3')); }); - test('clicking the `done` button on a filter group hides the options for the group', async () => { + test('clicking the `close` button on a filter group hides the options for the group', async () => { tableBuilderService.getSubjectMeta.mockResolvedValue(testSubjectMeta); - render( + const { user } = render( { onSave={noop} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - const filtersList = screen.getByTestId('reorder-list'); - await userEvent.click( - screen.getAllByRole('button', { - name: 'Reorder options within this group', - })[1], - ); - - await waitFor(() => { - expect(screen.getByText('Category 1 Group 2')); - }); + expect( + await screen.findByText('Reorder filters for Subject Name'), + ).toBeInTheDocument(); - const groupsList = within(filtersList).getByRole('list'); - const groups = within(groupsList).getAllByRole('button'); + await user.click( + screen.getByRole('button', { name: 'Reorder options within Category 1' }), + ); + expect(await screen.findByText('Category 1 Group 2')); - await userEvent.click( - within(groups[0]).getByRole('button', { - name: 'Reorder options within this group', + await user.click( + screen.getByRole('button', { + name: 'Reorder options within Category 1 Group 2', }), ); + expect(await screen.findByText('Category 1 Group 2 Item 1')); - const options = within(groups[0]).getAllByRole('button'); - - expect(within(options[1]).getByText('Category 1 Group 2 Item 1')); - - // Waiting to make sure the button has changed, - // otherwise it's clicked too quickly and thinks it's a double-click. - await waitFor(() => { - expect(within(groups[0]).getByText('Done')).toBeInTheDocument(); - }); - - await userEvent.click( - within(groups[0]).getByRole('button', { name: 'Done' }), + await user.click( + screen.getByRole('button', { name: 'Close Category 1 Group 2' }), ); expect( @@ -374,7 +338,7 @@ describe('ReorderFiltersList', () => { test('if the filter only has one group, clicking the `reorder options` button on the filter shows the options for the group', async () => { tableBuilderService.getSubjectMeta.mockResolvedValue(testSubjectMeta); - render( + const { user } = render( { onSave={noop} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - const filtersList = screen.getByTestId('reorder-list'); - await userEvent.click( - screen.getAllByRole('button', { - name: 'Reorder options within this group', - })[0], + + expect( + await screen.findByText('Reorder filters for Subject Name'), + ).toBeInTheDocument(); + + await user.click( + screen.getByRole('button', { name: 'Reorder options within Category 2' }), ); - await waitFor(() => { - expect(screen.getByText('Category 2 Group 1 Item 1')); - }); + expect(await screen.findByText('Category 2 Group 1 Item 1')); - const optionsList = within(filtersList).getByRole('list'); - const options = within(optionsList).getAllByRole('button'); + const options = within( + screen.getByTestId('reorder-filters-children'), + ).getAllByRole('listitem'); expect(within(options[0]).getByText('Category 2 Group 1 Item 1')); expect(within(options[1]).getByText('Category 2 Group 1 Item 2')); }); @@ -405,7 +367,7 @@ describe('ReorderFiltersList', () => { test('clicking the `save` button calls handleSave with the reordered list formatted for the update request', async () => { const handleSave = jest.fn(); tableBuilderService.getSubjectMeta.mockResolvedValue(testSubjectMeta); - render( + const { user } = render( { onSave={handleSave} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - await userEvent.click(screen.getByRole('button', { name: 'Save order' })); + expect( + await screen.findByText('Reorder filters for Subject Name'), + ).toBeInTheDocument(); + await user.click(screen.getByRole('button', { name: 'Save order' })); - const expectedRequest = [ + const expectedRequest: UpdateFiltersRequest = [ { id: 'category-2-id', filterGroups: [ diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReorderIndicatorsList.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReorderIndicatorsList.test.tsx index 3cb253d6e30..089f4471259 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReorderIndicatorsList.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/__tests__/ReorderIndicatorsList.test.tsx @@ -1,10 +1,10 @@ import ReorderIndicatorsList from '@admin/pages/release/data/components/ReorderIndicatorsList'; +import render from '@common-test/render'; import _tableBuilderService, { Subject, SubjectMeta, } from '@common/services/tableBuilderService'; -import { render, screen, waitFor, within } from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; +import { screen, waitFor, within } from '@testing-library/react'; import noop from 'lodash/noop'; import React from 'react'; @@ -81,20 +81,11 @@ describe('ReorderIndicatorsList', () => { />, ); - expect( - screen.getByRole('heading', { - name: 'Reorder indicators for Subject Name', - }), - ); - - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); + expect(await screen.findByText('Reorder indicators for Subject Name')); - // the listitems are given role 'button' by react-dnd - const indicators = within(screen.getByTestId('reorder-list')).getAllByRole( - 'button', - ); + const indicators = within( + screen.getByTestId('reorder-indicators'), + ).getAllByRole('listitem'); expect(within(indicators[0]).getByText('Category 2')); expect(within(indicators[1]).getByText('Category 1')); }); @@ -124,8 +115,8 @@ describe('ReorderIndicatorsList', () => { await waitFor(() => { expect(screen.getByText('No indicators available.')); - expect(screen.queryByTestId('reorder-list')).not.toBeInTheDocument(); }); + expect(screen.queryByTestId('reorder-indicators')).not.toBeInTheDocument(); }); test('indicators have a `reorder options` button if they have more than one item', async () => { @@ -138,31 +129,25 @@ describe('ReorderIndicatorsList', () => { onSave={noop} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - const indicatorsList = screen.getByTestId('reorder-list'); - - const buttons = within(indicatorsList).getAllByRole('button'); + expect( + await screen.findByText('Reorder indicators for Subject Name'), + ).toBeInTheDocument(); - expect(within(buttons[0]).getByText('Category 2')); expect( - within(buttons[0]).queryByRole('button', { - name: 'Reorder options within this group', + screen.queryByRole('button', { + name: 'Reorder options within Category 2', }), ).not.toBeInTheDocument(); - - expect(within(buttons[1]).getByText('Category 1')); expect( - within(buttons[1]).getByRole('button', { - name: 'Reorder options within this group', + screen.getByRole('button', { + name: 'Reorder options within Category 1', }), - ); + ).toBeInTheDocument(); }); test('clicking the `reorder options` button on a indicator shows the items for the indicator', async () => { tableBuilderService.getSubjectMeta.mockResolvedValue(testSubjectMeta); - render( + const { user } = render( { onSave={noop} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - const indicatorsList = screen.getByTestId('reorder-list'); - await userEvent.click( - screen.getAllByRole('button', { - name: 'Reorder options within this group', - })[0], + expect( + await screen.findByText('Reorder indicators for Subject Name'), + ).toBeInTheDocument(); + + await user.click( + screen.getByRole('button', { name: 'Reorder options within Category 1' }), ); + expect(await screen.findByText('Category 1 Item 1')); - await waitFor(() => { - expect(screen.getByText('Category 1 Item 1')); - }); - const itemsList = within(indicatorsList).getByRole('list'); - const items = within(itemsList).getAllByRole('button'); + const items = within( + screen.getByTestId('reorder-indicators-children'), + ).getAllByRole('listitem'); expect(within(items[0]).getByText('Category 1 Item 1')); expect(within(items[1]).getByText('Category 1 Item 2')); expect(within(items[2]).getByText('Category 1 Item 3')); }); - test('clicking the `done` button on a indicator hides the items for the indicator', async () => { + test('clicking the `close` button on a indicator hides the items for the indicator', async () => { tableBuilderService.getSubjectMeta.mockResolvedValue(testSubjectMeta); - render( + const { user } = render( { onSave={noop} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - const indicatorsList = screen.getByTestId('reorder-list'); - await userEvent.click( - screen.getAllByRole('button', { - name: 'Reorder options within this group', - })[0], - ); + expect(await screen.findByText('Reorder indicators for Subject Name')); - await waitFor(() => { - expect(screen.getByText('Category 1 Item 1')); - }); + await user.click( + screen.getByRole('button', { name: 'Reorder options within Category 1' }), + ); + expect(await screen.findByText('Category 1 Item 1')); - expect(within(indicatorsList).getByRole('list')); - await userEvent.click(screen.getByRole('button', { name: 'Done' })); + await user.click(screen.getByRole('button', { name: 'Close Category 1' })); expect(screen.queryByText('Category 1 Item 1')).not.toBeInTheDocument(); - expect(within(indicatorsList).queryByRole('list')).not.toBeInTheDocument(); }); test('clicking the `save` button calls handleSave with the reordered list formatted for the update request', async () => { const handleSave = jest.fn(); tableBuilderService.getSubjectMeta.mockResolvedValue(testSubjectMeta); - render( + const { user } = render( { onSave={handleSave} />, ); - await waitFor(() => { - expect(screen.getByTestId('reorder-list')); - }); - await userEvent.click(screen.getByRole('button', { name: 'Save order' })); + expect( + await screen.findByText('Reorder indicators for Subject Name'), + ).toBeInTheDocument(); + await user.click(screen.getByRole('button', { name: 'Save order' })); const expectedRequest = [ { diff --git a/src/explore-education-statistics-common/src/components/ReorderableItem.module.scss b/src/explore-education-statistics-common/src/components/ReorderableItem.module.scss index c34535c1ff5..c1aa45b9220 100644 --- a/src/explore-education-statistics-common/src/components/ReorderableItem.module.scss +++ b/src/explore-education-statistics-common/src/components/ReorderableItem.module.scss @@ -41,8 +41,9 @@ } .controls { + min-height: 38px; // to make correct height if arrow buttons not present. position: absolute; - right: 5px; + right: govuk-spacing(1); top: govuk-spacing(2); } diff --git a/src/explore-education-statistics-common/src/components/ReorderableItem.tsx b/src/explore-education-statistics-common/src/components/ReorderableItem.tsx index e37d36a86b5..897e69e968a 100644 --- a/src/explore-education-statistics-common/src/components/ReorderableItem.tsx +++ b/src/explore-education-statistics-common/src/components/ReorderableItem.tsx @@ -4,14 +4,17 @@ import VisuallyHidden from '@common/components/VisuallyHidden'; import { ArrowLeft, ArrowRight } from '@common/components/ArrowIcons'; import Button from '@common/components/Button'; import ButtonGroup from '@common/components/ButtonGroup'; +import ButtonText from '@common/components/ButtonText'; import { mergeRefs } from '@common/utils/mergeRefs'; import classNames from 'classnames'; import React, { ReactNode, useEffect, useRef } from 'react'; import { DraggableProvided, DraggableStateSnapshot } from '@hello-pangea/dnd'; export interface ReorderableListItem { + childOptions?: ReorderableListItem[]; id: string; label: ReactNode | string; + parentId?: string; } export interface ReorderResult { @@ -23,10 +26,11 @@ interface Props { draggableProvided: DraggableProvided; draggableSnapshot: DraggableStateSnapshot; dropAreaActive: boolean; - focusItem: boolean; + focusItem?: boolean; index: number; - isLastItem: boolean; + isLastItem?: boolean; item: ReorderableListItem; + onExpandOptions?: (itemId: string, parentId?: string) => void; onMoveItem: ({ prevIndex, nextIndex }: ReorderResult) => void; } @@ -36,8 +40,9 @@ export default function ReorderableItem({ dropAreaActive, focusItem = false, index, - isLastItem, + isLastItem = false, item, + onExpandOptions, onMoveItem, }: Props) { const itemRef = useRef(null); @@ -46,6 +51,13 @@ export default function ReorderableItem({ itemRef.current?.focus(); } }); + const hasChildren = item.childOptions && item.childOptions.length > 1; + const canReorderOptions = + hasChildren || + (item.childOptions && + item.childOptions.length === 1 && + item.childOptions[0].childOptions && + item.childOptions[0].childOptions.length > 1); return (
        • @@ -70,11 +82,22 @@ export default function ReorderableItem({ {!dropAreaActive && !draggableSnapshot.isDragging && ( - + + {canReorderOptions && ( + onExpandOptions?.(item.id, item.parentId)} + > + Reorder options + within {item.label} + + )} {index !== 0 && ( diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/AncillaryFileForm.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/AncillaryFileForm.tsx index e47434e8d4a..777f8815f6a 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/AncillaryFileForm.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/AncillaryFileForm.tsx @@ -76,8 +76,11 @@ export default function AncillaryFileForm({ f => f.title.toUpperCase() !== value.toUpperCase(), ); }, - }), - summary: Yup.string().required('Enter a summary'), + }) + .max(120, 'Title must be 120 characters or less'), + summary: Yup.string() + .required('Enter a summary') + .max(250, 'Summary must be 250 characters or less'), file: Yup.file() .minSize(0, 'Choose a file that is not empty') .maxSize(MAX_FILE_SIZE, 'Choose a file that is under 2GB') @@ -117,6 +120,7 @@ export default function AncillaryFileForm({ disabled={formState.isSubmitting} label="Title" name="title" + maxLength={120} /> @@ -124,6 +128,7 @@ export default function AncillaryFileForm({ disabled={formState.isSubmitting} label="Summary" name="summary" + maxLength={250} /> diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx index 59df8a9c1d6..b188f279ea0 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx @@ -83,6 +83,7 @@ function baseErrorMappings( ZipContainsUnusedFiles: 'ZipContainsUnusedFiles', DataReplacementAlreadyInProgress: 'Data replacement already in progress', + DatasetTitleTooLong: 'DatasetTitleTooLong', }, }), ]; @@ -179,21 +180,24 @@ export default function DataFileUploadForm({ is: (uploadType: FileType) => uploadType === 'csv' || uploadType === 'zip', then: s => - s.required('Enter a subject title').test({ - name: 'unique', - message: 'Enter a unique subject title', - test(value: string) { - if (!value) { - return true; - } + s + .required('Enter a subject title') + .test({ + name: 'unique', + message: 'Enter a unique subject title', + test(value: string) { + if (!value) { + return true; + } - return ( - dataFiles?.find( - f => f.title.toUpperCase() === value.toUpperCase(), - ) === undefined - ); - }, - }), + return ( + dataFiles?.find( + f => f.title.toUpperCase() === value.toUpperCase(), + ) === undefined + ); + }, + }) + .max(120, 'Subject title must be 120 characters or less'), }), }); } @@ -233,6 +237,7 @@ export default function DataFileUploadForm({ name="subjectTitle" label="Subject title" className="govuk-!-width-two-thirds" + maxLength={120} /> )} diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataGuidanceSection.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataGuidanceSection.tsx index efcc4dc3dae..90cd76ab81d 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataGuidanceSection.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataGuidanceSection.tsx @@ -114,16 +114,21 @@ const ReleaseDataGuidanceSection = ({ releaseId, canUpdateRelease }: Props) => { dataSets: Yup.array().of( Yup.object({ id: Yup.string(), - content: Yup.string().required(params => { - const [, index] = toPath(params.path); - const dataSet = dataGuidance?.dataSets[Number(index)]; - - if (!dataSet) { - return null; - } - - return `Enter file guidance content for ${dataSet.name}`; - }), + content: Yup.string() + .required(params => { + const [, index] = toPath(params.path); + const dataSet = dataGuidance?.dataSets[Number(index)]; + + if (!dataSet) { + return null; + } + + return `Enter file guidance content for ${dataSet.name}`; + }) + .max( + 250, + 'File guidance content must be 250 characters or less', + ), }), ), })} @@ -179,6 +184,7 @@ const ReleaseDataGuidanceSection = ({ releaseId, canUpdateRelease }: Props) => { label="File guidance content" name={`dataSets.${index}.content`} rows={3} + maxLength={250} /> ) : ( >(() => { return Yup.object({ name: Yup.string().required('Enter a data block name'), - heading: Yup.string().required('Enter a table title'), + heading: Yup.string() + .required('Enter a table title') + .max(120, 'Table title must be 120 characters or less'), source: Yup.string(), highlightName: Yup.string().when('isHighlight', { is: true, - then: s => s.required('Enter a featured table name'), + then: s => + s + .required('Enter a featured table name') + .max(120, 'Featured table name must be 120 characters or less'), }), highlightDescription: Yup.string().when('isHighlight', { is: true, - then: s => s.required('Enter a featured table description'), + then: s => + s + .required('Enter a featured table description') + .max( + 200, + 'Featured table description must be 200 characters or less', + ), }), isHighlight: Yup.boolean(), }); @@ -100,6 +111,7 @@ const DataBlockDetailsForm = ({ onBlur={() => { onTitleChange?.(getValues('heading')); }} + maxLength={120} /> @@ -125,12 +137,14 @@ const DataBlockDetailsForm = ({ label="Featured table name" hint="We will show this name to table builder users as a featured table" className="govuk-!-width-two-thirds" + maxLength={120} /> name="highlightDescription" label="Featured table description" hint="Describe the contents of this featured table to table builder users" className="govuk-!-width-two-thirds" + maxLength={200} /> } From 6dd4b389c866463ade20eca5cb6fef36fd60e1ac Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Wed, 11 Dec 2024 13:32:49 +0000 Subject: [PATCH 111/144] EES-5047: Move input/textarea useWatch usage to parent components and update tests. --- .../src/components/form/FormFieldTextArea.tsx | 28 ++- .../components/form/FormFieldTextInput.tsx | 28 ++- .../src/components/form/FormTextArea.tsx | 32 +-- .../src/components/form/FormTextInput.tsx | 18 -- .../form/__tests__/FormFieldTextArea.test.tsx | 136 ++++++++++++ .../form/__tests__/FormTextArea.test.tsx | 210 +++--------------- 6 files changed, 221 insertions(+), 231 deletions(-) create mode 100644 src/explore-education-statistics-common/src/components/form/__tests__/FormFieldTextArea.test.tsx diff --git a/src/explore-education-statistics-common/src/components/form/FormFieldTextArea.tsx b/src/explore-education-statistics-common/src/components/form/FormFieldTextArea.tsx index 246ba3837e1..72369488537 100644 --- a/src/explore-education-statistics-common/src/components/form/FormFieldTextArea.tsx +++ b/src/explore-education-statistics-common/src/components/form/FormFieldTextArea.tsx @@ -4,15 +4,35 @@ import FormField, { import FormTextArea from '@common/components/form/FormTextArea'; import { FormTextAreaProps } from '@common/components/form/FormBaseTextArea'; import React from 'react'; -import { FieldValues } from 'react-hook-form'; +import { FieldValues, useWatch } from 'react-hook-form'; +import FormCharacterCount from '@common/components/form/FormCharacterCount'; +import FormGroup from './FormGroup'; type Props = FormFieldComponentProps< FormTextAreaProps, TFormValues >; -export default function FormFieldTextArea( - props: Props, -) { +export default function FormFieldTextArea({ + maxLength, + ...props +}: Props) { + const watchedValue = useWatch({ name: props.name }); + + if (!!maxLength && maxLength > 0) { + return ( +
          + + + + +
          + ); + } + return ; } diff --git a/src/explore-education-statistics-common/src/components/form/FormFieldTextInput.tsx b/src/explore-education-statistics-common/src/components/form/FormFieldTextInput.tsx index abc58788276..90aba48798d 100644 --- a/src/explore-education-statistics-common/src/components/form/FormFieldTextInput.tsx +++ b/src/explore-education-statistics-common/src/components/form/FormFieldTextInput.tsx @@ -5,15 +5,35 @@ import FormTextInput, { FormTextInputProps, } from '@common/components/form/FormTextInput'; import React from 'react'; -import { FieldValues } from 'react-hook-form'; +import { FieldValues, useWatch } from 'react-hook-form'; +import FormCharacterCount from '@common/components/form/FormCharacterCount'; +import FormGroup from './FormGroup'; type Props = FormFieldComponentProps< FormTextInputProps, TFormValues >; -export default function FormFieldTextInput( - props: Props, -) { +export default function FormFieldTextInput({ + maxLength, + ...props +}: Props) { + const watchedValue = useWatch({ name: props.name }); + + if (!!maxLength && maxLength > 0) { + return ( +
          + + + + +
          + ); + } + return ; } diff --git a/src/explore-education-statistics-common/src/components/form/FormTextArea.tsx b/src/explore-education-statistics-common/src/components/form/FormTextArea.tsx index 6aa385bf9bf..2ebf6b27e5f 100644 --- a/src/explore-education-statistics-common/src/components/form/FormTextArea.tsx +++ b/src/explore-education-statistics-common/src/components/form/FormTextArea.tsx @@ -1,36 +1,8 @@ -import FormGroup from '@common/components/form/FormGroup'; -import FormCharacterCount from '@common/components/form/FormCharacterCount'; import FormBaseTextArea, { FormTextAreaProps, } from '@common/components/form/FormBaseTextArea'; import React from 'react'; -import { useWatch } from 'react-hook-form'; -export default function FormTextArea({ - id, - maxLength, - name, - ...props -}: FormTextAreaProps) { - const value = useWatch({ name }); - - if (!!maxLength && maxLength > 0) { - return ( -
          - - - - -
          - ); - } - - return ( - - ); +export default function FormTextArea(props: FormTextAreaProps) { + return ; } diff --git a/src/explore-education-statistics-common/src/components/form/FormTextInput.tsx b/src/explore-education-statistics-common/src/components/form/FormTextInput.tsx index 81e8bc82b37..0108fa1f1e0 100644 --- a/src/explore-education-statistics-common/src/components/form/FormTextInput.tsx +++ b/src/explore-education-statistics-common/src/components/form/FormTextInput.tsx @@ -1,8 +1,6 @@ import FormBaseInput, { FormBaseInputProps, } from '@common/components/form/FormBaseInput'; -import FormCharacterCount from '@common/components/form/FormCharacterCount'; -import FormGroup from '@common/components/form/FormGroup'; import React from 'react'; export interface FormTextInputProps extends FormBaseInputProps { @@ -13,24 +11,8 @@ export interface FormTextInputProps extends FormBaseInputProps { export default function FormTextInput({ id, - maxLength, value, ...props }: FormTextInputProps) { - if (!!maxLength && maxLength > 0) { - return ( -
          - - - - -
          - ); - } return ; } diff --git a/src/explore-education-statistics-common/src/components/form/__tests__/FormFieldTextArea.test.tsx b/src/explore-education-statistics-common/src/components/form/__tests__/FormFieldTextArea.test.tsx new file mode 100644 index 00000000000..733ba544071 --- /dev/null +++ b/src/explore-education-statistics-common/src/components/form/__tests__/FormFieldTextArea.test.tsx @@ -0,0 +1,136 @@ +import FormFieldTextArea from '@common/components/form/FormFieldTextArea'; +import FormProvider from '@common/components/form/FormProvider'; +import { render, screen } from '@testing-library/react'; +import noop from 'lodash/noop'; +import React from 'react'; + +describe('FormFieldTextArea', () => { + describe('maxLength', () => { + test('shows a character count message when `maxLength` is above 0', () => { + render( + + + , + ); + + expect( + screen.getByText('You have 10 characters remaining'), + ).toBeInTheDocument(); + }); + + test('aria-describedby contains the character count message id when `maxLength` is above 0', () => { + render( + + + , + ); + + const ariaDescribedBy = screen + .getByLabelText('Test input') + .getAttribute('aria-describedby'); + + expect( + screen.getByText('You have 10 characters remaining'), + ).toHaveAttribute('id', 'test-input-info'); + expect(ariaDescribedBy).toContain('test-input-info'); + }); + + test('does not show a character count message when `maxLength` is below 0', () => { + render( + + + , + ); + + expect( + screen.queryByText(/You have .+ characters remaining/), + ).not.toBeInTheDocument(); + }); + + test('does not show a character count message when `maxLength` is 0', () => { + render( + + + , + ); + + expect( + screen.queryByText(/You have .+ characters remaining/), + ).not.toBeInTheDocument(); + }); + + test('shows correct character count message when difference to `maxLength` is 1', () => { + render( + + + , + ); + + expect( + screen.getByText('You have 1 character remaining'), + ).toBeInTheDocument(); + }); + + test('shows correct character count message when difference to `maxLength` is 0', () => { + render( + + + , + ); + + expect( + screen.getByText('You have 0 characters remaining'), + ).toBeInTheDocument(); + }); + + test('shows correct character count message when difference to `maxLength` is -1', () => { + render( + + + , + ); + + expect( + screen.getByText('You have 1 character too many'), + ).toBeInTheDocument(); + }); + }); +}); diff --git a/src/explore-education-statistics-common/src/components/form/__tests__/FormTextArea.test.tsx b/src/explore-education-statistics-common/src/components/form/__tests__/FormTextArea.test.tsx index 9b86fd200ab..138eb0a23d7 100644 --- a/src/explore-education-statistics-common/src/components/form/__tests__/FormTextArea.test.tsx +++ b/src/explore-education-statistics-common/src/components/form/__tests__/FormTextArea.test.tsx @@ -1,5 +1,4 @@ import FormTextArea from '@common/components/form/FormTextArea'; -import FormProvider from '@common/components/form/FormProvider'; import { render, screen } from '@testing-library/react'; import noop from 'lodash/noop'; import React from 'react'; @@ -7,9 +6,7 @@ import React from 'react'; describe('FormTextArea', () => { test('renders correctly with required props', () => { const { container } = render( - - - , + , ); expect(screen.getByLabelText('Test input')).toBeDefined(); @@ -18,14 +15,12 @@ describe('FormTextArea', () => { test('renders correctly with hint', () => { const { container } = render( - - - , + , ); const hint = screen.getByText('Fill me in'); @@ -36,15 +31,13 @@ describe('FormTextArea', () => { test('renders correctly with error', () => { const { container } = render( - - - , + , ); const error = screen.getByText('Field is required'); @@ -55,15 +48,13 @@ describe('FormTextArea', () => { test('aria-describedby is equal to the hint id', () => { render( - - - , + , ); expect(screen.getByText('Fill me in')).toHaveAttribute( @@ -78,15 +69,13 @@ describe('FormTextArea', () => { test('aria-describedby is equal to the error id', () => { render( - - - , + , ); expect(screen.getByText('Field is required')).toHaveAttribute( @@ -101,15 +90,13 @@ describe('FormTextArea', () => { test('aria-describedby contains both hint and error ids', () => { render( - - - , + , ); expect(screen.getByText('Fill me in')).toHaveAttribute( @@ -128,131 +115,4 @@ describe('FormTextArea', () => { expect(ariaDescribedBy).toContain('test-input-error'); expect(ariaDescribedBy).toContain('test-input-hint'); }); - - test('shows a character count message when `maxLength` is above 0', () => { - render( - - - , - ); - - expect( - screen.getByText('You have 10 characters remaining'), - ).toBeInTheDocument(); - }); - - test('aria-describedby contains the character count message id when `maxLength` is above 0', () => { - render( - - - , - ); - - const ariaDescribedBy = screen - .getByLabelText('Test input') - .getAttribute('aria-describedby'); - - expect( - screen.getByText('You have 10 characters remaining'), - ).toHaveAttribute('id', 'test-input-info'); - expect(ariaDescribedBy).toContain('test-input-info'); - }); - - test('does not show a character count message when `maxLength` is below 0', () => { - render( - - - , - ); - - expect( - screen.queryByText(/You have .+ characters remaining/), - ).not.toBeInTheDocument(); - }); - - test('does not show a character count message when `maxLength` is 0', () => { - render( - - - , - ); - - expect( - screen.queryByText(/You have .+ characters remaining/), - ).not.toBeInTheDocument(); - }); - - test('shows correct character count message when difference to `maxLength` is 1', () => { - render( - - - , - ); - - expect( - screen.getByText('You have 1 character remaining'), - ).toBeInTheDocument(); - }); - - test('shows correct character count message when difference to `maxLength` is 0', () => { - render( - - - , - ); - - expect( - screen.getByText('You have 0 characters remaining'), - ).toBeInTheDocument(); - }); - - test('shows correct character count message when difference to `maxLength` is -1', () => { - render( - - - , - ); - - expect( - screen.getByText('You have 1 character too many'), - ).toBeInTheDocument(); - }); }); From 2c3c9867e2a24c07c4e60cac3f032dc6481b5196 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Fri, 13 Dec 2024 12:54:22 +0000 Subject: [PATCH 112/144] EES-5047: Move string length and message definitions into constants, update naming convention for "DataSet" in validation messaging. --- .../DataArchiveValidationServiceTests.cs | 12 ++--- .../Controllers/Api/ReleasesController.cs | 5 +- .../Requests/DataBlockRequests.cs | 13 ++--- .../Requests/DataGuidanceUpdateRequest.cs | 36 +++++++++++--- .../Requests/FeaturedTableRequests.cs | 42 ++++++++++++---- .../Requests/ReleaseFileRequests.cs | 41 +++++++++++----- .../Services/DataArchiveValidationService.cs | 17 +++---- .../Validators/ValidationMessages.cs | 48 +++++++++---------- .../Constants/ValidationConstants.cs | 25 ++++++++++ .../release/data/ReleaseDataFilePage.tsx | 9 +++- .../data/components/AncillaryFileForm.tsx | 16 +++++-- .../data/components/DataFileUploadForm.tsx | 26 +++++----- .../components/ReleaseDataGuidanceSection.tsx | 7 +-- .../components/DataBlockDetailsForm.tsx | 22 ++++++--- 14 files changed, 218 insertions(+), 101 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Common/Constants/ValidationConstants.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs index 27fca5a4bca..11c6f987170 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/DataArchiveValidationServiceTests.cs @@ -238,8 +238,8 @@ public async Task ValidateBulkDataArchiveFiles_IndexFileMissing_ReturnsValidatio .AssertBadRequestWithValidationErrors([ new ErrorViewModel { - Code = ValidationMessages.BulkDataZipMustContainDatasetNamesCsv.Code, - Message = ValidationMessages.BulkDataZipMustContainDatasetNamesCsv.Message, + Code = ValidationMessages.BulkDataZipMustContainDataSetNamesCsv.Code, + Message = ValidationMessages.BulkDataZipMustContainDataSetNamesCsv.Message, } ]); } @@ -267,8 +267,8 @@ public async Task ValidateBulkDataArchiveFiles_IndexFileHasIncorrectHeaders_Retu .AssertBadRequestWithValidationErrors([ new ErrorViewModel { - Code = ValidationMessages.DatasetNamesCsvIncorrectHeaders.Code, - Message = ValidationMessages.DatasetNamesCsvIncorrectHeaders.Message, + Code = ValidationMessages.DataSetNamesCsvIncorrectHeaders.Code, + Message = ValidationMessages.DataSetNamesCsvIncorrectHeaders.Message, }, ]); } @@ -319,7 +319,7 @@ public async Task ValidateBulkDataArchiveFiles_DuplicateDataSetTitlesAndFileName .AssertLeft() .AssertBadRequestWithValidationErrors([ ValidationMessages.GenerateErrorDataSetTitleShouldBeUnique("Duplicate title"), - ValidationMessages.GenerateErrorDatasetNamesCsvFilenamesShouldBeUnique("one"), + ValidationMessages.GenerateErrorDataSetNamesCsvFilenamesShouldBeUnique("one"), ]); } } @@ -342,7 +342,7 @@ public async Task ValidateBulkDataArchiveFiles_Fail_DataSetNamesCsvFilesnamesSho result .AssertLeft() .AssertBadRequestWithValidationErrors([ - ValidationMessages.GenerateErrorDatasetNamesCsvFilenamesShouldNotEndDotCsv("one.csv") + ValidationMessages.GenerateErrorDataSetNamesCsvFilenamesShouldNotEndDotCsv("one.csv") ]); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs index 75d34de36fd..7f1003b4f63 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs @@ -14,6 +14,7 @@ using System.ComponentModel.DataAnnotations; using System.Threading; using System.Threading.Tasks; +using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Controllers.Api { @@ -108,7 +109,7 @@ public async Task>> ReorderDataFiles(Guid releas public async Task> UploadDataSet(Guid releaseVersionId, [FromQuery(Name = "replacingFileId")] Guid? replacingFileId, [FromQuery(Name = "title")] - [MaxLength(120, ErrorMessage = "Subject title must be 120 characters or less")] + [MaxLength(SubjectTitleMaxLength, ErrorMessage = SubjectTitleMaxLengthMessage)] string title, IFormFile file, IFormFile metaFile) @@ -128,7 +129,7 @@ public async Task> UploadDataSet(Guid releaseVersionI public async Task> UploadDataSetAsZip(Guid releaseVersionId, [FromQuery(Name = "replacingFileId")] Guid? replacingFileId, [FromQuery(Name = "title")] - [MaxLength(120, ErrorMessage = "Subject title must be 120 characters or less")] + [MaxLength(SubjectTitleMaxLength, ErrorMessage = SubjectTitleMaxLengthMessage)] string title, IFormFile zipFile) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataBlockRequests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataBlockRequests.cs index 88f1809d9df..6d2fe6767ce 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataBlockRequests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataBlockRequests.cs @@ -4,6 +4,7 @@ using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Requests; using System.Collections.Generic; +using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests; @@ -17,7 +18,7 @@ public record DataBlockCreateRequest public FullTableQueryRequest Query { get; init; } = null!; - public List Charts { get; init; } = new(); + public List Charts { get; init; } = []; public TableBuilderConfiguration Table { get; init; } = null!; @@ -27,8 +28,8 @@ public Validator() { RuleFor(request => request.Heading) .NotEmpty() - .MaximumLength(120) - .WithMessage("Table title must be 120 characters or less"); + .MaximumLength(TableTitleMaxLength) + .WithMessage(TableTitleMaxLengthMessage); RuleFor(request => request.Name) .NotEmpty(); @@ -49,7 +50,7 @@ public record DataBlockUpdateRequest public FullTableQueryRequest Query { get; init; } = null!; - public List Charts { get; init; } = new(); + public List Charts { get; init; } = []; public TableBuilderConfiguration Table { get; init; } = null!; @@ -59,8 +60,8 @@ public Validator() { RuleFor(request => request.Heading) .NotEmpty() - .MaximumLength(120) - .WithMessage("Table title must be 120 characters or less"); + .MaximumLength(TableTitleMaxLength) + .WithMessage(TableTitleMaxLengthMessage); RuleFor(request => request.Name) .NotEmpty(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataGuidanceUpdateRequest.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataGuidanceUpdateRequest.cs index d8bf0dd14b7..b057848d576 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataGuidanceUpdateRequest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataGuidanceUpdateRequest.cs @@ -1,25 +1,47 @@ #nullable enable +using FluentValidation; using System; using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; +using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests; public record DataGuidanceUpdateRequest { - [Required] public string Content { get; init; } = string.Empty; - [MinLength(1)] - public List DataSets { get; init; } = new(); + public List DataSets { get; init; } = []; + + public class Validator : AbstractValidator + { + public Validator() + { + RuleFor(request => request.Content) + .NotEmpty(); + + RuleFor(request => request.DataSets) + .NotEmpty(); + } + } } public record DataGuidanceDataSetUpdateRequest { - [Required] public Guid FileId { get; init; } - [Required] - [MaxLength(250, ErrorMessage = "File guidance content must be 250 characters or less")] public string Content { get; init; } = string.Empty; + + public class Validator : AbstractValidator + { + public Validator() + { + RuleFor(request => request.FileId) + .NotEmpty(); + + RuleFor(request => request.Content) + .NotEmpty() + .MaximumLength(FileGuidanceContentMaxLength) + .WithMessage(FileGuidanceContentMaxLengthMessage); + } + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/FeaturedTableRequests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/FeaturedTableRequests.cs index ae9430d871e..fe6ead1a189 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/FeaturedTableRequests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/FeaturedTableRequests.cs @@ -1,30 +1,54 @@ #nullable enable +using FluentValidation; using System; -using System.ComponentModel.DataAnnotations; +using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests; public record FeaturedTableCreateRequest { - [Required] - [MaxLength(120, ErrorMessage = "Featured table name must be 120 characters or less")] public string Name { get; init; } = string.Empty; - [Required] - [MaxLength(200, ErrorMessage = "Featured table description must be 200 characters or less")] public string Description { get; set; } = string.Empty; public Guid DataBlockId { get; set; } + public class Validator : AbstractValidator + { + public Validator() + { + RuleFor(request => request.Name) + .NotEmpty() + .MaximumLength(FeaturedTableNameMaxLength) + .WithMessage(FeaturedTableNameMaxLengthMessage); + + RuleFor(request => request.Description) + .NotEmpty() + .MaximumLength(FeaturedTableDescriptionMaxLength) + .WithMessage(FeaturedTableDescriptionMaxLengthMessage); + } + } } public record FeaturedTableUpdateRequest { - [Required] - [MaxLength(120, ErrorMessage = "Featured table name must be 120 characters or less")] public string Name { get; init; } = string.Empty; - [Required] - [MaxLength(200, ErrorMessage = "Featured table description must be 200 characters or less")] public string Description { get; set; } = string.Empty; + + public class Validator : AbstractValidator + { + public Validator() + { + RuleFor(request => request.Name) + .NotEmpty() + .MaximumLength(FeaturedTableNameMaxLength) + .WithMessage(FeaturedTableNameMaxLengthMessage); + + RuleFor(request => request.Description) + .NotEmpty() + .MaximumLength(FeaturedTableDescriptionMaxLength) + .WithMessage(FeaturedTableDescriptionMaxLengthMessage); + } + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseFileRequests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseFileRequests.cs index b07f72f1f95..f9bf20a2a86 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseFileRequests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseFileRequests.cs @@ -1,7 +1,7 @@ #nullable enable using FluentValidation; using Microsoft.AspNetCore.Http; -using System.ComponentModel.DataAnnotations; +using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests; @@ -16,20 +16,18 @@ public class Validator : AbstractValidator public Validator() { RuleFor(request => request.Title) - .MaximumLength(120) - .WithMessage("Subject title must be 120 characters or less"); + .MaximumLength(SubjectTitleMaxLength) + .WithMessage(SubjectTitleMaxLengthMessage); } } } public record ReleaseAncillaryFileUploadRequest { - [Required] public string Title { get; set; } = string.Empty; - [Required] public string Summary { get; set; } = string.Empty; + public string Summary { get; set; } = string.Empty; - [Required] public IFormFile File { get; set; } = null!; public class Validator : AbstractValidator @@ -37,23 +35,42 @@ public class Validator : AbstractValidator public Validator() { RuleFor(request => request.Title) - .MaximumLength(120) - .WithMessage("Title must be 120 characters or less"); + .NotEmpty() + .MaximumLength(TitleMaxLength) + .WithMessage(TitleMaxLengthMessage); RuleFor(request => request.Summary) - .MaximumLength(250) - .WithMessage("Summary must be 250 characters or less"); + .NotEmpty() + .MaximumLength(SummaryMaxLength) + .WithMessage(SummaryMaxLengthMessage); + + RuleFor(request => request.File) + .NotEmpty(); } } } public record ReleaseAncillaryFileUpdateRequest { - [Required] public string Title { get; set; } = string.Empty; - [Required] public string Summary { get; set; } = string.Empty; public IFormFile? File { get; set; } + + public class Validator : AbstractValidator + { + public Validator() + { + RuleFor(request => request.Title) + .NotEmpty() + .MaximumLength(TitleMaxLength) + .WithMessage(TitleMaxLengthMessage); + + RuleFor(request => request.Summary) + .NotEmpty() + .MaximumLength(SummaryMaxLength) + .WithMessage(SummaryMaxLengthMessage); + } + } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs index cd476108e49..2731cf1b6ce 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs @@ -15,6 +15,7 @@ using System.IO.Compression; using System.Linq; using System.Threading.Tasks; +using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Services { @@ -103,8 +104,8 @@ public async Task>> ValidateBulkDa { return Common.Validators.ValidationUtils.ValidationResult(new ErrorViewModel { - Code = ValidationMessages.BulkDataZipMustContainDatasetNamesCsv.Code, - Message = ValidationMessages.BulkDataZipMustContainDatasetNamesCsv.Message, + Code = ValidationMessages.BulkDataZipMustContainDataSetNamesCsv.Code, + Message = ValidationMessages.BulkDataZipMustContainDataSetNamesCsv.Message, }); } @@ -120,8 +121,8 @@ public async Task>> ValidateBulkDa { return Common.Validators.ValidationUtils.ValidationResult(new ErrorViewModel { - Code = ValidationMessages.DatasetNamesCsvIncorrectHeaders.Code, - Message = ValidationMessages.DatasetNamesCsvIncorrectHeaders.Message, + Code = ValidationMessages.DataSetNamesCsvIncorrectHeaders.Code, + Message = ValidationMessages.DataSetNamesCsvIncorrectHeaders.Message, }); } @@ -142,9 +143,9 @@ public async Task>> ValidateBulkDa var filename = row[fileNameIndex]; var datasetName = row[datasetNameIndex].Trim(); - if (datasetName.Length > 120) + if (datasetName.Length > SubjectTitleMaxLength) { - errors.Add(ValidationMessages.GenerateErrorDatasetTitleTooLong(datasetName)); + errors.Add(ValidationMessages.GenerateErrorDataSetTitleTooLong(datasetName)); } dataSetNamesCsvEntries.Add((BaseFilename: filename, Title: datasetName)); @@ -156,7 +157,7 @@ public async Task>> ValidateBulkDa .ToList() .ForEach(baseFilename => { - errors.Add(ValidationMessages.GenerateErrorDatasetNamesCsvFilenamesShouldNotEndDotCsv(baseFilename)); + errors.Add(ValidationMessages.GenerateErrorDataSetNamesCsvFilenamesShouldNotEndDotCsv(baseFilename)); }); // Check for duplicate data set titles - because the bulk zip itself may contain duplicates! @@ -180,7 +181,7 @@ public async Task>> ValidateBulkDa .ForEach(duplicateFilename => { errors.Add(ValidationMessages - .GenerateErrorDatasetNamesCsvFilenamesShouldBeUnique(duplicateFilename)); + .GenerateErrorDataSetNamesCsvFilenamesShouldBeUnique(duplicateFilename)); }); if (errors.Count > 0) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs index aedf5d4d042..9354e84ff7a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Validators/ValidationMessages.cs @@ -106,27 +106,27 @@ public static ErrorViewModel GenerateErrorMustBeCsvFile(string fullFilename) }; } - public static readonly LocalizableMessage BulkDataZipMustContainDatasetNamesCsv = new( - Code: nameof(BulkDataZipMustContainDatasetNamesCsv), + public static readonly LocalizableMessage BulkDataZipMustContainDataSetNamesCsv = new( + Code: nameof(BulkDataZipMustContainDataSetNamesCsv), Message: "For bulk imports, the ZIP must include dataset_names.csv" ); - public static readonly LocalizableMessage DatasetNamesCsvReaderException = new( - Code: nameof(DatasetNamesCsvReaderException), + public static readonly LocalizableMessage DataSetNamesCsvReaderException = new( + Code: nameof(DataSetNamesCsvReaderException), Message: "Failed to read dataset_names.csv. Exception: {0}" ); - public static ErrorViewModel GenerateErrorDatasetNamesCsvReaderException(string exception) + public static ErrorViewModel GenerateErrorDataSetNamesCsvReaderException(string exception) { return new ErrorViewModel { - Code = DatasetNamesCsvReaderException.Code, - Message = string.Format(DatasetNamesCsvReaderException.Message, exception), + Code = DataSetNamesCsvReaderException.Code, + Message = string.Format(DataSetNamesCsvReaderException.Message, exception), }; } - public static readonly LocalizableMessage DatasetNamesCsvIncorrectHeaders = new( - Code: nameof(DatasetNamesCsvIncorrectHeaders), + public static readonly LocalizableMessage DataSetNamesCsvIncorrectHeaders = new( + Code: nameof(DataSetNamesCsvIncorrectHeaders), Message: "dataset_names.csv has incorrect headers. It should have 'file_name' and 'dataset_name' only." ); @@ -158,22 +158,22 @@ public static ErrorViewModel GenerateErrorDataSetTitleShouldBeUnique(string dupl }; } - public static readonly LocalizableMessage DatasetNamesCsvFilenamesShouldBeUnique = new( - Code: nameof(DatasetNamesCsvFilenamesShouldBeUnique), + public static readonly LocalizableMessage DataSetNamesCsvFilenamesShouldBeUnique = new( + Code: nameof(DataSetNamesCsvFilenamesShouldBeUnique), Message: "In dataset_names.csv, all filenames should be unique. Duplicate filename: '{0}'." ); - public static ErrorViewModel GenerateErrorDatasetNamesCsvFilenamesShouldBeUnique(string duplicate) + public static ErrorViewModel GenerateErrorDataSetNamesCsvFilenamesShouldBeUnique(string duplicate) { return new ErrorViewModel { - Code = DatasetNamesCsvFilenamesShouldBeUnique.Code, - Message = string.Format(DatasetNamesCsvFilenamesShouldBeUnique.Message, duplicate), + Code = DataSetNamesCsvFilenamesShouldBeUnique.Code, + Message = string.Format(DataSetNamesCsvFilenamesShouldBeUnique.Message, duplicate), }; } - public static readonly LocalizableMessage DatasetNamesCsvFilenamesShouldNotEndDotCsv = new( - Code: nameof(DatasetNamesCsvFilenamesShouldNotEndDotCsv), + public static readonly LocalizableMessage DataSetNamesCsvFilenamesShouldNotEndDotCsv = new( + Code: nameof(DataSetNamesCsvFilenamesShouldNotEndDotCsv), Message: "Inside dataset_names.csv, file_name cell entries should not end in '.csv' i.e. should be 'filename' not 'filename.csv'. Filename found with extension: '{0}'." ); @@ -205,26 +205,26 @@ public static ErrorViewModel GenerateErrorDataReplacementAlreadyInProgress() }; } - public static readonly LocalizableMessage DatasetTitleTooLong = new( - Code: nameof(DatasetTitleTooLong), + public static readonly LocalizableMessage DataSetTitleTooLong = new( + Code: nameof(DataSetTitleTooLong), Message: "Subject title '{0}' must be 120 characters or less" ); - public static ErrorViewModel GenerateErrorDatasetTitleTooLong(string title) + public static ErrorViewModel GenerateErrorDataSetTitleTooLong(string title) { return new ErrorViewModel { - Code = DatasetTitleTooLong.Code, - Message = string.Format(DatasetTitleTooLong.Message, title), + Code = DataSetTitleTooLong.Code, + Message = string.Format(DataSetTitleTooLong.Message, title), }; } - public static ErrorViewModel GenerateErrorDatasetNamesCsvFilenamesShouldNotEndDotCsv(string filename) + public static ErrorViewModel GenerateErrorDataSetNamesCsvFilenamesShouldNotEndDotCsv(string filename) { return new ErrorViewModel { - Code = DatasetNamesCsvFilenamesShouldNotEndDotCsv.Code, - Message = string.Format(DatasetNamesCsvFilenamesShouldNotEndDotCsv.Message, filename), + Code = DataSetNamesCsvFilenamesShouldNotEndDotCsv.Code, + Message = string.Format(DataSetNamesCsvFilenamesShouldNotEndDotCsv.Message, filename), }; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Constants/ValidationConstants.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Constants/ValidationConstants.cs new file mode 100644 index 00000000000..69bf57be261 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Constants/ValidationConstants.cs @@ -0,0 +1,25 @@ +namespace GovUk.Education.ExploreEducationStatistics.Common.Constants; + +public class ValidationConstants +{ + public const int TitleMaxLength = 120; + public const string TitleMaxLengthMessage = "Title must be 120 characters or less"; + + public const int SummaryMaxLength = 250; + public const string SummaryMaxLengthMessage = "Summary must be 250 characters or less"; + + public const int SubjectTitleMaxLength = 120; + public const string SubjectTitleMaxLengthMessage = "Subject title must be 120 characters or less"; + + public const int TableTitleMaxLength = 120; + public const string TableTitleMaxLengthMessage = "Table title must be 120 characters or less"; + + public const int FeaturedTableNameMaxLength = 120; + public const string FeaturedTableNameMaxLengthMessage = "Featured table name must be 120 characters or less"; + + public const int FeaturedTableDescriptionMaxLength = 200; + public const string FeaturedTableDescriptionMaxLengthMessage = "Featured table description must be 200 characters or less"; + + public const int FileGuidanceContentMaxLength = 250; + public const string FileGuidanceContentMaxLengthMessage = "File guidance content must be 250 characters or less"; +} diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataFilePage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataFilePage.tsx index eb04b5c51dc..f189c26ab73 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataFilePage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseDataFilePage.tsx @@ -55,6 +55,8 @@ export default function ReleaseDataFilePage({ ); }; + const titleMaxLength = 120; + return ( <> ({ title: Yup.string() .required('Enter a title') - .max(120, 'Subject title must be 120 characters or less'), + .max( + titleMaxLength, + `Subject title must be ${titleMaxLength} characters or less`, + ), })} > @@ -86,7 +91,7 @@ export default function ReleaseDataFilePage({ className="govuk-!-width-two-thirds" label="Title" name="title" - maxLength={120} + maxLength={titleMaxLength} /> diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/AncillaryFileForm.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/AncillaryFileForm.tsx index 777f8815f6a..f8b58a3595a 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/AncillaryFileForm.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/AncillaryFileForm.tsx @@ -21,6 +21,8 @@ export interface AncillaryFileFormValues { const formId = 'ancillaryFileForm'; const MAX_FILE_SIZE = 2147483647; // 2GB +const titleMaxLength = 120; +const summaryMaxLength = 250; const errorMappings = [ mapFieldErrors({ @@ -77,10 +79,16 @@ export default function AncillaryFileForm({ ); }, }) - .max(120, 'Title must be 120 characters or less'), + .max( + titleMaxLength, + `Title must be ${titleMaxLength} characters or less`, + ), summary: Yup.string() .required('Enter a summary') - .max(250, 'Summary must be 250 characters or less'), + .max( + summaryMaxLength, + `Summary must be ${summaryMaxLength} characters or less`, + ), file: Yup.file() .minSize(0, 'Choose a file that is not empty') .maxSize(MAX_FILE_SIZE, 'Choose a file that is under 2GB') @@ -120,7 +128,7 @@ export default function AncillaryFileForm({ disabled={formState.isSubmitting} label="Title" name="title" - maxLength={120} + maxLength={titleMaxLength} /> @@ -128,7 +136,7 @@ export default function AncillaryFileForm({ disabled={formState.isSubmitting} label="Summary" name="summary" - maxLength={250} + maxLength={summaryMaxLength} /> diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx index b188f279ea0..ac728bd6ba2 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/DataFileUploadForm.tsx @@ -29,6 +29,7 @@ export interface DataFileUploadFormValues { } const MAX_FILENAME_SIZE = 150; +const titleMaxLength = 120; const subjectErrorMappings = [ mapFieldErrors({ @@ -71,19 +72,19 @@ function baseErrorMappings( ...fileErrorMappings, ZipFilenameMustEndDotZip: 'ZipFilenameMustEndDotZip', MustBeZipFile: 'MustBeZipFile', - BulkDataZipMustContainDatasetNamesCsv: - 'BulkDataZipMustContainDatasetNamesCsv', - DatasetNamesCsvReaderException: 'DatasetNamesCsvReaderException', - DatasetNamesCsvIncorrectHeaders: 'DatasetNamesCsvIncorrectHeaders', - DatasetNamesCsvFilenamesShouldNotEndDotCsv: - 'DatasetNamesCsvFilenamesShouldNotEndDotCsv', - DatasetNamesCsvFilenamesShouldBeUnique: - 'DatasetNamesCsvFilenamesShouldBeUnique', + BulkDataZipMustContainDataSetNamesCsv: + 'BulkDataZipMustContainDataSetNamesCsv', + DataSetNamesCsvReaderException: 'DataSetNamesCsvReaderException', + DataSetNamesCsvIncorrectHeaders: 'DataSetNamesCsvIncorrectHeaders', + DataSetNamesCsvFilenamesShouldNotEndDotCsv: + 'DataSetNamesCsvFilenamesShouldNotEndDotCsv', + DataSetNamesCsvFilenamesShouldBeUnique: + 'DataSetNamesCsvFilenamesShouldBeUnique', FileNotFoundInZip: 'FileNotFoundInZip', ZipContainsUnusedFiles: 'ZipContainsUnusedFiles', DataReplacementAlreadyInProgress: 'Data replacement already in progress', - DatasetTitleTooLong: 'DatasetTitleTooLong', + DataSetTitleTooLong: 'DataSetTitleTooLong', }, }), ]; @@ -197,7 +198,10 @@ export default function DataFileUploadForm({ ); }, }) - .max(120, 'Subject title must be 120 characters or less'), + .max( + titleMaxLength, + `Subject title must be ${titleMaxLength} characters or less`, + ), }), }); } @@ -237,7 +241,7 @@ export default function DataFileUploadForm({ name="subjectTitle" label="Subject title" className="govuk-!-width-two-thirds" - maxLength={120} + maxLength={titleMaxLength} /> )} diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataGuidanceSection.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataGuidanceSection.tsx index 90cd76ab81d..80146f9349c 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataGuidanceSection.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ReleaseDataGuidanceSection.tsx @@ -44,6 +44,7 @@ interface Props { canUpdateRelease: boolean; } +const contentMaxLength = 250; const formId = 'dataGuidanceForm'; const ReleaseDataGuidanceSection = ({ releaseId, canUpdateRelease }: Props) => { @@ -126,8 +127,8 @@ const ReleaseDataGuidanceSection = ({ releaseId, canUpdateRelease }: Props) => { return `Enter file guidance content for ${dataSet.name}`; }) .max( - 250, - 'File guidance content must be 250 characters or less', + contentMaxLength, + `File guidance content must be ${contentMaxLength} characters or less`, ), }), ), @@ -184,7 +185,7 @@ const ReleaseDataGuidanceSection = ({ releaseId, canUpdateRelease }: Props) => { label="File guidance content" name={`dataSets.${index}.content`} rows={3} - maxLength={250} + maxLength={contentMaxLength} /> ) : ( ; +const titleMaxLength = 120; +const descriptionMaxLength = 200; const formId = 'dataBlockDetailsForm'; interface Props { @@ -58,14 +60,20 @@ const DataBlockDetailsForm = ({ name: Yup.string().required('Enter a data block name'), heading: Yup.string() .required('Enter a table title') - .max(120, 'Table title must be 120 characters or less'), + .max( + titleMaxLength, + `Table title must be ${titleMaxLength} characters or less`, + ), source: Yup.string(), highlightName: Yup.string().when('isHighlight', { is: true, then: s => s .required('Enter a featured table name') - .max(120, 'Featured table name must be 120 characters or less'), + .max( + titleMaxLength, + `Featured table name must be ${titleMaxLength} characters or less`, + ), }), highlightDescription: Yup.string().when('isHighlight', { is: true, @@ -73,8 +81,8 @@ const DataBlockDetailsForm = ({ s .required('Enter a featured table description') .max( - 200, - 'Featured table description must be 200 characters or less', + descriptionMaxLength, + `Featured table description must be ${descriptionMaxLength} characters or less`, ), }), isHighlight: Yup.boolean(), @@ -111,7 +119,7 @@ const DataBlockDetailsForm = ({ onBlur={() => { onTitleChange?.(getValues('heading')); }} - maxLength={120} + maxLength={titleMaxLength} /> @@ -137,14 +145,14 @@ const DataBlockDetailsForm = ({ label="Featured table name" hint="We will show this name to table builder users as a featured table" className="govuk-!-width-two-thirds" - maxLength={120} + maxLength={titleMaxLength} /> name="highlightDescription" label="Featured table description" hint="Describe the contents of this featured table to table builder users" className="govuk-!-width-two-thirds" - maxLength={200} + maxLength={descriptionMaxLength} /> } From dd4f0b83a226355c5ff6ca8abf8e83bbc5cb16ea Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Wed, 18 Dec 2024 11:02:22 +0000 Subject: [PATCH 113/144] EES-5047: Remove shared backend validation messaging. --- .../Controllers/Api/ReleasesController.cs | 5 ++-- .../Requests/DataBlockRequests.cs | 7 ++---- .../Requests/DataGuidanceUpdateRequest.cs | 4 +-- .../Requests/FeaturedTableRequests.cs | 13 +++------- .../Requests/ReleaseFileRequests.cs | 16 ++++-------- .../Services/DataArchiveValidationService.cs | 3 +-- .../Constants/ValidationConstants.cs | 25 ------------------- 7 files changed, 15 insertions(+), 58 deletions(-) delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Common/Constants/ValidationConstants.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs index 7f1003b4f63..87440621bcb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/ReleasesController.cs @@ -14,7 +14,6 @@ using System.ComponentModel.DataAnnotations; using System.Threading; using System.Threading.Tasks; -using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Controllers.Api { @@ -109,7 +108,7 @@ public async Task>> ReorderDataFiles(Guid releas public async Task> UploadDataSet(Guid releaseVersionId, [FromQuery(Name = "replacingFileId")] Guid? replacingFileId, [FromQuery(Name = "title")] - [MaxLength(SubjectTitleMaxLength, ErrorMessage = SubjectTitleMaxLengthMessage)] + [MaxLength(120)] string title, IFormFile file, IFormFile metaFile) @@ -129,7 +128,7 @@ public async Task> UploadDataSet(Guid releaseVersionI public async Task> UploadDataSetAsZip(Guid releaseVersionId, [FromQuery(Name = "replacingFileId")] Guid? replacingFileId, [FromQuery(Name = "title")] - [MaxLength(SubjectTitleMaxLength, ErrorMessage = SubjectTitleMaxLengthMessage)] + [MaxLength(120)] string title, IFormFile zipFile) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataBlockRequests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataBlockRequests.cs index 6d2fe6767ce..d81e49514cd 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataBlockRequests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataBlockRequests.cs @@ -4,7 +4,6 @@ using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Requests; using System.Collections.Generic; -using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests; @@ -28,8 +27,7 @@ public Validator() { RuleFor(request => request.Heading) .NotEmpty() - .MaximumLength(TableTitleMaxLength) - .WithMessage(TableTitleMaxLengthMessage); + .MaximumLength(120); RuleFor(request => request.Name) .NotEmpty(); @@ -60,8 +58,7 @@ public Validator() { RuleFor(request => request.Heading) .NotEmpty() - .MaximumLength(TableTitleMaxLength) - .WithMessage(TableTitleMaxLengthMessage); + .MaximumLength(120); RuleFor(request => request.Name) .NotEmpty(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataGuidanceUpdateRequest.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataGuidanceUpdateRequest.cs index b057848d576..e854959c8ee 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataGuidanceUpdateRequest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/DataGuidanceUpdateRequest.cs @@ -2,7 +2,6 @@ using FluentValidation; using System; using System.Collections.Generic; -using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests; @@ -40,8 +39,7 @@ public Validator() RuleFor(request => request.Content) .NotEmpty() - .MaximumLength(FileGuidanceContentMaxLength) - .WithMessage(FileGuidanceContentMaxLengthMessage); + .MaximumLength(250); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/FeaturedTableRequests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/FeaturedTableRequests.cs index fe6ead1a189..43933bc1241 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/FeaturedTableRequests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/FeaturedTableRequests.cs @@ -1,7 +1,6 @@ #nullable enable using FluentValidation; using System; -using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests; @@ -19,13 +18,11 @@ public Validator() { RuleFor(request => request.Name) .NotEmpty() - .MaximumLength(FeaturedTableNameMaxLength) - .WithMessage(FeaturedTableNameMaxLengthMessage); + .MaximumLength(120); RuleFor(request => request.Description) .NotEmpty() - .MaximumLength(FeaturedTableDescriptionMaxLength) - .WithMessage(FeaturedTableDescriptionMaxLengthMessage); + .MaximumLength(200); } } } @@ -42,13 +39,11 @@ public Validator() { RuleFor(request => request.Name) .NotEmpty() - .MaximumLength(FeaturedTableNameMaxLength) - .WithMessage(FeaturedTableNameMaxLengthMessage); + .MaximumLength(120); RuleFor(request => request.Description) .NotEmpty() - .MaximumLength(FeaturedTableDescriptionMaxLength) - .WithMessage(FeaturedTableDescriptionMaxLengthMessage); + .MaximumLength(200); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseFileRequests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseFileRequests.cs index f9bf20a2a86..4f8c11a184c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseFileRequests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Requests/ReleaseFileRequests.cs @@ -1,7 +1,6 @@ #nullable enable using FluentValidation; using Microsoft.AspNetCore.Http; -using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Requests; @@ -16,8 +15,7 @@ public class Validator : AbstractValidator public Validator() { RuleFor(request => request.Title) - .MaximumLength(SubjectTitleMaxLength) - .WithMessage(SubjectTitleMaxLengthMessage); + .MaximumLength(120); } } } @@ -36,13 +34,11 @@ public Validator() { RuleFor(request => request.Title) .NotEmpty() - .MaximumLength(TitleMaxLength) - .WithMessage(TitleMaxLengthMessage); + .MaximumLength(120); RuleFor(request => request.Summary) .NotEmpty() - .MaximumLength(SummaryMaxLength) - .WithMessage(SummaryMaxLengthMessage); + .MaximumLength(250); RuleFor(request => request.File) .NotEmpty(); @@ -64,13 +60,11 @@ public Validator() { RuleFor(request => request.Title) .NotEmpty() - .MaximumLength(TitleMaxLength) - .WithMessage(TitleMaxLengthMessage); + .MaximumLength(120); RuleFor(request => request.Summary) .NotEmpty() - .MaximumLength(SummaryMaxLength) - .WithMessage(SummaryMaxLengthMessage); + .MaximumLength(250); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs index 2731cf1b6ce..ae3b51bcc35 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/DataArchiveValidationService.cs @@ -15,7 +15,6 @@ using System.IO.Compression; using System.Linq; using System.Threading.Tasks; -using static GovUk.Education.ExploreEducationStatistics.Common.Constants.ValidationConstants; namespace GovUk.Education.ExploreEducationStatistics.Admin.Services { @@ -143,7 +142,7 @@ public async Task>> ValidateBulkDa var filename = row[fileNameIndex]; var datasetName = row[datasetNameIndex].Trim(); - if (datasetName.Length > SubjectTitleMaxLength) + if (datasetName.Length > 120) { errors.Add(ValidationMessages.GenerateErrorDataSetTitleTooLong(datasetName)); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Constants/ValidationConstants.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Constants/ValidationConstants.cs deleted file mode 100644 index 69bf57be261..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Constants/ValidationConstants.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace GovUk.Education.ExploreEducationStatistics.Common.Constants; - -public class ValidationConstants -{ - public const int TitleMaxLength = 120; - public const string TitleMaxLengthMessage = "Title must be 120 characters or less"; - - public const int SummaryMaxLength = 250; - public const string SummaryMaxLengthMessage = "Summary must be 250 characters or less"; - - public const int SubjectTitleMaxLength = 120; - public const string SubjectTitleMaxLengthMessage = "Subject title must be 120 characters or less"; - - public const int TableTitleMaxLength = 120; - public const string TableTitleMaxLengthMessage = "Table title must be 120 characters or less"; - - public const int FeaturedTableNameMaxLength = 120; - public const string FeaturedTableNameMaxLengthMessage = "Featured table name must be 120 characters or less"; - - public const int FeaturedTableDescriptionMaxLength = 200; - public const string FeaturedTableDescriptionMaxLengthMessage = "Featured table description must be 200 characters or less"; - - public const int FileGuidanceContentMaxLength = 250; - public const string FileGuidanceContentMaxLengthMessage = "File guidance content must be 250 characters or less"; -} From 00c9dd0b9f93fcb058ba57ac8c75f3a19974d956 Mon Sep 17 00:00:00 2001 From: dfe-sdt Date: Thu, 19 Dec 2024 09:44:11 +0000 Subject: [PATCH 114/144] chore(tests): update test snapshots --- tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json | 2 +- tests/robot-tests/tests/snapshots/find_statistics_snapshot.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json index eeeed40583d..0c55c613d6a 100644 --- a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json +++ b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json @@ -1,5 +1,5 @@ { - "num_datasets": "926 data sets", + "num_datasets": "927 data sets", "themes": [ { "publications": [ diff --git a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json index ef9c78c9aed..7dcf47b800a 100644 --- a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json +++ b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json @@ -527,7 +527,7 @@ { "publication_summary": "Pupil attendance and absence data including termly national statistics and fortnightly statistics in development derived from DfE\u2019s regular attendance data", "publication_title": "Pupil attendance in schools", - "published": "5 Dec 2024", + "published": "19 Dec 2024", "release_type": "Official statistics in development", "theme": "Pupils and schools" }, From 7372f24894f9ae3594d04ba22dca7f88f01e3738 Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 19 Dec 2024 11:54:52 +0000 Subject: [PATCH 115/144] EES-5580 validate EditableContentForm on submit after reinitialisation --- .../src/components/editable/EditableContentForm.tsx | 1 + .../src/components/form/FormFieldEditor.tsx | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/explore-education-statistics-admin/src/components/editable/EditableContentForm.tsx b/src/explore-education-statistics-admin/src/components/editable/EditableContentForm.tsx index 658e0aaaf4f..329fa85c165 100644 --- a/src/explore-education-statistics-admin/src/components/editable/EditableContentForm.tsx +++ b/src/explore-education-statistics-admin/src/components/editable/EditableContentForm.tsx @@ -289,6 +289,7 @@ const EditableContentForm = ({ onChange={setElements} onCancelComment={toggleCommentAddForm.off} onClickAddComment={toggleCommentAddForm.on} + onElementsReady={setElements} onImageUpload={onImageUpload} onImageUploadCancel={onImageUploadCancel} /> diff --git a/src/explore-education-statistics-admin/src/components/form/FormFieldEditor.tsx b/src/explore-education-statistics-admin/src/components/form/FormFieldEditor.tsx index bc969779e6f..678339f261f 100644 --- a/src/explore-education-statistics-admin/src/components/form/FormFieldEditor.tsx +++ b/src/explore-education-statistics-admin/src/components/form/FormFieldEditor.tsx @@ -42,6 +42,7 @@ export default function FormFieldEditor({ testId, onBlur, onChange, + onElementsReady, ...props }: Props) { const { @@ -86,7 +87,10 @@ export default function FormFieldEditor({ } }} onElementsChange={handleElements} - onElementsReady={handleElements} + onElementsReady={els => { + handleElements(els); + onElementsReady?.(els); + }} onChange={nextValue => { setValue( name, From 898a54f80eff80805ac74cb797a07004288a3a18 Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 19 Dec 2024 13:39:59 +0000 Subject: [PATCH 116/144] EES-5457 show extra create data block link when more than 5 --- .../datablocks/ReleaseDataBlocksPage.tsx | 2 +- .../__tests__/ReleaseDataBlocksPage.test.tsx | 242 +++++++++++------- 2 files changed, 146 insertions(+), 98 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/ReleaseDataBlocksPage.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/ReleaseDataBlocksPage.tsx index 2c27275ff62..94545abcda5 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/ReleaseDataBlocksPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/ReleaseDataBlocksPage.tsx @@ -128,7 +128,7 @@ const ReleaseDataBlocksPage = ({

          - {canUpdateRelease && filteredDataBlocks.length > 5 && ( + {canUpdateRelease && dataBlocks.length > 5 && ( Create data block )} diff --git a/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseDataBlocksPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseDataBlocksPage.test.tsx index 11d757e419d..675d8b950c7 100644 --- a/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseDataBlocksPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/datablocks/__tests__/ReleaseDataBlocksPage.test.tsx @@ -14,7 +14,6 @@ import _permissionService from '@admin/services/permissionService'; import render from '@common-test/render'; import { waitFor } from '@testing-library/dom'; import { screen, within } from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; import React from 'react'; import { generatePath, MemoryRouter } from 'react-router'; import { Route } from 'react-router-dom'; @@ -218,6 +217,10 @@ describe('ReleaseDataBlocksPage', () => { name: 'Delete block', }), ).toBeInTheDocument(); + + expect( + screen.getByRole('link', { name: 'Create data block' }), + ).toBeInTheDocument(); }); test('renders page correctly when release cannot be updated', async () => { @@ -334,9 +337,28 @@ describe('ReleaseDataBlocksPage', () => { ).toBeInTheDocument(); }); - test('clicking `Delete block` button shows modal', async () => { - dataBlockService.listDataBlocks.mockResolvedValue(testDataBlocks); - dataBlockService.getDeleteBlockPlan.mockResolvedValue(testBlock1DeletePlan); + test('renders an extra "Create data block" button when there are more than 5 blocks', async () => { + dataBlockService.listDataBlocks.mockResolvedValue([ + ...testDataBlocks, + { + id: 'block-5', + name: 'Block 5', + created: '2021-02-01T15:00:00.0000000', + heading: 'Block 5 heading', + source: 'Block 5 source', + inContent: false, + chartsCount: 0, + }, + { + id: 'block-6', + name: 'Block 6', + created: '2021-02-01T15:00:00.0000000', + heading: 'Block 6 heading', + source: 'Block 6 source', + inContent: false, + chartsCount: 0, + }, + ]); featuredTableService.listFeaturedTables.mockResolvedValue( testFeaturedTables, ); @@ -347,122 +369,148 @@ describe('ReleaseDataBlocksPage', () => { expect(screen.getByTestId('dataBlocks')).toBeInTheDocument(); }); - expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); - expect(dataBlockService.getDeleteBlockPlan).toHaveBeenCalledTimes(0); - - const buttons = screen.getAllByRole('button', { name: 'Delete block' }); - - await userEvent.click(buttons[0]); - - await waitFor(() => { - expect(screen.getByRole('dialog')).toBeInTheDocument(); - }); - - expect(dataBlockService.getDeleteBlockPlan).toHaveBeenCalledTimes(1); - - const modal = within(screen.getByRole('dialog')); - - expect(modal.getByTestId('deleteDataBlock-name')).toHaveTextContent( - 'Block 1', - ); expect( - modal.getByTestId('deleteDataBlock-contentSectionHeading'), - ).toHaveTextContent('Section 1'); + screen.getAllByRole('link', { name: 'Create data block' }), + ).toHaveLength(2); }); - test('clicking `Cancel` button hides modal', async () => { - dataBlockService.listDataBlocks.mockResolvedValue(testDataBlocks); - dataBlockService.getDeleteBlockPlan.mockResolvedValue(testBlock1DeletePlan); - featuredTableService.listFeaturedTables.mockResolvedValue( - testFeaturedTables, - ); + describe('deleting a data block', () => { + test('clicking `Delete block` button shows modal', async () => { + dataBlockService.listDataBlocks.mockResolvedValue(testDataBlocks); + dataBlockService.getDeleteBlockPlan.mockResolvedValue( + testBlock1DeletePlan, + ); + featuredTableService.listFeaturedTables.mockResolvedValue( + testFeaturedTables, + ); - renderPage(); + const { user } = renderPage(); - await waitFor(() => { - expect(screen.getByTestId('dataBlocks')).toBeInTheDocument(); - }); + await waitFor(() => { + expect(screen.getByTestId('dataBlocks')).toBeInTheDocument(); + }); - const buttons = screen.getAllByRole('button', { name: 'Delete block' }); + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + expect(dataBlockService.getDeleteBlockPlan).toHaveBeenCalledTimes(0); - await userEvent.click(buttons[0]); + const buttons = screen.getAllByRole('button', { name: 'Delete block' }); - await waitFor(() => { - expect(screen.getByRole('dialog')).toBeInTheDocument(); - }); + await user.click(buttons[0]); - const modal = within(screen.getByRole('dialog')); + await waitFor(() => { + expect(screen.getByRole('dialog')).toBeInTheDocument(); + }); - await userEvent.click(modal.getByRole('button', { name: 'Cancel' })); + expect(dataBlockService.getDeleteBlockPlan).toHaveBeenCalledTimes(1); - await waitFor(() => { - expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + const modal = within(screen.getByRole('dialog')); + + expect(modal.getByTestId('deleteDataBlock-name')).toHaveTextContent( + 'Block 1', + ); + expect( + modal.getByTestId('deleteDataBlock-contentSectionHeading'), + ).toHaveTextContent('Section 1'); }); - }); - test('clicking `Confirm` button hides modal and deletes data block', async () => { - dataBlockService.listDataBlocks.mockResolvedValue(testDataBlocks); - dataBlockService.getDeleteBlockPlan.mockResolvedValue(testBlock1DeletePlan); - featuredTableService.listFeaturedTables.mockResolvedValue( - testFeaturedTables, - ); + test('clicking `Cancel` button hides modal', async () => { + dataBlockService.listDataBlocks.mockResolvedValue(testDataBlocks); + dataBlockService.getDeleteBlockPlan.mockResolvedValue( + testBlock1DeletePlan, + ); + featuredTableService.listFeaturedTables.mockResolvedValue( + testFeaturedTables, + ); - renderPage(); + const { user } = renderPage(); - await waitFor(() => { - expect(screen.getByTestId('dataBlocks')).toBeInTheDocument(); - }); + await waitFor(() => { + expect(screen.getByTestId('dataBlocks')).toBeInTheDocument(); + }); - const buttons = screen.getAllByRole('button', { name: 'Delete block' }); + const buttons = screen.getAllByRole('button', { name: 'Delete block' }); - await userEvent.click(buttons[0]); + await user.click(buttons[0]); - await waitFor(() => { - expect(screen.getByRole('dialog')).toBeInTheDocument(); - }); + await waitFor(() => { + expect(screen.getByRole('dialog')).toBeInTheDocument(); + }); - const modal = within(screen.getByRole('dialog')); + const modal = within(screen.getByRole('dialog')); - await userEvent.click(modal.getByRole('button', { name: 'Confirm' })); + await user.click(modal.getByRole('button', { name: 'Cancel' })); - await waitFor(() => { - expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + await waitFor(() => { + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + }); }); - expect(dataBlockService.deleteDataBlock).toHaveBeenCalledTimes(1); - expect(dataBlockService.deleteDataBlock).toHaveBeenCalledWith( - 'release-1', - 'block-1', - ); - - const featuredTablesTable = within(screen.getByTestId('featuredTables')); - const featuredTablesRows = featuredTablesTable.getAllByRole('row'); - expect(featuredTablesRows).toHaveLength(2); - - const featuredTablesRow1Cells = within(featuredTablesRows[1]).getAllByRole( - 'cell', - ); - expect(featuredTablesRow1Cells).toHaveLength(6); - expect(featuredTablesRow1Cells[0]).toHaveTextContent('Block 3'); - expect(featuredTablesRow1Cells[1]).toHaveTextContent('No'); - expect(featuredTablesRow1Cells[2]).toHaveTextContent('No'); - expect(featuredTablesRow1Cells[3]).toHaveTextContent('Featured 3'); - expect(featuredTablesRow1Cells[4]).toHaveTextContent( - '1 January 2021 15:00', - ); - expect( - within(featuredTablesRow1Cells[5]).getByRole('link', { - name: 'Edit block', - }), - ).toHaveAttribute( - 'href', - '/publication/publication-1/release/release-1/data-blocks/block-3', - ); - expect( - within(featuredTablesRow1Cells[5]).getByRole('button', { - name: 'Delete block', - }), - ).toBeInTheDocument(); + test('clicking `Confirm` button hides modal and deletes data block', async () => { + dataBlockService.listDataBlocks.mockResolvedValue(testDataBlocks); + dataBlockService.getDeleteBlockPlan.mockResolvedValue( + testBlock1DeletePlan, + ); + featuredTableService.listFeaturedTables.mockResolvedValue( + testFeaturedTables, + ); + + const { user } = renderPage(); + + await waitFor(() => { + expect(screen.getByTestId('dataBlocks')).toBeInTheDocument(); + }); + + const buttons = screen.getAllByRole('button', { name: 'Delete block' }); + + await user.click(buttons[0]); + + await waitFor(() => { + expect(screen.getByRole('dialog')).toBeInTheDocument(); + }); + + const modal = within(screen.getByRole('dialog')); + + await user.click(modal.getByRole('button', { name: 'Confirm' })); + + await waitFor(() => { + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + }); + + expect(dataBlockService.deleteDataBlock).toHaveBeenCalledTimes(1); + expect(dataBlockService.deleteDataBlock).toHaveBeenCalledWith( + 'release-1', + 'block-1', + ); + + const featuredTablesTable = within(screen.getByTestId('featuredTables')); + const featuredTablesRows = featuredTablesTable.getAllByRole('row'); + expect(featuredTablesRows).toHaveLength(2); + + const featuredTablesRow1Cells = within( + featuredTablesRows[1], + ).getAllByRole('cell'); + expect(featuredTablesRow1Cells).toHaveLength(6); + expect(featuredTablesRow1Cells[0]).toHaveTextContent('Block 3'); + expect(featuredTablesRow1Cells[1]).toHaveTextContent('No'); + expect(featuredTablesRow1Cells[2]).toHaveTextContent('No'); + expect(featuredTablesRow1Cells[3]).toHaveTextContent('Featured 3'); + expect(featuredTablesRow1Cells[4]).toHaveTextContent( + '1 January 2021 15:00', + ); + expect( + within(featuredTablesRow1Cells[5]).getByRole('link', { + name: 'Edit block', + }), + ).toHaveAttribute( + 'href', + '/publication/publication-1/release/release-1/data-blocks/block-3', + ); + expect( + within(featuredTablesRow1Cells[5]).getByRole('button', { + name: 'Delete block', + }), + ).toBeInTheDocument(); + }); }); const renderPage = () => { From 0778874667513128d715535d8173303317697ec8 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Fri, 13 Dec 2024 11:13:35 +0000 Subject: [PATCH 117/144] EES-5738 Create new DataSetFileGeographicLevels table --- ...taSetFileGeographicLevelsTable.Designer.cs | 2272 +++++++++++++++++ ..._CreateDataSetFileGeographicLevelsTable.cs | 43 + .../ContentDbContextModelSnapshot.cs | 29 + .../DataSetFileGeographicLevels.cs | 17 + .../Database/ContentDbContext.cs | 18 +- .../File.cs | 3 + 6 files changed, 2380 insertions(+), 2 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs new file mode 100644 index 00000000000..dcc66a1e236 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs @@ -0,0 +1,2272 @@ +// +using System; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + [DbContext(typeof(ContentDbContext))] + [Migration("20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable")] + partial class EES5738_CreateDataSetFileGeographicLevelsTable + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContactTelNo") + .HasColumnType("nvarchar(max)"); + + b.Property("TeamEmail") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TeamName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Contacts"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Common.Model.FreeTextRank", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Rank") + .HasColumnType("int"); + + b.ToTable((string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("LegacyCreatedBy") + .HasColumnType("nvarchar(max)"); + + b.Property("Resolved") + .HasColumnType("datetime2"); + + b.Property("ResolvedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ResolvedById"); + + b.ToTable("Comment"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentSectionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Locked") + .HasColumnType("datetime2"); + + b.Property("LockedById") + .IsConcurrencyToken() + .HasColumnType("uniqueidentifier"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContentSectionId"); + + b.HasIndex("LockedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentBlock", (string)null); + + b.HasDiscriminator("Type").HasValue("ContentBlock"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Caption") + .HasColumnType("nvarchar(max)"); + + b.Property("Heading") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("Type"); + + b.ToTable("ContentSections"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestDraftVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("LatestDraftVersionId") + .IsUnique() + .HasFilter("[LatestDraftVersionId] IS NOT NULL"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("DataBlocks", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier") + .HasColumnName("DataBlockId"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ContentBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("DataBlockVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("ExpectedImportedRows") + .HasColumnType("int"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("GeographicLevels") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ImportedRows") + .HasColumnType("int"); + + b.Property("LastProcessedRowIndex") + .HasColumnType("int"); + + b.Property("MetaFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("StagePercentageComplete") + .HasColumnType("int"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("TotalRows") + .HasColumnType("int"); + + b.Property("ZipFileId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId") + .IsUnique(); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("FileId"), new[] { "Status" }); + + b.HasIndex("MetaFileId") + .IsUnique(); + + b.HasIndex("ZipFileId"); + + b.ToTable("DataImports"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataImportId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("DataImportId"); + + b.ToTable("DataImportErrors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileGeographicLevel", b => + { + b.Property("DataSetFileVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("GeographicLevel") + .HasColumnType("int"); + + b.HasKey("DataSetFileVersionId", "GeographicLevel"); + + b.ToTable("DataSetFileGeographicLevels"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("EmbedBlocks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DataBlockId") + .IsUnique(); + + b.HasIndex("DataBlockParentId"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("FeaturedTables"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContentLength") + .HasColumnType("bigint"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataSetFileMeta") + .HasColumnType("nvarchar(max)"); + + b.Property("DataSetFileVersion") + .HasColumnType("int"); + + b.Property("Filename") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReplacedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReplacingId") + .HasColumnType("uniqueidentifier"); + + b.Property("RootPath") + .HasColumnType("uniqueidentifier"); + + b.Property("SourceId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(25) + .HasColumnType("nvarchar(25)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReplacedById") + .IsUnique() + .HasFilter("[ReplacedById] IS NOT NULL"); + + b.HasIndex("ReplacingId") + .IsUnique() + .HasFilter("[ReplacingId] IS NOT NULL"); + + b.HasIndex("SourceId"); + + b.HasIndex("Type"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.ToTable("GlossaryEntries"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("GuidanceText") + .HasColumnType("nvarchar(max)"); + + b.Property("GuidanceTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Trend") + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("KeyStatistics"); + + b.UseTptMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("OwningPublicationSlug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OwningPublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("LatestPublishedVersionId") + .IsUnique() + .HasFilter("[LatestPublishedVersionId] IS NOT NULL"); + + b.ToTable("Methodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DisplayDate") + .HasColumnType("datetime2"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("UpdatedById") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.HasIndex("UpdatedById"); + + b.ToTable("MethodologyNotes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("MethodologyVersionId", "Slug"); + + b.ToTable("MethodologyRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyVersionId"); + + b.ToTable("MethodologyStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("AlternativeSlug") + .HasColumnType("nvarchar(max)"); + + b.Property("AlternativeTitle") + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("PublishingStrategy") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ScheduledWithReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Status") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("MethodologyId"); + + b.HasIndex("PreviousVersionId"); + + b.HasIndex("ScheduledWithReleaseVersionId"); + + b.ToTable("MethodologyVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.Property("MethodologyVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Annexes") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Content") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("MethodologyVersionId"); + + b.ToTable("MethodologyVersions", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Permalink", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("DataSetTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MigratedFromLegacy") + .HasColumnType("bit"); + + b.Property("PublicationTitle") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("SubjectId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("SubjectId"); + + b.ToTable("Permalinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ContactId") + .HasColumnType("uniqueidentifier"); + + b.Property("LatestPublishedReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseSeries") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .IsRequired() + .HasMaxLength(160) + .HasColumnType("nvarchar(160)"); + + b.Property("SupersededById") + .HasColumnType("uniqueidentifier"); + + b.Property("ThemeId") + .HasColumnType("uniqueidentifier"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("ContactId"); + + b.HasIndex("LatestPublishedReleaseVersionId") + .IsUnique() + .HasFilter("[LatestPublishedReleaseVersionId] IS NOT NULL"); + + b.HasIndex("SupersededById"); + + b.HasIndex("ThemeId"); + + b.ToTable("Publications"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("MethodologyId") + .HasColumnType("uniqueidentifier"); + + b.Property("Owner") + .HasColumnType("bit"); + + b.HasKey("PublicationId", "MethodologyId"); + + b.HasIndex("MethodologyId"); + + b.ToTable("PublicationMethodologies"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("PublicationId", "Slug"); + + b.ToTable("PublicationRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("Label") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(81) + .HasColumnType("nvarchar(81)"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("nvarchar(5)"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.Property("Year") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("PublicationId", "Year", "TimePeriodCoverage", "Label") + .IsUnique(); + + b.ToTable("Releases"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .HasColumnType("uniqueidentifier"); + + b.Property("FilterSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("IndicatorSequence") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Order") + .HasColumnType("int"); + + b.Property("PublicApiDataSetId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicApiDataSetVersion") + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("FileId"); + + b.HasIndex("ReleaseVersionId", "FileId") + .IsUnique(); + + b.HasIndex("ReleaseVersionId", "PublicApiDataSetId", "PublicApiDataSetVersion") + .IsUnique() + .HasFilter("[PublicApiDataSetId] IS NOT NULL AND [PublicApiDataSetVersion] IS NOT NULL"); + + b.ToTable("ReleaseFiles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseRedirect", b => + { + b.Property("ReleaseId") + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(450)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.HasKey("ReleaseId", "Slug"); + + b.ToTable("ReleaseRedirects"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("InternalReleaseNote") + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("ReleaseStatus"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ApprovalStatus") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("DataGuidance") + .HasColumnType("nvarchar(max)"); + + b.Property("NextReleaseDate") + .HasColumnType("nvarchar(max)"); + + b.Property("NotifiedOn") + .HasColumnType("datetime2"); + + b.Property("NotifySubscribers") + .HasColumnType("bit"); + + b.Property("PreReleaseAccessList") + .HasColumnType("nvarchar(max)"); + + b.Property("PreviousVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("PublishScheduled") + .HasColumnType("datetime2"); + + b.Property("Published") + .HasColumnType("datetime2"); + + b.Property("RelatedInformation") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseName") + .HasColumnType("nvarchar(max)"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("TimePeriodCoverage") + .IsRequired() + .HasMaxLength(6) + .HasColumnType("nvarchar(6)"); + + b.Property("Type") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("UpdatePublishedDate") + .HasColumnType("bit"); + + b.Property("Version") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("ReleaseId"); + + b.HasIndex("Type"); + + b.HasIndex("PreviousVersionId", "Version"); + + b.ToTable("ReleaseVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Slug") + .HasColumnType("nvarchar(max)"); + + b.Property("Summary") + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Themes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("On") + .HasColumnType("datetime2"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("Update"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .HasColumnType("nvarchar(max)"); + + b.Property("FirstName") + .HasColumnType("nvarchar(max)"); + + b.Property("LastName") + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("DeletedById"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("PublicationId"); + + b.ToTable("UserPublicationInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("PublicationId"); + + b.HasIndex("UserId"); + + b.ToTable("UserPublicationRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Email") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EmailSent") + .HasColumnType("bit"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("Updated") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ReleaseVersionId"); + + b.ToTable("UserReleaseInvites"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Deleted") + .HasColumnType("datetime2"); + + b.Property("DeletedById") + .HasColumnType("uniqueidentifier"); + + b.Property("ReleaseVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("Role") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SoftDeleted") + .HasColumnType("bit"); + + b.Property("UserId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("DeletedById"); + + b.HasIndex("ReleaseVersionId"); + + b.HasIndex("UserId"); + + b.ToTable("UserReleaseRoles"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Charts") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Charts"); + + b.Property("Heading") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Heading"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Query") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Query"); + + b.Property("Source") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Table") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("DataBlock_Table"); + + b.HasDiscriminator().HasValue("DataBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("EmbedBlockId") + .HasColumnType("uniqueidentifier") + .HasColumnName("EmbedBlockId"); + + b.HasIndex("EmbedBlockId") + .IsUnique() + .HasFilter("[EmbedBlockId] IS NOT NULL"); + + b.HasDiscriminator().HasValue("EmbedBlockLink"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.HtmlBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("HtmlBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MarkDownBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock"); + + b.Property("Body") + .IsRequired() + .ValueGeneratedOnUpdateSometimes() + .HasColumnType("nvarchar(max)") + .HasColumnName("Body"); + + b.HasDiscriminator().HasValue("MarkDownBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("DataBlockId") + .HasColumnType("uniqueidentifier"); + + b.Property("DataBlockParentId") + .HasColumnType("uniqueidentifier"); + + b.HasIndex("DataBlockId"); + + b.HasIndex("DataBlockParentId"); + + b.ToTable("KeyStatisticsDataBlock", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic"); + + b.Property("Statistic") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.ToTable("KeyStatisticsText", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Comment", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", "ContentBlock") + .WithMany("Comments") + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "ResolvedBy") + .WithMany() + .HasForeignKey("ResolvedById"); + + b.Navigation("ContentBlock"); + + b.Navigation("CreatedBy"); + + b.Navigation("ResolvedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", "ContentSection") + .WithMany("Content") + .HasForeignKey("ContentSectionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "LockedBy") + .WithMany() + .HasForeignKey("LockedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentSection"); + + b.Navigation("LockedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Content") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestDraftVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestDraftVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "LatestPublishedVersionId"); + + b.Navigation("LatestDraftVersion"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "ContentBlock") + .WithMany() + .HasForeignKey("ContentBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("DataBlockVersions") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ContentBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "FileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "MetaFile") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "MetaFileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ZipFile") + .WithMany() + .HasForeignKey("ZipFileId") + .OnDelete(DeleteBehavior.Restrict); + + b.Navigation("File"); + + b.Navigation("MetaFile"); + + b.Navigation("ZipFile"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImportError", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", "DataImport") + .WithMany("Errors") + .HasForeignKey("DataImportId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataImport"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileGeographicLevel", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "DataSetFileVersion") + .WithMany("DataSetFileGeographicLevels") + .HasForeignKey("DataSetFileVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetFileVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", "DataBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("FeaturedTables") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedBy") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Replacing") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "ReplacingId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "Source") + .WithMany() + .HasForeignKey("SourceId"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReplacedBy"); + + b.Navigation("Replacing"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.GlossaryEntry", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("CreatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("KeyStatistics") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById"); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "LatestPublishedVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "LatestPublishedVersionId"); + + b.Navigation("LatestPublishedVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyNote", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany("Notes") + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "UpdatedBy") + .WithMany() + .HasForeignKey("UpdatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + + b.Navigation("UpdatedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany("MethodologyRedirects") + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "MethodologyVersion") + .WithMany() + .HasForeignKey("MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("MethodologyVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Versions") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ScheduledWithReleaseVersion") + .WithMany() + .HasForeignKey("ScheduledWithReleaseVersionId"); + + b.Navigation("CreatedBy"); + + b.Navigation("Methodology"); + + b.Navigation("PreviousVersion"); + + b.Navigation("ScheduledWithReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", null) + .WithOne("MethodologyContent") + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersionContent", "MethodologyVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Common.Model.Contact", "Contact") + .WithMany() + .HasForeignKey("ContactId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "LatestPublishedReleaseVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "LatestPublishedReleaseVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "SupersededBy") + .WithMany() + .HasForeignKey("SupersededById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", "Theme") + .WithMany("Publications") + .HasForeignKey("ThemeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ExternalMethodology", "ExternalMethodology", b1 => + { + b1.Property("PublicationId") + .HasColumnType("uniqueidentifier"); + + b1.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b1.HasKey("PublicationId"); + + b1.ToTable("ExternalMethodology", (string)null); + + b1.WithOwner() + .HasForeignKey("PublicationId"); + }); + + b.Navigation("Contact"); + + b.Navigation("ExternalMethodology"); + + b.Navigation("LatestPublishedReleaseVersion"); + + b.Navigation("SupersededBy"); + + b.Navigation("Theme"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationMethodology", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", "Methodology") + .WithMany("Publications") + .HasForeignKey("MethodologyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Methodologies") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Methodology"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.PublicationRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("PublicationRedirects") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("Releases") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseFile", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "File") + .WithMany() + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.Navigation("File"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseRedirect", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") + .WithMany("ReleaseRedirects") + .HasForeignKey("ReleaseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Release"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseStatus", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("ReleaseStatuses") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "PreviousVersion") + .WithMany() + .HasForeignKey("PreviousVersionId") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany("ReleaseVersions") + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", "Release") + .WithMany("Versions") + .HasForeignKey("ReleaseId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("PreviousVersion"); + + b.Navigation("Publication"); + + b.Navigation("Release"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Update", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany("Updates") + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.User", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.Navigation("DeletedBy"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("Publication"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserPublicationRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.NoAction); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", "Publication") + .WithMany() + .HasForeignKey("PublicationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("Publication"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ReleaseVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.UserReleaseRole", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "DeletedBy") + .WithMany() + .HasForeignKey("DeletedById"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", "ReleaseVersion") + .WithMany() + .HasForeignKey("ReleaseVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("DeletedBy"); + + b.Navigation("ReleaseVersion"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", "EmbedBlock") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlockLink", "EmbedBlockId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("EmbedBlock"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlock", "DataBlock") + .WithMany() + .HasForeignKey("DataBlockId") + .OnDelete(DeleteBehavior.NoAction) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.DataBlockParent", "DataBlockParent") + .WithMany() + .HasForeignKey("DataBlockParentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticDataBlock", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataBlock"); + + b.Navigation("DataBlockParent"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatistic", null) + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Content.Model.KeyStatisticText", "Id") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentBlock", b => + { + b.Navigation("Comments"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ContentSection", b => + { + b.Navigation("Content"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataImport", b => + { + b.Navigation("Errors"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.Navigation("DataSetFileGeographicLevels"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => + { + b.Navigation("Publications"); + + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.MethodologyVersion", b => + { + b.Navigation("MethodologyContent") + .IsRequired(); + + b.Navigation("MethodologyRedirects"); + + b.Navigation("Notes"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Publication", b => + { + b.Navigation("Methodologies"); + + b.Navigation("PublicationRedirects"); + + b.Navigation("ReleaseVersions"); + + b.Navigation("Releases"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Release", b => + { + b.Navigation("ReleaseRedirects"); + + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion", b => + { + b.Navigation("Content"); + + b.Navigation("DataBlockVersions"); + + b.Navigation("FeaturedTables"); + + b.Navigation("KeyStatistics"); + + b.Navigation("ReleaseStatuses"); + + b.Navigation("Updates"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Theme", b => + { + b.Navigation("Publications"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.cs new file mode 100644 index 00000000000..b4a23fa687c --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.cs @@ -0,0 +1,43 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations +{ + /// + public partial class EES5738_CreateDataSetFileGeographicLevelsTable : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "DataSetFileGeographicLevels", + columns: table => new + { + DataSetFileVersionId = table.Column(type: "uniqueidentifier", nullable: false), + GeographicLevel = table.Column(type: "int", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_DataSetFileGeographicLevels", x => new { x.DataSetFileVersionId, x.GeographicLevel }); + table.ForeignKey( + name: "FK_DataSetFileGeographicLevels_Files_DataSetFileVersionId", + column: x => x.DataSetFileVersionId, + principalTable: "Files", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.Sql("GRANT SELECT ON dbo.DataSetFileGeographicLevels TO [content];"); + migrationBuilder.Sql("GRANT INSERT ON dbo.DataSetFileGeographicLevels TO [importer];"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("REVOKE SELECT ON dbo.DataSetFileGeographicLevels TO [content];"); + migrationBuilder.Sql("REVOKE INSERT ON dbo.DataSetFileGeographicLevels TO [importer];"); + migrationBuilder.DropTable( + name: "DataSetFileGeographicLevels"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs index 2877ed7a1a1..75b3a97870b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs @@ -328,6 +328,19 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("DataImportErrors"); }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileGeographicLevel", b => + { + b.Property("DataSetFileVersionId") + .HasColumnType("uniqueidentifier"); + + b.Property("GeographicLevel") + .HasColumnType("int"); + + b.HasKey("DataSetFileVersionId", "GeographicLevel"); + + b.ToTable("DataSetFileGeographicLevels"); + }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", b => { b.Property("Id") @@ -1594,6 +1607,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("DataImport"); }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileGeographicLevel", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "DataSetFileVersion") + .WithMany("DataSetFileGeographicLevels") + .HasForeignKey("DataSetFileVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetFileVersion"); + }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.FeaturedTable", b => { b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.User", "CreatedBy") @@ -2180,6 +2204,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Errors"); }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => + { + b.Navigation("DataSetFileGeographicLevels"); + }); + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => { b.Navigation("Publications"); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs new file mode 100644 index 00000000000..ae83fb59cbc --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs @@ -0,0 +1,17 @@ +#nullable enable +using System; +using GovUk.Education.ExploreEducationStatistics.Common.Converters; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using Newtonsoft.Json; + +namespace GovUk.Education.ExploreEducationStatistics.Content.Model; + +public class DataSetFileGeographicLevel +{ + public Guid DataSetFileVersionId { get; set; } // Currently Files.Id, but will become DataSetFileVersion.Id in EES-5105 + + public File DataSetFileVersion { get; set; } = null!; + + [JsonConverter(typeof(EnumToEnumValueJsonConverter))] + public GeographicLevel GeographicLevel { get; set; } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs index a86f10300d4..61c71e4f880 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs @@ -55,6 +55,7 @@ private void Configure(bool updateTimestamps = true) public virtual DbSet ReleaseStatus { get; set; } public virtual DbSet ReleaseFiles { get; set; } public virtual DbSet Files { get; set; } + public virtual DbSet DataSetFileGeographicLevels { get; set; } public virtual DbSet ContentSections { get; set; } public virtual DbSet ContentBlocks { get; set; } public virtual DbSet KeyStatistics { get; set; } @@ -110,6 +111,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) ConfigureReleaseStatus(modelBuilder); ConfigureReleaseFile(modelBuilder); ConfigureFile(modelBuilder); + ConfigureDataSetFileGeographicLevel(modelBuilder); ConfigureContentBlock(modelBuilder); ConfigureContentSection(modelBuilder); ConfigureReleaseVersion(modelBuilder); @@ -452,13 +454,25 @@ private static void ConfigureFile(ModelBuilder modelBuilder) v => v.HasValue ? DateTime.SpecifyKind(v.Value, DateTimeKind.Utc) : null); - entity.Property(p => p.DataSetFileMeta) - .HasConversion( // You might want to use EF8 JSON support instead of this + entity.Property(p => p.DataSetFileMeta) // EES-5666 + .HasConversion( v => JsonConvert.SerializeObject(v), v => JsonConvert.DeserializeObject(v)); }); } + private static void ConfigureDataSetFileGeographicLevel(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.HasKey(gl => new + { + gl.DataSetFileVersionId, + gl.GeographicLevel + }); + }); + } + private static void ConfigureContentBlock(ModelBuilder modelBuilder) { modelBuilder.Entity(entity => diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs index f9a9ac9fd22..8fc1696a46a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs @@ -1,5 +1,6 @@ #nullable enable using System; +using System.Collections.Generic; using GovUk.Education.ExploreEducationStatistics.Common.Model; namespace GovUk.Education.ExploreEducationStatistics.Content.Model @@ -22,6 +23,8 @@ public class File : ICreatedTimestamp public int? DataSetFileVersion { get; set; } + public List? DataSetFileGeographicLevels { get; set; } + public DataSetFileMeta? DataSetFileMeta { get; set; } public Guid? ReplacedById { get; set; } From 6e32d38808bdb82be843d0b2634f074632316fb3 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Fri, 13 Dec 2024 11:40:51 +0000 Subject: [PATCH 118/144] EES-5738 Migration to move DataSetFileMeta geog lvls to new table --- ...FileGeographicLevelsMigrationController.cs | 83 +++++++++++++++++++ ...aSetFileGeographicLevelsTable.Designer.cs} | 6 +- ...CreateDataSetFileGeographicLevelsTable.cs} | 2 +- .../ContentDbContextModelSnapshot.cs | 4 +- .../DataSetFileGeographicLevels.cs | 3 - .../Database/ContentDbContext.cs | 3 + 6 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataSetFileGeographicLevelsMigrationController.cs rename src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/{20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs => 20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs} (99%) rename src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/{20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.cs => 20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.cs} (94%) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataSetFileGeographicLevelsMigrationController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataSetFileGeographicLevelsMigrationController.cs new file mode 100644 index 00000000000..f1adf49fd34 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataSetFileGeographicLevelsMigrationController.cs @@ -0,0 +1,83 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Admin.Models; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Content.Model; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Controllers.Api; + +[Route("api")] +[ApiController] +[Authorize(Roles = GlobalRoles.RoleNames.BauUser)] +public class DataSetFileMetaMigrationController(ContentDbContext contentDbContext) : ControllerBase +{ + public class MigrationResult + { + public bool IsDryRun; + public int Processed; + public string Errors; + } + + [HttpPut("bau/migrate-datasetfile-geographiclevels")] + public async Task DataSetFileGeographicLevelsMigration( + [FromQuery] bool isDryRun = true, + [FromQuery] int? num = null, + CancellationToken cancellationToken = default) + { + var queryable = contentDbContext.Files + .Where(f => f.DataSetFileMeta != null + && f.DataSetFileGeographicLevels.Count == 0); + + if (num != null) + { + queryable = queryable.Take(num.Value); + } + + var files = queryable.ToList(); + + var numProcessed = 0; + List errors = []; + + foreach (var file in files) + { + var meta = file.DataSetFileMeta; + + if (meta == null) + { + errors.Add($"No DataSetFileMeta found for File {file.Id}"); + continue; + } + + var dataSetFileGeographicLevels = meta!.GeographicLevels + .Distinct() + .Select(gl => new DataSetFileGeographicLevel + { + DataSetFileVersionId = file.Id, + GeographicLevel = gl, + }) + .ToList(); + + contentDbContext.DataSetFileGeographicLevels.AddRange(dataSetFileGeographicLevels); + + numProcessed++; + } + + if (!isDryRun) + { + await contentDbContext.SaveChangesAsync(cancellationToken); + } + + return new MigrationResult + { + IsDryRun = isDryRun, + Processed = numProcessed, + Errors = errors.IsNullOrEmpty() ? "No errors" : errors.JoinToString("\n"), + }; + } + +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs similarity index 99% rename from src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs rename to src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs index dcc66a1e236..9c4bfa60215 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs @@ -12,7 +12,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations { [DbContext(typeof(ContentDbContext))] - [Migration("20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable")] + [Migration("20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable")] partial class EES5738_CreateDataSetFileGeographicLevelsTable { /// @@ -336,8 +336,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("DataSetFileVersionId") .HasColumnType("uniqueidentifier"); - b.Property("GeographicLevel") - .HasColumnType("int"); + b.Property("GeographicLevel") + .HasColumnType("nvarchar(450)"); b.HasKey("DataSetFileVersionId", "GeographicLevel"); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.cs similarity index 94% rename from src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.cs rename to src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.cs index b4a23fa687c..1434036a2a3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213112036_EES5738_CreateDataSetFileGeographicLevelsTable.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.cs @@ -14,7 +14,7 @@ protected override void Up(MigrationBuilder migrationBuilder) columns: table => new { DataSetFileVersionId = table.Column(type: "uniqueidentifier", nullable: false), - GeographicLevel = table.Column(type: "int", nullable: false) + GeographicLevel = table.Column(type: "nvarchar(450)", nullable: false) }, constraints: table => { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs index 75b3a97870b..bc677140783 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs @@ -333,8 +333,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("DataSetFileVersionId") .HasColumnType("uniqueidentifier"); - b.Property("GeographicLevel") - .HasColumnType("int"); + b.Property("GeographicLevel") + .HasColumnType("nvarchar(450)"); b.HasKey("DataSetFileVersionId", "GeographicLevel"); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs index ae83fb59cbc..2b23db07a4a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs @@ -1,8 +1,6 @@ #nullable enable using System; -using GovUk.Education.ExploreEducationStatistics.Common.Converters; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; -using Newtonsoft.Json; namespace GovUk.Education.ExploreEducationStatistics.Content.Model; @@ -12,6 +10,5 @@ public class DataSetFileGeographicLevel public File DataSetFileVersion { get; set; } = null!; - [JsonConverter(typeof(EnumToEnumValueJsonConverter))] public GeographicLevel GeographicLevel { get; set; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs index 61c71e4f880..2cec48a76d7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs @@ -470,6 +470,9 @@ private static void ConfigureDataSetFileGeographicLevel(ModelBuilder modelBuilde gl.DataSetFileVersionId, gl.GeographicLevel }); + + entity.Property(gl => gl.GeographicLevel) + .HasConversion(new EnumToEnumValueConverter()); }); } From 47a3c8e99ae957cfe938442c1bffa27cd0d2f322 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Fri, 13 Dec 2024 12:54:04 +0000 Subject: [PATCH 119/144] EES-5738 Create DataSetFileGL entries in WriteDataSetFileMeta --- .../Services/DataImportServiceTests.cs | 2 +- .../Services/FileImportServiceTests.cs | 5 ++++- .../Services/DataImportService.cs | 11 ++++++++++- .../Services/FileImportService.cs | 4 ++-- .../Services/Interfaces/IDataImportService.cs | 2 +- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs index f0aabc50536..5aa9ee86309 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs @@ -284,7 +284,7 @@ public async Task WriteDataSetMetaFile_Success() contentDbContextId, statisticsDbContextId); - await service.WriteDataSetFileMeta(subject.Id); + await service.WriteDataSetFileMeta(file.Id, subject.Id); await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/FileImportServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/FileImportServiceTests.cs index c33c514b8a8..eaaa1f0c9f7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/FileImportServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/FileImportServiceTests.cs @@ -60,6 +60,7 @@ public async Task CompleteImport() dataImportService .Setup(s => s.WriteDataSetFileMeta( + import.FileId, import.SubjectId)) .Returns(Task.CompletedTask); @@ -228,7 +229,9 @@ await FinishedStatuses var dataImportService = new Mock(Strict); if (finishedStatus == COMPLETE) { - dataImportService.Setup(mock => mock.WriteDataSetFileMeta(import.SubjectId)) + dataImportService.Setup(mock => mock.WriteDataSetFileMeta( + import.FileId, + import.SubjectId)) .Returns(Task.CompletedTask); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs index 301d89cd76a..acb15120605 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs @@ -151,7 +151,7 @@ public async Task UpdateStatus(Guid id, DataImportStatus newStatus, double perce await context.SaveChangesAsync(); } - public async Task WriteDataSetFileMeta(Guid subjectId) + public async Task WriteDataSetFileMeta(Guid fileId, Guid subjectId) { await using var contentDbContext = _dbContextSupplier.CreateDbContext(); await using var statisticsDbContext = _dbContextSupplier.CreateDbContext(); @@ -220,6 +220,15 @@ public async Task WriteDataSetFileMeta(Guid subjectId) .Single(f => f.Type == FileType.Data && f.SubjectId == subjectId); file.DataSetFileMeta = dataSetFileMeta; + + var dataSetFileGeographicLevels = geographicLevels + .Select(gl => new DataSetFileGeographicLevel + { + DataSetFileVersionId = fileId, + GeographicLevel = gl, + }).ToList(); + await contentDbContext.DataSetFileGeographicLevels.AddRangeAsync(dataSetFileGeographicLevels); + await contentDbContext.SaveChangesAsync(); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/FileImportService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/FileImportService.cs index 44daca0a939..c2c9550cdd0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/FileImportService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/FileImportService.cs @@ -105,7 +105,7 @@ public async Task CompleteImport(DataImport import, StatisticsDbContext context) if (import.Status == COMPLETE) { - await _dataImportService.WriteDataSetFileMeta(import.SubjectId); + await _dataImportService.WriteDataSetFileMeta(import.FileId, import.SubjectId); } return; @@ -139,7 +139,7 @@ await _dataImportService.FailImport(import.Id, if (import.Errors.Count == 0) { await _dataImportService.UpdateStatus(import.Id, COMPLETE, 100); - await _dataImportService.WriteDataSetFileMeta(import.SubjectId); + await _dataImportService.WriteDataSetFileMeta(import.FileId, import.SubjectId); } else { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/Interfaces/IDataImportService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/Interfaces/IDataImportService.cs index e6ea8ba07ba..1ece36d23a5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/Interfaces/IDataImportService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/Interfaces/IDataImportService.cs @@ -19,7 +19,7 @@ public interface IDataImportService Task UpdateStatus(Guid id, DataImportStatus newStatus, double percentageComplete); - Task WriteDataSetFileMeta(Guid subjectId); + Task WriteDataSetFileMeta(Guid fileId, Guid subjectId); Task Update( Guid id, From c614926726b51b09f43646ab980ac92eb68de86b Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Fri, 13 Dec 2024 14:29:24 +0000 Subject: [PATCH 120/144] EES-5738 Data Catalogue page Geog lvl filtering --- .../FunctionsIntegrationTest.cs | 1 - .../DataSetFilesControllerCachingTests.cs | 2 + .../Controllers/DataSetFilesController.cs | 2 + .../DataSetFileListRequest.cs | 12 ++++ .../DataSetFileService.cs | 62 ++++++++++--------- .../Interfaces/IDataSetFileService.cs | 2 + .../data-catalogue/DataCataloguePage.tsx | 5 ++ .../data-catalogue/components/Filters.tsx | 36 +++++++++++ .../utils/createDataSetFileListRequest.ts | 3 + .../utils/dataSetFileFilters.ts | 1 + .../src/services/dataSetFileService.ts | 2 + 11 files changed, 97 insertions(+), 31 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/FunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/FunctionsIntegrationTest.cs index 0a7cda58edf..9516e767bbc 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/FunctionsIntegrationTest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/FunctionsIntegrationTest.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Linq.Expressions; using System.Threading.Tasks; using Azure.Data.Tables; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerCachingTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerCachingTests.cs index 651ba30ac9d..efa82dcb7ee 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerCachingTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerCachingTests.cs @@ -31,6 +31,7 @@ public class ListDataSetsTests : DataSetFilesControllerCachingTests ThemeId: Guid.NewGuid(), PublicationId: Guid.NewGuid(), ReleaseId: Guid.NewGuid(), + GeographicLevel: GeographicLevel.Country.GetEnumValue(), LatestOnly: true, DataSetType: DataSetType.Api, SearchTerm: "term", @@ -116,6 +117,7 @@ public async Task NoCachedEntryExists_CreatesCache() _query.ThemeId, _query.PublicationId, _query.ReleaseId, + _query.GeographicLevelEnum, _query.LatestOnly, _query.DataSetType, _query.SearchTerm, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/DataSetFilesController.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/DataSetFilesController.cs index 015a79b28da..db48b8fc809 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/DataSetFilesController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/DataSetFilesController.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Cache; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using GovUk.Education.ExploreEducationStatistics.Content.Api.Cache; using GovUk.Education.ExploreEducationStatistics.Content.Requests; @@ -39,6 +40,7 @@ public async Task + GeographicLevel == null + ? null + : EnumUtil.GetFromEnumValue(GeographicLevel); + public class Validator : AbstractValidator { public Validator() @@ -23,6 +32,9 @@ public Validator() .MinimumLength(3); RuleFor(request => request.ReleaseId).NotEmpty() .When(request => request.Sort == DataSetsListRequestSortBy.Natural); + RuleFor(request => request.GeographicLevel) + .AllowedValue(EnumUtil.GetEnumValues()) + .When(request => request.GeographicLevel != null); RuleFor(request => request.SearchTerm).NotEmpty() .When(request => request.Sort == DataSetsListRequestSortBy.Relevance); RuleFor(request => request.Page) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index ab71aad330e..4a0775ccddd 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -26,35 +26,25 @@ using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using static GovUk.Education.ExploreEducationStatistics.Common.Model.SortDirection; using static GovUk.Education.ExploreEducationStatistics.Content.Requests.DataSetsListRequestSortBy; using ReleaseVersion = GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion; namespace GovUk.Education.ExploreEducationStatistics.Content.Services; -public class DataSetFileService : IDataSetFileService +public class DataSetFileService( + ContentDbContext contentDbContext, + IReleaseVersionRepository releaseVersionRepository, + IPublicBlobStorageService publicBlobStorageService, + IFootnoteRepository footnoteRepository) + : IDataSetFileService { - private readonly ContentDbContext _contentDbContext; - private readonly IReleaseVersionRepository _releaseVersionRepository; - private readonly IPublicBlobStorageService _publicBlobStorageService; - private readonly IFootnoteRepository _footnoteRepository; - - public DataSetFileService( - ContentDbContext contentDbContext, - IReleaseVersionRepository releaseVersionRepository, - IPublicBlobStorageService publicBlobStorageService, - IFootnoteRepository footnoteRepository) - { - _contentDbContext = contentDbContext; - _releaseVersionRepository = releaseVersionRepository; - _publicBlobStorageService = publicBlobStorageService; - _footnoteRepository = footnoteRepository; - } - public async Task>> ListDataSetFiles( Guid? themeId, Guid? publicationId, Guid? releaseVersionId, + GeographicLevel? geographicLevel, bool? latestOnly, DataSetType? dataSetType, string? searchTerm, @@ -73,18 +63,19 @@ public async Task rf.Id, searchTerm); + .JoinFreeText(contentDbContext.ReleaseFilesFreeTextTable, rf => rf.Id, searchTerm); var results = await query .OrderBy(sort.Value, sortDirection.Value) @@ -144,10 +135,10 @@ private static Expression, DataSetFileSumm public async Task>> ListSitemapItems( CancellationToken cancellationToken = default) { - var latestReleaseVersions = _contentDbContext.ReleaseVersions + var latestReleaseVersions = contentDbContext.ReleaseVersions .LatestReleaseVersions(publishedOnly: true); - var latestReleaseFiles = _contentDbContext.ReleaseFiles + var latestReleaseFiles = contentDbContext.ReleaseFiles .AsNoTracking() .OfFileType(FileType.Data) .HavingNoDataReplacementInProgress() @@ -176,7 +167,7 @@ private static async Task> ChangeSummaryHtmlTo public async Task> GetDataSetFile(Guid dataSetFileId) { - var releaseFile = await _contentDbContext.ReleaseFiles + var releaseFile = await contentDbContext.ReleaseFiles .Include(rf => rf.ReleaseVersion.Publication.Theme) .Include(rf => rf.ReleaseVersion.Publication.SupersededBy) .Include(rf => rf.File) @@ -188,7 +179,7 @@ public async Task> GetDataSetFile(Gui .FirstOrDefaultAsync(); if (releaseFile == null - || !await _releaseVersionRepository.IsLatestPublishedReleaseVersion( + || !await releaseVersionRepository.IsLatestPublishedReleaseVersion( releaseFile.ReleaseVersionId)) { return new NotFoundResult(); @@ -198,7 +189,7 @@ public async Task> GetDataSetFile(Gui var variables = GetVariables(releaseFile.File.DataSetFileMeta!); - var footnotes = await _footnoteRepository.GetFootnotes( + var footnotes = await footnoteRepository.GetFootnotes( releaseFile.ReleaseVersionId, releaseFile.File.SubjectId); @@ -249,7 +240,7 @@ public async Task> GetDataSetFile(Gui public async Task DownloadDataSetFile( Guid dataSetFileId) { - var releaseFile = await _contentDbContext.ReleaseFiles + var releaseFile = await contentDbContext.ReleaseFiles .Include(rf => rf.File) .Where(rf => rf.File.DataSetFileId == dataSetFileId @@ -259,13 +250,13 @@ public async Task DownloadDataSetFile( .FirstOrDefaultAsync(); if (releaseFile == null - || !await _releaseVersionRepository.IsLatestPublishedReleaseVersion( + || !await releaseVersionRepository.IsLatestPublishedReleaseVersion( releaseFile.ReleaseVersionId)) { return new NotFoundResult(); } - var stream = await _publicBlobStorageService.StreamBlob( + var stream = await publicBlobStorageService.StreamBlob( containerName: BlobContainers.PublicReleaseFiles, path: releaseFile.PublicPath()); @@ -306,7 +297,7 @@ private static DataSetFileMetaViewModel BuildDataSetFileMetaViewModel( private async Task GetDataCsvPreview(ReleaseFile releaseFile) { - var datafileStreamProvider = () => _publicBlobStorageService.StreamBlob( + var datafileStreamProvider = () => publicBlobStorageService.StreamBlob( containerName: BlobContainers.PublicReleaseFiles, path: releaseFile.PublicPath()); @@ -491,6 +482,17 @@ internal static IQueryable HavingReleaseVersionId( return releaseVersionId.HasValue ? query.Where(rf => rf.ReleaseVersionId == releaseVersionId.Value) : query; } + internal static IQueryable HavingGeographicLevel( // @MarkFix write tests + this IQueryable query, + GeographicLevel? geographicLevel) + { + return geographicLevel.HasValue + ? query.Where(rf => rf.File.DataSetFileGeographicLevels!.Any( // @MarkFix null allowing + gl => gl.GeographicLevel == geographicLevel + && rf.FileId == gl.DataSetFileVersionId)) + : query; + } + internal static IQueryable OfDataSetType( this IQueryable query, DataSetType dataSetType) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IDataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IDataSetFileService.cs index 1b5cb2a3372..0a16d4cb669 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IDataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/Interfaces/IDataSetFileService.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; using GovUk.Education.ExploreEducationStatistics.Content.Requests; using GovUk.Education.ExploreEducationStatistics.Content.ViewModels; @@ -17,6 +18,7 @@ Task>> Guid? themeId, Guid? publicationId, Guid? releaseVersionId, + GeographicLevel? geographicLevel, bool? latestOnly, DataSetType? dataSetType, string? searchTerm, diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePage.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePage.tsx index e551420f4d2..c14872aa2e0 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePage.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePage.tsx @@ -52,6 +52,7 @@ import omit from 'lodash/omit'; import Head from 'next/head'; import { useRouter } from 'next/router'; import { ParsedUrlQuery } from 'querystring'; +import { GeographicLevelCode } from '@common/utils/locationLevelsMap'; const defaultPageTitle = 'Data catalogue'; @@ -61,6 +62,7 @@ export interface DataCataloguePageQuery { page?: number; publicationId?: string; releaseId?: string; + geographicLevel?: GeographicLevelCode; searchTerm?: string; sortBy?: DataSetFileSortOption; sortDirection?: SortDirection; @@ -85,6 +87,7 @@ const DataCataloguePage: NextPage = ({ showTypeFilter }) => { sortBy, publicationId, releaseId, + geographicLevel, searchTerm, themeId, } = getParamsFromQuery(router.query); @@ -131,6 +134,7 @@ const DataCataloguePage: NextPage = ({ showTypeFilter }) => { searchTerm, selectedTheme?.title, selectedPublication?.title, + geographicLevel, ]).join(', '); const updateQueryParams = async (nextQuery: DataCataloguePageQuery) => { @@ -327,6 +331,7 @@ const DataCataloguePage: NextPage = ({ showTypeFilter }) => { publicationId={publicationId} publications={publications} releaseId={releaseId} + geographicLevel={geographicLevel} releases={releases} showResetFiltersButton={!isMobileMedia && isFiltered} showTypeFilter={showTypeFilter} diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/Filters.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/Filters.tsx index 1dc33f5d1ed..c564cbc8fd5 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/Filters.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/Filters.tsx @@ -13,6 +13,10 @@ import styles from '@frontend/modules/data-catalogue/components/Filters.module.s import { DataSetFileFilter } from '@frontend/modules/data-catalogue/utils/dataSetFileFilters'; import React from 'react'; import classNames from 'classnames'; +import locationLevelsMap, { + GeographicLevelCode, +} from '@common/utils/locationLevelsMap'; +import typedKeys from '@common/utils/object/typedKeys'; const formId = 'filters-form'; @@ -23,6 +27,7 @@ interface Props { publications?: PublicationTreeSummary[]; releaseId?: string; releases?: ReleaseSummary[]; + geographicLevel?: GeographicLevelCode; showResetFiltersButton?: boolean; showTypeFilter?: boolean; themeId?: string; @@ -44,6 +49,7 @@ export default function Filters({ publicationId, releaseId, releases = [], + geographicLevel, showResetFiltersButton, showTypeFilter, themeId, @@ -140,6 +146,36 @@ export default function Filters({ /> + + + Filter by Geographic level + + } + name="geographicLevel" + options={[ + { label: 'All', value: 'all' }, + ...typedKeys(locationLevelsMap).map(key => { + return { + label: locationLevelsMap[key].label, + value: locationLevelsMap[key].code, + }; + }), + ]} + value={geographicLevel} + order={[]} + onChange={e => { + onChange({ + filterType: 'geographicLevel', + nextValue: e.target.value, + }); + }} + /> + + {showResetFiltersButton && ( Date: Fri, 13 Dec 2024 16:35:11 +0000 Subject: [PATCH 121/144] EES-5738 Add tests --- .../DataSetFilesControllerTests.cs | 47 +++++++++++++++++ .../Fixtures/DataImportGeneratorExtensions.cs | 34 ++++++++++++- ...tFileGeographicLevelGeneratorExtensions.cs | 50 +++++++++++++++++++ .../Fixtures/FileGeneratorExtensions.cs | 23 ++++++++- .../File.cs | 2 +- .../DataSetFileService.cs | 4 +- .../Functions/ProcessorStage3Tests.cs | 21 ++++---- .../Services/DataImportServiceTests.cs | 1 + .../Services/DataImportService.cs | 1 + .../data-catalogue/__data__/testDataSets.ts | 2 +- .../__tests__/DataCataloguePage.test.tsx | 26 ++++++++++ .../__tests__/DataSetSummary.test.tsx | 4 +- 12 files changed, 195 insertions(+), 20 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileGeographicLevelGeneratorExtensions.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index 91a6b434769..b69ddb03ce6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -95,6 +95,51 @@ await TestApp.AddTestData(context => AssertResultsForExpectedReleaseFiles(publication1Release1Version1Files, pagedResult.Results); } + [Fact] + public async Task FilterByGeographicLevel_Success() + { + var (publication1, publication2) = _fixture + .DefaultPublication() + // Publications each have a published release version + .WithReleases(_fixture.DefaultRelease(publishedVersions: 1) + .Generate(1)) + .WithTheme(_fixture.DefaultTheme()) + .GenerateTuple2(); + + var publication1Release1Version1Files = _fixture.DefaultReleaseFile() + .WithReleaseVersion(publication1.ReleaseVersions[0]) + .WithFiles(_fixture.DefaultFile(FileType.Data) + .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() + .WithGeographicLevels( + [GeographicLevel.Country, GeographicLevel.LocalAuthority, GeographicLevel.Institution])) + .WithDataSetFileGeographicLevels( + [GeographicLevel.Country, GeographicLevel.LocalAuthority, GeographicLevel.Institution]) + .GenerateList(1)) + .GenerateList(); + + var publication2Release1Version1Files = GenerateDataSetFilesForReleaseVersion(publication2.ReleaseVersions[0]); + + await TestApp.AddTestData(context => + { + context.ReleaseFiles.AddRange(publication1Release1Version1Files); + context.ReleaseFiles.AddRange(publication2Release1Version1Files); + }); + + MemoryCacheService + .SetupNotFoundForAnyKey>(); + + var query = new DataSetFileListRequest(GeographicLevel: GeographicLevel.Institution.GetEnumValue()); + var response = await ListDataSetFiles(query); + + MockUtils.VerifyAllMocks(MemoryCacheService); + + var pagedResult = response.AssertOk>(); + + pagedResult.AssertHasExpectedPagingAndResultCount( + expectedTotalResults: 1); + AssertResultsForExpectedReleaseFiles(publication1Release1Version1Files, pagedResult.Results); + } + [Fact] public async Task FilterByPublicationId_Success() { @@ -1675,6 +1720,7 @@ private async Task ListDataSetFiles( { "themeId", request.ThemeId?.ToString() }, { "publicationId", request.PublicationId?.ToString() }, { "releaseId", request.ReleaseId?.ToString() }, + { "geographicLevel", request.GeographicLevel }, { "latestOnly", request.LatestOnly?.ToString() }, { "searchTerm", request.SearchTerm }, { "sort", request.Sort?.ToString() }, @@ -1737,6 +1783,7 @@ private List GenerateDataSetFilesForReleaseVersion( .WithReleaseVersion(releaseVersion) .WithFiles(_fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta()) + .WithDataSetFileGeographicLevels([GeographicLevel.Country]) .GenerateList(numberOfDataSets)) .GenerateList(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs index 7a02d7410cf..0eab870e422 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs @@ -19,8 +19,15 @@ public static Generator WithSubjectId( public static Generator WithDefaultFiles( this Generator generator, - string dataFileName) - => generator.ForInstance(s => s.SetDefaultFiles(dataFileName)); + string dataFileName, + bool metaSet = true) + { + if (metaSet) + { + return generator.ForInstance(s => s.SetDefaultFiles(dataFileName)); + } + return generator.ForInstance(s => s.SetDefaultFilesWithoutMeta(dataFileName)); + } public static Generator WithFile( this Generator generator, @@ -98,6 +105,29 @@ public static InstanceSetters SetDefaultFiles( ) .Set(d => d.MetaFileId, (_, d) => d.MetaFile.Id); + public static InstanceSetters SetDefaultFilesWithoutMeta( + this InstanceSetters setters, + string dataFileName) + => setters + .Set( + d => d.File, + (_, d, context) => context.Fixture + .DefaultFile(FileType.Data) + .WithDataSetFileMeta(null) + .WithDataSetFileGeographicLevels([]) + .WithFilename($"{dataFileName}.csv") + .WithSubjectId(d.SubjectId) + ) + .Set(d => d.FileId, (_, d) => d.File.Id) + .Set( + d => d.MetaFile, + (_, d, context) => context.Fixture + .DefaultFile(FileType.Metadata) + .WithFilename($"{dataFileName}.meta.csv") + .WithSubjectId(d.SubjectId) + ) + .Set(d => d.MetaFileId, (_, d) => d.MetaFile.Id); + public static InstanceSetters SetFile( this InstanceSetters setters, File file) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileGeographicLevelGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileGeographicLevelGeneratorExtensions.cs new file mode 100644 index 00000000000..b9c7d83a357 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileGeographicLevelGeneratorExtensions.cs @@ -0,0 +1,50 @@ +using System; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; + +namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; + +public static class DataSetFileGeographicLevelGeneratorExtensions +{ + public static Generator DefaultDataSetFileGeographicLevel(this DataFixture fixture) + => fixture.Generator().WithDefaults(); + + public static Generator WithDefaults(this Generator generator) + => generator.ForInstance(d => d.SetDefaults()); + + public static InstanceSetters SetDefaults( + this InstanceSetters setters) + => setters + .SetDataSetFileVersionId(Guid.NewGuid()) + .SetGeographicLevel(GeographicLevel.Country); + + public static Generator WithDataSetFileVersion( + this Generator generator, + File dataSetFileVersion) + => generator.ForInstance(s => s.SetDataSetFileVersion(dataSetFileVersion)); + + public static Generator WithDataSetFileVersionId( + this Generator generator, + Guid dataSetFileVersionId) + => generator.ForInstance(s => s.SetDataSetFileVersionId(dataSetFileVersionId)); + + public static Generator WithGeographicLevel( + this Generator generator, + GeographicLevel geographicLevel) + => generator.ForInstance(s => s.SetGeographicLevel(geographicLevel)); + + public static InstanceSetters SetDataSetFileVersion( + this InstanceSetters setters, + File dataSetFileVersion) + => setters.Set(f => f.DataSetFileVersion, dataSetFileVersion); + + public static InstanceSetters SetDataSetFileVersionId( + this InstanceSetters setters, + Guid dataSetFileVersionId) + => setters.Set(f => f.DataSetFileVersionId, dataSetFileVersionId); + + public static InstanceSetters SetGeographicLevel( + this InstanceSetters setters, + GeographicLevel geographicLevel) + => setters.Set(f => f.GeographicLevel, geographicLevel); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs index f45b0e99adc..b60fb4e6798 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs @@ -1,8 +1,9 @@ using System; +using System.Collections.Generic; +using System.Linq; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; -using Semver; namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; @@ -84,6 +85,11 @@ public static Generator WithDataSetFileMeta( DataSetFileMeta? dataSetFileMeta) => generator.ForInstance(s => s.SetDataSetFileMeta(dataSetFileMeta)); + public static Generator WithDataSetFileGeographicLevels( + this Generator generator, + List geographicLevels) + => generator.ForInstance(s => s.SetDataSetFileGeographicLevels(geographicLevels)); + public static InstanceSetters SetDefaults(this InstanceSetters setters, FileType? fileType) => fileType switch { @@ -110,7 +116,8 @@ public static InstanceSetters SetDataFileDefaults(this InstanceSetters f.SubjectId) .SetContentType("text/csv") .SetDefault(f => f.DataSetFileId) - .Set(f => f.DataSetFileMeta, (_, _, context) => context.Fixture.DefaultDataSetFileMeta()); + .Set(f => f.DataSetFileMeta, (_, _, context) => context.Fixture.DefaultDataSetFileMeta()) + .SetDataSetFileGeographicLevels([GeographicLevel.Country, GeographicLevel.LocalAuthority, GeographicLevel.LocalAuthorityDistrict]); public static InstanceSetters SetMetaFileDefaults(this InstanceSetters setters) => setters @@ -200,4 +207,16 @@ public static InstanceSetters SetDataSetFileMeta( this InstanceSetters setters, DataSetFileMeta? dataSetFileMeta) => setters.Set(f => f.DataSetFileMeta, dataSetFileMeta); + + public static InstanceSetters SetDataSetFileGeographicLevels( + this InstanceSetters setters, + List geographicLevels) + => setters.Set( + file => file.DataSetFileGeographicLevels, + (_, file) => geographicLevels.Select( + gl => new DataSetFileGeographicLevel + { + DataSetFileVersionId = file.Id, + GeographicLevel = gl, + }).ToList()); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs index 8fc1696a46a..42c6f75b9dd 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs @@ -23,7 +23,7 @@ public class File : ICreatedTimestamp public int? DataSetFileVersion { get; set; } - public List? DataSetFileGeographicLevels { get; set; } + public List DataSetFileGeographicLevels { get; set; } = []; public DataSetFileMeta? DataSetFileMeta { get; set; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index 4a0775ccddd..b743fd1198e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -482,12 +482,12 @@ internal static IQueryable HavingReleaseVersionId( return releaseVersionId.HasValue ? query.Where(rf => rf.ReleaseVersionId == releaseVersionId.Value) : query; } - internal static IQueryable HavingGeographicLevel( // @MarkFix write tests + internal static IQueryable HavingGeographicLevel( this IQueryable query, GeographicLevel? geographicLevel) { return geographicLevel.HasValue - ? query.Where(rf => rf.File.DataSetFileGeographicLevels!.Any( // @MarkFix null allowing + ? query.Where(rf => rf.File.DataSetFileGeographicLevels.Any( gl => gl.GeographicLevel == geographicLevel && rf.FileId == gl.DataSetFileVersionId)) : query; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage3Tests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage3Tests.cs index 36f3fdc9e87..965c0c781e1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage3Tests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Functions/ProcessorStage3Tests.cs @@ -23,6 +23,7 @@ using GovUk.Education.ExploreEducationStatistics.Data.Processor.Services; using GovUk.Education.ExploreEducationStatistics.Data.Processor.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests.Services; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Migrations; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -106,7 +107,7 @@ public async Task ProcessStage3() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithDefaultFiles("small-csv") + .WithDefaultFiles("small-csv", metaSet: false) .WithStatus(STAGE_3) .WithTotalRows(16) .WithExpectedImportedRows(16) @@ -325,7 +326,7 @@ public async Task ProcessStage3_FailsImportIfRowCountsDontMatch() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithDefaultFiles("small-csv") + .WithDefaultFiles("small-csv", metaSet: false) .WithStatus(STAGE_3) .WithTotalRows(16) .WithExpectedImportedRows(16) @@ -454,7 +455,7 @@ public async Task ProcessStage3_PartiallyImportedAlready() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithDefaultFiles("small-csv") + .WithDefaultFiles("small-csv", metaSet: false) .WithStatus(STAGE_3) .WithTotalRows(16) .WithExpectedImportedRows(16) @@ -604,7 +605,7 @@ public async Task ProcessStage3_PartiallyImportedAlready_BatchSizeChanged() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithDefaultFiles("small-csv") + .WithDefaultFiles("small-csv", metaSet: false) .WithStatus(STAGE_3) .WithTotalRows(16) .WithExpectedImportedRows(16) @@ -749,7 +750,7 @@ public async Task ProcessStage3_IgnoredRows() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithDefaultFiles("ignored-school-rows") + .WithDefaultFiles("ignored-school-rows", metaSet: false) .WithStatus(STAGE_3) .WithTotalRows(16) .WithExpectedImportedRows(8) @@ -894,7 +895,7 @@ public async Task ProcessStage3_IgnoredRows_PartiallyImported() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithDefaultFiles("ignored-school-rows") + .WithDefaultFiles("ignored-school-rows", metaSet: false) .WithStatus(STAGE_3) .WithTotalRows(16) .WithExpectedImportedRows(8) @@ -1052,7 +1053,7 @@ public async Task ProcessStage3_Cancelling() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithDefaultFiles("small-csv") + .WithDefaultFiles("small-csv", metaSet: false) .WithStatus(STAGE_3) .WithTotalRows(16) .WithExpectedImportedRows(16) @@ -1190,7 +1191,7 @@ public async Task ProcessStage3_CancelledAlready() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithDefaultFiles("small-csv") + .WithDefaultFiles("small-csv", metaSet: false) .WithStatus(CANCELLED) .WithTotalRows(16) .WithExpectedImportedRows(16) @@ -1324,7 +1325,7 @@ public async Task ProcessStage3_AdditionalFiltersAndIndicators() var import = _fixture .DefaultDataImport() .WithSubjectId(_subject.Id) - .WithDefaultFiles("additional-filters-and-indicators") + .WithDefaultFiles("additional-filters-and-indicators", metaSet: false) .WithStatus(STAGE_3) .WithTotalRows(16) .WithExpectedImportedRows(16) @@ -1493,7 +1494,7 @@ public async Task ProcessStage3_SpecialFilterItemAndIndicatorValues() var import = _fixture .DefaultDataImport() .WithSubjectId(subject.Id) - .WithDefaultFiles("small-csv-with-special-data") + .WithDefaultFiles("small-csv-with-special-data", metaSet: false) .WithStatus(STAGE_3) .WithTotalRows(5) .WithExpectedImportedRows(5) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs index 5aa9ee86309..8463051e0f5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs @@ -220,6 +220,7 @@ public async Task WriteDataSetMetaFile_Success() var file = _fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(null) + .WithDataSetFileGeographicLevels([]) .WithSubjectId(subject.Id) .Generate(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs index acb15120605..162c826776d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs @@ -222,6 +222,7 @@ public async Task WriteDataSetFileMeta(Guid fileId, Guid subjectId) file.DataSetFileMeta = dataSetFileMeta; var dataSetFileGeographicLevels = geographicLevels + .Distinct() .Select(gl => new DataSetFileGeographicLevel { DataSetFileVersionId = fileId, diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts index c73c983240c..c6a2e4a13ca 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__data__/testDataSets.ts @@ -77,7 +77,7 @@ export const testDataSetFileSummaries: DataSetFileSummary[] = [ to: '2020', }, filters: ['Filter 1', 'Filter 2'], - geographicLevels: ['National', 'Regional'], + geographicLevels: ['National', 'Regional', 'Local authority'], indicators: ['Indicator 1', 'Indicator 2'], }, latestData: true, diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataCataloguePage.test.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataCataloguePage.test.tsx index 4026e9df8c6..e6b684958f7 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataCataloguePage.test.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataCataloguePage.test.tsx @@ -1195,6 +1195,32 @@ describe('DataCataloguePage', () => { ).toBeInTheDocument(); }); + test('filters by geographic level', async () => { + mockRouter.setCurrentUrl('/data-catalogue?geographicLevel=LA'); + + dataSetService.listDataSetFiles.mockResolvedValueOnce({ + results: [testDataSetFileSummaries[1], testDataSetFileSummaries[2]], + paging: { ...testPaging, totalPages: 1, totalResults: 1 }, + }); + publicationService.getPublicationTree.mockResolvedValue(testThemes); + publicationService.listReleases.mockResolvedValue(testReleases); + + render(); + + await waitFor(() => { + expect(screen.getByText('1 data set')).toBeInTheDocument(); + }); + + expect(mockRouter).toMatchObject({ + pathname: '/data-catalogue', + query: { geographicLevel: 'LA' }, + }); + + expect(screen.getByLabelText('Filter by Geographic level')).toHaveValue( + 'LA', + ); + }); + test('filters by search term', async () => { mockRouter.setCurrentUrl('/data-catalogue?searchTerm=find+me'); diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetSummary.test.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetSummary.test.tsx index ddf431535b5..fd464e53a87 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetSummary.test.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetSummary.test.tsx @@ -50,7 +50,7 @@ describe('DataSetFileSummary', () => { expect( within(screen.getByTestId('Geographic levels')).getByText( - 'National, Regional', + 'Local authority, National, Regional', ), ).toBeInTheDocument(); expect( @@ -83,7 +83,7 @@ describe('DataSetFileSummary', () => { expect( within(screen.getByTestId('Geographic levels')).getByText( - 'National, Regional', + 'Local authority, National, Regional', ), ).toBeInTheDocument(); expect( From 1d65b4b418c6ff75ea282e1fb3ba8c0799af53d4 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Tue, 17 Dec 2024 12:23:51 +0000 Subject: [PATCH 122/144] EES-5738 Changes in response to PR comments --- ...FileGeographicLevelsMigrationController.cs | 14 ++--- ...aSetFileGeographicLevelsTable.Designer.cs} | 15 +++--- ...CreateDataSetFileGeographicLevelsTable.cs} | 22 +++++--- .../ContentDbContextModelSnapshot.cs | 13 ++--- .../DataSetFilesControllerTests.cs | 22 +++----- .../Fixtures/DataImportGeneratorExtensions.cs | 2 +- ...tFileGeographicLevelGeneratorExtensions.cs | 50 ------------------ ...rsionGeographicLevelGeneratorExtensions.cs | 51 +++++++++++++++++++ .../Fixtures/FileGeneratorExtensions.cs | 13 ++--- ...s => DataSetFileVersionGeographicLevel.cs} | 2 +- .../Database/ContentDbContext.cs | 11 ++-- .../File.cs | 2 +- .../DataSetFileService.cs | 5 +- .../Services/DataImportServiceTests.cs | 2 +- .../Services/DataImportService.cs | 8 +-- .../src/utils/locationLevelsMap.ts | 19 +++++++ .../data-catalogue/DataCataloguePage.tsx | 22 +++++++- .../data-catalogue/components/Filters.tsx | 4 +- .../admin_and_public/bau/data_catalogue.robot | 16 ++++++ 19 files changed, 177 insertions(+), 116 deletions(-) rename src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/{20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs => 20241219093820_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs} (99%) rename src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/{20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.cs => 20241219093820_EES5738_CreateDataSetFileGeographicLevelsTable.cs} (62%) delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileGeographicLevelGeneratorExtensions.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileVersionGeographicLevelGeneratorExtensions.cs rename src/GovUk.Education.ExploreEducationStatistics.Content.Model/{DataSetFileGeographicLevels.cs => DataSetFileVersionGeographicLevel.cs} (89%) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataSetFileGeographicLevelsMigrationController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataSetFileGeographicLevelsMigrationController.cs index f1adf49fd34..b4ceec2eb98 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataSetFileGeographicLevelsMigrationController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/DataSetFileGeographicLevelsMigrationController.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Models; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using Microsoft.AspNetCore.Authorization; @@ -24,14 +25,14 @@ public class MigrationResult } [HttpPut("bau/migrate-datasetfile-geographiclevels")] - public async Task DataSetFileGeographicLevelsMigration( + public async Task DataSetFileVersionGeographicLevelsMigration( [FromQuery] bool isDryRun = true, [FromQuery] int? num = null, CancellationToken cancellationToken = default) { var queryable = contentDbContext.Files - .Where(f => f.DataSetFileMeta != null - && f.DataSetFileGeographicLevels.Count == 0); + .Where(f => f.Type == FileType.Data + && f.DataSetFileVersionGeographicLevels.Count == 0); if (num != null) { @@ -53,16 +54,17 @@ public async Task DataSetFileGeographicLevelsMigration( continue; } - var dataSetFileGeographicLevels = meta!.GeographicLevels + var dataSetFileVersionGeographicLevels = meta!.GeographicLevels .Distinct() - .Select(gl => new DataSetFileGeographicLevel + .Select(gl => new DataSetFileVersionGeographicLevel { DataSetFileVersionId = file.Id, GeographicLevel = gl, }) .ToList(); - contentDbContext.DataSetFileGeographicLevels.AddRange(dataSetFileGeographicLevels); + contentDbContext.DataSetFileVersionGeographicLevels.AddRange( + dataSetFileVersionGeographicLevels); numProcessed++; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241219093820_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs similarity index 99% rename from src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs rename to src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241219093820_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs index 9c4bfa60215..739bb0e01e7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241219093820_EES5738_CreateDataSetFileGeographicLevelsTable.Designer.cs @@ -12,7 +12,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations { [DbContext(typeof(ContentDbContext))] - [Migration("20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable")] + [Migration("20241219093820_EES5738_CreateDataSetFileGeographicLevelsTable")] partial class EES5738_CreateDataSetFileGeographicLevelsTable { /// @@ -331,17 +331,18 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("DataImportErrors"); }); - modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileGeographicLevel", b => + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileVersionGeographicLevel", b => { b.Property("DataSetFileVersionId") .HasColumnType("uniqueidentifier"); b.Property("GeographicLevel") - .HasColumnType("nvarchar(450)"); + .HasMaxLength(6) + .HasColumnType("nvarchar(6)"); b.HasKey("DataSetFileVersionId", "GeographicLevel"); - b.ToTable("DataSetFileGeographicLevels"); + b.ToTable("DataSetFileVersionGeographicLevels"); }); modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", b => @@ -1610,10 +1611,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Navigation("DataImport"); }); - modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileGeographicLevel", b => + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileVersionGeographicLevel", b => { b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "DataSetFileVersion") - .WithMany("DataSetFileGeographicLevels") + .WithMany("DataSetFileVersionGeographicLevels") .HasForeignKey("DataSetFileVersionId") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); @@ -2209,7 +2210,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => { - b.Navigation("DataSetFileGeographicLevels"); + b.Navigation("DataSetFileVersionGeographicLevels"); }); modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241219093820_EES5738_CreateDataSetFileGeographicLevelsTable.cs similarity index 62% rename from src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.cs rename to src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241219093820_EES5738_CreateDataSetFileGeographicLevelsTable.cs index 1434036a2a3..7a652fbef83 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241213123442_EES5738_CreateDataSetFileGeographicLevelsTable.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/20241219093820_EES5738_CreateDataSetFileGeographicLevelsTable.cs @@ -1,6 +1,8 @@ using System; using Microsoft.EntityFrameworkCore.Migrations; +#nullable disable + namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.ContentMigrations { /// @@ -10,25 +12,27 @@ public partial class EES5738_CreateDataSetFileGeographicLevelsTable : Migration protected override void Up(MigrationBuilder migrationBuilder) { migrationBuilder.CreateTable( - name: "DataSetFileGeographicLevels", + name: "DataSetFileVersionGeographicLevels", columns: table => new { DataSetFileVersionId = table.Column(type: "uniqueidentifier", nullable: false), - GeographicLevel = table.Column(type: "nvarchar(450)", nullable: false) + GeographicLevel = table.Column(type: "nvarchar(6)", maxLength: 6, nullable: false) }, constraints: table => { - table.PrimaryKey("PK_DataSetFileGeographicLevels", x => new { x.DataSetFileVersionId, x.GeographicLevel }); + table.PrimaryKey("PK_DataSetFileVersionGeographicLevels", x => new { x.DataSetFileVersionId, x.GeographicLevel }); table.ForeignKey( - name: "FK_DataSetFileGeographicLevels_Files_DataSetFileVersionId", + name: "FK_DataSetFileVersionGeographicLevels_Files_DataSetFileVersionId", column: x => x.DataSetFileVersionId, principalTable: "Files", principalColumn: "Id", onDelete: ReferentialAction.Cascade); }); + migrationBuilder.Sql("GRANT SELECT ON dbo.DataSetFileVersionGeographicLevels TO [content];"); + migrationBuilder.Sql("GRANT INSERT ON dbo.DataSetFileVersionGeographicLevels TO [importer];"); - migrationBuilder.Sql("GRANT SELECT ON dbo.DataSetFileGeographicLevels TO [content];"); - migrationBuilder.Sql("GRANT INSERT ON dbo.DataSetFileGeographicLevels TO [importer];"); + migrationBuilder.Sql("GRANT SELECT ON dbo.DataSetFileVersionGeographicLevels TO [data];"); + migrationBuilder.Sql("GRANT SELECT ON dbo.DataSetFileVersionGeographicLevels TO [publisher];"); } /// @@ -36,8 +40,12 @@ protected override void Down(MigrationBuilder migrationBuilder) { migrationBuilder.Sql("REVOKE SELECT ON dbo.DataSetFileGeographicLevels TO [content];"); migrationBuilder.Sql("REVOKE INSERT ON dbo.DataSetFileGeographicLevels TO [importer];"); + + migrationBuilder.Sql("REVOKE SELECT ON dbo.DataSetFileGeographicLevels TO [data];"); + migrationBuilder.Sql("REVOKE SELECT ON dbo.DataSetFileGeographicLevels TO [publisher];"); + migrationBuilder.DropTable( - name: "DataSetFileGeographicLevels"); + name: "DataSetFileVersionGeographicLevels"); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs index bc677140783..292daec9d7f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/ContentMigrations/ContentDbContextModelSnapshot.cs @@ -328,17 +328,18 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("DataImportErrors"); }); - modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileGeographicLevel", b => + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileVersionGeographicLevel", b => { b.Property("DataSetFileVersionId") .HasColumnType("uniqueidentifier"); b.Property("GeographicLevel") - .HasColumnType("nvarchar(450)"); + .HasMaxLength(6) + .HasColumnType("nvarchar(6)"); b.HasKey("DataSetFileVersionId", "GeographicLevel"); - b.ToTable("DataSetFileGeographicLevels"); + b.ToTable("DataSetFileVersionGeographicLevels"); }); modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.EmbedBlock", b => @@ -1607,10 +1608,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("DataImport"); }); - modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileGeographicLevel", b => + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.DataSetFileVersionGeographicLevel", b => { b.HasOne("GovUk.Education.ExploreEducationStatistics.Content.Model.File", "DataSetFileVersion") - .WithMany("DataSetFileGeographicLevels") + .WithMany("DataSetFileVersionGeographicLevels") .HasForeignKey("DataSetFileVersionId") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); @@ -2206,7 +2207,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.File", b => { - b.Navigation("DataSetFileGeographicLevels"); + b.Navigation("DataSetFileVersionGeographicLevels"); }); modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Content.Model.Methodology", b => diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs index b69ddb03ce6..b32dde93656 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs @@ -101,27 +101,21 @@ public async Task FilterByGeographicLevel_Success() var (publication1, publication2) = _fixture .DefaultPublication() // Publications each have a published release version - .WithReleases(_fixture.DefaultRelease(publishedVersions: 1) - .Generate(1)) + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]) .WithTheme(_fixture.DefaultTheme()) .GenerateTuple2(); - var publication1Release1Version1Files = _fixture.DefaultReleaseFile() + ReleaseFile publication1Release1Version1File = _fixture.DefaultReleaseFile() .WithReleaseVersion(publication1.ReleaseVersions[0]) - .WithFiles(_fixture.DefaultFile(FileType.Data) - .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta() - .WithGeographicLevels( - [GeographicLevel.Country, GeographicLevel.LocalAuthority, GeographicLevel.Institution])) - .WithDataSetFileGeographicLevels( - [GeographicLevel.Country, GeographicLevel.LocalAuthority, GeographicLevel.Institution]) - .GenerateList(1)) - .GenerateList(); + .WithFile(_fixture.DefaultFile(FileType.Data) + .WithDataSetFileVersionGeographicLevels( + [GeographicLevel.Country, GeographicLevel.LocalAuthority, GeographicLevel.Institution])); var publication2Release1Version1Files = GenerateDataSetFilesForReleaseVersion(publication2.ReleaseVersions[0]); await TestApp.AddTestData(context => { - context.ReleaseFiles.AddRange(publication1Release1Version1Files); + context.ReleaseFiles.Add(publication1Release1Version1File); context.ReleaseFiles.AddRange(publication2Release1Version1Files); }); @@ -137,7 +131,7 @@ await TestApp.AddTestData(context => pagedResult.AssertHasExpectedPagingAndResultCount( expectedTotalResults: 1); - AssertResultsForExpectedReleaseFiles(publication1Release1Version1Files, pagedResult.Results); + AssertResultsForExpectedReleaseFiles([publication1Release1Version1File], pagedResult.Results); } [Fact] @@ -1783,7 +1777,7 @@ private List GenerateDataSetFilesForReleaseVersion( .WithReleaseVersion(releaseVersion) .WithFiles(_fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta()) - .WithDataSetFileGeographicLevels([GeographicLevel.Country]) + .WithDataSetFileVersionGeographicLevels([GeographicLevel.Country]) .GenerateList(numberOfDataSets)) .GenerateList(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs index 0eab870e422..5d13f4f4164 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataImportGeneratorExtensions.cs @@ -114,7 +114,7 @@ public static InstanceSetters SetDefaultFilesWithoutMeta( (_, d, context) => context.Fixture .DefaultFile(FileType.Data) .WithDataSetFileMeta(null) - .WithDataSetFileGeographicLevels([]) + .WithDataSetFileVersionGeographicLevels([]) .WithFilename($"{dataFileName}.csv") .WithSubjectId(d.SubjectId) ) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileGeographicLevelGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileGeographicLevelGeneratorExtensions.cs deleted file mode 100644 index b9c7d83a357..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileGeographicLevelGeneratorExtensions.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; -using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; -using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; - -namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; - -public static class DataSetFileGeographicLevelGeneratorExtensions -{ - public static Generator DefaultDataSetFileGeographicLevel(this DataFixture fixture) - => fixture.Generator().WithDefaults(); - - public static Generator WithDefaults(this Generator generator) - => generator.ForInstance(d => d.SetDefaults()); - - public static InstanceSetters SetDefaults( - this InstanceSetters setters) - => setters - .SetDataSetFileVersionId(Guid.NewGuid()) - .SetGeographicLevel(GeographicLevel.Country); - - public static Generator WithDataSetFileVersion( - this Generator generator, - File dataSetFileVersion) - => generator.ForInstance(s => s.SetDataSetFileVersion(dataSetFileVersion)); - - public static Generator WithDataSetFileVersionId( - this Generator generator, - Guid dataSetFileVersionId) - => generator.ForInstance(s => s.SetDataSetFileVersionId(dataSetFileVersionId)); - - public static Generator WithGeographicLevel( - this Generator generator, - GeographicLevel geographicLevel) - => generator.ForInstance(s => s.SetGeographicLevel(geographicLevel)); - - public static InstanceSetters SetDataSetFileVersion( - this InstanceSetters setters, - File dataSetFileVersion) - => setters.Set(f => f.DataSetFileVersion, dataSetFileVersion); - - public static InstanceSetters SetDataSetFileVersionId( - this InstanceSetters setters, - Guid dataSetFileVersionId) - => setters.Set(f => f.DataSetFileVersionId, dataSetFileVersionId); - - public static InstanceSetters SetGeographicLevel( - this InstanceSetters setters, - GeographicLevel geographicLevel) - => setters.Set(f => f.GeographicLevel, geographicLevel); -} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileVersionGeographicLevelGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileVersionGeographicLevelGeneratorExtensions.cs new file mode 100644 index 00000000000..eb49d2897ec --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/DataSetFileVersionGeographicLevelGeneratorExtensions.cs @@ -0,0 +1,51 @@ +using System; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; + +namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; + +public static class DataSetFileVersionGeographicLevelGeneratorExtensions +{ + public static Generator DefaultDataSetFileVersionGeographicLevel(this DataFixture fixture) + => fixture.Generator().WithDefaults(); + + public static Generator WithDefaults(this Generator generator) + => generator.ForInstance(d => d.SetDefaults()); + + public static InstanceSetters SetDefaults( + this InstanceSetters setters) + => setters + .SetDefault(p => p.DataSetFileVersionId); + + public static Generator WithDataSetFileVersion( + this Generator generator, + File dataSetFileVersion) + => generator.ForInstance(s => s.SetDataSetFileVersion(dataSetFileVersion)); + + public static Generator WithDataSetFileVersionId( + this Generator generator, + Guid dataSetFileVersionId) + => generator.ForInstance(s => s.SetDataSetFileVersionId(dataSetFileVersionId)); + + public static Generator WithGeographicLevel( + this Generator generator, + GeographicLevel geographicLevel) + => generator.ForInstance(s => s.SetGeographicLevel(geographicLevel)); + + public static InstanceSetters SetDataSetFileVersion( + this InstanceSetters setters, + File dataSetFileVersion) + => setters + .Set(f => f.DataSetFileVersion, dataSetFileVersion) + .Set(f => f.DataSetFileVersionId, dataSetFileVersion.Id); + + public static InstanceSetters SetDataSetFileVersionId( + this InstanceSetters setters, + Guid dataSetFileVersionId) + => setters.Set(f => f.DataSetFileVersionId, dataSetFileVersionId); + + public static InstanceSetters SetGeographicLevel( + this InstanceSetters setters, + GeographicLevel geographicLevel) + => setters.Set(f => f.GeographicLevel, geographicLevel); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs index b60fb4e6798..320c2950442 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Fixtures/FileGeneratorExtensions.cs @@ -85,10 +85,10 @@ public static Generator WithDataSetFileMeta( DataSetFileMeta? dataSetFileMeta) => generator.ForInstance(s => s.SetDataSetFileMeta(dataSetFileMeta)); - public static Generator WithDataSetFileGeographicLevels( + public static Generator WithDataSetFileVersionGeographicLevels( this Generator generator, List geographicLevels) - => generator.ForInstance(s => s.SetDataSetFileGeographicLevels(geographicLevels)); + => generator.ForInstance(s => s.SetDataSetFileVersionGeographicLevels(geographicLevels)); public static InstanceSetters SetDefaults(this InstanceSetters setters, FileType? fileType) => fileType switch @@ -117,7 +117,7 @@ public static InstanceSetters SetDataFileDefaults(this InstanceSetters f.DataSetFileId) .Set(f => f.DataSetFileMeta, (_, _, context) => context.Fixture.DefaultDataSetFileMeta()) - .SetDataSetFileGeographicLevels([GeographicLevel.Country, GeographicLevel.LocalAuthority, GeographicLevel.LocalAuthorityDistrict]); + .SetDataSetFileVersionGeographicLevels([GeographicLevel.Country, GeographicLevel.LocalAuthority, GeographicLevel.LocalAuthorityDistrict]); public static InstanceSetters SetMetaFileDefaults(this InstanceSetters setters) => setters @@ -208,15 +208,16 @@ public static InstanceSetters SetDataSetFileMeta( DataSetFileMeta? dataSetFileMeta) => setters.Set(f => f.DataSetFileMeta, dataSetFileMeta); - public static InstanceSetters SetDataSetFileGeographicLevels( + public static InstanceSetters SetDataSetFileVersionGeographicLevels( this InstanceSetters setters, List geographicLevels) => setters.Set( - file => file.DataSetFileGeographicLevels, + file => file.DataSetFileVersionGeographicLevels, (_, file) => geographicLevels.Select( - gl => new DataSetFileGeographicLevel + gl => new DataSetFileVersionGeographicLevel { DataSetFileVersionId = file.Id, + DataSetFileVersion = file, GeographicLevel = gl, }).ToList()); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileVersionGeographicLevel.cs similarity index 89% rename from src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs rename to src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileVersionGeographicLevel.cs index 2b23db07a4a..d677283dbe6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileGeographicLevels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileVersionGeographicLevel.cs @@ -4,7 +4,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Model; -public class DataSetFileGeographicLevel +public class DataSetFileVersionGeographicLevel { public Guid DataSetFileVersionId { get; set; } // Currently Files.Id, but will become DataSetFileVersion.Id in EES-5105 diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs index 2cec48a76d7..8521a3ede07 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Database/ContentDbContext.cs @@ -55,7 +55,7 @@ private void Configure(bool updateTimestamps = true) public virtual DbSet ReleaseStatus { get; set; } public virtual DbSet ReleaseFiles { get; set; } public virtual DbSet Files { get; set; } - public virtual DbSet DataSetFileGeographicLevels { get; set; } + public virtual DbSet DataSetFileVersionGeographicLevels { get; set; } public virtual DbSet ContentSections { get; set; } public virtual DbSet ContentBlocks { get; set; } public virtual DbSet KeyStatistics { get; set; } @@ -111,7 +111,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) ConfigureReleaseStatus(modelBuilder); ConfigureReleaseFile(modelBuilder); ConfigureFile(modelBuilder); - ConfigureDataSetFileGeographicLevel(modelBuilder); + ConfigureDataSetFileVersionGeographicLevel(modelBuilder); ConfigureContentBlock(modelBuilder); ConfigureContentSection(modelBuilder); ConfigureReleaseVersion(modelBuilder); @@ -454,16 +454,16 @@ private static void ConfigureFile(ModelBuilder modelBuilder) v => v.HasValue ? DateTime.SpecifyKind(v.Value, DateTimeKind.Utc) : null); - entity.Property(p => p.DataSetFileMeta) // EES-5666 + entity.Property(p => p.DataSetFileMeta) .HasConversion( v => JsonConvert.SerializeObject(v), v => JsonConvert.DeserializeObject(v)); }); } - private static void ConfigureDataSetFileGeographicLevel(ModelBuilder modelBuilder) + private static void ConfigureDataSetFileVersionGeographicLevel(ModelBuilder modelBuilder) { - modelBuilder.Entity(entity => + modelBuilder.Entity(entity => { entity.HasKey(gl => new { @@ -472,6 +472,7 @@ private static void ConfigureDataSetFileGeographicLevel(ModelBuilder modelBuilde }); entity.Property(gl => gl.GeographicLevel) + .HasMaxLength(6) .HasConversion(new EnumToEnumValueConverter()); }); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs index 42c6f75b9dd..7ada31a8934 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/File.cs @@ -23,7 +23,7 @@ public class File : ICreatedTimestamp public int? DataSetFileVersion { get; set; } - public List DataSetFileGeographicLevels { get; set; } = []; + public List DataSetFileVersionGeographicLevels { get; set; } = []; public DataSetFileMeta? DataSetFileMeta { get; set; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs index b743fd1198e..e6d0e9d85d9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/DataSetFileService.cs @@ -487,9 +487,8 @@ internal static IQueryable HavingGeographicLevel( GeographicLevel? geographicLevel) { return geographicLevel.HasValue - ? query.Where(rf => rf.File.DataSetFileGeographicLevels.Any( - gl => gl.GeographicLevel == geographicLevel - && rf.FileId == gl.DataSetFileVersionId)) + ? query.Where(rf => rf.File.DataSetFileVersionGeographicLevels.Any( + gl => gl.GeographicLevel == geographicLevel)) : query; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs index 8463051e0f5..de19c60ad62 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor.Tests/Services/DataImportServiceTests.cs @@ -220,7 +220,7 @@ public async Task WriteDataSetMetaFile_Success() var file = _fixture.DefaultFile(FileType.Data) .WithDataSetFileMeta(null) - .WithDataSetFileGeographicLevels([]) + .WithDataSetFileVersionGeographicLevels([]) .WithSubjectId(subject.Id) .Generate(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs index 162c826776d..5b29fd83184 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Processor/Services/DataImportService.cs @@ -221,14 +221,14 @@ public async Task WriteDataSetFileMeta(Guid fileId, Guid subjectId) && f.SubjectId == subjectId); file.DataSetFileMeta = dataSetFileMeta; - var dataSetFileGeographicLevels = geographicLevels - .Distinct() - .Select(gl => new DataSetFileGeographicLevel + var dataSetFileVersionGeographicLevels = geographicLevels + .Select(gl => new DataSetFileVersionGeographicLevel { DataSetFileVersionId = fileId, GeographicLevel = gl, }).ToList(); - await contentDbContext.DataSetFileGeographicLevels.AddRangeAsync(dataSetFileGeographicLevels); + contentDbContext.DataSetFileVersionGeographicLevels.AddRange( + dataSetFileVersionGeographicLevels); await contentDbContext.SaveChangesAsync(); } diff --git a/src/explore-education-statistics-common/src/utils/locationLevelsMap.ts b/src/explore-education-statistics-common/src/utils/locationLevelsMap.ts index 6f54b04f835..947dd7b3049 100644 --- a/src/explore-education-statistics-common/src/utils/locationLevelsMap.ts +++ b/src/explore-education-statistics-common/src/utils/locationLevelsMap.ts @@ -22,6 +22,7 @@ export type GeographicLevelCode = | 'WARD'; export interface LocationLevelDetails { + filterLabel: string; // used in the dropdown that filters data sets by geographic level label: string; plural: string; prefix: string; @@ -30,108 +31,126 @@ export interface LocationLevelDetails { const locationLevelsMap = { country: { + filterLabel: 'National', label: 'Country', plural: 'Countries', prefix: 'a', code: 'NAT', }, englishDevolvedArea: { + filterLabel: 'English Devolved Area', label: 'English Devolved Area', plural: 'English Devolved Areas', prefix: 'an', code: 'EDA', }, institution: { + filterLabel: 'Institution', label: 'Institution', plural: 'Institutions', prefix: 'an', code: 'INST', }, localAuthority: { + filterLabel: 'Local Authority', label: 'Local Authority', plural: 'Local Authorities', prefix: 'a', code: 'LA', }, localAuthorityDistrict: { + filterLabel: 'Local Authority District', label: 'Local Authority District', plural: 'Local Authority Districts', prefix: 'a', code: 'LAD', }, localEnterprisePartnership: { + filterLabel: 'Local Enterprise Partnership', label: 'Local Enterprise Partnership', plural: 'Local Enterprise Partnerships', prefix: 'a', code: 'LEP', }, localSkillsImprovementPlanArea: { + filterLabel: 'Local Skills Improvement Plan Area', label: 'Local Skills Improvement Plan Area', plural: 'Local Skills Improvement Plan Areas', prefix: 'a', code: 'LSIP', }, mayoralCombinedAuthority: { + filterLabel: 'Mayoral Combined Authority', label: 'Mayoral Combined Authority', plural: 'Mayoral Combined Authorities', prefix: 'a', code: 'MCA', }, multiAcademyTrust: { + filterLabel: 'Multi Academy Trust', label: 'Multi Academy Trust', plural: 'Multi Academy Trusts', prefix: 'a', code: 'MAT', }, opportunityArea: { + filterLabel: 'Opportunity Area', label: 'Opportunity Area', plural: 'Opportunity Areas', prefix: 'an', code: 'OA', }, parliamentaryConstituency: { + filterLabel: 'Parliamentary Constituency', label: 'Parliamentary Constituency', plural: 'Parliamentary Constituencies', prefix: 'a', code: 'PCON', }, planningArea: { + filterLabel: 'Planning Area', label: 'Planning Area', plural: 'Planning Areas', prefix: 'a', code: 'PA', }, provider: { + filterLabel: 'Provider', label: 'Provider', plural: 'Providers', prefix: 'a', code: 'PROV', }, region: { + filterLabel: 'Region', label: 'Region', plural: 'Regions', prefix: 'a', code: 'REG', }, rscRegion: { + filterLabel: 'RSC Region', label: 'RSC Region', plural: 'RSC Regions', prefix: 'an', code: 'RSC', }, school: { + filterLabel: 'School', label: 'School', plural: 'Schools', prefix: 'a', code: 'SCH', }, sponsor: { + filterLabel: 'Sponsor', label: 'Sponsor', plural: 'Sponsors', prefix: 'a', code: 'SPON', }, ward: { + filterLabel: 'Ward', label: 'Ward', plural: 'Wards', prefix: 'a', diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePage.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePage.tsx index c14872aa2e0..039b1dd1c12 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePage.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataCataloguePage.tsx @@ -52,7 +52,11 @@ import omit from 'lodash/omit'; import Head from 'next/head'; import { useRouter } from 'next/router'; import { ParsedUrlQuery } from 'querystring'; -import { GeographicLevelCode } from '@common/utils/locationLevelsMap'; +import locationLevelsMap, { + GeographicLevelCode, + geographicLevelCodesMap, + LocationLevelKey, +} from '@common/utils/locationLevelsMap'; const defaultPageTitle = 'Data catalogue'; @@ -124,11 +128,16 @@ const DataCataloguePage: NextPage = ({ showTypeFilter }) => { const selectedRelease = releases.find(release => release.id === releaseId); + const selectedGeographicLevel = geographicLevel + ? geographicLevelCodesMap[geographicLevel] + : undefined; + const { paging, results: dataSets = [] } = dataSetsData ?? {}; const { page, totalPages, totalResults = 0 } = paging ?? {}; const [showAllDetails, toggleAllDetails] = useToggle(false); - const isFiltered = !!publicationId || !!searchTerm || !!themeId; + const isFiltered = + !!publicationId || !!searchTerm || !!themeId || !!geographicLevel; const filteredByString = compact([ searchTerm, @@ -420,6 +429,15 @@ const DataCataloguePage: NextPage = ({ showTypeFilter }) => { } /> )} + {selectedGeographicLevel && ( + + handleResetFilter({ filterType: 'geographicLevel' }) + } + /> + )} )} diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/Filters.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/Filters.tsx index c564cbc8fd5..a5e34d16eb0 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/Filters.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/Filters.tsx @@ -160,12 +160,12 @@ export default function Filters({ { label: 'All', value: 'all' }, ...typedKeys(locationLevelsMap).map(key => { return { - label: locationLevelsMap[key].label, + label: locationLevelsMap[key].filterLabel, value: locationLevelsMap[key].code, }; }), ]} - value={geographicLevel} + value={geographicLevel ?? 'all'} order={[]} onChange={e => { onChange({ diff --git a/tests/robot-tests/tests/admin_and_public/bau/data_catalogue.robot b/tests/robot-tests/tests/admin_and_public/bau/data_catalogue.robot index d1781d21f42..ceff983c154 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/data_catalogue.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/data_catalogue.robot @@ -212,6 +212,22 @@ Remove release filter user checks page does not contain button ${PUPIL_ABSENCE_RELEASE_NAME} user checks selected option label id:filters-form-release All releases +Filter by geographic level + user wait for option to be available and select it css:select[id="filters-form-geographic-level"] + ... Local Authority District + + user checks page contains button Local Authority District + user checks testid element contains total-results 1 data set + +Remove geographic level filter + user clicks button Local Authority District + + user checks page does not contain button Local Authority District + user checks selected option label id:filters-form-geographic-level All + + user checks page contains button ${PUPILS_AND_SCHOOLS_THEME_TITLE} + user checks page contains button ${PUPIL_ABSENCE_PUBLICATION_TITLE} + Reset all filters user clicks element id:searchForm-search user presses keys pupil From 1f7d7164e26fe710e80ed7f3205c4b00b6d0eb41 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Tue, 10 Dec 2024 10:30:50 +0000 Subject: [PATCH 123/144] EES-5656 Change PublicationService to use primary constructor --- .../Services/PublicationService.cs | 214 ++++++++---------- 1 file changed, 95 insertions(+), 119 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs index 1341e018e56..f3cb0a8689d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs @@ -31,59 +31,35 @@ namespace GovUk.Education.ExploreEducationStatistics.Admin.Services { - public class PublicationService : IPublicationService + public class PublicationService( + ContentDbContext context, + IMapper mapper, + IPersistenceHelper persistenceHelper, + IUserService userService, + IPublicationRepository publicationRepository, + IReleaseVersionRepository releaseVersionRepository, + IMethodologyService methodologyService, + IPublicationCacheService publicationCacheService, + IMethodologyCacheService methodologyCacheService, + IRedirectsCacheService redirectsCacheService) + : IPublicationService { - private readonly ContentDbContext _context; - private readonly IMapper _mapper; - private readonly IPersistenceHelper _persistenceHelper; - private readonly IUserService _userService; - private readonly IPublicationRepository _publicationRepository; - private readonly IReleaseVersionRepository _releaseVersionRepository; - private readonly IMethodologyService _methodologyService; - private readonly IPublicationCacheService _publicationCacheService; - private readonly IMethodologyCacheService _methodologyCacheService; - private readonly IRedirectsCacheService _redirectsCacheService; - - public PublicationService( - ContentDbContext context, - IMapper mapper, - IPersistenceHelper persistenceHelper, - IUserService userService, - IPublicationRepository publicationRepository, - IReleaseVersionRepository releaseVersionRepository, - IMethodologyService methodologyService, - IPublicationCacheService publicationCacheService, - IMethodologyCacheService methodologyCacheService, - IRedirectsCacheService redirectsCacheService) - { - _context = context; - _mapper = mapper; - _persistenceHelper = persistenceHelper; - _userService = userService; - _publicationRepository = publicationRepository; - _releaseVersionRepository = releaseVersionRepository; - _methodologyService = methodologyService; - _publicationCacheService = publicationCacheService; - _methodologyCacheService = methodologyCacheService; - _redirectsCacheService = redirectsCacheService; - } - public async Task>> ListPublications( Guid? themeId = null) { - return await _userService + return await userService .CheckCanAccessSystem() - .OnSuccess(_ => _userService.CheckCanViewAllPublications() + .OnSuccess(_ => userService.CheckCanViewAllPublications() .OnSuccess(async () => { var hydratedPublication = HydratePublication( - _publicationRepository.QueryPublicationsForTheme(themeId)); + publicationRepository.QueryPublicationsForTheme(themeId)); return await hydratedPublication.ToListAsync(); }) .OrElse(() => { - var userId = _userService.GetUserId(); - return _publicationRepository.ListPublicationsForUser(userId, themeId); + var userId = userService.GetUserId(); + return publicationRepository.ListPublicationsForUser(userId, themeId); }) ) .OnSuccess(async publications => @@ -98,11 +74,11 @@ public async Task>> ListPublicat public async Task>> ListPublicationSummaries() { - return await _userService + return await userService .CheckCanViewAllPublications() .OnSuccess(_ => { - return _context.Publications + return context.Publications .Select(publication => new PublicationSummaryViewModel(publication)) .ToList(); }); @@ -115,7 +91,7 @@ public async Task> CreatePublic .OnSuccess(_ => ValidatePublicationSlug(publication.Slug)) .OnSuccess(async _ => { - var contact = await _context.Contacts.AddAsync(new Contact + var contact = await context.Contacts.AddAsync(new Contact { ContactName = publication.Contact.ContactName, ContactTelNo = string.IsNullOrWhiteSpace(publication.Contact.ContactTelNo) @@ -125,7 +101,7 @@ public async Task> CreatePublic TeamEmail = publication.Contact.TeamEmail }); - var saved = await _context.Publications.AddAsync(new Publication + var saved = await context.Publications.AddAsync(new Publication { Contact = contact.Entity, Title = publication.Title, @@ -134,9 +110,9 @@ public async Task> CreatePublic Slug = publication.Slug, }); - await _context.SaveChangesAsync(); + await context.SaveChangesAsync(); - return await _persistenceHelper + return await persistenceHelper .CheckEntityExists(saved.Entity.Id, HydratePublication) .OnSuccess(GeneratePublicationCreateViewModel); }); @@ -146,14 +122,14 @@ public async Task> UpdatePublication( Guid publicationId, PublicationSaveRequest updatedPublication) { - return await _persistenceHelper + return await persistenceHelper .CheckEntityExists(publicationId) - .OnSuccess(_userService.CheckCanUpdatePublicationSummary) + .OnSuccess(userService.CheckCanUpdatePublicationSummary) .OnSuccessDo(async publication => { if (publication.Title != updatedPublication.Title) { - return await _userService.CheckCanUpdatePublication(); + return await userService.CheckCanUpdatePublication(); } return Unit.Instance; @@ -162,7 +138,7 @@ public async Task> UpdatePublication( { if (publication.SupersededById != updatedPublication.SupersededById) { - return await _userService.CheckCanUpdatePublication(); + return await userService.CheckCanUpdatePublication(); } return Unit.Instance; @@ -180,7 +156,7 @@ public async Task> UpdatePublication( { if (publication.ThemeId != updatedPublication.ThemeId) { - return await _userService.CheckCanUpdatePublication(); + return await userService.CheckCanUpdatePublication(); } return Unit.Instance; @@ -206,7 +182,7 @@ public async Task> UpdatePublication( publication.Slug = updatedPublication.Slug; if (publication.Live - && _context.PublicationRedirects.All(pr => + && context.PublicationRedirects.All(pr => !(pr.PublicationId == publicationId && pr.Slug == originalSlug))) // don't create duplicate redirect { var publicationRedirect = new PublicationRedirect @@ -215,16 +191,16 @@ public async Task> UpdatePublication( Publication = publication, PublicationId = publication.Id, }; - _context.PublicationRedirects.Add(publicationRedirect); + context.PublicationRedirects.Add(publicationRedirect); } // If there is an existing redirects for the new slug, they're redundant. Remove them - var redundantRedirects = await _context.PublicationRedirects + var redundantRedirects = await context.PublicationRedirects .Where(pr => pr.Slug == updatedPublication.Slug) .ToListAsync(); if (redundantRedirects.Count > 0) { - _context.PublicationRedirects.RemoveRange(redundantRedirects); + context.PublicationRedirects.RemoveRange(redundantRedirects); } } @@ -234,13 +210,13 @@ public async Task> UpdatePublication( publication.Updated = DateTime.UtcNow; publication.SupersededById = updatedPublication.SupersededById; - _context.Publications.Update(publication); + context.Publications.Update(publication); - await _context.SaveChangesAsync(); + await context.SaveChangesAsync(); if (titleChanged || slugChanged) { - await _methodologyService.PublicationTitleOrSlugChanged(publicationId, + await methodologyService.PublicationTitleOrSlugChanged(publicationId, originalSlug, publication.Title, publication.Slug); @@ -248,14 +224,14 @@ await _methodologyService.PublicationTitleOrSlugChanged(publicationId, if (publication.Live) { - await _methodologyCacheService.UpdateSummariesTree(); - await _publicationCacheService.UpdatePublicationTree(); - await _publicationCacheService.UpdatePublication(publication.Slug); + await methodologyCacheService.UpdateSummariesTree(); + await publicationCacheService.UpdatePublicationTree(); + await publicationCacheService.UpdatePublication(publication.Slug); if (slugChanged) { - await _publicationCacheService.RemovePublication(originalSlug); - await _redirectsCacheService.UpdateRedirects(); + await publicationCacheService.RemovePublication(originalSlug); + await redirectsCacheService.UpdateRedirects(); } await UpdateCachedSupersededPublications(publication); @@ -269,42 +245,42 @@ private async Task UpdateCachedSupersededPublications(Publication publication) { // NOTE: When a publication is updated, any publication that is superseded by it can be affected, so // update any superseded publications that are cached - var supersededPublications = await _context.Publications + var supersededPublications = await context.Publications .Where(p => p.SupersededById == publication.Id) .ToListAsync(); await supersededPublications .ToAsyncEnumerable() - .ForEachAwaitAsync(p => _publicationCacheService.UpdatePublication(p.Slug)); + .ForEachAwaitAsync(p => publicationCacheService.UpdatePublication(p.Slug)); } private async Task> ValidateSelectedTheme(Guid themeId) { - var theme = await _context.Themes.FindAsync(themeId); + var theme = await context.Themes.FindAsync(themeId); if (theme is null) { return ValidationActionResult(ThemeDoesNotExist); } - return await _userService.CheckCanCreatePublicationForTheme(theme) + return await userService.CheckCanCreatePublicationForTheme(theme) .OnSuccess(_ => Unit.Instance); } public async Task> GetPublication( Guid publicationId, bool includePermissions = false) { - return await _persistenceHelper + return await persistenceHelper .CheckEntityExists(publicationId, HydratePublication) - .OnSuccess(_userService.CheckCanViewPublication) + .OnSuccess(userService.CheckCanViewPublication) .OnSuccess(publication => GeneratePublicationViewModel(publication, includePermissions)); } public async Task> GetExternalMethodology(Guid publicationId) { - return await _persistenceHelper + return await persistenceHelper .CheckEntityExists(publicationId) - .OnSuccessDo(_userService.CheckCanViewPublication) + .OnSuccessDo(userService.CheckCanViewPublication) .OnSuccess(publication => publication.ExternalMethodology != null ? new ExternalMethodologyViewModel(publication.ExternalMethodology) : NotFound()); @@ -313,19 +289,19 @@ public async Task> GetExterna public async Task> UpdateExternalMethodology( Guid publicationId, ExternalMethodologySaveRequest updatedExternalMethodology) { - return await _persistenceHelper + return await persistenceHelper .CheckEntityExists(publicationId) - .OnSuccessDo(_userService.CheckCanManageExternalMethodologyForPublication) + .OnSuccessDo(userService.CheckCanManageExternalMethodologyForPublication) .OnSuccess(async publication => { - _context.Update(publication); + context.Update(publication); publication.ExternalMethodology ??= new ExternalMethodology(); publication.ExternalMethodology.Title = updatedExternalMethodology.Title; publication.ExternalMethodology.Url = updatedExternalMethodology.Url; - await _context.SaveChangesAsync(); + await context.SaveChangesAsync(); // Update publication cache because ExternalMethodology is in Content.Services.ViewModels.PublicationViewModel - await _publicationCacheService.UpdatePublication(publication.Slug); + await publicationCacheService.UpdatePublication(publication.Slug); return new ExternalMethodologyViewModel(publication.ExternalMethodology); }); @@ -334,17 +310,17 @@ public async Task> UpdateExte public async Task> RemoveExternalMethodology( Guid publicationId) { - return await _persistenceHelper + return await persistenceHelper .CheckEntityExists(publicationId) - .OnSuccessDo(_userService.CheckCanManageExternalMethodologyForPublication) + .OnSuccessDo(userService.CheckCanManageExternalMethodologyForPublication) .OnSuccess(async publication => { - _context.Update(publication); + context.Update(publication); publication.ExternalMethodology = null; - await _context.SaveChangesAsync(); + await context.SaveChangesAsync(); // Clear cache because ExternalMethodology is in Content.Services.ViewModels.PublicationViewModel - await _publicationCacheService.UpdatePublication(publication.Slug); + await publicationCacheService.UpdatePublication(publication.Slug); return Unit.Instance; }); @@ -352,24 +328,24 @@ public async Task> RemoveExternalMethodology( public async Task> GetContact(Guid publicationId) { - return await _persistenceHelper + return await persistenceHelper .CheckEntityExists(publicationId, query => query.Include(p => p.Contact)) - .OnSuccessDo(_userService.CheckCanViewPublication) - .OnSuccess(publication => _mapper.Map(publication.Contact)); + .OnSuccessDo(userService.CheckCanViewPublication) + .OnSuccess(publication => mapper.Map(publication.Contact)); } public async Task> UpdateContact(Guid publicationId, ContactSaveRequest updatedContact) { - return await _persistenceHelper + return await persistenceHelper .CheckEntityExists(publicationId, query => query.Include(p => p.Contact)) - .OnSuccessDo(_userService.CheckCanUpdateContact) + .OnSuccessDo(userService.CheckCanUpdateContact) .OnSuccess(async publication => { // Replace existing contact that is shared with another publication with a new // contact, as we want each publication to have its own contact. - if (_context.Publications + if (context.Publications .Any(p => p.ContactId == publication.ContactId && p.Id != publication.Id)) { publication.Contact = new Contact(); @@ -381,12 +357,12 @@ public async Task> UpdateContact(Guid pub : updatedContact.ContactTelNo; publication.Contact.TeamName = updatedContact.TeamName; publication.Contact.TeamEmail = updatedContact.TeamEmail; - await _context.SaveChangesAsync(); + await context.SaveChangesAsync(); // Clear cache because Contact is in Content.Services.ViewModels.PublicationViewModel - await _publicationCacheService.UpdatePublication(publication.Slug); + await publicationCacheService.UpdatePublication(publication.Slug); - return _mapper.Map(publication.Contact); + return mapper.Map(publication.Contact); }); } @@ -415,14 +391,14 @@ public async Task>> ListLates bool? live = null, bool includePermissions = false) { - return await _persistenceHelper + return await persistenceHelper .CheckEntityExists(publicationId) - .OnSuccess(_userService.CheckCanViewPublication) + .OnSuccess(userService.CheckCanViewPublication) .OnSuccess(async () => { // Note the 'live' filter is applied after the latest release versions are retrieved. // A published release with a current draft version is deliberately not returned when 'live' is true. - var releaseVersions = (await _releaseVersionRepository.ListLatestReleaseVersions(publicationId)) + var releaseVersions = (await releaseVersionRepository.ListLatestReleaseVersions(publicationId)) .Where(rv => live == null || rv.Live == live) .ToList(); @@ -436,9 +412,9 @@ public async Task>> ListLates public async Task>> GetReleaseSeries( Guid publicationId) { - return await _context.Publications + return await context.Publications .FirstOrNotFoundAsync(p => p.Id == publicationId) - .OnSuccess(_userService.CheckCanViewPublication) + .OnSuccess(userService.CheckCanViewPublication) .OnSuccess(async publication => { var result = new List(); @@ -455,10 +431,10 @@ public async Task>> } else { - var release = await _context.Releases + var release = await context.Releases .SingleAsync(r => r.Id == seriesItem.ReleaseId); - var latestPublishedReleaseVersion = await _context.ReleaseVersions + var latestPublishedReleaseVersion = await context.ReleaseVersions .LatestReleaseVersion(releaseId: seriesItem.ReleaseId!.Value, publishedOnly: true) .SingleOrDefaultAsync(); @@ -483,9 +459,9 @@ public async Task>> Guid publicationId, ReleaseSeriesLegacyLinkAddRequest newLegacyLink) { - return await _context.Publications + return await context.Publications .FirstOrNotFoundAsync(p => p.Id == publicationId) - .OnSuccess(_userService.CheckCanManageReleaseSeries) + .OnSuccess(userService.CheckCanManageReleaseSeries) .OnSuccess(async publication => { publication.ReleaseSeries.Add(new ReleaseSeriesItem @@ -495,10 +471,10 @@ public async Task>> LegacyLinkUrl = newLegacyLink.Url, }); - _context.Publications.Update(publication); - await _context.SaveChangesAsync(); + context.Publications.Update(publication); + await context.SaveChangesAsync(); - await _publicationCacheService.UpdatePublication(publication.Slug); + await publicationCacheService.UpdatePublication(publication.Slug); return await GetReleaseSeries(publication.Id); }); @@ -508,9 +484,9 @@ public async Task>> Guid publicationId, List updatedReleaseSeriesItems) { - return await _context.Publications + return await context.Publications .FirstOrNotFoundAsync(p => p.Id == publicationId) - .OnSuccess(_userService.CheckCanManageReleaseSeries) + .OnSuccess(userService.CheckCanManageReleaseSeries) .OnSuccess(async publication => { // Check new series items details are correct @@ -530,7 +506,7 @@ public async Task>> } // Check all publication releases are included in updatedReleaseSeriesItems - var publicationReleaseIds = await _context.Releases + var publicationReleaseIds = await context.Releases .Where(r => r.PublicationId == publicationId) .Select(r => r.Id) .ToListAsync(); @@ -556,9 +532,9 @@ public async Task>> LegacyLinkUrl = request.LegacyLinkUrl, }).ToList(); - await _context.SaveChangesAsync(); + await context.SaveChangesAsync(); - await _publicationCacheService.UpdatePublication(publication.Slug); + await publicationCacheService.UpdatePublication(publication.Slug); return await GetReleaseSeries(publication.Id); }); @@ -567,7 +543,7 @@ public async Task>> private async Task> ValidatePublicationSlug( string newSlug, Guid? publicationId = null) { - if (await _context.Publications + if (await context.Publications .AnyAsync(publication => publication.Id != publicationId && publication.Slug == newSlug)) @@ -575,7 +551,7 @@ private async Task> ValidatePublicationSlug( return ValidationActionResult(PublicationSlugNotUnique); } - var hasRedirect = await _context.PublicationRedirects + var hasRedirect = await context.PublicationRedirects .AnyAsync(pr => pr.PublicationId != publicationId // If publication previously used this slug, can change it back && pr.Slug == newSlug); @@ -586,7 +562,7 @@ private async Task> ValidatePublicationSlug( } if (publicationId.HasValue && - _context.PublicationMethodologies.Any(pm => + context.PublicationMethodologies.Any(pm => pm.Publication.Id == publicationId && pm.Owner) // Strictly, we should also check whether the owned methodology inherits the publication slug - we don't @@ -594,7 +570,7 @@ private async Task> ValidatePublicationSlug( // this check is expensive and an unlikely edge case, so doesn't seem worth it. ) { - var methodologySlugValidation = await _methodologyService + var methodologySlugValidation = await methodologyService .ValidateMethodologySlug(newSlug); if (methodologySlugValidation.IsLeft) { @@ -614,14 +590,14 @@ public static IQueryable HydratePublication(IQueryable private async Task GeneratePublicationViewModel(Publication publication, bool includePermissions = false) { - var publicationViewModel = _mapper.Map(publication); + var publicationViewModel = mapper.Map(publication); - publicationViewModel.IsSuperseded = await _publicationRepository.IsSuperseded(publication.Id); + publicationViewModel.IsSuperseded = await publicationRepository.IsSuperseded(publication.Id); if (includePermissions) { publicationViewModel.Permissions = - await PermissionsUtils.GetPublicationPermissions(_userService, publication); + await PermissionsUtils.GetPublicationPermissions(userService, publication); } return publicationViewModel; @@ -629,9 +605,9 @@ private async Task GeneratePublicationViewModel(Publicatio private async Task GeneratePublicationCreateViewModel(Publication publication) { - var publicationCreateViewModel = _mapper.Map(publication); + var publicationCreateViewModel = mapper.Map(publication); - publicationCreateViewModel.IsSuperseded = await _publicationRepository.IsSuperseded(publication.Id); + publicationCreateViewModel.IsSuperseded = await publicationRepository.IsSuperseded(publication.Id); return publicationCreateViewModel; } @@ -639,11 +615,11 @@ private async Task GeneratePublicationCreateViewMode private async Task HydrateReleaseListItemViewModel(ReleaseVersion releaseVersion, bool includePermissions) { - var viewModel = _mapper.Map(releaseVersion); + var viewModel = mapper.Map(releaseVersion); if (includePermissions) { - viewModel.Permissions = await PermissionsUtils.GetReleasePermissions(_userService, releaseVersion); + viewModel.Permissions = await PermissionsUtils.GetReleasePermissions(userService, releaseVersion); } return viewModel; From 3043196c08ad068ac2702c469f5083ee0880067a Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Tue, 10 Dec 2024 09:40:52 +0000 Subject: [PATCH 124/144] EES-5656 Set the latest published release version for publication based on the latest release with a published version --- .../PublicationServicePermissionTests.cs | 2 + .../Services/PublicationServiceTests.cs | 169 ++++++++++++++++-- .../Services/PublicationService.cs | 29 +++ 3 files changed, 184 insertions(+), 16 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServicePermissionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServicePermissionTests.cs index 75477c2195e..62cc49678fc 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServicePermissionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServicePermissionTests.cs @@ -627,6 +627,7 @@ private static PublicationService BuildPublicationService( IReleaseVersionRepository? releaseVersionRepository = null, IMethodologyService? methodologyService = null, IPublicationCacheService? publicationCacheService = null, + IReleaseCacheService? releaseCacheService = null, IMethodologyCacheService? methodologyCacheService = null, IRedirectsCacheService? redirectsCacheService = null) { @@ -641,6 +642,7 @@ private static PublicationService BuildPublicationService( releaseVersionRepository ?? Mock.Of(Strict), methodologyService ?? Mock.Of(Strict), publicationCacheService ?? Mock.Of(Strict), + releaseCacheService ?? Mock.Of(Strict), methodologyCacheService ?? Mock.Of(Strict), redirectsCacheService ?? Mock.Of(Strict)); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs index 39fb7872c3f..768316817bd 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs @@ -2942,7 +2942,7 @@ public async Task UpdateReleaseSeries() _dataFixture .DefaultRelease(publishedVersions: 1, year: 2020), _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), + .DefaultRelease(publishedVersions: 1, draftVersion: true, year: 2021), _dataFixture .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022) ]) @@ -2953,6 +2953,16 @@ public async Task UpdateReleaseSeries() var release2021 = publication.Releases.Single(r => r.Year == 2021); var release2022 = publication.Releases.Single(r => r.Year == 2022); + // Check the publication's latest published release version in the generated test data setup + Assert.Equal(release2022.Versions[1].Id, publication.LatestPublishedReleaseVersionId); + + // Check the expected order of the release series items in the generated test data setup + Assert.Equal(4, publication.ReleaseSeries.Count); + Assert.Equal(release2022.Id, publication.ReleaseSeries[0].ReleaseId); + Assert.Equal(release2021.Id, publication.ReleaseSeries[1].ReleaseId); + Assert.Equal(release2020.Id, publication.ReleaseSeries[2].ReleaseId); + Assert.True(publication.ReleaseSeries[3].IsLegacyLink); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) { @@ -2978,11 +2988,11 @@ public async Task UpdateReleaseSeries() new ReleaseSeriesItemUpdateRequest { LegacyLinkDescription = "Legacy link new", - LegacyLinkUrl = "https://test.com/new", + LegacyLinkUrl = "https://test.com/new" }, new ReleaseSeriesItemUpdateRequest { - ReleaseId = release2021.Id + ReleaseId = release2022.Id }, new ReleaseSeriesItemUpdateRequest { @@ -2990,14 +3000,13 @@ public async Task UpdateReleaseSeries() }, new ReleaseSeriesItemUpdateRequest { - ReleaseId = release2022.Id + ReleaseId = release2021.Id } ]); VerifyAllMocks(publicationCacheService); var viewModels = result.AssertRight(); - Assert.Equal(4, viewModels.Count); Assert.True(viewModels[0].IsLegacyLink); @@ -3009,11 +3018,11 @@ public async Task UpdateReleaseSeries() Assert.Equal("https://test.com/new", viewModels[0].LegacyLinkUrl); Assert.False(viewModels[1].IsLegacyLink); - Assert.Equal(release2021.Title, viewModels[1].Description); - Assert.Equal(release2021.Id, viewModels[1].ReleaseId); - Assert.Equal(release2021.Slug, viewModels[1].ReleaseSlug); - Assert.False(viewModels[1].IsLatest); - Assert.False(viewModels[1].IsPublished); + Assert.Equal(release2022.Title, viewModels[1].Description); + Assert.Equal(release2022.Id, viewModels[1].ReleaseId); + Assert.Equal(release2022.Slug, viewModels[1].ReleaseSlug); + Assert.True(viewModels[1].IsLatest); + Assert.True(viewModels[1].IsPublished); Assert.Null(viewModels[1].LegacyLinkUrl); Assert.False(viewModels[2].IsLegacyLink); @@ -3025,10 +3034,10 @@ public async Task UpdateReleaseSeries() Assert.Null(viewModels[2].LegacyLinkUrl); Assert.False(viewModels[3].IsLegacyLink); - Assert.Equal(release2022.Title, viewModels[3].Description); - Assert.Equal(release2022.Id, viewModels[3].ReleaseId); - Assert.Equal(release2022.Slug, viewModels[3].ReleaseSlug); - Assert.True(viewModels[3].IsLatest); + Assert.Equal(release2021.Title, viewModels[3].Description); + Assert.Equal(release2021.Id, viewModels[3].ReleaseId); + Assert.Equal(release2021.Slug, viewModels[3].ReleaseSlug); + Assert.False(viewModels[3].IsLatest); Assert.True(viewModels[3].IsPublished); Assert.Null(viewModels[3].LegacyLinkUrl); } @@ -3044,9 +3053,135 @@ public async Task UpdateReleaseSeries() Assert.Equal("Legacy link new", actualReleaseSeries[0].LegacyLinkDescription); Assert.Equal("https://test.com/new", actualReleaseSeries[0].LegacyLinkUrl); - Assert.Equal(release2021.Id, actualReleaseSeries[1].ReleaseId); + Assert.Equal(release2022.Id, actualReleaseSeries[1].ReleaseId); Assert.Equal(release2020.Id, actualReleaseSeries[2].ReleaseId); - Assert.Equal(release2022.Id, actualReleaseSeries[3].ReleaseId); + Assert.Equal(release2021.Id, actualReleaseSeries[3].ReleaseId); + + // The publication's latest published release version should be unchanged as 2022 was positioned + // as the first release after the legacy link + Assert.Equal(release2022.Versions[1].Id, publication.LatestPublishedReleaseVersionId); + } + } + + [Fact] + public async Task UpdateReleaseSeries_UpdatesLatestPublishedReleaseVersion() + { + Publication publication = _dataFixture + .DefaultPublication() + .WithReleases([ + _dataFixture + .DefaultRelease(publishedVersions: 1, year: 2020), + _dataFixture + .DefaultRelease(publishedVersions: 1, draftVersion: true, year: 2021), + _dataFixture + .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022) + ]) + .WithTheme(_dataFixture.DefaultTheme()); + + var release2020 = publication.Releases.Single(r => r.Year == 2020); + var release2021 = publication.Releases.Single(r => r.Year == 2021); + var release2022 = publication.Releases.Single(r => r.Year == 2022); + + var expectedLatestPublishedReleaseVersionId = release2021.Versions[0].Id; + + // Check the publication's latest published release version in the generated test data setup + Assert.Equal(release2022.Versions[1].Id, publication.LatestPublishedReleaseVersionId); + + // Check the expected order of the release series items in the generated test data setup + Assert.Equal(3, publication.ReleaseSeries.Count); + Assert.Equal(release2022.Id, publication.ReleaseSeries[0].ReleaseId); + Assert.Equal(release2021.Id, publication.ReleaseSeries[1].ReleaseId); + Assert.Equal(release2020.Id, publication.ReleaseSeries[2].ReleaseId); + + var contentDbContextId = Guid.NewGuid().ToString(); + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + contentDbContext.Publications.Add(publication); + await contentDbContext.SaveChangesAsync(); + } + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + var publicationCacheService = new Mock(Strict); + publicationCacheService.Setup(mock => + mock.UpdatePublication(publication.Slug)) + .ReturnsAsync(new PublicationCacheViewModel()); + + var releaseCacheService = new Mock(Strict); + releaseCacheService.Setup(mock => mock.UpdateRelease( + expectedLatestPublishedReleaseVersionId, + publication.Slug, + null)) + .ReturnsAsync(new ReleaseCacheViewModel(expectedLatestPublishedReleaseVersionId)); + + var publicationService = BuildPublicationService( + contentDbContext, + publicationCacheService: publicationCacheService.Object, + releaseCacheService: releaseCacheService.Object); + + var result = await publicationService.UpdateReleaseSeries( + publication.Id, + updatedReleaseSeriesItems: + [ + new ReleaseSeriesItemUpdateRequest + { + ReleaseId = release2021.Id + }, + new ReleaseSeriesItemUpdateRequest + { + ReleaseId = release2020.Id + }, + new ReleaseSeriesItemUpdateRequest + { + ReleaseId = release2022.Id + } + ]); + + VerifyAllMocks(publicationCacheService, releaseCacheService); + + var viewModels = result.AssertRight(); + Assert.Equal(3, viewModels.Count); + + Assert.False(viewModels[0].IsLegacyLink); + Assert.Equal(release2021.Title, viewModels[0].Description); + Assert.Equal(release2021.Id, viewModels[0].ReleaseId); + Assert.Equal(release2021.Slug, viewModels[0].ReleaseSlug); + Assert.True(viewModels[0].IsLatest); + Assert.True(viewModels[0].IsPublished); + Assert.Null(viewModels[0].LegacyLinkUrl); + + Assert.False(viewModels[1].IsLegacyLink); + Assert.Equal(release2020.Title, viewModels[1].Description); + Assert.Equal(release2020.Id, viewModels[1].ReleaseId); + Assert.Equal(release2020.Slug, viewModels[1].ReleaseSlug); + Assert.False(viewModels[1].IsLatest); + Assert.True(viewModels[1].IsPublished); + Assert.Null(viewModels[1].LegacyLinkUrl); + + Assert.False(viewModels[2].IsLegacyLink); + Assert.Equal(release2022.Title, viewModels[2].Description); + Assert.Equal(release2022.Id, viewModels[2].ReleaseId); + Assert.Equal(release2022.Slug, viewModels[2].ReleaseSlug); + Assert.False(viewModels[2].IsLatest); + Assert.True(viewModels[2].IsPublished); + Assert.Null(viewModels[2].LegacyLinkUrl); + } + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + var actualPublication = await contentDbContext.Publications + .SingleAsync(p => p.Id == publication.Id); + + var actualReleaseSeries = actualPublication.ReleaseSeries; + Assert.Equal(3, actualReleaseSeries.Count); + + Assert.Equal(release2021.Id, actualReleaseSeries[0].ReleaseId); + Assert.Equal(release2020.Id, actualReleaseSeries[1].ReleaseId); + Assert.Equal(release2022.Id, actualReleaseSeries[2].ReleaseId); + + // The latest published version of 2021 should now be the publication's latest published release + // version since it was positioned as the first release + Assert.Equal(release2022.Versions[1].Id, publication.LatestPublishedReleaseVersionId); } } @@ -3246,6 +3381,7 @@ private static PublicationService BuildPublicationService( IReleaseVersionRepository? releaseVersionRepository = null, IMethodologyService? methodologyService = null, IPublicationCacheService? publicationCacheService = null, + IReleaseCacheService? releaseCacheService = null, IMethodologyCacheService? methodologyCacheService = null, IRedirectsCacheService? redirectsCacheService = null) { @@ -3258,6 +3394,7 @@ private static PublicationService BuildPublicationService( releaseVersionRepository ?? new ReleaseVersionRepository(context), methodologyService ?? Mock.Of(Strict), publicationCacheService ?? Mock.Of(Strict), + releaseCacheService ?? Mock.Of(Strict), methodologyCacheService ?? Mock.Of(Strict), redirectsCacheService ?? Mock.Of(Strict)); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs index f3cb0a8689d..43b256454ac 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/PublicationService.cs @@ -40,6 +40,7 @@ public class PublicationService( IReleaseVersionRepository releaseVersionRepository, IMethodologyService methodologyService, IPublicationCacheService publicationCacheService, + IReleaseCacheService releaseCacheService, IMethodologyCacheService methodologyCacheService, IRedirectsCacheService redirectsCacheService) : IPublicationService @@ -523,6 +524,24 @@ public async Task>> publicationReleaseIds.JoinToString(",")); } + // Work out the publication's new latest published release version (if any). + // This is the latest published version of the first release which has a published version + Guid? latestPublishedReleaseVersionId = null; + foreach (var releaseId in updatedSeriesReleaseIds) + { + latestPublishedReleaseVersionId = (await context.ReleaseVersions + .LatestReleaseVersion(releaseId: releaseId, publishedOnly: true) + .SingleOrDefaultAsync())?.Id; + + if (latestPublishedReleaseVersionId != null) + { + break; + } + } + + var oldLatestPublishedReleaseVersionId = publication.LatestPublishedReleaseVersionId; + publication.LatestPublishedReleaseVersionId = latestPublishedReleaseVersionId; + publication.ReleaseSeries = updatedReleaseSeriesItems .Select(request => new ReleaseSeriesItem { @@ -534,8 +553,18 @@ public async Task>> await context.SaveChangesAsync(); + // Update the cached publication await publicationCacheService.UpdatePublication(publication.Slug); + // If the publication's latest published release version has changed, + // update the publication's cached latest release version + if (oldLatestPublishedReleaseVersionId != latestPublishedReleaseVersionId && + latestPublishedReleaseVersionId.HasValue) + { + await releaseCacheService.UpdateRelease(releaseVersionId: latestPublishedReleaseVersionId.Value, + publicationSlug: publication.Slug); + } + return await GetReleaseSeries(publication.Id); }); } From a12a20037f8ad81852eeb58a3126f7cd4c85eb5f Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Thu, 12 Dec 2024 09:46:04 +0000 Subject: [PATCH 125/144] EES-5656 Add UI tests to cover changing a publication's latest published release --- .../bau/release_reordering.robot | 75 ++++++++++++++++++- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot b/tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot index 54fcece6839..9daf295d23d 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot @@ -136,8 +136,13 @@ Validate first release has latest release status in publication release order user checks table cell contains 3 4 Edit user checks table cell contains 3 4 Delete -Validate other releases section on public frontend +Navigate to first published release on public frontend user navigates to public frontend ${PUBLIC_RELEASE_1_URL} + +Validate first published release on public frontend is the latest data + user checks page contains This is the latest data + +Validate other releases section of first published release includes legacy releases user checks number of other releases is correct 2 ${view_releases}= user opens details dropdown View releases (2) @@ -230,8 +235,13 @@ Validate reordered publication releases user checks table cell contains 3 2 ${PUBLIC_RELEASE_1_URL} user checks table cell contains 3 3 Latest release -Validate other releases section on public frontend includes updated legacy release with expected order +Navigate to first published release on public frontend after reordering user navigates to public frontend ${PUBLIC_RELEASE_1_URL} + +Validate first published release is the latest data after reordering + user checks page contains This is the latest data + +Validate other releases section of first published release contains updated legacy release in expected order user checks number of other releases is correct 2 ${view_releases}= user opens details dropdown View releases (2) @@ -317,8 +327,13 @@ Validate second release has latest release status in publication release order user checks table cell contains 4 2 ${PUBLIC_RELEASE_1_URL} user checks table cell does not contain 4 3 Latest release -Validate other releases section on public frontend includes first release with expected order +Navigate to second published release on public frontend user navigates to public frontend ${PUBLIC_RELEASE_2_URL} + +Validate second published release is the latest data + user checks page contains This is the latest data + +Validate other releases section of second published release includes first release with expected order user checks number of other releases is correct 3 ${view_releases}= user opens details dropdown View releases (3) @@ -392,10 +407,62 @@ Validate first legacy release is deleted from publication release order user checks table cell contains 3 2 ${PUBLIC_RELEASE_1_URL} user checks table cell does not contain 3 3 Latest release -Validate other releases section on public frontend does not include first legacy release +Navigate to second published release on public frontend after deleting legacy release user navigates to public frontend ${PUBLIC_RELEASE_2_URL} + +Validate other releases section of second published release does not include first legacy release user checks number of other releases is correct 2 ${view_releases}= user opens details dropdown View releases (2) user checks other release is shown in position ${LEGACY_RELEASE_2_DESCRIPTION} 1 user checks other release is shown in position ${RELEASE_1_NAME} 2 + +Reorder the publication releases so the first release is the latest release + user navigates to publication page from dashboard ${PUBLICATION_NAME} + user clicks link Release order + user waits until h2 is visible Release order + + user clicks button Reorder releases + ${modal}= user waits until modal is visible Reorder releases + user clicks button OK ${modal} + user waits until modal is not visible Reorder releases + user waits until page contains button Confirm order + + click element xpath://div[text()="${RELEASE_1_NAME}"] CTRL + user presses keys ${SPACE} + user presses keys ARROW_UP + user presses keys ARROW_UP + user presses keys ${SPACE} + + user clicks button Confirm order + sleep 2 + +Validate first release has latest release status in publication release order after reordering + user waits until page contains button Reorder releases + user checks table body has x rows 3 testid:release-series + + user checks table cell contains 1 1 ${RELEASE_1_NAME} + user checks table cell contains 1 2 ${PUBLIC_RELEASE_1_URL} + user checks table cell contains 1 3 Latest release + + user checks table cell contains 2 1 ${RELEASE_2_NAME} + user checks table cell contains 2 2 ${PUBLIC_RELEASE_2_URL} + user checks table cell does not contain 2 3 Latest release + + user checks table cell contains 3 1 ${LEGACY_RELEASE_2_DESCRIPTION} + user checks table cell contains 3 2 ${LEGACY_RELEASE_2_URL} + user checks table cell contains 3 3 Legacy release + user checks table cell contains 3 4 Edit + user checks table cell contains 3 4 Delete + +Navigate to first published release on public frontend after changing the latest release + user navigates to public frontend ${PUBLIC_RELEASE_1_URL} + +Validate first published release is the latest data after changing the latest release + user checks page contains This is the latest data + +Navigate to second published release on public frontend after changing the latest release + user navigates to public frontend ${PUBLIC_RELEASE_2_URL} + +Validate second published release is not the latest data after changing the latest release + user checks page contains This is not the latest data From a15bf4fb93cd13cfe670ade6b8f874836898e70d Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Thu, 12 Dec 2024 15:43:10 +0000 Subject: [PATCH 126/144] EES-5656 Correct mistake in UpdateReleaseSeries_UpdatesLatestPublishedReleaseVersion --- .../Services/PublicationServiceTests.cs | 45 +++++-------------- 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs index 768316817bd..7af28748d0e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs @@ -2990,18 +2990,9 @@ public async Task UpdateReleaseSeries() LegacyLinkDescription = "Legacy link new", LegacyLinkUrl = "https://test.com/new" }, - new ReleaseSeriesItemUpdateRequest - { - ReleaseId = release2022.Id - }, - new ReleaseSeriesItemUpdateRequest - { - ReleaseId = release2020.Id - }, - new ReleaseSeriesItemUpdateRequest - { - ReleaseId = release2021.Id - } + new ReleaseSeriesItemUpdateRequest { ReleaseId = release2022.Id }, + new ReleaseSeriesItemUpdateRequest { ReleaseId = release2020.Id }, + new ReleaseSeriesItemUpdateRequest { ReleaseId = release2021.Id } ]); VerifyAllMocks(publicationCacheService); @@ -3059,7 +3050,7 @@ public async Task UpdateReleaseSeries() // The publication's latest published release version should be unchanged as 2022 was positioned // as the first release after the legacy link - Assert.Equal(release2022.Versions[1].Id, publication.LatestPublishedReleaseVersionId); + Assert.Equal(release2022.Versions[1].Id, actualPublication.LatestPublishedReleaseVersionId); } } @@ -3123,18 +3114,9 @@ public async Task UpdateReleaseSeries_UpdatesLatestPublishedReleaseVersion() publication.Id, updatedReleaseSeriesItems: [ - new ReleaseSeriesItemUpdateRequest - { - ReleaseId = release2021.Id - }, - new ReleaseSeriesItemUpdateRequest - { - ReleaseId = release2020.Id - }, - new ReleaseSeriesItemUpdateRequest - { - ReleaseId = release2022.Id - } + new ReleaseSeriesItemUpdateRequest { ReleaseId = release2021.Id }, + new ReleaseSeriesItemUpdateRequest { ReleaseId = release2020.Id }, + new ReleaseSeriesItemUpdateRequest { ReleaseId = release2022.Id } ]); VerifyAllMocks(publicationCacheService, releaseCacheService); @@ -3181,7 +3163,8 @@ public async Task UpdateReleaseSeries_UpdatesLatestPublishedReleaseVersion() // The latest published version of 2021 should now be the publication's latest published release // version since it was positioned as the first release - Assert.Equal(release2022.Versions[1].Id, publication.LatestPublishedReleaseVersionId); + Assert.Equal(expectedLatestPublishedReleaseVersionId, + actualPublication.LatestPublishedReleaseVersionId); } } @@ -3288,14 +3271,8 @@ public async Task UpdateReleaseSeries_SetDuplicateRelease() publication.Id, updatedReleaseSeriesItems: [ - new ReleaseSeriesItemUpdateRequest - { - ReleaseId = release.Id - }, - new ReleaseSeriesItemUpdateRequest - { - ReleaseId = release.Id - } + new ReleaseSeriesItemUpdateRequest { ReleaseId = release.Id }, + new ReleaseSeriesItemUpdateRequest { ReleaseId = release.Id } ])); Assert.Equal($"Missing or duplicate release in new release series. Expected ReleaseIds: {release.Id}", From 9787a3dc726c343f9071fe06bfe41cee48dd6f00 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Thu, 12 Dec 2024 16:38:40 +0000 Subject: [PATCH 127/144] EES-5656 Add additional test UpdateReleaseSeries_UpdatesLatestPublishedReleaseVersion_SkipsUnpublishedReleases and reduce level of detail checked in UpdateReleaseSeries_UpdatesLatestPublishedReleaseVersion --- .../Services/PublicationServiceTests.cs | 114 ++++++++++++++++-- 1 file changed, 102 insertions(+), 12 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs index 7af28748d0e..de4e453532c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/PublicationServiceTests.cs @@ -3124,29 +3124,17 @@ public async Task UpdateReleaseSeries_UpdatesLatestPublishedReleaseVersion() var viewModels = result.AssertRight(); Assert.Equal(3, viewModels.Count); - Assert.False(viewModels[0].IsLegacyLink); - Assert.Equal(release2021.Title, viewModels[0].Description); Assert.Equal(release2021.Id, viewModels[0].ReleaseId); - Assert.Equal(release2021.Slug, viewModels[0].ReleaseSlug); Assert.True(viewModels[0].IsLatest); Assert.True(viewModels[0].IsPublished); - Assert.Null(viewModels[0].LegacyLinkUrl); - Assert.False(viewModels[1].IsLegacyLink); - Assert.Equal(release2020.Title, viewModels[1].Description); Assert.Equal(release2020.Id, viewModels[1].ReleaseId); - Assert.Equal(release2020.Slug, viewModels[1].ReleaseSlug); Assert.False(viewModels[1].IsLatest); Assert.True(viewModels[1].IsPublished); - Assert.Null(viewModels[1].LegacyLinkUrl); - Assert.False(viewModels[2].IsLegacyLink); - Assert.Equal(release2022.Title, viewModels[2].Description); Assert.Equal(release2022.Id, viewModels[2].ReleaseId); - Assert.Equal(release2022.Slug, viewModels[2].ReleaseSlug); Assert.False(viewModels[2].IsLatest); Assert.True(viewModels[2].IsPublished); - Assert.Null(viewModels[2].LegacyLinkUrl); } await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) @@ -3168,6 +3156,108 @@ public async Task UpdateReleaseSeries_UpdatesLatestPublishedReleaseVersion() } } + [Fact] + public async Task UpdateReleaseSeries_UpdatesLatestPublishedReleaseVersion_SkipsUnpublishedReleases() + { + Publication publication = _dataFixture + .DefaultPublication() + .WithReleases([ + _dataFixture + .DefaultRelease(publishedVersions: 1, year: 2020), + _dataFixture + .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), + _dataFixture + .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022) + ]) + .WithTheme(_dataFixture.DefaultTheme()); + + var release2020 = publication.Releases.Single(r => r.Year == 2020); + var release2021 = publication.Releases.Single(r => r.Year == 2021); + var release2022 = publication.Releases.Single(r => r.Year == 2022); + + var expectedLatestPublishedReleaseVersionId = release2020.Versions[0].Id; + + // Check the publication's latest published release version in the generated test data setup + Assert.Equal(release2022.Versions[1].Id, publication.LatestPublishedReleaseVersionId); + + // Check the expected order of the release series items in the generated test data setup + Assert.Equal(3, publication.ReleaseSeries.Count); + Assert.Equal(release2022.Id, publication.ReleaseSeries[0].ReleaseId); + Assert.Equal(release2021.Id, publication.ReleaseSeries[1].ReleaseId); + Assert.Equal(release2020.Id, publication.ReleaseSeries[2].ReleaseId); + + var contentDbContextId = Guid.NewGuid().ToString(); + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + contentDbContext.Publications.Add(publication); + await contentDbContext.SaveChangesAsync(); + } + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + var publicationCacheService = new Mock(Strict); + publicationCacheService.Setup(mock => + mock.UpdatePublication(publication.Slug)) + .ReturnsAsync(new PublicationCacheViewModel()); + + var releaseCacheService = new Mock(Strict); + releaseCacheService.Setup(mock => mock.UpdateRelease( + expectedLatestPublishedReleaseVersionId, + publication.Slug, + null)) + .ReturnsAsync(new ReleaseCacheViewModel(expectedLatestPublishedReleaseVersionId)); + + var publicationService = BuildPublicationService( + contentDbContext, + publicationCacheService: publicationCacheService.Object, + releaseCacheService: releaseCacheService.Object); + + var result = await publicationService.UpdateReleaseSeries( + publication.Id, + updatedReleaseSeriesItems: + [ + new ReleaseSeriesItemUpdateRequest { ReleaseId = release2021.Id }, // Unpublished + new ReleaseSeriesItemUpdateRequest { ReleaseId = release2020.Id }, + new ReleaseSeriesItemUpdateRequest { ReleaseId = release2022.Id } + ]); + + VerifyAllMocks(publicationCacheService, releaseCacheService); + + var viewModels = result.AssertRight(); + Assert.Equal(3, viewModels.Count); + + Assert.Equal(release2021.Id, viewModels[0].ReleaseId); + Assert.False(viewModels[0].IsLatest); + Assert.False(viewModels[0].IsPublished); + + Assert.Equal(release2020.Id, viewModels[1].ReleaseId); + Assert.True(viewModels[1].IsLatest); + Assert.True(viewModels[1].IsPublished); + + Assert.Equal(release2022.Id, viewModels[2].ReleaseId); + Assert.False(viewModels[2].IsLatest); + Assert.True(viewModels[2].IsPublished); + } + + await using (var contentDbContext = InMemoryApplicationDbContext(contentDbContextId)) + { + var actualPublication = await contentDbContext.Publications + .SingleAsync(p => p.Id == publication.Id); + + var actualReleaseSeries = actualPublication.ReleaseSeries; + Assert.Equal(3, actualReleaseSeries.Count); + + Assert.Equal(release2021.Id, actualReleaseSeries[0].ReleaseId); + Assert.Equal(release2020.Id, actualReleaseSeries[1].ReleaseId); + Assert.Equal(release2022.Id, actualReleaseSeries[2].ReleaseId); + + // The latest published version of 2020 should now be the publication's latest published release + // version since it was positioned as the next release after 2021 which is unpublished + Assert.Equal(expectedLatestPublishedReleaseVersionId, + actualPublication.LatestPublishedReleaseVersionId); + } + } + [Fact] public async Task UpdateReleaseSeries_SetEmpty() { From 5fba23dee9d17b7590074b06b6b78d6826d8d41f Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Thu, 12 Dec 2024 17:05:07 +0000 Subject: [PATCH 128/144] EES-5656 Remove incorrect mention of ordering in method comments --- .../Repository/Interfaces/IReleaseVersionRepository.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs index f17b7e734cb..7333c2e4be9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs @@ -65,7 +65,7 @@ Task IsLatestReleaseVersion( CancellationToken cancellationToken = default); /// - /// Retrieves the latest published release version id's associated with a publication in reverse chronological order. + /// Retrieves the latest published release version id's associated with a publication. /// /// The unique identifier of the publication. /// A to observe while waiting for the task to complete. @@ -85,7 +85,7 @@ Task> ListLatestPublishedReleaseVersions( CancellationToken cancellationToken = default); /// - /// Retrieves the latest version id's of all releases associated with a publication in reverse chronological order. + /// Retrieves the latest version id's of all releases associated with a publication. /// /// The unique identifier of the publication. /// A to observe while waiting for the task to complete. From 303e0a1203717b55a43c6dce2bc304746173bf52 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Thu, 12 Dec 2024 17:05:21 +0000 Subject: [PATCH 129/144] EES-5656 Remove mention of 'published' in method comments because it's dependent on the value of `publishedOnly` --- .../Predicates/ReleaseVersionPredicates.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Predicates/ReleaseVersionPredicates.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Predicates/ReleaseVersionPredicates.cs index 90fd93d88a5..57560809186 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Predicates/ReleaseVersionPredicates.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Predicates/ReleaseVersionPredicates.cs @@ -11,7 +11,7 @@ public static class ReleaseVersionPredicates { /// /// Filters a sequence of of type to only include the latest - /// published versions of each release. + /// versions of each release. /// /// The source of type to filter. /// Unique identifier of a publication to filter by. @@ -50,20 +50,20 @@ public static IQueryable LatestReleaseVersions(this IQueryable /// Filters a sequence of of type to only include the latest - /// published version of the release. + /// version of the release. /// - /// The source of type to filter. + /// The source of type to filter. /// Unique identifier of a release to filter by. /// Flag to only include published release versions. /// An of type that contains elements from the input /// sequence filtered to only include the latest version of the release. - public static IQueryable LatestReleaseVersion(this IQueryable releaseVersionsQueryable, + public static IQueryable LatestReleaseVersion(this IQueryable releaseVersions, Guid releaseId, bool publishedOnly = false) { - return releaseVersionsQueryable + return releaseVersions .Where(releaseVersion => releaseVersion.ReleaseId == releaseId) - .Where(releaseVersion => releaseVersion.Version == releaseVersionsQueryable + .Where(releaseVersion => releaseVersion.Version == releaseVersions .Where(latestVersion => latestVersion.ReleaseId == releaseId) .Where(latestVersion => !publishedOnly || latestVersion.Published.HasValue) .Select(latestVersion => (int?)latestVersion.Version) From 2bb26de3ae678b3952a2f44e1962014890749611 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Thu, 12 Dec 2024 17:08:19 +0000 Subject: [PATCH 130/144] EES-5656 Check latest release link has correct release name on non-latest release page --- .../tests/admin_and_public/bau/release_reordering.robot | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot b/tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot index 9daf295d23d..2aaa4188822 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/release_reordering.robot @@ -466,3 +466,4 @@ Navigate to second published release on public frontend after changing the lates Validate second published release is not the latest data after changing the latest release user checks page contains This is not the latest data + user waits until page contains link View latest data: ${RELEASE_1_NAME} From 7ff9fe3deb4c1407bd09b78b54c77ee034f5f253 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Fri, 13 Dec 2024 17:54:08 +0000 Subject: [PATCH 131/144] EES-5656 Change ManageContentPageService.GetManageContentPageViewModel to not use method ListLatestPublishedReleaseVersions --- .../ManageContentPageServiceTests.cs | 9 +-- .../ManageContent/ManageContentPageService.cs | 74 ++++++++++--------- .../Startup.cs | 1 + .../Interfaces/IReleaseRepository.cs | 30 ++++++++ .../Repository/ReleaseRepository.cs | 40 ++++++++++ 5 files changed, 113 insertions(+), 41 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseRepository.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseRepository.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs index 7d7b01ece26..f4dea4b73af 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs @@ -17,6 +17,7 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository; using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using Microsoft.AspNetCore.Mvc; @@ -26,10 +27,6 @@ using static GovUk.Education.ExploreEducationStatistics.Common.Model.TimeIdentifier; using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; using HtmlBlockViewModel = GovUk.Education.ExploreEducationStatistics.Admin.ViewModels.HtmlBlockViewModel; -using IReleaseVersionRepository = - GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces.IReleaseVersionRepository; -using ReleaseVersionRepository = - GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.ReleaseVersionRepository; namespace GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.ManageContent { @@ -678,7 +675,7 @@ private static ManageContentPageService SetupManageContentPageService( IDataBlockService? dataBlockService = null, IMethodologyVersionRepository? methodologyVersionRepository = null, IReleaseFileService? releaseFileService = null, - IReleaseVersionRepository? releaseVersionRepository = null, + IReleaseRepository? releaseRepository = null, IUserService? userService = null) { return new( @@ -688,7 +685,7 @@ private static ManageContentPageService SetupManageContentPageService( dataBlockService ?? new Mock().Object, methodologyVersionRepository ?? new Mock().Object, releaseFileService ?? new Mock().Object, - releaseVersionRepository ?? new ReleaseVersionRepository(contentDbContext), + releaseRepository ?? new ReleaseRepository(contentDbContext), userService ?? MockUtils.AlwaysTrueUserService().Object ); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ManageContent/ManageContentPageService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ManageContent/ManageContentPageService.cs index f6c6c038a9c..84ba3798b60 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ManageContent/ManageContentPageService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/ManageContent/ManageContentPageService.cs @@ -19,7 +19,6 @@ using System.Linq; using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Common.Model.FileType; -using IReleaseVersionRepository = GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces.IReleaseVersionRepository; namespace GovUk.Education.ExploreEducationStatistics.Admin.Services.ManageContent { @@ -31,7 +30,7 @@ public class ManageContentPageService : IManageContentPageService private readonly IDataBlockService _dataBlockService; private readonly IMethodologyVersionRepository _methodologyVersionRepository; private readonly IReleaseFileService _releaseFileService; - private readonly IReleaseVersionRepository _releaseVersionRepository; + private readonly IReleaseRepository _releaseRepository; private readonly IUserService _userService; public ManageContentPageService( @@ -41,7 +40,7 @@ public ManageContentPageService( IDataBlockService dataBlockService, IMethodologyVersionRepository methodologyVersionRepository, IReleaseFileService releaseFileService, - IReleaseVersionRepository releaseVersionRepository, + IReleaseRepository releaseRepository, IUserService userService) { _contentDbContext = contentDbContext; @@ -50,7 +49,7 @@ public ManageContentPageService( _dataBlockService = dataBlockService; _methodologyVersionRepository = methodologyVersionRepository; _releaseFileService = releaseFileService; - _releaseVersionRepository = releaseVersionRepository; + _releaseRepository = releaseRepository; _userService = userService; } @@ -101,38 +100,12 @@ public async Task> GetManageCon var releaseViewModel = _mapper.Map(releaseVersion); - // Hydrate Publication.ReleaseSeries - var publishedVersions = - await _releaseVersionRepository - .ListLatestPublishedReleaseVersions(releaseVersion.PublicationId); - var filteredReleaseSeries = releaseVersion.Publication.ReleaseSeries - .Where(rsi => // only show items for legacy links and published releases - rsi.IsLegacyLink - || publishedVersions - .Any(rv => rsi.ReleaseId == rv.ReleaseId) - ).ToList(); - releaseViewModel.Publication.ReleaseSeries = filteredReleaseSeries - .Select(rsi => - { - if (rsi.IsLegacyLink) - { - return new ReleaseSeriesItemViewModel - { - Description = rsi.LegacyLinkDescription!, - LegacyLinkUrl = rsi.LegacyLinkUrl, - }; - } - - var latestReleaseVersion = publishedVersions - .Single(rv => rv.ReleaseId == rsi.ReleaseId); + var publishedReleases = + await _releaseRepository.ListPublishedReleases(releaseVersion.PublicationId); - return new ReleaseSeriesItemViewModel - { - Description = latestReleaseVersion.Title, - ReleaseId = latestReleaseVersion.ReleaseId, - ReleaseSlug = latestReleaseVersion.Slug, - }; - }).ToList(); + // Hydrate Publication.ReleaseSeries + releaseViewModel.Publication.ReleaseSeries = + BuildReleaseSeriesItemViewModels(releaseVersion.Publication, publishedReleases); releaseViewModel.DownloadFiles = files.ToList(); releaseViewModel.Publication.Methodologies = @@ -146,6 +119,37 @@ await _releaseVersionRepository }); } + private static List BuildReleaseSeriesItemViewModels( + Publication publication, + List publishedReleases) + { + var publishedReleasesById = publishedReleases.ToDictionary(r => r.Id); + return publication.ReleaseSeries + // Only include release series items for legacy links and published releases + .Where(rsi => rsi.IsLegacyLink || publishedReleasesById.ContainsKey(rsi.ReleaseId!.Value)) + .Select(rsi => + { + if (rsi.IsLegacyLink) + { + return new ReleaseSeriesItemViewModel + { + Description = rsi.LegacyLinkDescription!, + LegacyLinkUrl = rsi.LegacyLinkUrl + }; + } + + var release = publishedReleasesById[rsi.ReleaseId!.Value]; + + return new ReleaseSeriesItemViewModel + { + Description = release.Title, + ReleaseId = release.Id, + ReleaseSlug = release.Slug + }; + }) + .ToList(); + } + private IQueryable HydrateReleaseQuery(IQueryable queryable) { // Using `AsSplitQuery` as the generated SQL without it is incredibly diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs index b04002a6bbd..f41641b645a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Startup.cs @@ -552,6 +552,7 @@ public virtual void ConfigureServices(IServiceCollection services) services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddTransient + /// Retrieves the published releases that are associated with a publication. + /// + /// The unique identifier of the publication. + /// A to observe while waiting for the task to complete. + /// A collection of the published releases associated with the publication. + Task> ListPublishedReleases( + Guid publicationId, + CancellationToken cancellationToken = default); + + /// + /// Retrieves the id's of all published releases that are associated with a publication. + /// + /// The unique identifier of the publication. + /// A to observe while waiting for the task to complete. + /// A collection of the id's of all published releases associated with the publication. + Task> ListPublishedReleaseIds( + Guid publicationId, + CancellationToken cancellationToken = default); +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseRepository.cs new file mode 100644 index 00000000000..fc9bc10a6b1 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseRepository.cs @@ -0,0 +1,40 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Repository; + +public class ReleaseRepository(ContentDbContext context) : IReleaseRepository +{ + public Task> ListPublishedReleases( + Guid publicationId, + CancellationToken cancellationToken = default) + { + return QueryPublishedReleases(publicationId) + .ToListAsync(cancellationToken); + } + + public Task> ListPublishedReleaseIds( + Guid publicationId, + CancellationToken cancellationToken = default) + { + return QueryPublishedReleases(publicationId) + .Select(r => r.Id) + .ToListAsync(cancellationToken); + } + + private IQueryable QueryPublishedReleases(Guid publicationId) + { + // For simplicity, we only query releases that have ANY version that has been published. + // In future this may need to change if release versions can be recalled/unpublished. + return context.Releases + .Where(r => r.PublicationId == publicationId) + .Where(r => r.Versions.Any(rv => rv.Published != null)); + } +} From da82791b157ba444bce12c384bb7c927de713833 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Fri, 13 Dec 2024 18:09:06 +0000 Subject: [PATCH 132/144] EES-5656 Change PublicationService.Get to not use method ListLatestPublishedReleaseVersions --- .../Startup.cs | 1 + .../Interfaces/IReleaseRepository.cs | 4 +- .../Repository/ReleaseRepository.cs | 26 +++-- .../Cache/PublicationCacheServiceTests.cs | 8 +- .../PublicationServiceTests.cs | 107 ++++++++++-------- .../PublicationService.cs | 104 ++++++++--------- .../PublicationCacheViewModel.cs | 2 +- .../PublicationViewModel.cs | 4 +- .../ReleaseTitleViewModel.cs | 10 ++ .../ReleaseVersionTitleViewModel.cs | 10 -- .../PublisherHostBuilderExtensions.cs | 1 + 11 files changed, 153 insertions(+), 124 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/ReleaseTitleViewModel.cs delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/ReleaseVersionTitleViewModel.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Startup.cs index f926c6e179e..b75ee88371f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Api/Startup.cs @@ -162,6 +162,7 @@ public virtual void ConfigureServices(IServiceCollection services) services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseRepository.cs index ab46e1d668e..9a7bbbef143 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseRepository.cs @@ -9,11 +9,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.In public interface IReleaseRepository { /// - /// Retrieves the published releases that are associated with a publication. + /// Retrieves the published releases in release series order that are associated with a publication. /// /// The unique identifier of the publication. /// A to observe while waiting for the task to complete. - /// A collection of the published releases associated with the publication. + /// A collection of the published releases in release series order associated with the publication. Task> ListPublishedReleases( Guid publicationId, CancellationToken cancellationToken = default); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseRepository.cs index fc9bc10a6b1..ce4e1aa401e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseRepository.cs @@ -5,26 +5,38 @@ using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using Microsoft.EntityFrameworkCore; namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Repository; -public class ReleaseRepository(ContentDbContext context) : IReleaseRepository +public class ReleaseRepository(ContentDbContext contentDbContext) : IReleaseRepository { - public Task> ListPublishedReleases( + public async Task> ListPublishedReleases( Guid publicationId, CancellationToken cancellationToken = default) { - return QueryPublishedReleases(publicationId) - .ToListAsync(cancellationToken); + var publication = await contentDbContext.Publications + .SingleAsync(p => p.Id == publicationId, cancellationToken: cancellationToken); + + var publicationReleaseSeriesReleaseIds = publication.ReleaseSeries.ReleaseIds(); + + var releaseIdIndexMap = publicationReleaseSeriesReleaseIds + .Select((releaseId, index) => (releaseId, index)) + .ToDictionary(tuple => tuple.releaseId, tuple => tuple.index); + + return (await QueryPublishedReleases(publicationId) + .ToListAsync(cancellationToken)) + .OrderBy(r => releaseIdIndexMap[r.Id]) + .ToList(); } - public Task> ListPublishedReleaseIds( + public async Task> ListPublishedReleaseIds( Guid publicationId, CancellationToken cancellationToken = default) { - return QueryPublishedReleases(publicationId) + return await QueryPublishedReleases(publicationId) .Select(r => r.Id) .ToListAsync(cancellationToken); } @@ -33,7 +45,7 @@ private IQueryable QueryPublishedReleases(Guid publicationId) { // For simplicity, we only query releases that have ANY version that has been published. // In future this may need to change if release versions can be recalled/unpublished. - return context.Releases + return contentDbContext.Releases .Where(r => r.PublicationId == publicationId) .Where(r => r.Versions.Any(rv => rv.Published != null)); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/Cache/PublicationCacheServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/Cache/PublicationCacheServiceTests.cs index 55bd9df9411..ccf19513798 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/Cache/PublicationCacheServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/Cache/PublicationCacheServiceTests.cs @@ -43,15 +43,15 @@ public class PublicationCacheServiceTests : CacheServiceTestFixture Url = "" }, LatestReleaseId = Guid.NewGuid(), - Releases = new List - { - new() + Releases = + [ + new ReleaseTitleViewModel { Id = Guid.NewGuid(), Slug = "", Title = "" } - }, + ], ReleaseSeries = [ new ReleaseSeriesItemViewModel diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/PublicationServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/PublicationServiceTests.cs index 6821c31bde4..862a979d2ed 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/PublicationServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/PublicationServiceTests.cs @@ -122,32 +122,39 @@ public class GetTests : PublicationServiceTests Url = "https://external.methodology.com", }; - private readonly List _legacyLinks = new() - { - new ReleaseSeriesItem - { - Id = Guid.NewGuid(), - LegacyLinkDescription = "Legacy release description", - LegacyLinkUrl = "https://legacy.release.com", - }, - }; - [Fact] public async Task Success() { + ReleaseSeriesItem legacyLink = _dataFixture.DefaultLegacyReleaseSeriesItem(); + Publication publication = _dataFixture .DefaultPublication() - .WithReleases(ListOf( + .WithReleases([ _dataFixture - .DefaultRelease(publishedVersions: 1, year: 2020), + .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022), _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), + .DefaultRelease(publishedVersions: 1, year: 2020), _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022))) + .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021) + ]) .WithContact(_contact) .WithExternalMethodology(_externalMethodology) - .WithLegacyLinks(_legacyLinks) - .WithTheme(_dataFixture.DefaultTheme()); + .WithLegacyLinks([legacyLink]) + .WithTheme(_dataFixture.DefaultTheme()) + .FinishWith(p => + { + // Adjust the generated LatestPublishedReleaseVersion to make 2020 the latest published release + var release2020Version0 = p.Releases.Single(r => r.Year == 2020).Versions[0]; + p.LatestPublishedReleaseVersion = release2020Version0; + p.LatestPublishedReleaseVersionId = release2020Version0.Id; + + // Apply a different release series order rather than using the default + p.ReleaseSeries = + [.. GenerateReleaseSeries(p.Releases, 2021, 2020, 2022), legacyLink]; + }); + + var release2020 = publication.Releases.Single(r => r.Year == 2020); + var release2022 = publication.Releases.Single(r => r.Year == 2022); var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) @@ -158,11 +165,6 @@ public async Task Success() await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { - var expectedReleaseVersion1 = publication.ReleaseVersions - .Single(rv => rv is { Year: 2022, Version: 1 }); - var expectedReleaseVersion2 = publication.ReleaseVersions - .Single(rv => rv is { Year: 2020, Version: 0 }); - var service = SetupPublicationService(contentDbContext); var result = await service.Get(publication.Slug); @@ -175,41 +177,41 @@ public async Task Success() Assert.False(publicationViewModel.IsSuperseded); Assert.Equal(2, publicationViewModel.Releases.Count); - Assert.Equal(expectedReleaseVersion1.Id, publicationViewModel.LatestReleaseId); - Assert.Equal(expectedReleaseVersion1.Id, publicationViewModel.Releases[0].Id); - Assert.Equal(expectedReleaseVersion1.Slug, publicationViewModel.Releases[0].Slug); - Assert.Equal(expectedReleaseVersion1.Title, publicationViewModel.Releases[0].Title); + Assert.Equal(release2020.Versions[0].Id, publicationViewModel.LatestReleaseId); + + Assert.Equal(release2020.Id, publicationViewModel.Releases[0].Id); + Assert.Equal(release2020.Slug, publicationViewModel.Releases[0].Slug); + Assert.Equal(release2020.Title, publicationViewModel.Releases[0].Title); - Assert.Equal(expectedReleaseVersion2.Id, publicationViewModel.Releases[1].Id); - Assert.Equal(expectedReleaseVersion2.Slug, publicationViewModel.Releases[1].Slug); - Assert.Equal(expectedReleaseVersion2.Title, publicationViewModel.Releases[1].Title); + Assert.Equal(release2022.Id, publicationViewModel.Releases[1].Id); + Assert.Equal(release2022.Slug, publicationViewModel.Releases[1].Slug); + Assert.Equal(release2022.Title, publicationViewModel.Releases[1].Title); - Assert.Equal(3, publicationViewModel.ReleaseSeries.Count); + var releaseSeries = publicationViewModel.ReleaseSeries; - var releaseSeriesItem1 = publicationViewModel.ReleaseSeries[0]; - Assert.False(releaseSeriesItem1.IsLegacyLink); - Assert.Equal(expectedReleaseVersion1.ReleaseId, releaseSeriesItem1.ReleaseId); - Assert.Equal(expectedReleaseVersion1.Title, releaseSeriesItem1.Description); - Assert.Equal(expectedReleaseVersion1.Slug, releaseSeriesItem1.ReleaseSlug); - Assert.Null(releaseSeriesItem1.LegacyLinkUrl); + Assert.Equal(3, releaseSeries.Count); + + Assert.False(releaseSeries[0].IsLegacyLink); + Assert.Equal(release2020.Id, releaseSeries[0].ReleaseId); + Assert.Equal(release2020.Title, releaseSeries[0].Description); + Assert.Equal(release2020.Slug, releaseSeries[0].ReleaseSlug); + Assert.Null(releaseSeries[0].LegacyLinkUrl); // NOTE: 2021 release does exist in the database's publication.ReleaseSeries, but is filtered out // because it's unpublished - var releaseSeriesItem2 = publicationViewModel.ReleaseSeries[1]; - Assert.False(releaseSeriesItem2.IsLegacyLink); - Assert.Equal(expectedReleaseVersion2.ReleaseId, releaseSeriesItem2.ReleaseId); - Assert.Equal(expectedReleaseVersion2.Title, releaseSeriesItem2.Description); - Assert.Equal(expectedReleaseVersion2.Slug, releaseSeriesItem2.ReleaseSlug); - Assert.Null(releaseSeriesItem2.LegacyLinkUrl); + Assert.False(releaseSeries[1].IsLegacyLink); + Assert.Equal(release2022.Id, releaseSeries[1].ReleaseId); + Assert.Equal(release2022.Title, releaseSeries[1].Description); + Assert.Equal(release2022.Slug, releaseSeries[1].ReleaseSlug); + Assert.Null(releaseSeries[1].LegacyLinkUrl); - var releaseSeriesItem3 = publicationViewModel.ReleaseSeries[2]; - Assert.True(releaseSeriesItem3.IsLegacyLink); - Assert.Null(releaseSeriesItem3.ReleaseId); - Assert.Equal(_legacyLinks[0].LegacyLinkDescription, releaseSeriesItem3.Description); - Assert.Null(releaseSeriesItem3.ReleaseSlug); - Assert.Equal(_legacyLinks[0].LegacyLinkUrl, releaseSeriesItem3.LegacyLinkUrl); + Assert.True(releaseSeries[2].IsLegacyLink); + Assert.Null(releaseSeries[2].ReleaseId); + Assert.Equal(legacyLink.LegacyLinkDescription, releaseSeries[2].Description); + Assert.Null(releaseSeries[2].ReleaseSlug); + Assert.Equal(legacyLink.LegacyLinkUrl, releaseSeries[2].LegacyLinkUrl); Assert.Equal(publication.Theme.Id, publicationViewModel.Theme.Id); Assert.Equal(publication.Theme.Slug, publicationViewModel.Theme.Slug); @@ -1932,9 +1934,19 @@ public async Task ListSitemapItems() } } + private List GenerateReleaseSeries(IReadOnlyList releases, params int[] years) + { + return years.Select(year => + { + var release = releases.Single(r => r.Year == year); + return _dataFixture.DefaultReleaseSeriesItem().WithReleaseId(release.Id).Generate(); + }).ToList(); + } + private static PublicationService SetupPublicationService( ContentDbContext? contentDbContext = null, IPublicationRepository? publicationRepository = null, + IReleaseRepository? releaseRepository = null, IReleaseVersionRepository? releaseVersionRepository = null) { contentDbContext ??= InMemoryContentDbContext(); @@ -1943,6 +1955,7 @@ private static PublicationService SetupPublicationService( contentDbContext, new PersistenceHelper(contentDbContext), publicationRepository ?? new PublicationRepository(contentDbContext), + releaseRepository ?? new ReleaseRepository(contentDbContext), releaseVersionRepository ?? new ReleaseVersionRepository(contentDbContext) ); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs index 7792a5f2b93..25554190db8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs @@ -27,17 +27,20 @@ public class PublicationService : IPublicationService private readonly ContentDbContext _contentDbContext; private readonly IPersistenceHelper _contentPersistenceHelper; private readonly IPublicationRepository _publicationRepository; + private readonly IReleaseRepository _releaseRepository; private readonly IReleaseVersionRepository _releaseVersionRepository; public PublicationService( ContentDbContext contentDbContext, IPersistenceHelper contentPersistenceHelper, IPublicationRepository publicationRepository, + IReleaseRepository releaseRepository, IReleaseVersionRepository releaseVersionRepository) { _contentDbContext = contentDbContext; _contentPersistenceHelper = contentPersistenceHelper; _publicationRepository = publicationRepository; + _releaseRepository = releaseRepository; _releaseVersionRepository = releaseVersionRepository; } @@ -81,42 +84,9 @@ public async Task> Get(string pu return new Either(new NotFoundResult()); } - var publishedReleaseVersions = await _releaseVersionRepository.ListLatestPublishedReleaseVersions(publication.Id); - + var publishedReleases = await _releaseRepository.ListPublishedReleases(publication.Id); var isSuperseded = await _publicationRepository.IsSuperseded(publication.Id); - - // Only show legacy links and published releases in ReleaseSeries - var filteredReleaseSeries = publication.ReleaseSeries - .Where(rsi => - rsi.IsLegacyLink - || publishedReleaseVersions - .Any(rv => rsi.ReleaseId == rv.ReleaseId) - ).ToList(); - - var releaseSeriesItemViewModels = filteredReleaseSeries - .Select(rsi => - { - if (rsi.IsLegacyLink) - { - return new ReleaseSeriesItemViewModel - { - Description = rsi.LegacyLinkDescription!, - LegacyLinkUrl = rsi.LegacyLinkUrl, - }; - } - - var latestReleaseVersion = publishedReleaseVersions - .Single(rv => rv.ReleaseId == rsi.ReleaseId); - - return new ReleaseSeriesItemViewModel - { - Description = latestReleaseVersion.Title, - ReleaseId = latestReleaseVersion.ReleaseId, - ReleaseSlug = latestReleaseVersion.Slug, - }; - }).ToList(); - - return BuildPublicationViewModel(publication, publishedReleaseVersions, isSuperseded, releaseSeriesItemViewModels); + return BuildPublicationViewModel(publication, publishedReleases, isSuperseded); }); } @@ -222,23 +192,24 @@ public async Task releaseVersions, - bool isSuperseded, - List releaseSeries) + List releases, + bool isSuperseded) { - var theme = new ThemeViewModel( - publication.Theme.Id, - publication.Theme.Slug, - publication.Theme.Title, - publication.Theme.Summary - ); + var theme = publication.Theme; + + var releaseSeriesItemViewModels = BuildReleaseSeriesItemViewModels(publication, releases); return new PublicationCacheViewModel { Id = publication.Id, Title = publication.Title, Slug = publication.Slug, - Theme = theme, + Theme = new ThemeViewModel( + theme.Id, + theme.Slug, + theme.Title, + theme.Summary + ), Contact = new ContactViewModel(publication.Contact), ExternalMethodology = publication.ExternalMethodology != null ? new ExternalMethodologyViewModel(publication.ExternalMethodology) @@ -253,18 +224,49 @@ private static PublicationCacheViewModel BuildPublicationViewModel( Title = publication.SupersededBy.Title } : null, - Releases = releaseVersions - .Select(releaseVersion => new ReleaseVersionTitleViewModel + Releases = releases + .Select(r => new ReleaseTitleViewModel { - Id = releaseVersion.Id, - Slug = releaseVersion.Slug, - Title = releaseVersion.Title, + Id = r.Id, + Slug = r.Slug, + Title = r.Title }) .ToList(), - ReleaseSeries = releaseSeries, + ReleaseSeries = releaseSeriesItemViewModels }; } + private static List BuildReleaseSeriesItemViewModels( + Publication publication, + List releases) + { + var publishedReleasesById = releases.ToDictionary(r => r.Id); + return publication.ReleaseSeries + // Only include release series items for legacy links and published releases + .Where(rsi => rsi.IsLegacyLink || publishedReleasesById.ContainsKey(rsi.ReleaseId!.Value)) + .Select(rsi => + { + if (rsi.IsLegacyLink) + { + return new ReleaseSeriesItemViewModel + { + Description = rsi.LegacyLinkDescription!, + LegacyLinkUrl = rsi.LegacyLinkUrl + }; + } + + var release = publishedReleasesById[rsi.ReleaseId!.Value]; + + return new ReleaseSeriesItemViewModel + { + Description = release.Title, + ReleaseId = release.Id, + ReleaseSlug = release.Slug + }; + }) + .ToList(); + } + private async Task BuildPublicationTreeTheme(Theme theme) { var publications = await theme.Publications diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/PublicationCacheViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/PublicationCacheViewModel.cs index 56e581c8ad8..fd047c0df75 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/PublicationCacheViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/PublicationCacheViewModel.cs @@ -16,7 +16,7 @@ public record PublicationCacheViewModel public PublicationSupersededByViewModel? SupersededBy { get; init; } = new(); - public List Releases { get; init; } = []; + public List Releases { get; init; } = []; public List ReleaseSeries { get; init; } = []; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/PublicationViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/PublicationViewModel.cs index 4a83b1455b5..100a88072fd 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/PublicationViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/PublicationViewModel.cs @@ -16,9 +16,9 @@ public record PublicationViewModel public PublicationSupersededByViewModel? SupersededBy { get; init; } = new(); - public List Releases { get; init; } = new(); + public List Releases { get; init; } = []; - public List ReleaseSeries { get; init; } = new(); + public List ReleaseSeries { get; init; } = []; public ThemeViewModel Theme { get; init; } = null!; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/ReleaseTitleViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/ReleaseTitleViewModel.cs new file mode 100644 index 00000000000..862370facf0 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/ReleaseTitleViewModel.cs @@ -0,0 +1,10 @@ +namespace GovUk.Education.ExploreEducationStatistics.Content.ViewModels; + +public record ReleaseTitleViewModel +{ + public required Guid Id { get; init; } + + public required string Slug { get; init; } + + public required string Title { get; init; } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/ReleaseVersionTitleViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/ReleaseVersionTitleViewModel.cs deleted file mode 100644 index 1a6017b82c8..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.ViewModels/ReleaseVersionTitleViewModel.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace GovUk.Education.ExploreEducationStatistics.Content.ViewModels; - -public record ReleaseVersionTitleViewModel -{ - public Guid Id { get; set; } - - public string Slug { get; set; } = string.Empty; - - public string Title { get; set; } = string.Empty; -} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs index 3fc0b16ebdd..395fe578313 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs @@ -123,6 +123,7 @@ public static IHostBuilder ConfigurePublisherHostBuilder(this IHostBuilder hostB .AddScoped() .AddScoped() .AddScoped() + .AddScoped() .AddScoped() .AddScoped() .AddScoped() From 0d82a73c57a8c6b443e96b64dcf2f4b587291ea3 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 16 Dec 2024 09:33:05 +0000 Subject: [PATCH 133/144] EES-5656 Consolidate methods ListLatestReleaseVersionIds/ListLatestPublishedReleaseVersionIds and ListLatestReleaseVersions/ListLatestPublishedReleaseVersions with new publishedOnly flag. --- .../ReleaseVersionRepositoryTests.cs | 540 ++++++++++-------- .../Interfaces/IReleaseVersionRepository.cs | 24 +- .../Repository/ReleaseVersionRepository.cs | 27 +- .../ReleaseServiceTests.cs | 2 +- .../PublicationService.cs | 3 +- .../ReleaseService.cs | 2 +- 6 files changed, 327 insertions(+), 271 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs index 8cf258824ce..d4446fa2cc7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs @@ -370,247 +370,333 @@ public async Task ReleaseVersionDoesNotExist_ReturnsFalse() } } - public class ListLatestPublishedReleaseVersionIdsTests : ReleaseVersionRepositoryTests + public class ListLatestReleaseVersionsTests : ReleaseVersionRepositoryTests { - [Fact] - public async Task Success() - { - var publications = _dataFixture - .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - var result = await repository.ListLatestPublishedReleaseVersionIds(publications[0].Id); - - // Expect the result to contain the highest published version of each release for the specified publication - AssertIdsAreEqualIgnoringOrder( - [ - publications[0].ReleaseVersions[2].Id, - publications[0].ReleaseVersions[5].Id - ], - result); - } - - [Fact] - public async Task PublicationHasNoPublishedReleaseVersions_ReturnsEmpty() - { - var publications = _dataFixture - .DefaultPublication() - // Index 0 has an unpublished release version - // Index 1 has a published release version - .ForIndex(0, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true) - .Generate(1))) - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestPublishedReleaseVersionIds(publications[0].Id)); - } - - [Fact] - public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() + public class AnyPublishedStateTests : ListLatestReleaseVersionIdsTests { - var publications = _dataFixture - .DefaultPublication() - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestPublishedReleaseVersionIds(publications[0].Id)); + [Fact] + public async Task Success() + { + var publications = _dataFixture + .DefaultPublication() + .WithReleases(_ => ListOf( + _dataFixture + .DefaultRelease(publishedVersions: 0, draftVersion: true), + _dataFixture + .DefaultRelease(publishedVersions: 2, draftVersion: true), + _dataFixture + .DefaultRelease(publishedVersions: 2))) + .GenerateList(2); + + var contextId = await AddTestData(publications); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + var result = await repository.ListLatestReleaseVersions(publications[0].Id, publishedOnly: false); + + // Expect the result to contain the highest version of each release for the specified publication + AssertIdsAreEqualIgnoringOrder( + [ + publications[0].ReleaseVersions[0].Id, + publications[0].ReleaseVersions[3].Id, + publications[0].ReleaseVersions[5].Id + ], + result); + } + + [Fact] + public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() + { + var publications = _dataFixture + .DefaultPublication() + // Index 0 has no release versions + // Index 1 has a published release version + .ForIndex(1, + p => p.SetReleases(_dataFixture + .DefaultRelease(publishedVersions: 1) + .Generate(1))) + .GenerateList(2); + + var contextId = await AddTestData(publications); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + Assert.Empty(await repository.ListLatestReleaseVersions(publications[0].Id, publishedOnly: false)); + } + + [Fact] + public async Task PublicationDoesNotExist_ReturnsEmpty() + { + Publication publication = _dataFixture + .DefaultPublication() + .WithReleases(_dataFixture + .DefaultRelease(publishedVersions: 1) + .Generate(1)); + + var contextId = await AddTestData(publication); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + Assert.Empty(await repository.ListLatestReleaseVersions(publicationId: Guid.NewGuid(), + publishedOnly: false)); + } } - [Fact] - public async Task PublicationDoesNotExist_ReturnsEmpty() + public class PublishedOnlyTests : ListLatestReleaseVersionsTests { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); - - var contextId = await AddTestData(publication); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestPublishedReleaseVersionIds(Guid.NewGuid())); - } - } - - public class ListLatestPublishedReleaseVersionsTests : ReleaseVersionRepositoryTests - { - [Fact] - public async Task Success() - { - var publications = _dataFixture - .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - var result = await repository.ListLatestPublishedReleaseVersions(publications[0].Id); - - // Expect the result to contain the highest published version of each release for the specified publication - AssertIdsAreEqualIgnoringOrder( - [ - publications[0].ReleaseVersions[2].Id, - publications[0].ReleaseVersions[5].Id - ], - result); - } - - [Fact] - public async Task PublicationHasNoPublishedReleaseVersions_ReturnsEmpty() - { - var publications = _dataFixture - .DefaultPublication() - // Index 0 has an unpublished release version - // Index 1 has a published release version - .ForIndex(0, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true) - .Generate(1))) - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestPublishedReleaseVersions(publications[0].Id)); - } - - [Fact] - public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() - { - var publications = _dataFixture - .DefaultPublication() - // Index 0 has no release versions - // Index 1 has a published release version - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestPublishedReleaseVersions(publications[0].Id)); - } - - [Fact] - public async Task PublicationDoesNotExist_ReturnsEmpty() - { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); - - var contextId = await AddTestData(publication); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestPublishedReleaseVersions(Guid.NewGuid())); + [Fact] + public async Task Success() + { + var publications = _dataFixture + .DefaultPublication() + .WithReleases(_ => ListOf( + _dataFixture + .DefaultRelease(publishedVersions: 0, draftVersion: true), + _dataFixture + .DefaultRelease(publishedVersions: 2, draftVersion: true), + _dataFixture + .DefaultRelease(publishedVersions: 2))) + .GenerateList(2); + + var contextId = await AddTestData(publications); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + var result = await repository.ListLatestReleaseVersions(publications[0].Id, publishedOnly: true); + + // Expect the result to contain the highest published version of each release for the specified publication + AssertIdsAreEqualIgnoringOrder( + [ + publications[0].ReleaseVersions[2].Id, + publications[0].ReleaseVersions[5].Id + ], + result); + } + + [Fact] + public async Task PublicationHasNoPublishedReleaseVersions_ReturnsEmpty() + { + var publications = _dataFixture + .DefaultPublication() + // Index 0 has an unpublished release version + // Index 1 has a published release version + .ForIndex(0, + p => p.SetReleases(_dataFixture + .DefaultRelease(publishedVersions: 0, draftVersion: true) + .Generate(1))) + .ForIndex(1, + p => p.SetReleases(_dataFixture + .DefaultRelease(publishedVersions: 1) + .Generate(1))) + .GenerateList(2); + + var contextId = await AddTestData(publications); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + Assert.Empty(await repository.ListLatestReleaseVersions(publications[0].Id, publishedOnly: true)); + } + + [Fact] + public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() + { + var publications = _dataFixture + .DefaultPublication() + // Index 0 has no release versions + // Index 1 has a published release version + .ForIndex(1, + p => p.SetReleases(_dataFixture + .DefaultRelease(publishedVersions: 1) + .Generate(1))) + .GenerateList(2); + + var contextId = await AddTestData(publications); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + Assert.Empty(await repository.ListLatestReleaseVersions(publications[0].Id, publishedOnly: true)); + } + + [Fact] + public async Task PublicationDoesNotExist_ReturnsEmpty() + { + Publication publication = _dataFixture + .DefaultPublication() + .WithReleases(_dataFixture + .DefaultRelease(publishedVersions: 1) + .Generate(1)); + + var contextId = await AddTestData(publication); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + Assert.Empty( + await repository.ListLatestReleaseVersions(publicationId: Guid.NewGuid(), publishedOnly: true)); + } } } public class ListLatestReleaseVersionIdsTests : ReleaseVersionRepositoryTests { - [Fact] - public async Task Success() + public class AnyPublishedStateTests : ListLatestReleaseVersionIdsTests { - var publications = _dataFixture - .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - var result = await repository.ListLatestReleaseVersionIds(publications[0].Id); - - // Expect the result to contain the highest version of each release for the specified publication - AssertIdsAreEqualIgnoringOrder( - [ - publications[0].ReleaseVersions[0].Id, - publications[0].ReleaseVersions[3].Id, - publications[0].ReleaseVersions[5].Id - ], - result); + [Fact] + public async Task Success() + { + var publications = _dataFixture + .DefaultPublication() + .WithReleases(_ => ListOf( + _dataFixture + .DefaultRelease(publishedVersions: 0, draftVersion: true), + _dataFixture + .DefaultRelease(publishedVersions: 2, draftVersion: true), + _dataFixture + .DefaultRelease(publishedVersions: 2))) + .GenerateList(2); + + var contextId = await AddTestData(publications); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + var result = await repository.ListLatestReleaseVersionIds(publications[0].Id, publishedOnly: false); + + // Expect the result to contain the highest version of each release for the specified publication + AssertIdsAreEqualIgnoringOrder( + [ + publications[0].ReleaseVersions[0].Id, + publications[0].ReleaseVersions[3].Id, + publications[0].ReleaseVersions[5].Id + ], + result); + } + + [Fact] + public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() + { + var publications = _dataFixture + .DefaultPublication() + // Index 0 has no release versions + // Index 1 has a published release version + .ForIndex(1, + p => p.SetReleases(_dataFixture + .DefaultRelease(publishedVersions: 1) + .Generate(1))) + .GenerateList(2); + + var contextId = await AddTestData(publications); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + Assert.Empty(await repository.ListLatestReleaseVersionIds(publications[0].Id, publishedOnly: false)); + } + + [Fact] + public async Task PublicationDoesNotExist_ReturnsEmpty() + { + Publication publication = _dataFixture + .DefaultPublication() + .WithReleases(_dataFixture + .DefaultRelease(publishedVersions: 1) + .Generate(1)); + + var contextId = await AddTestData(publication); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + Assert.Empty(await repository.ListLatestReleaseVersionIds(publicationId: Guid.NewGuid(), + publishedOnly: false)); + } } - [Fact] - public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() + public class PublishedOnlyTrueTests : ListLatestReleaseVersionIdsTests { - var publications = _dataFixture - .DefaultPublication() - // Index 0 has no release versions - // Index 1 has a published release version - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestReleaseVersionIds(publications[0].Id)); - } - - [Fact] - public async Task PublicationDoesNotExist_ReturnsEmpty() - { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); - - var contextId = await AddTestData(publication); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Empty(await repository.ListLatestReleaseVersionIds(Guid.NewGuid())); + [Fact] + public async Task Success() + { + var publications = _dataFixture + .DefaultPublication() + .WithReleases(_ => ListOf( + _dataFixture + .DefaultRelease(publishedVersions: 0, draftVersion: true), + _dataFixture + .DefaultRelease(publishedVersions: 2, draftVersion: true), + _dataFixture + .DefaultRelease(publishedVersions: 2))) + .GenerateList(2); + + var contextId = await AddTestData(publications); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + var result = await repository.ListLatestReleaseVersionIds(publications[0].Id, publishedOnly: true); + + // Expect the result to contain the highest published version of each release for the specified publication + AssertIdsAreEqualIgnoringOrder( + [ + publications[0].ReleaseVersions[2].Id, + publications[0].ReleaseVersions[5].Id + ], + result); + } + + [Fact] + public async Task PublicationHasNoPublishedReleaseVersions_ReturnsEmpty() + { + var publications = _dataFixture + .DefaultPublication() + // Index 0 has an unpublished release version + // Index 1 has a published release version + .ForIndex(0, + p => p.SetReleases(_dataFixture + .DefaultRelease(publishedVersions: 0, draftVersion: true) + .Generate(1))) + .ForIndex(1, + p => p.SetReleases(_dataFixture + .DefaultRelease(publishedVersions: 1) + .Generate(1))) + .GenerateList(2); + + var contextId = await AddTestData(publications); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + Assert.Empty(await repository.ListLatestReleaseVersionIds(publications[0].Id, publishedOnly: true)); + } + + [Fact] + public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() + { + var publications = _dataFixture + .DefaultPublication() + .ForIndex(1, + p => p.SetReleases(_dataFixture + .DefaultRelease(publishedVersions: 1) + .Generate(1))) + .GenerateList(2); + + var contextId = await AddTestData(publications); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + Assert.Empty(await repository.ListLatestReleaseVersionIds(publications[0].Id, publishedOnly: true)); + } + + [Fact] + public async Task PublicationDoesNotExist_ReturnsEmpty() + { + Publication publication = _dataFixture + .DefaultPublication() + .WithReleases(_dataFixture + .DefaultRelease(publishedVersions: 1) + .Generate(1)); + + var contextId = await AddTestData(publication); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + Assert.Empty( + await repository.ListLatestReleaseVersionIds(publicationId: Guid.NewGuid(), publishedOnly: true)); + } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs index 7333c2e4be9..8cc5dd3d21e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs @@ -64,43 +64,27 @@ Task IsLatestReleaseVersion( Guid releaseVersionId, CancellationToken cancellationToken = default); - /// - /// Retrieves the latest published release version id's associated with a publication. - /// - /// The unique identifier of the publication. - /// A to observe while waiting for the task to complete. - /// A collection of the latest published version id's of all releases associated with the publication. - Task> ListLatestPublishedReleaseVersionIds( - Guid publicationId, - CancellationToken cancellationToken = default); - - /// - /// Retrieves the latest published versions of all releases associated with a publication in reverse chronological order. - /// - /// The unique identifier of the publication. - /// A to observe while waiting for the task to complete. - /// A collection of the latest published versions of all releases associated with the publication. - Task> ListLatestPublishedReleaseVersions( - Guid publicationId, - CancellationToken cancellationToken = default); - /// /// Retrieves the latest version id's of all releases associated with a publication. /// /// The unique identifier of the publication. + /// Flag to only include published release version id's. /// A to observe while waiting for the task to complete. /// A collection of the latest version id's of all releases associated with the publication. Task> ListLatestReleaseVersionIds( Guid publicationId, + bool publishedOnly = false, CancellationToken cancellationToken = default); /// /// Retrieves the latest versions of all releases associated with a given publication in reverse chronological order. /// /// The unique identifier of the publication. + /// Flag to only include published release versions. /// A to observe while waiting for the task to complete. /// A collection of the latest version id's of all releases associated with the publication. Task> ListLatestReleaseVersions( Guid publicationId, + bool publishedOnly = false, CancellationToken cancellationToken = default); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs index 7d7071ae153..6a167df5e37 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs @@ -103,39 +103,24 @@ public async Task IsLatestReleaseVersion( cancellationToken: cancellationToken); } - public async Task> ListLatestPublishedReleaseVersions( - Guid publicationId, - CancellationToken cancellationToken = default) - { - return (await _contentDbContext.ReleaseVersions.LatestReleaseVersions(publicationId, publishedOnly: true) - .ToListAsync(cancellationToken: cancellationToken)) - .OrderByReverseChronologicalOrder() - .ToList(); - } - - public async Task> ListLatestPublishedReleaseVersionIds( - Guid publicationId, - CancellationToken cancellationToken = default) - { - return await _contentDbContext.ReleaseVersions.LatestReleaseVersions(publicationId, publishedOnly: true) - .Select(rv => rv.Id) - .ToListAsync(cancellationToken: cancellationToken); - } - public async Task> ListLatestReleaseVersionIds( Guid publicationId, + bool publishedOnly = false, CancellationToken cancellationToken = default) { - return await _contentDbContext.ReleaseVersions.LatestReleaseVersions(publicationId) + return await _contentDbContext.ReleaseVersions + .LatestReleaseVersions(publicationId, publishedOnly: publishedOnly) .Select(rv => rv.Id) .ToListAsync(cancellationToken: cancellationToken); } public async Task> ListLatestReleaseVersions( Guid publicationId, + bool publishedOnly = false, CancellationToken cancellationToken = default) { - return (await _contentDbContext.ReleaseVersions.LatestReleaseVersions(publicationId) + return (await _contentDbContext.ReleaseVersions + .LatestReleaseVersions(publicationId, publishedOnly: publishedOnly) .ToListAsync(cancellationToken: cancellationToken)) .OrderByReverseChronologicalOrder() .ToList(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs index d54156bf6b2..2ddc439be26 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs @@ -745,7 +745,7 @@ public async Task List() Assert.Equal(2, releases.Count); - // Ordered from most newest to oldest + // Ordered from newest to oldest Assert.Equal(release2Version1.Id, releases[0].Id); Assert.Equal(release2Version1.Title, releases[0].Title); Assert.Equal(release2Version1.Slug, releases[0].Slug); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs index 25554190db8..8548db4a29c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/PublicationService.cs @@ -294,7 +294,8 @@ private async Task BuildPublicationTreePubl var latestReleaseHasData = latestPublishedReleaseVersionId.HasValue && await HasAnyDataFiles(latestPublishedReleaseVersionId.Value); - var publishedReleaseVersionIds = await _releaseVersionRepository.ListLatestPublishedReleaseVersionIds(publication.Id); + var publishedReleaseVersionIds = + await _releaseVersionRepository.ListLatestReleaseVersionIds(publication.Id, publishedOnly: true); var anyLiveReleaseHasData = await publishedReleaseVersionIds .ToAsyncEnumerable() .AnyAwaitAsync(async id => await HasAnyDataFiles(id)); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs index 97d0b74d267..076fa2592b6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs @@ -115,7 +115,7 @@ public async Task>> List(stri .OnSuccess(async publication => { var publishedReleaseVersions = - await _releaseVersionRepository.ListLatestPublishedReleaseVersions(publication.Id); + await _releaseVersionRepository.ListLatestReleaseVersions(publication.Id, publishedOnly: true); return publishedReleaseVersions .Select(releaseVersion => new ReleaseSummaryViewModel(releaseVersion, latestPublishedRelease: releaseVersion.Id == publication.LatestPublishedReleaseVersionId)) From cbbeb43a47b9b76ad0c0f2b1d7f8c063c0c4c54f Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 16 Dec 2024 11:14:51 +0000 Subject: [PATCH 134/144] EES-5656 Change ReleaseService to not use method GetLatestPublishedReleaseVersion(Guid publicationId) --- .../ReleaseServiceTests.cs | 2 - .../ReleaseService.cs | 47 +++++++++---------- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs index 2ddc439be26..bc7f6e58e37 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services.Tests/ReleaseServiceTests.cs @@ -7,7 +7,6 @@ using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; -using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository; @@ -851,7 +850,6 @@ private static ReleaseService SetupReleaseService( { return new( contentDbContext, - new PersistenceHelper(contentDbContext), releaseFileRepository ?? new ReleaseFileRepository(contentDbContext), releaseVersionRepository ?? new ReleaseVersionRepository(contentDbContext), userService ?? AlwaysTrueUserService().Object, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs index 076fa2592b6..68bc9328b6a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs @@ -8,7 +8,6 @@ using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; -using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; @@ -25,7 +24,6 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Services public class ReleaseService : IReleaseService { private readonly ContentDbContext _contentDbContext; - private readonly IPersistenceHelper _persistenceHelper; private readonly IReleaseFileRepository _releaseFileRepository; private readonly IReleaseVersionRepository _releaseVersionRepository; private readonly IUserService _userService; @@ -36,42 +34,41 @@ public class ReleaseService : IReleaseService public ReleaseService( ContentDbContext contentDbContext, - IPersistenceHelper persistenceHelper, IReleaseFileRepository releaseFileRepository, IReleaseVersionRepository releaseVersionRepository, IUserService userService, IMapper mapper) { _contentDbContext = contentDbContext; - _persistenceHelper = persistenceHelper; _releaseFileRepository = releaseFileRepository; _releaseVersionRepository = releaseVersionRepository; _userService = userService; _mapper = mapper; } - public async Task> GetRelease(string publicationSlug, + public async Task> GetRelease( + string publicationSlug, string? releaseSlug = null) { - return await _persistenceHelper.CheckEntityExists(q => - q.Where(p => p.Slug == publicationSlug)) + return await _contentDbContext.Publications + .SingleOrNotFoundAsync(p => p.Slug == publicationSlug) .OnSuccess(async publication => { - // If no release is requested get the latest published release version - if (releaseSlug == null) - { - return await _releaseVersionRepository.GetLatestPublishedReleaseVersion(publication.Id) - .OrNotFound(); - } - - // Otherwise get the latest published version of the requested release - return await _releaseVersionRepository.GetLatestPublishedReleaseVersion(publication.Id, releaseSlug) - .OrNotFound(); - }) - .OnSuccess(releaseVersion => GetRelease(releaseVersion.Id)); + // If no release is requested use the publication's latest published release version, + // otherwise use the latest published version of the requested release + var latestReleaseVersionId = releaseSlug == null + ? publication.LatestPublishedReleaseVersionId + : (await _releaseVersionRepository.GetLatestPublishedReleaseVersion(publication.Id, + releaseSlug))?.Id; + + return latestReleaseVersionId.HasValue + ? await GetRelease(latestReleaseVersionId.Value) + : new NotFoundResult(); + }); } - public async Task> GetRelease(Guid releaseVersionId, + public async Task> GetRelease( + Guid releaseVersionId, DateTime? expectedPublishDate = null) { // Note this method is allowed to return a view model for an unpublished release version so that Publisher @@ -107,10 +104,8 @@ public async Task> GetRelease(Guid r public async Task>> List(string publicationSlug) { - return await _persistenceHelper.CheckEntityExists( - q => q - .Where(p => p.Slug == publicationSlug) - ) + return await _contentDbContext.Publications + .SingleOrNotFoundAsync(p => p.Slug == publicationSlug) .OnSuccess(_userService.CheckCanViewPublication) .OnSuccess(async publication => { @@ -140,9 +135,9 @@ private static void FilterContentBlock(IContentBlockViewModel block) private async Task> GetDownloadFiles(ReleaseVersion releaseVersion) { var files = await _releaseFileRepository.GetByFileType( - releaseVersion.Id, + releaseVersion.Id, types: [FileType.Ancillary, FileType.Data]); - + return files .Select(rf => rf.ToPublicFileInfo()) .OrderBy(file => file.Name) From 65a1bb8fab066accd152b7816258393cf5797595 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 16 Dec 2024 15:18:36 +0000 Subject: [PATCH 135/144] EES-5656 Change PermalinkService to not use method GetLatestPublishedReleaseVersion(Guid publicationId) --- .../Services/PermalinkServiceTests.cs | 479 ++++++------------ ...tePermalink_WithReleaseVersionId_csv.snap} | 0 ...ePermalink_WithReleaseVersionId_json.snap} | 0 ...ermalink_WithoutReleaseVersionId_csv.snap} | 0 ...rmalink_WithoutReleaseVersionId_json.snap} | 0 .../Services/PermalinkService.cs | 28 +- 6 files changed, 175 insertions(+), 332 deletions(-) rename src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/{PermalinkServiceTests.CreatePermalink_WithReleaseId_csv.snap => PermalinkServiceTests.CreatePermalink_WithReleaseVersionId_csv.snap} (100%) rename src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/{PermalinkServiceTests.CreatePermalink_WithReleaseId_json.snap => PermalinkServiceTests.CreatePermalink_WithReleaseVersionId_json.snap} (100%) rename src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/{PermalinkServiceTests.CreatePermalink_WithoutReleaseId_csv.snap => PermalinkServiceTests.CreatePermalink_WithoutReleaseVersionId_csv.snap} (100%) rename src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/{PermalinkServiceTests.CreatePermalink_WithoutReleaseId_json.snap => PermalinkServiceTests.CreatePermalink_WithoutReleaseVersionId_json.snap} (100%) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/PermalinkServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/PermalinkServiceTests.cs index b3ca52e070e..e55fa71c113 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/PermalinkServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/PermalinkServiceTests.cs @@ -1,4 +1,11 @@ #nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -12,11 +19,13 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository; using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Data.Api.Models; using GovUk.Education.ExploreEducationStatistics.Data.Api.Requests; using GovUk.Education.ExploreEducationStatistics.Data.Api.Services; using GovUk.Education.ExploreEducationStatistics.Data.Api.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.Api.ViewModels; +using GovUk.Education.ExploreEducationStatistics.Data.Model; using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Data.Model.Utils; @@ -29,18 +38,12 @@ using Newtonsoft.Json.Linq; using Snapshooter; using Snapshooter.Xunit; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; using Xunit; using static GovUk.Education.ExploreEducationStatistics.Common.Model.TimeIdentifier; using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; using static GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Utils.ContentDbUtils; using File = GovUk.Education.ExploreEducationStatistics.Content.Model.File; +using ReleaseVersion = GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion; namespace GovUk.Education.ExploreEducationStatistics.Data.Api.Tests.Services { @@ -55,8 +58,6 @@ public class PermalinkServiceTests } }; - private readonly Guid _publicationId = Guid.NewGuid(); - private readonly PermalinkTableViewModel _frontendTableResponse = new() { Caption = "Admission Numbers for 'Sample publication' in North East between 2022 and 2023", @@ -70,6 +71,10 @@ public class PermalinkServiceTests [Fact] public async Task CreatePermalink_LatestPublishedReleaseForSubjectNotFound() { + Publication publication = _fixture.DefaultPublication() + .WithReleases([_fixture.DefaultRelease(publishedVersions: 0, draftVersion: true)]) + .WithTheme(_fixture.DefaultTheme()); + var request = new PermalinkCreateRequest { Query = @@ -78,57 +83,64 @@ public async Task CreatePermalink_LatestPublishedReleaseForSubjectNotFound() } }; - var releaseVersionRepository = new Mock(MockBehavior.Strict); var subjectRepository = new Mock(MockBehavior.Strict); - releaseVersionRepository - .Setup(s => s.GetLatestPublishedReleaseVersion(_publicationId, default)) - .ReturnsAsync((ReleaseVersion?)null); - subjectRepository .Setup(s => s.FindPublicationIdForSubject(request.Query.SubjectId, default)) - .ReturnsAsync(_publicationId); + .ReturnsAsync(publication.Id); - var service = BuildService(releaseVersionRepository: releaseVersionRepository.Object, - subjectRepository: subjectRepository.Object); + var contentDbContextId = Guid.NewGuid().ToString(); + await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) + { + contentDbContext.Publications.Add(publication); + await contentDbContext.SaveChangesAsync(); + } + + await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) + { + var service = BuildService( + contentDbContext: contentDbContext, + subjectRepository: subjectRepository.Object + ); - var result = await service.CreatePermalink(request); + var result = await service.CreatePermalink(request); - MockUtils.VerifyAllMocks( - releaseVersionRepository, - subjectRepository); + MockUtils.VerifyAllMocks(subjectRepository); - result.AssertNotFound(); + result.AssertNotFound(); + } } [Fact] - public async Task CreatePermalink_WithoutReleaseId() + public async Task CreatePermalink_WithoutReleaseVersionId() { - var subject = _fixture + Subject subject = _fixture .DefaultSubject() .WithFilters(_fixture.DefaultFilter() - .ForIndex(0, s => - s.SetGroupCsvColumn("filter_0_grouping") - .SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 1) - .ForInstance(s => s.Set( - fg => fg.Label, - (_, _, context) => $"Filter group {context.FixtureTypeIndex}")) - .Generate(2))) - .ForIndex(1, s => - s.SetGroupCsvColumn("filter_1_grouping") - .SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 1) - .ForInstance(s => s.Set( - fg => fg.Label, - (_, _, context) => $"Filter group {context.FixtureTypeIndex}")) - .Generate(2))) - .ForIndex(2, s => - s.SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 2) - .Generate(1))) + .ForIndex(0, + s => + s.SetGroupCsvColumn("filter_0_grouping") + .SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 1) + .ForInstance(s => s.Set( + fg => fg.Label, + (_, _, context) => $"Filter group {context.FixtureTypeIndex}")) + .Generate(2))) + .ForIndex(1, + s => + s.SetGroupCsvColumn("filter_1_grouping") + .SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 1) + .ForInstance(s => s.Set( + fg => fg.Label, + (_, _, context) => $"Filter group {context.FixtureTypeIndex}")) + .Generate(2))) + .ForIndex(2, + s => + s.SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 2) + .Generate(1))) .GenerateList()) .WithIndicatorGroups(_fixture .DefaultIndicatorGroup(indicatorCount: 1) - .Generate(3)) - .Generate(); + .Generate(3)); var indicators = subject .IndicatorGroups @@ -201,17 +213,11 @@ public async Task CreatePermalink_WithoutReleaseId() Filters = FiltersMetaViewModelBuilder.BuildFilters(subject.Filters), Indicators = IndicatorsMetaViewModelBuilder.BuildIndicators(indicators), Footnotes = footnoteViewModels, - TimePeriodRange = new List - { - new(2022, AcademicYear) - { - Label = "2022/23" - }, - new(2023, AcademicYear) - { - Label = "2023/24" - } - } + TimePeriodRange = + [ + new TimePeriodMetaViewModel(2022, AcademicYear) { Label = "2022/23" }, + new TimePeriodMetaViewModel(2023, AcademicYear) { Label = "2023/24" } + ] }, Results = observations .Select(o => @@ -219,20 +225,13 @@ public async Task CreatePermalink_WithoutReleaseId() .ToList() }; - var releaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2000", - TimePeriodCoverage = AcademicYear, - Published = DateTime.UtcNow, - }; + Publication publication = _fixture.DefaultPublication() + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]) + .WithTheme(_fixture.DefaultTheme()); - var publication = new Publication - { - Id = _publicationId, - LatestPublishedReleaseVersion = releaseVersion - }; + var releaseVersion = publication.Releases.Single().Versions.Single(); + + var releaseDataFile = ReleaseDataFile(releaseVersion, subject.Id); var csvMeta = new PermalinkCsvMetaViewModel { @@ -242,8 +241,8 @@ public async Task CreatePermalink_WithoutReleaseId() .Select(i => new IndicatorCsvMetaViewModel(i)) .ToDictionary(i => i.Name), Locations = locations.ToDictionary(l => l.Id, l => l.GetCsvValues()), - Headers = new List - { + Headers = + [ "time_period", "time_identifier", "geographic_level", @@ -262,7 +261,7 @@ public async Task CreatePermalink_WithoutReleaseId() indicators[0].Name, indicators[1].Name, indicators[2].Name - } + ] }; var request = new PermalinkCreateRequest @@ -349,17 +348,11 @@ public async Task CreatePermalink_WithoutReleaseId() It.IsAny())) .ReturnsAsync(csvMeta); - var releaseVersionRepository = new Mock(MockBehavior.Strict); - - releaseVersionRepository - .Setup(s => s.GetLatestPublishedReleaseVersion(_publicationId, default)) - .ReturnsAsync(releaseVersion); - var subjectRepository = new Mock(MockBehavior.Strict); subjectRepository .Setup(s => s.FindPublicationIdForSubject(subject.Id, default)) - .ReturnsAsync(_publicationId); + .ReturnsAsync(publication.Id); var tableBuilderService = new Mock(MockBehavior.Strict); @@ -375,8 +368,7 @@ public async Task CreatePermalink_WithoutReleaseId() await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { contentDbContext.Publications.Add(publication); - contentDbContext.ReleaseVersions.Add(releaseVersion); - contentDbContext.ReleaseFiles.AddRange(ReleaseDataFile(releaseVersion, subject.Id)); + contentDbContext.ReleaseFiles.Add(releaseDataFile); await contentDbContext.SaveChangesAsync(); } @@ -388,7 +380,6 @@ public async Task CreatePermalink_WithoutReleaseId() publicBlobStorageService: publicBlobStorageService.Object, frontendService: frontendService.Object, permalinkCsvMetaService: permalinkCsvMetaService.Object, - releaseVersionRepository: releaseVersionRepository.Object, subjectRepository: subjectRepository.Object, tableBuilderService: tableBuilderService.Object); @@ -398,7 +389,6 @@ public async Task CreatePermalink_WithoutReleaseId() publicBlobStorageService, frontendService, permalinkCsvMetaService, - releaseVersionRepository, subjectRepository, tableBuilderService); @@ -439,33 +429,35 @@ public async Task CreatePermalink_WithoutReleaseId() } [Fact] - public async Task CreatePermalink_WithReleaseId() + public async Task CreatePermalink_WithReleaseVersionId() { - var subject = _fixture + Subject subject = _fixture .DefaultSubject() .WithFilters(_fixture.DefaultFilter() - .ForIndex(0, s => - s.SetGroupCsvColumn("filter_0_grouping") - .SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 1) - .ForInstance(s => s.Set( - fg => fg.Label, - (_, _, context) => $"Filter group {context.FixtureTypeIndex}")) - .Generate(2))) - .ForIndex(1, s => - s.SetGroupCsvColumn("filter_1_grouping") - .SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 1) - .ForInstance(s => s.Set( - fg => fg.Label, - (_, _, context) => $"Filter group {context.FixtureTypeIndex}")) - .Generate(2))) - .ForIndex(2, s => - s.SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 2) - .Generate(1))) + .ForIndex(0, + s => + s.SetGroupCsvColumn("filter_0_grouping") + .SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 1) + .ForInstance(s => s.Set( + fg => fg.Label, + (_, _, context) => $"Filter group {context.FixtureTypeIndex}")) + .Generate(2))) + .ForIndex(1, + s => + s.SetGroupCsvColumn("filter_1_grouping") + .SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 1) + .ForInstance(s => s.Set( + fg => fg.Label, + (_, _, context) => $"Filter group {context.FixtureTypeIndex}")) + .Generate(2))) + .ForIndex(2, + s => + s.SetFilterGroups(_fixture.DefaultFilterGroup(filterItemCount: 2) + .Generate(1))) .GenerateList()) .WithIndicatorGroups(_fixture .DefaultIndicatorGroup(indicatorCount: 1) - .Generate(3)) - .Generate(); + .Generate(3)); var indicators = subject .IndicatorGroups @@ -537,17 +529,11 @@ public async Task CreatePermalink_WithReleaseId() Filters = FiltersMetaViewModelBuilder.BuildFilters(subject.Filters), Indicators = IndicatorsMetaViewModelBuilder.BuildIndicators(indicators), Footnotes = footnoteViewModels, - TimePeriodRange = new List - { - new(2022, AcademicYear) - { - Label = "2022/23" - }, - new(2023, AcademicYear) - { - Label = "2023/24" - } - } + TimePeriodRange = + [ + new TimePeriodMetaViewModel(2022, AcademicYear) { Label = "2022/23" }, + new TimePeriodMetaViewModel(2023, AcademicYear) { Label = "2023/24" } + ] }, Results = observations .Select(o => @@ -555,20 +541,13 @@ public async Task CreatePermalink_WithReleaseId() .ToList() }; - var releaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2000", - TimePeriodCoverage = AcademicYear, - Published = DateTime.UtcNow, - }; + Publication publication = _fixture.DefaultPublication() + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]) + .WithTheme(_fixture.DefaultTheme()); - var publication = new Publication - { - Id = _publicationId, - LatestPublishedReleaseVersion = releaseVersion - }; + var releaseVersion = publication.Releases.Single().Versions.Single(); + + var releaseDataFile = ReleaseDataFile(releaseVersion, subject.Id); var csvMeta = new PermalinkCsvMetaViewModel { @@ -578,8 +557,8 @@ public async Task CreatePermalink_WithReleaseId() .Select(i => new IndicatorCsvMetaViewModel(i)) .ToDictionary(i => i.Name), Locations = locations.ToDictionary(l => l.Id, l => l.GetCsvValues()), - Headers = new List - { + Headers = + [ "time_period", "time_identifier", "geographic_level", @@ -598,7 +577,7 @@ public async Task CreatePermalink_WithReleaseId() indicators[0].Name, indicators[1].Name, indicators[2].Name - } + ] }; var request = new PermalinkCreateRequest @@ -699,8 +678,7 @@ public async Task CreatePermalink_WithReleaseId() await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { contentDbContext.Publications.Add(publication); - contentDbContext.ReleaseVersions.Add(releaseVersion); - contentDbContext.ReleaseFiles.AddRange(ReleaseDataFile(releaseVersion, subject.Id)); + contentDbContext.ReleaseFiles.Add(releaseDataFile); await contentDbContext.SaveChangesAsync(); } @@ -761,20 +739,11 @@ public async Task CreatePermalink_WithReleaseId() [Fact] public async Task GetPermalink() { - var releaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - ReleaseName = "2000", - PublicationId = _publicationId, - TimePeriodCoverage = AcademicYear, - Published = DateTime.UtcNow, - }; + Publication publication = _fixture.DefaultPublication() + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]) + .WithTheme(_fixture.DefaultTheme()); - var publication = new Publication - { - Id = _publicationId, - LatestPublishedReleaseVersion = releaseVersion - }; + var releaseVersion = publication.Releases.Single().Versions.Single(); var permalink = new Permalink { @@ -785,6 +754,8 @@ public async Task GetPermalink() SubjectId = Guid.NewGuid() }; + var releaseDataFile = ReleaseDataFile(releaseVersion, permalink.SubjectId); + var table = _frontendTableResponse with { Footnotes = FootnotesViewModelBuilder.BuildFootnotes(_fixture @@ -797,8 +768,7 @@ public async Task GetPermalink() { contentDbContext.Permalinks.Add(permalink); contentDbContext.Publications.Add(publication); - contentDbContext.ReleaseVersions.Add(releaseVersion); - contentDbContext.ReleaseFiles.AddRange(ReleaseDataFile(releaseVersion, permalink.SubjectId)); + contentDbContext.ReleaseFiles.Add(releaseDataFile); await contentDbContext.SaveChangesAsync(); } @@ -873,7 +843,7 @@ public async Task GetPermalink_BlobNotFound() } [Fact] - public async Task GetPermalink_ReleaseWithSubjectNotFound() + public async Task GetPermalink_ReleaseVersionWithSubjectNotFound() { var permalink = new Permalink { @@ -912,47 +882,29 @@ public async Task GetPermalink_ReleaseWithSubjectNotFound() [Fact] public async Task GetPermalink_SubjectIsForMultipleReleaseVersions() { - var previousVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2000", - TimePeriodCoverage = AcademicYear, - Published = DateTime.UtcNow, - PreviousVersionId = null - }; - - var latestVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2000", - TimePeriodCoverage = AcademicYear, - Published = DateTime.UtcNow, - PreviousVersionId = previousVersion.Id - }; + Publication publication = _fixture.DefaultPublication() + .WithReleases([_fixture.DefaultRelease(publishedVersions: 2)]) + .WithTheme(_fixture.DefaultTheme()); - var publication = new Publication - { - Id = _publicationId, - LatestPublishedReleaseVersion = latestVersion - }; + var (previousReleaseVersion, latestReleaseVersion) = publication.Releases.Single().Versions.ToTuple2(); var permalink = new Permalink { SubjectId = Guid.NewGuid() }; + ReleaseFile[] releaseDataFiles = + [ + ReleaseDataFile(previousReleaseVersion, permalink.SubjectId), + ReleaseDataFile(latestReleaseVersion, permalink.SubjectId) + ]; + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { contentDbContext.Permalinks.Add(permalink); contentDbContext.Publications.Add(publication); - contentDbContext.ReleaseVersions.AddRange(previousVersion, latestVersion); - contentDbContext.ReleaseFiles.AddRange( - ReleaseDataFile(previousVersion, permalink.SubjectId), - ReleaseDataFile(latestVersion, permalink.SubjectId) - ); + contentDbContext.ReleaseFiles.AddRange(releaseDataFiles); await contentDbContext.SaveChangesAsync(); } @@ -980,109 +932,37 @@ public async Task GetPermalink_SubjectIsForMultipleReleaseVersions() } [Fact] - public async Task GetPermalink_SubjectIsNotForLatestRelease_OlderByYear() + public async Task GetPermalink_SubjectIsNotForPublicationsLatestRelease() { - var releaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2000", - TimePeriodCoverage = AcademicYear, - Published = DateTime.UtcNow, - }; - - var latestReleaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2001", - TimePeriodCoverage = AcademicYear, - Published = DateTime.UtcNow, - }; - - var publication = new Publication - { - Id = _publicationId, - LatestPublishedReleaseVersion = latestReleaseVersion - }; + Publication publication = _fixture + .DefaultPublication() + .WithReleases([ + _fixture + .DefaultRelease(publishedVersions: 1, year: 2020), + _fixture + .DefaultRelease(publishedVersions: 1, year: 2021) + ]) + .WithTheme(_fixture.DefaultTheme()); + + var release2020 = publication.Releases.Single(r => r.Year == 2020); + var release2021 = publication.Releases.Single(r => r.Year == 2021); + + // Check the publication's latest published release version in the generated test data setup + Assert.Equal(release2021.Versions[0].Id, publication.LatestPublishedReleaseVersionId); var permalink = new Permalink { SubjectId = Guid.NewGuid() }; - var contentDbContextId = Guid.NewGuid().ToString(); - await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) - { - contentDbContext.Permalinks.Add(permalink); - contentDbContext.Publications.Add(publication); - contentDbContext.ReleaseVersions.AddRange(releaseVersion, latestReleaseVersion); - contentDbContext.ReleaseFiles.AddRange(ReleaseDataFile(releaseVersion, permalink.SubjectId)); - - await contentDbContext.SaveChangesAsync(); - } - - var publicBlobStorageService = new Mock(MockBehavior.Strict); - - publicBlobStorageService.SetupGetDeserializedJson( - container: BlobContainers.PermalinkSnapshots, - path: $"{permalink.Id}.json.zst", - value: new PermalinkTableViewModel()); - - await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) - { - var service = BuildService( - contentDbContext: contentDbContext, - publicBlobStorageService: publicBlobStorageService.Object); - - var result = (await service.GetPermalink(permalink.Id)).AssertRight(); - - MockUtils.VerifyAllMocks(publicBlobStorageService); - - Assert.Equal(permalink.Id, result.Id); - Assert.Equal(PermalinkStatus.NotForLatestRelease, result.Status); - } - } - - [Fact] - public async Task GetPermalink_SubjectIsNotForLatestRelease_OlderByTimePeriod() - { - var releaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2000", - TimePeriodCoverage = January, - Published = DateTime.UtcNow, - }; - - var latestReleaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2000", - TimePeriodCoverage = February, - Published = DateTime.UtcNow, - }; - - var publication = new Publication - { - Id = _publicationId, - LatestPublishedReleaseVersion = latestReleaseVersion - }; - - var permalink = new Permalink - { - SubjectId = Guid.NewGuid() - }; + var releaseDataFile = ReleaseDataFile(release2020.Versions[0], permalink.SubjectId); var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { contentDbContext.Permalinks.Add(permalink); contentDbContext.Publications.Add(publication); - contentDbContext.ReleaseVersions.AddRange(releaseVersion, latestReleaseVersion); - contentDbContext.ReleaseFiles.AddRange(ReleaseDataFile(releaseVersion, permalink.SubjectId)); + contentDbContext.ReleaseFiles.Add(releaseDataFile); await contentDbContext.SaveChangesAsync(); } @@ -1112,45 +992,24 @@ public async Task GetPermalink_SubjectIsNotForLatestRelease_OlderByTimePeriod() [Fact] public async Task GetPermalink_SubjectIsNotForLatestReleaseVersion() { - var previousVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2000", - TimePeriodCoverage = AcademicYear, - Published = DateTime.UtcNow, - PreviousVersionId = null - }; + Publication publication = _fixture.DefaultPublication() + .WithReleases([_fixture.DefaultRelease(publishedVersions: 2)]); - var latestVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2000", - TimePeriodCoverage = AcademicYear, - Published = DateTime.UtcNow, - PreviousVersionId = previousVersion.Id - }; - - var publication = new Publication - { - Id = _publicationId, - LatestPublishedReleaseVersion = latestVersion - }; + var previousReleaseVersion = publication.Releases.Single().Versions[0]; var permalink = new Permalink { SubjectId = Guid.NewGuid() }; + var releaseDataFile = ReleaseDataFile(previousReleaseVersion, permalink.SubjectId); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { contentDbContext.Permalinks.Add(permalink); contentDbContext.Publications.Add(publication); - contentDbContext.ReleaseVersions.AddRange(previousVersion, latestVersion); - contentDbContext.ReleaseFiles.AddRange(ReleaseDataFile(previousVersion, - permalink.SubjectId)); + contentDbContext.ReleaseFiles.Add(releaseDataFile); await contentDbContext.SaveChangesAsync(); } @@ -1180,37 +1039,27 @@ public async Task GetPermalink_SubjectIsNotForLatestReleaseVersion() [Fact] public async Task GetPermalink_SubjectIsFromSupersededPublication() { - var releaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - PublicationId = _publicationId, - ReleaseName = "2000", - TimePeriodCoverage = AcademicYear, - Published = DateTime.UtcNow, - }; + Publication publication = _fixture.DefaultPublication() + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]) + .WithSupersededBy(_fixture + .DefaultPublication() + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)])); - var publication = new Publication - { - Id = _publicationId, - LatestPublishedReleaseVersion = releaseVersion, - SupersededBy = new Publication - { - LatestPublishedReleaseVersionId = Guid.NewGuid() - } - }; + var releaseVersion = publication.Releases.Single().Versions.Single(); var permalink = new Permalink { SubjectId = Guid.NewGuid() }; + var releaseDataFile = ReleaseDataFile(releaseVersion, permalink.SubjectId); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { contentDbContext.Permalinks.Add(permalink); - contentDbContext.Publications.Add(publication); - contentDbContext.ReleaseVersions.Add(releaseVersion); - contentDbContext.ReleaseFiles.AddRange(ReleaseDataFile(releaseVersion, permalink.SubjectId)); + contentDbContext.Publications.AddRange(publication); + contentDbContext.ReleaseFiles.Add(releaseDataFile); await contentDbContext.SaveChangesAsync(); } @@ -1337,7 +1186,6 @@ private static PermalinkService BuildService( IPermalinkCsvMetaService? permalinkCsvMetaService = null, IPublicBlobStorageService? publicBlobStorageService = null, IFrontendService? frontendService = null, - IReleaseVersionRepository? releaseVersionRepository = null, ISubjectRepository? subjectRepository = null, IPublicationRepository? publicationRepository = null) { @@ -1350,8 +1198,7 @@ private static PermalinkService BuildService( publicBlobStorageService ?? Mock.Of(MockBehavior.Strict), frontendService ?? Mock.Of(MockBehavior.Strict), subjectRepository ?? Mock.Of(MockBehavior.Strict), - publicationRepository ?? new PublicationRepository(contentDbContext), - releaseVersionRepository ?? Mock.Of(MockBehavior.Strict) + publicationRepository ?? new PublicationRepository(contentDbContext) ); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithReleaseId_csv.snap b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithReleaseVersionId_csv.snap similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithReleaseId_csv.snap rename to src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithReleaseVersionId_csv.snap diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithReleaseId_json.snap b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithReleaseVersionId_json.snap similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithReleaseId_json.snap rename to src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithReleaseVersionId_json.snap diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithoutReleaseId_csv.snap b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithoutReleaseVersionId_csv.snap similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithoutReleaseId_csv.snap rename to src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithoutReleaseVersionId_csv.snap diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithoutReleaseId_json.snap b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithoutReleaseVersionId_json.snap similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithoutReleaseId_json.snap rename to src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Services/__snapshots__/PermalinkServiceTests.CreatePermalink_WithoutReleaseVersionId_json.snap diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Api/Services/PermalinkService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Api/Services/PermalinkService.cs index 5638c3496e0..e72b68fce1e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Api/Services/PermalinkService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Api/Services/PermalinkService.cs @@ -40,7 +40,6 @@ public class PermalinkService : IPermalinkService private readonly IFrontendService _frontendService; private readonly ISubjectRepository _subjectRepository; private readonly IPublicationRepository _publicationRepository; - private readonly IReleaseVersionRepository _releaseVersionRepository; public PermalinkService( ContentDbContext contentDbContext, @@ -49,8 +48,7 @@ public PermalinkService( IPublicBlobStorageService publicBlobStorageService, IFrontendService frontendService, ISubjectRepository subjectRepository, - IPublicationRepository publicationRepository, - IReleaseVersionRepository releaseVersionRepository) + IPublicationRepository publicationRepository) { _contentDbContext = contentDbContext; _tableBuilderService = tableBuilderService; @@ -59,7 +57,6 @@ public PermalinkService( _frontendService = frontendService; _subjectRepository = subjectRepository; _publicationRepository = publicationRepository; - _releaseVersionRepository = releaseVersionRepository; } public async Task> GetPermalink(Guid permalinkId, @@ -307,9 +304,11 @@ private async Task> FindLatestPublishedReleaseVersion { return await _subjectRepository.FindPublicationIdForSubject(subjectId) .OrNotFound() - .OnSuccess(publicationId => - _releaseVersionRepository.GetLatestPublishedReleaseVersion(publicationId).OrNotFound()) - .OnSuccess(releaseVersion => releaseVersion.Id); + .OnSuccess(async publicationId => await _contentDbContext.Publications + .Where(p => p.Id == publicationId) + .Select(p => p.LatestPublishedReleaseVersionId) + .SingleOrDefaultAsync() + .OrNotFound()); } private async Task GetPermalinkStatus(Guid subjectId) @@ -317,9 +316,7 @@ private async Task GetPermalinkStatus(Guid subjectId) // TODO EES-3339 This doesn't currently include a status to warn if the footnotes have been amended on a Release, // and will return 'Current' unless one of the other cases also applies. - var releasesWithSubject = await _contentDbContext.ReleaseFiles - .Include(rf => rf.File) - .Include(rf => rf.ReleaseVersion) + var releasesVersionsWithSubject = await _contentDbContext.ReleaseFiles .Where(rf => rf.File.SubjectId == subjectId && rf.File.Type == FileType.Data @@ -327,26 +324,25 @@ private async Task GetPermalinkStatus(Guid subjectId) .Select(rf => rf.ReleaseVersion) .ToListAsync(); - if (releasesWithSubject.Count == 0) + if (releasesVersionsWithSubject.Count == 0) { return PermalinkStatus.SubjectRemoved; } var publication = await _contentDbContext.Publications .Include(p => p.LatestPublishedReleaseVersion) - .SingleAsync(p => p.Id == releasesWithSubject.First().PublicationId); + .SingleAsync(p => p.Id == releasesVersionsWithSubject.First().PublicationId); var latestPublishedReleaseVersion = publication.LatestPublishedReleaseVersion; - if (latestPublishedReleaseVersion != null && releasesWithSubject.All(rv => - rv.Year != latestPublishedReleaseVersion.Year - || rv.TimePeriodCoverage != latestPublishedReleaseVersion.TimePeriodCoverage)) + if (latestPublishedReleaseVersion != null && releasesVersionsWithSubject.All(rv => + rv.ReleaseId != latestPublishedReleaseVersion.ReleaseId)) { return PermalinkStatus.NotForLatestRelease; } if (latestPublishedReleaseVersion != null - && releasesWithSubject.All(rv => rv.Id != latestPublishedReleaseVersion.Id)) + && releasesVersionsWithSubject.All(rv => rv.Id != latestPublishedReleaseVersion.Id)) { return PermalinkStatus.SubjectReplacedOrRemoved; } From 364dc0887710866afddbbe562c5414068e74e529 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 16 Dec 2024 16:24:16 +0000 Subject: [PATCH 136/144] EES-5656 Change TableBuilderService to not use method GetLatestPublishedReleaseVersion(Guid publicationId) --- .../TableBuilderServicePermissionTests.cs | 124 ++++++++--------- .../TableBuilderServiceTests.cs | 125 +++++++----------- ...ts.QueryToCsvStream_ReleaseVersionId.snap} | 0 ...CsvStream_ReleaseVersionId_NoFilters.snap} | 0 .../TableBuilderService.cs | 40 +++--- 5 files changed, 134 insertions(+), 155 deletions(-) rename src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/__snapshots__/{TableBuilderServiceTests.QueryToCsvStream_ReleaseId.snap => TableBuilderServiceTests.QueryToCsvStream_ReleaseVersionId.snap} (100%) rename src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/__snapshots__/{TableBuilderServiceTests.QueryToCsvStream_ReleaseId_NoFilters.snap => TableBuilderServiceTests.QueryToCsvStream_ReleaseVersionId_NoFilters.snap} (100%) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/TableBuilderServicePermissionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/TableBuilderServicePermissionTests.cs index 9ff053349b9..94b5d24c312 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/TableBuilderServicePermissionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/TableBuilderServicePermissionTests.cs @@ -1,110 +1,115 @@ #nullable enable +using System; +using System.Linq; +using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data.Query; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils; using GovUk.Education.ExploreEducationStatistics.Common.Utils; -using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Content.Model; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Data.Model; using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Data.Services.Options; using GovUk.Education.ExploreEducationStatistics.Data.Services.Security; using Microsoft.Extensions.Options; using Moq; -using System; -using System.Threading.Tasks; -using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; -using GovUk.Education.ExploreEducationStatistics.Data.Services.Options; using Xunit; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.PermissionTestUtils; +using static GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Utils.ContentDbUtils; using static Moq.MockBehavior; -using ReleaseVersion = GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseVersion; namespace GovUk.Education.ExploreEducationStatistics.Data.Services.Tests { public class TableBuilderServicePermissionTests { - private static readonly Guid PublicationId = Guid.NewGuid(); - private static readonly Guid ReleaseVersionId = Guid.NewGuid(); - private static readonly Guid SubjectId = Guid.NewGuid(); - - private readonly Subject _subject = new() - { - Id = Guid.NewGuid(), - }; - - private readonly ReleaseSubject _releaseSubject = new() - { - ReleaseVersionId = ReleaseVersionId, - SubjectId = SubjectId, - }; + private readonly DataFixture _dataFixture = new(); [Fact] public async Task Query_LatestRelease_CanViewSubjectData() { - await PolicyCheckBuilder() - .SetupResourceCheckToFail(_releaseSubject, DataSecurityPolicies.CanViewSubjectData) - .AssertForbidden( - async userService => - { - var statisticsPersistenceHelper = StatisticsPersistenceHelperMock(_subject); + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); - MockUtils.SetupCall(statisticsPersistenceHelper, _releaseSubject); + var releaseVersion = publication.Releases.Single().Versions.Single(); - var subjectRepository = new Mock(Strict); + var releaseSubject = new ReleaseSubject + { + ReleaseVersionId = releaseVersion.Id, + SubjectId = Guid.NewGuid(), + }; - subjectRepository - .Setup(s => s.FindPublicationIdForSubject(_subject.Id, default)) - .ReturnsAsync(PublicationId); + var statisticsPersistenceHelper = MockUtils.MockPersistenceHelper(); + MockUtils.SetupCall(statisticsPersistenceHelper, releaseSubject); - var releaseVersionRepository = new Mock(Strict); + await using var contextDbContext = InMemoryContentDbContext(); + contextDbContext.Publications.Add(publication); + await contextDbContext.SaveChangesAsync(); - releaseVersionRepository - .Setup(s => s.GetLatestPublishedReleaseVersion(PublicationId, default)) - .ReturnsAsync(new ReleaseVersion - { - Id = ReleaseVersionId - }); + var subjectRepository = new Mock(Strict); + subjectRepository + .Setup(s => s.FindPublicationIdForSubject(releaseSubject.SubjectId, default)) + .ReturnsAsync(publication.Id); + + await PolicyCheckBuilder() + .SetupResourceCheckToFail(releaseSubject, DataSecurityPolicies.CanViewSubjectData) + .AssertForbidden( + async userService => + { var service = BuildTableBuilderService( + contextDbContext, userService: userService.Object, subjectRepository: subjectRepository.Object, - releaseVersionRepository: releaseVersionRepository.Object, statisticsPersistenceHelper: statisticsPersistenceHelper.Object ); return await service.Query( - new FullTableQuery - { - SubjectId = _subject.Id - } + new FullTableQuery { SubjectId = releaseSubject.SubjectId } ); } ); } [Fact] - public async Task Query_ReleaseId_CanViewSubjectData() + public async Task Query_ReleaseVersionId_CanViewSubjectData() { + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var releaseVersion = publication.Releases.Single().Versions.Single(); + + var releaseSubject = new ReleaseSubject + { + ReleaseVersionId = releaseVersion.Id, + SubjectId = Guid.NewGuid(), + }; + + var statisticsPersistenceHelper = MockUtils.MockPersistenceHelper(); + MockUtils.SetupCall(statisticsPersistenceHelper, releaseSubject); + + await using var contextDbContext = InMemoryContentDbContext(); + contextDbContext.Publications.Add(publication); + await contextDbContext.SaveChangesAsync(); + await PolicyCheckBuilder() - .SetupResourceCheckToFail(_releaseSubject, DataSecurityPolicies.CanViewSubjectData) + .SetupResourceCheckToFail(releaseSubject, DataSecurityPolicies.CanViewSubjectData) .AssertForbidden( async userService => { - var statisticsPersistenceHelper = StatisticsPersistenceHelperMock(_subject); - - MockUtils.SetupCall(statisticsPersistenceHelper, _releaseSubject); - var service = BuildTableBuilderService( + contextDbContext, userService: userService.Object, statisticsPersistenceHelper: statisticsPersistenceHelper.Object ); return await service.Query( - ReleaseVersionId, - new FullTableQuery - { - SubjectId = _subject.Id - }, + releaseVersionId: releaseVersion.Id, + new FullTableQuery { SubjectId = releaseSubject.SubjectId }, boundaryLevelId: null ); } @@ -112,6 +117,7 @@ await PolicyCheckBuilder() } private TableBuilderService BuildTableBuilderService( + ContentDbContext contentDbContext, IFilterItemRepository? filterItemRepository = null, ILocationService? locationService = null, IObservationService? observationService = null, @@ -120,29 +126,23 @@ private TableBuilderService BuildTableBuilderService( ISubjectCsvMetaService? subjectCsvMetaService = null, ISubjectRepository? subjectRepository = null, IUserService? userService = null, - IReleaseVersionRepository? releaseVersionRepository = null, IOptions? tableBuilderOptions = null, IOptions? locationsOptions = null) { return new( Mock.Of(), + contentDbContext, filterItemRepository ?? Mock.Of(Strict), locationService ?? Mock.Of(Strict), observationService ?? Mock.Of(Strict), - statisticsPersistenceHelper ?? StatisticsPersistenceHelperMock(_subject).Object, + statisticsPersistenceHelper ?? MockUtils.MockPersistenceHelper().Object, subjectResultMetaService ?? Mock.Of(Strict), subjectCsvMetaService ?? Mock.Of(Strict), subjectRepository ?? Mock.Of(Strict), userService ?? Mock.Of(Strict), - releaseVersionRepository ?? Mock.Of(Strict), tableBuilderOptions ?? new TableBuilderOptions().ToOptionsWrapper(), locationsOptions ?? new LocationsOptions().ToOptionsWrapper() ); } - - private static Mock> StatisticsPersistenceHelperMock(Subject subject) - { - return MockUtils.MockPersistenceHelper(subject.Id, subject); - } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/TableBuilderServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/TableBuilderServiceTests.cs index 8ef7eba13c9..3b4c307a5e1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/TableBuilderServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/TableBuilderServiceTests.cs @@ -1,4 +1,9 @@ #nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data.Query; @@ -8,8 +13,6 @@ using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; -using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository; -using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Data.Model; using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; @@ -18,17 +21,12 @@ using GovUk.Education.ExploreEducationStatistics.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Data.Model.Utils; using GovUk.Education.ExploreEducationStatistics.Data.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Data.Services.Options; using GovUk.Education.ExploreEducationStatistics.Data.Services.Utils; using GovUk.Education.ExploreEducationStatistics.Data.ViewModels.Meta; using Microsoft.Extensions.Options; using Moq; using Snapshooter.Xunit; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using GovUk.Education.ExploreEducationStatistics.Data.Services.Options; using Xunit; using static GovUk.Education.ExploreEducationStatistics.Common.Model.TimeIdentifier; using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; @@ -48,9 +46,7 @@ public async Task Query_LatestRelease() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -244,9 +240,7 @@ public async Task Query_LatestRelease_SubjectNotFound() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -290,9 +284,7 @@ public async Task Query_LatestRelease_PublicationNotFound() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -334,7 +326,7 @@ public async Task Query_LatestRelease_PublicationNotFound() [Fact] public async Task Query_LatestRelease_ReleaseNotFound() { - // Set up a ReleaseSubject that references a non-existent release + // Set up a ReleaseSubject that references a non-existent release version ReleaseSubject releaseSubject = _fixture .DefaultReleaseSubject() .WithReleaseVersion(_fixture @@ -370,9 +362,7 @@ public async Task Query_LatestRelease_PredictedTableTooBig() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -453,9 +443,7 @@ public async Task Query_ReleaseId() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -640,13 +628,11 @@ public async Task Query_ReleaseId() } [Fact] - public async Task Query_ReleaseId_ReleaseNotFound() + public async Task Query_ReleaseVersionId_ReleaseVersionNotFound() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -677,21 +663,22 @@ public async Task Query_ReleaseId_ReleaseNotFound() SubjectId = releaseSubject.SubjectId }; - // Query using a non-existent release id - var result = await service.Query(Guid.NewGuid(), query, null); + // Query using a non-existent release version id + var result = await service.Query( + releaseVersionId: Guid.NewGuid(), + query, + boundaryLevelId: null); result.AssertNotFound(); } } [Fact] - public async Task Query_ReleaseId_SubjectNotFound() + public async Task Query_ReleaseVersionId_SubjectNotFound() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -723,20 +710,21 @@ public async Task Query_ReleaseId_SubjectNotFound() SubjectId = Guid.NewGuid(), }; - var result = await service.Query(releaseVersion.Id, query, null); + var result = await service.Query( + releaseVersionId: releaseVersion.Id, + query, + boundaryLevelId: null); result.AssertNotFound(); } } [Fact] - public async Task Query_ReleaseId_PredictedTableTooBig() + public async Task Query_ReleaseVersionId_PredictedTableTooBig() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -803,7 +791,10 @@ public async Task Query_ReleaseId_PredictedTableTooBig() tableBuilderOptions: options.ToOptionsWrapper() ); - var result = await service.Query(releaseSubject.ReleaseVersionId, query, null); + var result = await service.Query( + releaseVersionId: releaseSubject.ReleaseVersionId, + query, + boundaryLevelId: null); VerifyAllMocks(filterItemRepository); @@ -816,9 +807,7 @@ public async Task QueryToCsvStream_LatestRelease() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var filters = _fixture.DefaultFilter() .ForIndex(0, s => @@ -1011,7 +1000,7 @@ public async Task QueryToCsvStream_LatestRelease() } [Fact] - public async Task QueryToCsvStream_LatestRelease_ReleaseNotFound() + public async Task QueryToCsvStream_LatestRelease_ReleaseVersionNotFound() { // Set up a ReleaseSubject that references a non-existent release ReleaseSubject releaseSubject = _fixture @@ -1053,9 +1042,7 @@ public async Task QueryToCsvStream_LatestRelease_SubjectNotFound() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -1101,9 +1088,7 @@ public async Task QueryToCsvStream_LatestRelease_PredictedTableTooBig() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -1182,13 +1167,11 @@ public async Task QueryToCsvStream_LatestRelease_PredictedTableTooBig() } [Fact] - public async Task QueryToCsvStream_ReleaseId() + public async Task QueryToCsvStream_ReleaseVersionId() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -1368,13 +1351,11 @@ public async Task QueryToCsvStream_ReleaseId() } [Fact] - public async Task QueryToCsvStream_ReleaseId_NoFilters() + public async Task QueryToCsvStream_ReleaseVersionId_NoFilters() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -1504,13 +1485,11 @@ public async Task QueryToCsvStream_ReleaseId_NoFilters() } [Fact] - public async Task QueryToCsvStream_ReleaseId_ReleaseNotFound() + public async Task QueryToCsvStream_ReleaseVersionId_ReleaseVersionNotFound() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -1544,21 +1523,22 @@ public async Task QueryToCsvStream_ReleaseId_ReleaseNotFound() using var stream = new MemoryStream(); - // Query using a non-existent release id - var result = await service.QueryToCsvStream(Guid.NewGuid(), query, stream); + // Query using a non-existent release version id + var result = await service.QueryToCsvStream( + releaseVersionId: Guid.NewGuid(), + query, + stream); result.AssertNotFound(); } } [Fact] - public async Task QueryToCsvStream_ReleaseId_SubjectNotFound() + public async Task QueryToCsvStream_ReleaseVersionId_SubjectNotFound() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -1600,13 +1580,11 @@ public async Task QueryToCsvStream_ReleaseId_SubjectNotFound() } [Fact] - public async Task QueryToCsvStream_ReleaseId_PredictedTableTooBig() + public async Task QueryToCsvStream_ReleaseVersionId_PredictedTableTooBig() { Publication publication = _fixture .DefaultPublication() - .WithReleases(_fixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases([_fixture.DefaultRelease(publishedVersions: 1)]); var releaseVersion = publication.ReleaseVersions.Single(); @@ -1711,12 +1689,12 @@ private static TableBuilderService BuildTableBuilderService( ISubjectCsvMetaService? subjectCsvMetaService = null, ISubjectRepository? subjectRepository = null, IUserService? userService = null, - IReleaseVersionRepository? releaseVersionRepository = null, IOptions? tableBuilderOptions = null, IOptions? locationsOptions = null) { return new( statisticsDbContext, + contentDbContext ?? InMemoryContentDbContext(), filterItemRepository ?? Mock.Of(Strict), locationService ?? Mock.Of(Strict), observationService ?? Mock.Of(Strict), @@ -1725,7 +1703,6 @@ private static TableBuilderService BuildTableBuilderService( subjectCsvMetaService ?? Mock.Of(Strict), subjectRepository ?? new SubjectRepository(statisticsDbContext), userService ?? AlwaysTrueUserService().Object, - releaseVersionRepository ?? new ReleaseVersionRepository(contentDbContext ?? Mock.Of()), tableBuilderOptions ?? DefaultTableBuilderOptions(), locationsOptions ?? DefaultLocationOptions() ); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/__snapshots__/TableBuilderServiceTests.QueryToCsvStream_ReleaseId.snap b/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/__snapshots__/TableBuilderServiceTests.QueryToCsvStream_ReleaseVersionId.snap similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/__snapshots__/TableBuilderServiceTests.QueryToCsvStream_ReleaseId.snap rename to src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/__snapshots__/TableBuilderServiceTests.QueryToCsvStream_ReleaseVersionId.snap diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/__snapshots__/TableBuilderServiceTests.QueryToCsvStream_ReleaseId_NoFilters.snap b/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/__snapshots__/TableBuilderServiceTests.QueryToCsvStream_ReleaseVersionId_NoFilters.snap similarity index 100% rename from src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/__snapshots__/TableBuilderServiceTests.QueryToCsvStream_ReleaseId_NoFilters.snap rename to src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/__snapshots__/TableBuilderServiceTests.QueryToCsvStream_ReleaseVersionId_NoFilters.snap diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Services/TableBuilderService.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Services/TableBuilderService.cs index a3c0c22295a..ce7ae6d4af2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Services/TableBuilderService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Services/TableBuilderService.cs @@ -1,4 +1,12 @@ #nullable enable +using System; +using System.Collections.Generic; +using System.Dynamic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using CsvHelper; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; @@ -6,7 +14,7 @@ using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Common.Validators; -using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Data.Model; using GovUk.Education.ExploreEducationStatistics.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Data.Model.Repository.Interfaces; @@ -20,14 +28,6 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; -using System; -using System.Collections.Generic; -using System.Dynamic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Data.Services.Utils.TableBuilderUtils; using static GovUk.Education.ExploreEducationStatistics.Data.Services.ValidationErrorMessages; using Unit = GovUk.Education.ExploreEducationStatistics.Common.Model.Unit; @@ -36,7 +36,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Data.Services { public class TableBuilderService : ITableBuilderService { - private readonly StatisticsDbContext _context; + private readonly StatisticsDbContext _statisticsDbContext; + private readonly ContentDbContext _contentDbContext; private readonly IFilterItemRepository _filterItemRepository; private readonly ILocationService _locationService; private readonly IObservationService _observationService; @@ -45,12 +46,12 @@ public class TableBuilderService : ITableBuilderService private readonly ISubjectCsvMetaService _subjectCsvMetaService; private readonly ISubjectRepository _subjectRepository; private readonly IUserService _userService; - private readonly IReleaseVersionRepository _releaseVersionRepository; private readonly TableBuilderOptions _options; private readonly LocationsOptions _locationOptions; public TableBuilderService( - StatisticsDbContext context, + StatisticsDbContext statisticsDbContext, + ContentDbContext contentDbContext, IFilterItemRepository filterItemRepository, ILocationService locationService, IObservationService observationService, @@ -59,11 +60,11 @@ public TableBuilderService( ISubjectCsvMetaService subjectCsvMetaService, ISubjectRepository subjectRepository, IUserService userService, - IReleaseVersionRepository releaseVersionRepository, IOptions options, IOptions locationOptions) { - _context = context; + _statisticsDbContext = statisticsDbContext; + _contentDbContext = contentDbContext; _filterItemRepository = filterItemRepository; _locationService = locationService; _observationService = observationService; @@ -72,7 +73,6 @@ public TableBuilderService( _subjectCsvMetaService = subjectCsvMetaService; _subjectRepository = subjectRepository; _userService = userService; - _releaseVersionRepository = releaseVersionRepository; _options = options.Value; _locationOptions = locationOptions.Value; } @@ -189,7 +189,7 @@ private async Task>> ListQueryObservation (await _observationService.GetMatchedObservations(query, cancellationToken)) .Select(row => row.Id); - return await _context + return await _statisticsDbContext .Observation .AsNoTracking() .Include(o => o.Location) @@ -236,9 +236,11 @@ private async Task> FindLatestPublishedReleaseVersion { return await _subjectRepository.FindPublicationIdForSubject(subjectId) .OrNotFound() - .OnSuccess(publicationId => - _releaseVersionRepository.GetLatestPublishedReleaseVersion(publicationId).OrNotFound()) - .OnSuccess(releaseVersion => releaseVersion.Id); + .OnSuccess(async publicationId => await _contentDbContext.Publications + .Where(p => p.Id == publicationId) + .Select(p => p.LatestPublishedReleaseVersionId) + .SingleOrDefaultAsync() + .OrNotFound()); } private Task> CheckReleaseSubjectExists(Guid subjectId, From 2d81e797bbdd0620486d9453983354d62332f0cf Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 16 Dec 2024 17:36:09 +0000 Subject: [PATCH 137/144] EES-5656 Change TableBuilderController to not use method GetLatestPublishedReleaseVersion(Guid publicationId) --- .../TableBuilderControllerTests.cs | 457 +++++++++++------- .../Controllers/TableBuilderController.cs | 72 ++- 2 files changed, 318 insertions(+), 211 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/TableBuilderControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/TableBuilderControllerTests.cs index d182167b4b8..687c858841d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/TableBuilderControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Api.Tests/Controllers/TableBuilderControllerTests.cs @@ -1,4 +1,10 @@ #nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Model.Chart; @@ -9,7 +15,6 @@ using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; -using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Data.Api.Cache; using GovUk.Education.ExploreEducationStatistics.Data.Api.Controllers; @@ -23,15 +28,8 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Net.Http.Headers; using Moq; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using Xunit; using static GovUk.Education.ExploreEducationStatistics.Common.Model.TimeIdentifier; -using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; using static Moq.MockBehavior; @@ -40,83 +38,50 @@ namespace GovUk.Education.ExploreEducationStatistics.Data.Api.Tests.Controllers public class TableBuilderControllerTests(TestApplicationFactory testApp) : IntegrationTestFixture(testApp) { - private static readonly DataFixture Fixture = new(); - - private static readonly Release Release = Fixture.DefaultRelease(publishedVersions: 1) - .WithPublication(Fixture.DefaultPublication()); - - private static readonly ReleaseVersion ReleaseVersion = Release.Versions.Single(); - - private static readonly DataBlockParent DataBlockParent = Fixture - .DefaultDataBlockParent() - .WithLatestPublishedVersion(Fixture - .DefaultDataBlockVersion() - .WithReleaseVersion(ReleaseVersion) - .WithDates(published: DateTime.UtcNow.AddDays(-1)) - .WithQuery(new FullTableQuery - { - SubjectId = Guid.NewGuid(), - LocationIds = [Guid.NewGuid(),], - TimePeriod = new TimePeriodQuery - { - StartYear = 2021, - StartCode = CalendarYear, - EndYear = 2022, - EndCode = CalendarYear - }, - Filters = new List(), - Indicators = new List // use collection expression -> test failures - { - Guid.NewGuid(), - }, - }) - .WithTable(new TableBuilderConfiguration - { - TableHeaders = new TableHeaders - { - Rows = new List { new("table header 1", TableHeaderType.Filter) } - } - }) - .WithCharts(ListOf(new LineChart - { - Title = "Test chart", - Height = 400, - Width = 500, - })) - .Generate()) - .Generate(); - - private static readonly DataBlockParent DataBlockParentWithNoPublishedVersion = Fixture - .DefaultDataBlockParent() - .WithLatestDraftVersion(Fixture - .DefaultDataBlockVersion() - .WithReleaseVersion(ReleaseVersion) - .Generate()) - .Generate(); - - private static readonly Guid PublicationId = ReleaseVersion.PublicationId; - - private static readonly Guid ReleaseVersionId = ReleaseVersion.Id; + private readonly DataFixture _dataFixture = new(); - private static readonly Guid DataBlockId = DataBlockParent.LatestPublishedVersion!.Id; - - private static readonly Guid DataBlockParentId = DataBlockParent.Id; + private static readonly List Charts = + [ + new LineChart + { + Title = "Test chart", + Height = 400, + Width = 500 + } + ]; - private static readonly FullTableQuery FullTableQuery = - DataBlockParent.LatestPublishedVersion!.Query; + private static readonly FullTableQuery FullTableQuery = new() + { + SubjectId = Guid.NewGuid(), + LocationIds = [Guid.NewGuid()], + TimePeriod = new TimePeriodQuery + { + StartYear = 2021, + StartCode = CalendarYear, + EndYear = 2022, + EndCode = CalendarYear + }, + Filters = new List(), + Indicators = new List // use collection expression -> test failures + { + Guid.NewGuid() + } + }; - private static readonly TableBuilderConfiguration TableConfiguration = - DataBlockParent.LatestPublishedVersion!.Table; + private static readonly TableBuilderConfiguration TableConfiguration = new() + { + TableHeaders = new TableHeaders { Rows = [new TableHeader("table header 1", TableHeaderType.Filter)] } + }; private readonly TableBuilderResultViewModel _tableBuilderResults = new() { SubjectMeta = new SubjectResultMetaViewModel { - TimePeriodRange = new List - { - new(2020, AcademicYear), - new(2021, AcademicYear), - } + TimePeriodRange = + [ + new TimePeriodMetaViewModel(2020, AcademicYear), + new TimePeriodMetaViewModel(2021, AcademicYear) + ] }, Results = new List { @@ -164,8 +129,7 @@ public async Task Query_Csv() ) .ReturnsAsync(Unit.Instance) .Callback( - (_, stream, _) => { stream.WriteText("Test csv"); } - ); + (_, stream, _) => stream.WriteText("Test csv")); var client = SetupApp(tableBuilderService: tableBuilderService.Object).CreateClient(); @@ -181,17 +145,22 @@ public async Task Query_Csv() } [Fact] - public async Task Query_ReleaseId() + public async Task Query_ReleaseVersionId() { + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var releaseVersion = publication.Releases.Single().Versions.Single(); + await TestApp.AddTestData(context => - context.ReleaseVersions.Add(ReleaseVersion)); + context.Publications.Add(publication)); var tableBuilderService = new Mock(Strict); tableBuilderService .Setup( s => s.Query( - ReleaseVersionId, + releaseVersion.Id, ItIs.DeepEqualTo(FullTableQuery), It.IsAny(), It.IsAny() @@ -203,7 +172,7 @@ await TestApp.AddTestData(context => .CreateClient(); var response = await client - .PostAsync($"/api/tablebuilder/release/{ReleaseVersionId}", + .PostAsync($"/api/tablebuilder/release/{releaseVersion.Id}", new JsonNetContent(FullTableQuery)); VerifyAllMocks(tableBuilderService); @@ -212,17 +181,23 @@ await TestApp.AddTestData(context => } [Fact] - public async Task Query_ReleaseId_Csv() + public async Task Query_ReleaseVersionId_Csv() { + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); + await TestApp.AddTestData(context => - context.ReleaseVersions.Add(ReleaseVersion)); + context.Publications.Add(publication)); var tableBuilderService = new Mock(Strict); tableBuilderService .Setup( s => s.QueryToCsvStream( - ReleaseVersionId, + releaseVersion.Id, ItIs.DeepEqualTo(FullTableQuery), It.IsAny(), It.IsAny() @@ -230,14 +205,13 @@ await TestApp.AddTestData(context => ) .ReturnsAsync(Unit.Instance) .Callback( - (_, _, stream, _) => { stream.WriteText("Test csv"); } - ); + (_, _, stream, _) => stream.WriteText("Test csv")); var client = SetupApp(tableBuilderService: tableBuilderService.Object) .CreateClient(); var response = await client - .PostAsync($"/api/tablebuilder/release/{ReleaseVersionId}", + .PostAsync($"/api/tablebuilder/release/{releaseVersion.Id}", content: new JsonNetContent(FullTableQuery), headers: new Dictionary { { HeaderNames.Accept, ContentTypes.Csv } } ); @@ -251,12 +225,34 @@ await TestApp.AddTestData(context => [Fact] public async Task QueryForTableBuilderResult() { + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); + + DataBlockParent dataBlockParent = _dataFixture + .DefaultDataBlockParent() + .WithLatestPublishedVersion(_dataFixture + .DefaultDataBlockVersion() + .WithReleaseVersion(releaseVersion) + .WithDates(published: DateTime.UtcNow.AddDays(-1)) + .WithQuery(FullTableQuery) + .WithTable(TableConfiguration) + .WithCharts(Charts)); + + var dataBlockId = dataBlockParent.LatestPublishedVersion!.Id; + await TestApp.AddTestData(context => - context.DataBlockParents.Add(DataBlockParent)); + { + context.Publications.Add(publication); + context.DataBlockParents.Add(dataBlockParent); + }); - var cacheKey = new DataBlockTableResultCacheKey(publicationSlug: ReleaseVersion.Publication.Slug, - releaseSlug: ReleaseVersion.Slug, - DataBlockParentId); + var cacheKey = new DataBlockTableResultCacheKey( + publicationSlug: publication.Slug, + releaseSlug: release.Slug, + dataBlockParent.Id); BlobCacheService .Setup(s => s.GetItemAsync(cacheKey, typeof(TableBuilderResultViewModel))) @@ -269,7 +265,7 @@ await TestApp.AddTestData(context => var dataBlockService = new Mock(); dataBlockService - .Setup(s => s.GetDataBlockTableResult(ReleaseVersionId, DataBlockId, null)) + .Setup(s => s.GetDataBlockTableResult(releaseVersion.Id, dataBlockId, null)) .ReturnsAsync(_tableBuilderResults); var client = SetupApp(dataBlockService: dataBlockService.Object) @@ -277,7 +273,7 @@ await TestApp.AddTestData(context => var response = await client.GetAsync( - $"http://localhost/api/tablebuilder/release/{ReleaseVersionId}/data-block/{DataBlockParentId}"); + $"http://localhost/api/tablebuilder/release/{releaseVersion.Id}/data-block/{dataBlockParent.Id}"); VerifyAllMocks(BlobCacheService, dataBlockService); @@ -287,10 +283,18 @@ await client.GetAsync( [Fact] public async Task QueryForTableBuilderResult_NotFound() { + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); + + await TestApp.AddTestData(context => context.Publications.Add(publication)); + var client = SetupApp().CreateClient(); var response = - await client.GetAsync($"/api/tablebuilder/release/{ReleaseVersionId}/data-block/{DataBlockParentId}"); + await client.GetAsync($"/api/tablebuilder/release/{releaseVersion.Id}/data-block/{Guid.NewGuid()}"); response.AssertNotFound(); } @@ -298,13 +302,32 @@ public async Task QueryForTableBuilderResult_NotFound() [Fact] public async Task QueryForTableBuilderResult_NotModified() { + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); + + DataBlockParent dataBlockParent = _dataFixture + .DefaultDataBlockParent() + .WithLatestPublishedVersion(_dataFixture + .DefaultDataBlockVersion() + .WithReleaseVersion(releaseVersion) + .WithDates(published: DateTime.UtcNow.AddDays(-1)) + .WithQuery(FullTableQuery) + .WithTable(TableConfiguration) + .WithCharts(Charts)); + await TestApp.AddTestData(context => - context.DataBlockParents.Add(DataBlockParent)); + { + context.Publications.Add(publication); + context.DataBlockParents.Add(dataBlockParent); + }); var client = SetupApp() .CreateClient(); - var publishedDate = DataBlockParent + var publishedDate = dataBlockParent .LatestPublishedVersion! .Published! .Value; @@ -314,7 +337,7 @@ await TestApp.AddTestData(context => var ifModifiedSinceDate = publishedDate.AddSeconds(1); var response = await client.GetAsync( - $"/api/tablebuilder/release/{ReleaseVersionId}/data-block/{DataBlockParentId}", + $"/api/tablebuilder/release/{releaseVersion.Id}/data-block/{dataBlockParent.Id}", new Dictionary { { HeaderNames.IfModifiedSince, ifModifiedSinceDate.ToUniversalTime().ToString("R") }, @@ -330,12 +353,34 @@ await TestApp.AddTestData(context => [Fact] public async Task QueryForTableBuilderResult_ETagChanged() { + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); + + DataBlockParent dataBlockParent = _dataFixture + .DefaultDataBlockParent() + .WithLatestPublishedVersion(_dataFixture + .DefaultDataBlockVersion() + .WithReleaseVersion(releaseVersion) + .WithDates(published: DateTime.UtcNow.AddDays(-1)) + .WithQuery(FullTableQuery) + .WithTable(TableConfiguration) + .WithCharts(Charts)); + + var dataBlockId = dataBlockParent.LatestPublishedVersion!.Id; + await TestApp.AddTestData(context => - context.DataBlockParents.Add(DataBlockParent)); + { + context.Publications.Add(publication); + context.DataBlockParents.Add(dataBlockParent); + }); - var cacheKey = new DataBlockTableResultCacheKey(publicationSlug: ReleaseVersion.Publication.Slug, - releaseSlug: ReleaseVersion.Slug, - DataBlockParentId); + var cacheKey = new DataBlockTableResultCacheKey( + publicationSlug: publication.Slug, + releaseSlug: release.Slug, + dataBlockParent.Id); BlobCacheService .Setup(s => s.GetItemAsync(cacheKey, typeof(TableBuilderResultViewModel))) @@ -348,13 +393,13 @@ await TestApp.AddTestData(context => var dataBlockService = new Mock(Strict); dataBlockService - .Setup(s => s.GetDataBlockTableResult(ReleaseVersionId, DataBlockId, null)) + .Setup(s => s.GetDataBlockTableResult(releaseVersion.Id, dataBlockId, null)) .ReturnsAsync(_tableBuilderResults); var client = SetupApp(dataBlockService: dataBlockService.Object) .CreateClient(); - var publishedDate = DataBlockParent + var publishedDate = dataBlockParent .LatestPublishedVersion! .Published! .Value; @@ -365,7 +410,7 @@ await TestApp.AddTestData(context => var ifModifiedSinceDate = publishedDate.AddSeconds(1); var response = await client.GetAsync( - $"/api/tablebuilder/release/{ReleaseVersionId}/data-block/{DataBlockParentId}", + $"/api/tablebuilder/release/{releaseVersion.Id}/data-block/{dataBlockParent.Id}", new Dictionary { { HeaderNames.IfModifiedSince, ifModifiedSinceDate.ToUniversalTime().ToString("R") }, @@ -381,12 +426,34 @@ await TestApp.AddTestData(context => [Fact] public async Task QueryForTableBuilderResult_LastModifiedChanged() { + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); + + DataBlockParent dataBlockParent = _dataFixture + .DefaultDataBlockParent() + .WithLatestPublishedVersion(_dataFixture + .DefaultDataBlockVersion() + .WithReleaseVersion(releaseVersion) + .WithDates(published: DateTime.UtcNow.AddDays(-1)) + .WithQuery(FullTableQuery) + .WithTable(TableConfiguration) + .WithCharts(Charts)); + + var dataBlockId = dataBlockParent.LatestPublishedVersion!.Id; + await TestApp.AddTestData(context => - context.DataBlockParents.Add(DataBlockParent)); + { + context.Publications.Add(publication); + context.DataBlockParents.Add(dataBlockParent); + }); - var cacheKey = new DataBlockTableResultCacheKey(publicationSlug: ReleaseVersion.Publication.Slug, - releaseSlug: ReleaseVersion.Slug, - DataBlockParentId); + var cacheKey = new DataBlockTableResultCacheKey( + publicationSlug: publication.Slug, + releaseSlug: release.Slug, + dataBlockParent.Id); BlobCacheService .Setup(s => s.GetItemAsync(cacheKey, typeof(TableBuilderResultViewModel))) @@ -399,7 +466,7 @@ await TestApp.AddTestData(context => var dataBlockService = new Mock(Strict); dataBlockService - .Setup(s => s.GetDataBlockTableResult(ReleaseVersionId, DataBlockId, null)) + .Setup(s => s.GetDataBlockTableResult(releaseVersion.Id, dataBlockId, null)) .ReturnsAsync(_tableBuilderResults); var client = SetupApp(dataBlockService: dataBlockService.Object) @@ -407,14 +474,14 @@ await TestApp.AddTestData(context => // The latest published DataBlockVersion has been published since the caller last requested it, so we // consider this "Modified" by the published date alone. - var yearBeforePublishedDate = DataBlockParent + var yearBeforePublishedDate = dataBlockParent .LatestPublishedVersion! .Published! .Value .AddYears(-1); var response = await client.GetAsync( - $"/api/tablebuilder/release/{ReleaseVersionId}/data-block/{DataBlockParentId}", + $"/api/tablebuilder/release/{releaseVersion.Id}/data-block/{dataBlockParent.Id}", new Dictionary { { HeaderNames.IfModifiedSince, yearBeforePublishedDate.ToUniversalTime().ToString("R") }, @@ -430,20 +497,40 @@ await TestApp.AddTestData(context => [Fact] public async Task QueryForFastTrack() { - await TestApp.AddTestData(context => - context.DataBlockParents.Add(DataBlockParent)); + Publication publication = _dataFixture + .DefaultPublication() + .WithReleases([ + _dataFixture + .DefaultRelease(publishedVersions: 1, year: 2020), + _dataFixture + .DefaultRelease(publishedVersions: 1, year: 2021) + ]); + + var release = publication.Releases.Single(r => r.Year == 2021); + var releaseVersion = release.Versions.Single(); + + DataBlockParent dataBlockParent = _dataFixture + .DefaultDataBlockParent() + .WithLatestPublishedVersion(_dataFixture + .DefaultDataBlockVersion() + .WithReleaseVersion(releaseVersion) + .WithDates(published: DateTime.UtcNow.AddDays(-1)) + .WithQuery(FullTableQuery) + .WithTable(TableConfiguration) + .WithCharts(Charts)); + + var dataBlockId = dataBlockParent.LatestPublishedVersion!.Id; - var latestReleaseVersion = new ReleaseVersion + await TestApp.AddTestData(context => { - Id = ReleaseVersionId, - ReleaseName = "2020", - TimePeriodCoverage = AcademicYear - }; + context.Publications.Add(publication); + context.DataBlockParents.Add(dataBlockParent); + }); var cacheKey = new DataBlockTableResultCacheKey( - ReleaseVersion.Publication.Slug, - ReleaseVersion.Slug, - DataBlockParentId); + publicationSlug: publication.Slug, + releaseSlug: release.Slug, + dataBlockParent.Id); BlobCacheService .Setup(s => s.GetItemAsync(cacheKey, typeof(TableBuilderResultViewModel))) @@ -456,38 +543,34 @@ await TestApp.AddTestData(context => var dataBlockService = new Mock(Strict); dataBlockService - .Setup(s => s.GetDataBlockTableResult(ReleaseVersionId, DataBlockId, null)) + .Setup(s => s.GetDataBlockTableResult(releaseVersion.Id, dataBlockId, null)) .ReturnsAsync(_tableBuilderResults); - var releaseVersionRepository = new Mock(Strict); - - releaseVersionRepository - .Setup(s => s.GetLatestPublishedReleaseVersion(PublicationId, default)) - .ReturnsAsync(latestReleaseVersion); - var client = SetupApp( - dataBlockService: dataBlockService.Object, - releaseVersionRepository: releaseVersionRepository.Object + dataBlockService: dataBlockService.Object ) .CreateClient(); - var response = await client.GetAsync($"/api/tablebuilder/fast-track/{DataBlockParentId}"); + var response = await client.GetAsync($"/api/tablebuilder/fast-track/{dataBlockParent.Id}"); - VerifyAllMocks(BlobCacheService, dataBlockService, releaseVersionRepository); + VerifyAllMocks( + BlobCacheService, + dataBlockService + ); var viewModel = response.AssertOk(); - Assert.Equal(DataBlockParentId, viewModel.DataBlockParentId); - Assert.Equal(ReleaseVersionId, viewModel.ReleaseId); - Assert.Equal(ReleaseVersion.Slug, viewModel.ReleaseSlug); - Assert.Equal(ReleaseVersion.Type, viewModel.ReleaseType); + Assert.Equal(dataBlockParent.Id, viewModel.DataBlockParentId); + Assert.Equal(releaseVersion.Id, viewModel.ReleaseId); + Assert.Equal(release.Slug, viewModel.ReleaseSlug); + Assert.Equal(releaseVersion.Type, viewModel.ReleaseType); viewModel.Configuration.AssertDeepEqualTo(TableConfiguration); viewModel.FullTable.AssertDeepEqualTo(_tableBuilderResults); Assert.True(viewModel.LatestData); - Assert.Equal("Academic year 2020/21", viewModel.LatestReleaseTitle); + Assert.Equal(release.Title, viewModel.LatestReleaseTitle); var queryViewModel = viewModel.Query; Assert.NotNull(queryViewModel); - Assert.Equal(PublicationId, queryViewModel.PublicationId); + Assert.Equal(publication.Id, queryViewModel.PublicationId); Assert.Equal(FullTableQuery.SubjectId, viewModel.Query.SubjectId); Assert.Equal(FullTableQuery.TimePeriod, viewModel.Query.TimePeriod); Assert.Equal(FullTableQuery.Filters, viewModel.Query.Filters); @@ -498,14 +581,29 @@ await TestApp.AddTestData(context => [Fact] public async Task QueryForFastTrack_DataBlockNotYetPublished() { + Publication publication = _dataFixture.DefaultPublication() + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1)]); + + var release = publication.Releases.Single(); + var releaseVersion = release.Versions.Single(); + + DataBlockParent dataBlockParentWithNoPublishedVersion = _dataFixture + .DefaultDataBlockParent() + .WithLatestDraftVersion(_dataFixture + .DefaultDataBlockVersion() + .WithReleaseVersion(releaseVersion)); + await TestApp.AddTestData(context => - context.DataBlockParents.Add(DataBlockParentWithNoPublishedVersion)); + { + context.Publications.Add(publication); + context.DataBlockParents.Add(dataBlockParentWithNoPublishedVersion); + }); var client = SetupApp() .CreateClient(); var response = - await client.GetAsync($"/api/tablebuilder/fast-track/{DataBlockParentWithNoPublishedVersion.Id}"); + await client.GetAsync($"/api/tablebuilder/fast-track/{dataBlockParentWithNoPublishedVersion.Id}"); VerifyAllMocks(BlobCacheService); @@ -515,19 +613,43 @@ await TestApp.AddTestData(context => [Fact] public async Task QueryForFastTrack_NotLatestRelease() { - await TestApp.AddTestData(context => - context.DataBlockParents.Add(DataBlockParent)); + Publication publication = _dataFixture + .DefaultPublication() + .WithReleases([ + _dataFixture + .DefaultRelease(publishedVersions: 1, year: 2020), + _dataFixture + .DefaultRelease(publishedVersions: 1, year: 2021) + ]); + + var release2020 = publication.Releases.Single(r => r.Year == 2020); + var release2021 = publication.Releases.Single(r => r.Year == 2021); + + // Release version is from the 2020 release which is not the latest release for the publication + var releaseVersion = release2020.Versions.Single(); + + DataBlockParent dataBlockParent = _dataFixture + .DefaultDataBlockParent() + .WithLatestPublishedVersion(_dataFixture + .DefaultDataBlockVersion() + .WithReleaseVersion(releaseVersion) + .WithDates(published: DateTime.UtcNow.AddDays(-1)) + .WithQuery(FullTableQuery) + .WithTable(TableConfiguration) + .WithCharts(Charts)); + + var dataBlockId = dataBlockParent.LatestPublishedVersion!.Id; - var latestReleaseVersion = new ReleaseVersion + await TestApp.AddTestData(context => { - Id = Guid.NewGuid(), - ReleaseName = "2021", - TimePeriodCoverage = AcademicYear - }; + context.Publications.Add(publication); + context.DataBlockParents.Add(dataBlockParent); + }); - var cacheKey = new DataBlockTableResultCacheKey(publicationSlug: ReleaseVersion.Publication.Slug, - releaseSlug: ReleaseVersion.Slug, - DataBlockParentId); + var cacheKey = new DataBlockTableResultCacheKey( + publicationSlug: publication.Slug, + releaseSlug: release2020.Slug, + dataBlockParent.Id); BlobCacheService .Setup(s => s.GetItemAsync(cacheKey, typeof(TableBuilderResultViewModel))) @@ -540,37 +662,32 @@ await TestApp.AddTestData(context => var dataBlockService = new Mock(Strict); dataBlockService - .Setup(s => s.GetDataBlockTableResult(ReleaseVersionId, DataBlockId, null)) + .Setup(s => s.GetDataBlockTableResult(releaseVersion.Id, dataBlockId, null)) .ReturnsAsync(_tableBuilderResults); - var releaseVersionRepository = new Mock(Strict); - - releaseVersionRepository - .Setup(s => s.GetLatestPublishedReleaseVersion(PublicationId, default)) - .ReturnsAsync(latestReleaseVersion); - var client = SetupApp( - dataBlockService: dataBlockService.Object, - releaseVersionRepository: releaseVersionRepository.Object + dataBlockService: dataBlockService.Object ) .CreateClient(); - var response = await client.GetAsync($"/api/tablebuilder/fast-track/{DataBlockParentId}"); + var response = await client.GetAsync($"/api/tablebuilder/fast-track/{dataBlockParent.Id}"); - VerifyAllMocks(BlobCacheService, dataBlockService, releaseVersionRepository); + VerifyAllMocks( + BlobCacheService, + dataBlockService + ); var viewModel = response.AssertOk(); - Assert.Equal(DataBlockParentId, viewModel.DataBlockParentId); - Assert.Equal(ReleaseVersionId, viewModel.ReleaseId); - Assert.Equal(ReleaseVersion.Slug, viewModel.ReleaseSlug); - Assert.Equal(ReleaseVersion.Type, viewModel.ReleaseType); + Assert.Equal(dataBlockParent.Id, viewModel.DataBlockParentId); + Assert.Equal(releaseVersion.Id, viewModel.ReleaseId); + Assert.Equal(release2020.Slug, viewModel.ReleaseSlug); + Assert.Equal(releaseVersion.Type, viewModel.ReleaseType); Assert.False(viewModel.LatestData); - Assert.Equal("Academic year 2021/22", viewModel.LatestReleaseTitle); + Assert.Equal(release2021.Title, viewModel.LatestReleaseTitle); } private WebApplicationFactory SetupApp( IDataBlockService? dataBlockService = null, - IReleaseVersionRepository? releaseVersionRepository = null, ITableBuilderService? tableBuilderService = null) { return TestApp @@ -580,8 +697,6 @@ private WebApplicationFactory SetupApp( services.ReplaceService(BlobCacheService); services.AddTransient(_ => dataBlockService ?? Mock.Of(Strict)); - services.AddTransient(_ => - releaseVersionRepository ?? Mock.Of(Strict)); services.AddTransient(_ => tableBuilderService ?? Mock.Of(Strict)); } ); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Api/Controllers/TableBuilderController.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Api/Controllers/TableBuilderController.cs index 6f76f1c2af1..6f6e13bac06 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Api/Controllers/TableBuilderController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Api/Controllers/TableBuilderController.cs @@ -1,12 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Cache; using GovUk.Education.ExploreEducationStatistics.Common.Cancellation; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Requests; -using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; -using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.Api.Cache; using GovUk.Education.ExploreEducationStatistics.Data.Api.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Data.Api.ViewModels; @@ -15,39 +18,22 @@ using GovUk.Education.ExploreEducationStatistics.Data.ViewModels.Meta; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; using static GovUk.Education.ExploreEducationStatistics.Common.Cancellation.RequestTimeoutConfigurationKeys; namespace GovUk.Education.ExploreEducationStatistics.Data.Api.Controllers { [Route("api")] [ApiController] - public class TableBuilderController : ControllerBase + public class TableBuilderController( + ContentDbContext contextDbContext, + IDataBlockService dataBlockService, + ITableBuilderService tableBuilderService) + : ControllerBase { // Change this whenever there is a breaking change // that requires cache invalidation. public const string ApiVersion = "1"; - private readonly IPersistenceHelper _contentPersistenceHelper; - private readonly IDataBlockService _dataBlockService; - private readonly IReleaseVersionRepository _releaseVersionRepository; - private readonly ITableBuilderService _tableBuilderService; - - public TableBuilderController( - IPersistenceHelper contentPersistenceHelper, - IDataBlockService dataBlockService, - IReleaseVersionRepository releaseVersionRepository, - ITableBuilderService tableBuilderService) - { - _contentPersistenceHelper = contentPersistenceHelper; - _dataBlockService = dataBlockService; - _releaseVersionRepository = releaseVersionRepository; - _tableBuilderService = tableBuilderService; - } - [HttpPost("tablebuilder")] [Produces("application/json", "text/csv")] [CancellationTokenTimeout(TableBuilderQuery)] @@ -59,7 +45,7 @@ public async Task Query( { Response.ContentDispositionAttachment(ContentTypes.Csv); - return await _tableBuilderService.QueryToCsvStream( + return await tableBuilderService.QueryToCsvStream( query: request.AsFullTableQuery(), stream: Response.BodyWriter.AsStream(), cancellationToken: cancellationToken @@ -67,7 +53,7 @@ public async Task Query( .HandleFailuresOrNoOp(); } - return await _tableBuilderService + return await tableBuilderService .Query(request.AsFullTableQuery(), cancellationToken) .HandleFailuresOr(Ok); } @@ -86,7 +72,7 @@ public async Task Query( contentType: ContentTypes.Csv, filename: $"{releaseVersionId}.csv"); - return await _tableBuilderService.QueryToCsvStream( + return await tableBuilderService.QueryToCsvStream( releaseVersionId: releaseVersionId, query: request.AsFullTableQuery(), stream: Response.BodyWriter.AsStream(), @@ -95,7 +81,7 @@ public async Task Query( .HandleFailuresOrNoOp(); } - return await _tableBuilderService + return await tableBuilderService .Query(releaseVersionId, request.AsFullTableQuery(), boundaryLevelId: null, cancellationToken) .HandleFailuresOr(Ok); } @@ -147,9 +133,14 @@ public async Task> QueryForFastTrack(Guid dataB { return await GetLatestPublishedDataBlockVersion(dataBlockParentId) .OnSuccessCombineWith(dataBlockVersion => GetDataBlockTableResult(dataBlockVersion)) - .OnSuccessCombineWith(tuple => - _releaseVersionRepository.GetLatestPublishedReleaseVersion(tuple.Item1.ReleaseVersion.PublicationId) - .OrNotFound()) + .OnSuccessCombineWith(async tuple => + { + var (dataBlockVersion, _) = tuple; + return await contextDbContext.Publications + .Where(p => p.Id == dataBlockVersion.ReleaseVersion.PublicationId) + .Select(p => p.LatestPublishedReleaseVersion) + .SingleOrNotFoundAsync(); + }) .OnSuccess(tuple => { var (dataBlockVersion, tableResult, latestReleaseVersion) = tuple; @@ -163,7 +154,7 @@ private Task> GetDataBlockTabl DataBlockVersion dataBlockVersion, long? boundaryLevelId = null) { - return _dataBlockService.GetDataBlockTableResult( + return dataBlockService.GetDataBlockTableResult( releaseVersionId: dataBlockVersion.ReleaseVersionId, dataBlockVersionId: dataBlockVersion.Id, boundaryLevelId); @@ -174,7 +165,7 @@ private Task> GetLatestPublishedDataBlockVersion(Guid dataBlockParentId) + private async Task> GetLatestPublishedDataBlockVersion( + Guid dataBlockParentId) { - return _contentPersistenceHelper - .CheckEntityExists(dataBlockParentId, q => q - .Include(dataBlockParent => dataBlockParent.LatestPublishedVersion) - .ThenInclude(dataBlockVersion => dataBlockVersion.ReleaseVersion) - .ThenInclude(releaseVersion => releaseVersion.Publication)) - .OnSuccess(dataBlock => dataBlock.LatestPublishedVersion) + return await contextDbContext.DataBlockParents + .Include(dbp => dbp.LatestPublishedVersion) + .ThenInclude(dbv => dbv.ReleaseVersion) + .ThenInclude(rv => rv.Publication) + .SingleOrNotFoundAsync(dbp => dbp.Id == dataBlockParentId) + .OnSuccess(dbp => dbp.LatestPublishedVersion) .OrNotFound(); } } From 2d04ca5559609378f62fe659fe0424fa4f8bcc5f Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 16 Dec 2024 17:43:05 +0000 Subject: [PATCH 138/144] EES-5656 On completing publishing, set the latest published release version for publication based on the latest release with a published version --- .../ReleaseVersionRepositoryTests.cs | 133 +++--------------- .../Interfaces/IReleaseVersionRepository.cs | 10 -- .../Repository/ReleaseVersionRepository.cs | 10 -- .../Services/PublishingCompletionService.cs | 41 ++++-- 4 files changed, 49 insertions(+), 145 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs index d4446fa2cc7..3d410fc849a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs @@ -21,92 +21,6 @@ public class GetLatestPublishedReleaseVersionTests : ReleaseVersionRepositoryTes { [Fact] public async Task Success() - { - var publications = _dataFixture - .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2022), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2021), - _dataFixture - .DefaultRelease(publishedVersions: 2, year: 2020))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - var result = await repository.GetLatestPublishedReleaseVersion(publications[0].Id); - - // Expect the result to be the latest published version taken from releases of the specified publication in - // reverse chronological order - var expectedReleaseVersion = publications[0].ReleaseVersions - .Single(rv => rv is { Published: not null, Year: 2021, Version: 1 }); - - Assert.NotNull(result); - Assert.Equal(expectedReleaseVersion.Id, result.Id); - } - - [Fact] - public async Task PublicationHasNoPublishedReleaseVersions_ReturnsNull() - { - var publications = _dataFixture - .DefaultPublication() - // Index 0 has an unpublished release version - // Index 1 has a published release version - .ForIndex(0, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true) - .Generate(1))) - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publications[0].Id)); - } - - [Fact] - public async Task PublicationHasNoReleaseVersions_ReturnsNull() - { - var publications = _dataFixture - .DefaultPublication() - // Index 0 has no release versions - // Index 1 has a published release version - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publications[0].Id)); - } - - [Fact] - public async Task PublicationDoesNotExist_ReturnsNull() - { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); - - var contextId = await AddTestData(publication); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.Null(await repository.GetLatestPublishedReleaseVersion(Guid.NewGuid())); - } - - [Fact] - public async Task SpecificReleaseSlug_Success() { var publications = _dataFixture .DefaultPublication() @@ -121,7 +35,7 @@ public async Task SpecificReleaseSlug_Success() await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var result = await repository.GetLatestPublishedReleaseVersion(publications[0].Id, "2021-22"); + var result = await repository.GetLatestPublishedReleaseVersion(publications[0].Id, releaseSlug: "2021-22"); // Expect the result to be the latest published version for the 2021-22 release of the specified publication var expectedReleaseVersion = publications[0].ReleaseVersions @@ -132,76 +46,70 @@ public async Task SpecificReleaseSlug_Success() } [Fact] - public async Task SpecificReleaseSlug_PublicationHasNoPublishedReleaseVersions_ReturnsNull() + public async Task PublicationHasNoPublishedReleaseVersions_ReturnsNull() { var publications = _dataFixture .DefaultPublication() // Index 0 has an unpublished release version // Index 1 has a published release version - .ForIndex(0, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021) - .Generate(1))) - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1, year: 2021) - .Generate(1))) + .ForIndex(0, p => p.SetReleases([ + _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021) + ])) + .ForIndex(1, p => p.SetReleases([ + _dataFixture.DefaultRelease(publishedVersions: 1, year: 2021) + ])) .GenerateList(2); var contextId = await AddTestData(publications); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publications[0].Id, "2021-22")); + Assert.Null(await repository.GetLatestPublishedReleaseVersion(publications[0].Id, releaseSlug: "2021-22")); } [Fact] - public async Task SpecificReleaseSlug_PublicationHasNoPublishedReleaseVersionsMatchingSlug_ReturnsNull() + public async Task PublicationHasNoPublishedReleaseVersionsMatchingSlug_ReturnsNull() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1, year: 2020) - .Generate(1)); + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2020)]); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publication.Id, "2021-22")); + Assert.Null(await repository.GetLatestPublishedReleaseVersion(publication.Id, releaseSlug: "2021-22")); } [Fact] - public async Task SpecificReleaseSlug_PublicationHasNoReleaseVersions_ReturnsNull() + public async Task PublicationHasNoReleaseVersions_ReturnsNull() { var publications = _dataFixture .DefaultPublication() // Index 0 has no release versions // Index 1 has a published release version - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1, year: 2021) - .Generate(1))) + .ForIndex(1, p => p.SetReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2021)])) .GenerateList(2); var contextId = await AddTestData(publications); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publications[0].Id, "2021-22")); + Assert.Null(await repository.GetLatestPublishedReleaseVersion(publications[0].Id, releaseSlug: "2021-22")); } [Fact] - public async Task SpecificReleaseSlug_PublicationDoesNotExist_ReturnsNull() + public async Task PublicationDoesNotExist_ReturnsNull() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1, year: 2021) - .Generate(1)); + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2021)]); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(Guid.NewGuid(), "2021-22")); + Assert.Null(await repository.GetLatestPublishedReleaseVersion(Guid.NewGuid(), releaseSlug: "2021-22")); } } @@ -243,9 +151,8 @@ public async Task PublicationHasNoReleaseVersions_ReturnsNull() .DefaultPublication() // Index 0 has no release versions // Index 1 has a release version - .ForIndex(1, p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) + .ForIndex(1, p => p.SetReleases([ + _dataFixture.DefaultRelease(publishedVersions: 1)])) .GenerateList(2); var contextId = await AddTestData(publications); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs index 8cc5dd3d21e..8a01061242c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs @@ -12,16 +12,6 @@ Task GetPublishedDate( Guid releaseVersionId, DateTime actualPublishedDate); - /// - /// Retrieves the latest published version from all releases in reverse chronological order that are associated with a publication. - /// - /// The unique identifier of the publication. - /// A to observe while waiting for the task to complete. - /// The latest published version from all releases in reverse chronological order that are associated with a publication. - Task GetLatestPublishedReleaseVersion( - Guid publicationId, - CancellationToken cancellationToken = default); - /// /// Retrieves the latest published version of a release matching a given slug associated with a publication. /// diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs index 6a167df5e37..f07444f4340 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs @@ -54,16 +54,6 @@ await _contentDbContext.Entry(releaseVersion) return releaseVersion.PreviousVersion.Published.Value; } - public async Task GetLatestPublishedReleaseVersion( - Guid publicationId, - CancellationToken cancellationToken = default) - { - return (await _contentDbContext.ReleaseVersions.LatestReleaseVersions(publicationId, publishedOnly: true) - .ToListAsync(cancellationToken: cancellationToken)) - .OrderByReverseChronologicalOrder() - .FirstOrDefault(); - } - public async Task GetLatestReleaseVersion( Guid publicationId, CancellationToken cancellationToken = default) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs index 2269a2b9363..48728b8416b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; -using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Predicates; using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.Cache; using GovUk.Education.ExploreEducationStatistics.Publisher.Model; using GovUk.Education.ExploreEducationStatistics.Publisher.Services.Interfaces; @@ -18,7 +18,6 @@ public class PublishingCompletionService( INotificationsService notificationsService, IReleasePublishingStatusService releasePublishingStatusService, IPublicationCacheService publicationCacheService, - IReleaseVersionRepository releaseVersionRepository, IReleaseService releaseService, IRedirectsCacheService redirectsCacheService, IDataSetPublishingService dataSetPublishingService) @@ -29,10 +28,7 @@ public async Task CompletePublishingIfAllPriorStagesComplete( { var releaseStatuses = await releasePublishingKeys .ToAsyncEnumerable() - .SelectAwait(async key => - { - return await releasePublishingStatusService.Get(key); - }) + .SelectAwait(async key => await releasePublishingStatusService.Get(key)) .ToListAsync(); var prePublishingStagesComplete = releaseStatuses @@ -72,7 +68,6 @@ await releaseVersionIdsToUpdate foreach (var methodologyVersion in methodologyVersions) { - // WARN: This must be called before PublicationRepository#UpdateLatestPublishedRelease if (await methodologyService.IsBeingPublishedAlongsideRelease(methodologyVersion, releaseVersion)) { await methodologyService.Publish(methodologyVersion); @@ -89,7 +84,7 @@ await releaseVersionIdsToUpdate await directlyRelatedPublicationIds .ToAsyncEnumerable() - .ForEachAwaitAsync(UpdateLatestPublishedRelease); + .ForEachAwaitAsync(UpdateLatestPublishedReleaseVersionForPublication); // Update the cached publication and any cached superseded publications. // If this is the first live release of the publication, the superseding is now enforced @@ -124,14 +119,36 @@ await releasePublishingStatusService .UpdatePublishingStage(status.AsTableRowKey(), ReleasePublishingStatusPublishingStage.Complete)); } - private async Task UpdateLatestPublishedRelease(Guid publicationId) + private async Task UpdateLatestPublishedReleaseVersionForPublication(Guid publicationId) { var publication = await contentDbContext.Publications .SingleAsync(p => p.Id == publicationId); - var latestPublishedReleaseVersion = - await releaseVersionRepository.GetLatestPublishedReleaseVersion(publicationId); - publication.LatestPublishedReleaseVersionId = latestPublishedReleaseVersion!.Id; + // Get the publications release id's by the order they appear in the release series + var releaseSeriesReleaseIds = publication.ReleaseSeries + .Where(rsi => !rsi.IsLegacyLink) + .Select(rs => rs.ReleaseId!.Value) + .ToList(); + + // Work out the publication's new latest published release version. + // This is the latest published version of the first release which has a published version + Guid? latestPublishedReleaseVersionId = null; + foreach (var releaseId in releaseSeriesReleaseIds) + { + latestPublishedReleaseVersionId = (await contentDbContext.ReleaseVersions + .LatestReleaseVersion(releaseId: releaseId, publishedOnly: true) + .SingleOrDefaultAsync())?.Id; + + if (latestPublishedReleaseVersionId != null) + { + break; + } + } + + publication.LatestPublishedReleaseVersionId = + latestPublishedReleaseVersionId ?? + throw new InvalidOperationException( + $"No latest published release version found for publication {publicationId}"); contentDbContext.Update(publication); await contentDbContext.SaveChangesAsync(); From f3012b1a60743c5929bb03d5169a7e1a7baf7786 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Tue, 17 Dec 2024 16:05:04 +0000 Subject: [PATCH 139/144] EES-5656 Refactor logic to extract release id's from a release series to be reusable. --- .../Extensions/ReleaseSeriesItemExtensions.cs | 23 +++++++++++++++++++ .../Services/PublishingCompletionService.cs | 6 ++--- 2 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Content.Model/Extensions/ReleaseSeriesItemExtensions.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Extensions/ReleaseSeriesItemExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Extensions/ReleaseSeriesItemExtensions.cs new file mode 100644 index 00000000000..3581291c11a --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Extensions/ReleaseSeriesItemExtensions.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; + +public static class ReleaseSeriesItemExtensions +{ + /// + /// Retrieves the release id's by the order they appear in a of type , + /// ignoring any legacy links. + /// + /// + /// A of type containing the release id's in the order + /// they appear in the release series. + public static IReadOnlyList ReleaseIds(this List releaseSeriesItems) + { + return releaseSeriesItems + .Where(rsi => !rsi.IsLegacyLink) + .Select(rs => rs.ReleaseId!.Value) + .ToList(); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs index 48728b8416b..9bd2031ff20 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; using GovUk.Education.ExploreEducationStatistics.Content.Model.Predicates; using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.Cache; using GovUk.Education.ExploreEducationStatistics.Publisher.Model; @@ -125,10 +126,7 @@ private async Task UpdateLatestPublishedReleaseVersionForPublication(Guid public .SingleAsync(p => p.Id == publicationId); // Get the publications release id's by the order they appear in the release series - var releaseSeriesReleaseIds = publication.ReleaseSeries - .Where(rsi => !rsi.IsLegacyLink) - .Select(rs => rs.ReleaseId!.Value) - .ToList(); + var releaseSeriesReleaseIds = publication.ReleaseSeries.ReleaseIds(); // Work out the publication's new latest published release version. // This is the latest published version of the first release which has a published version From 9c873761b94c160a4eccf1151cd9b54d477536bc Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Wed, 18 Dec 2024 08:26:29 +0000 Subject: [PATCH 140/144] EES-5656 Update methods of ReleaseVersionRepository to use release series --- .../ReleaseVersionRepositoryTests.cs | 618 +++++++++--------- .../Extensions/ReleaseSeriesItemExtensions.cs | 26 +- .../Interfaces/IReleaseVersionRepository.cs | 8 +- .../Repository/ReleaseVersionRepository.cs | 41 +- 4 files changed, 375 insertions(+), 318 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs index 3d410fc849a..88ecdddf474 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs @@ -7,7 +7,6 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository; using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using Xunit; -using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; using static GovUk.Education.ExploreEducationStatistics.Common.Utils.ComparerUtils; using static GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Utils.ContentDbUtils; @@ -22,57 +21,54 @@ public class GetLatestPublishedReleaseVersionTests : ReleaseVersionRepositoryTes [Fact] public async Task Success() { - var publications = _dataFixture + Publication publication = _dataFixture .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 1, year: 2022), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2021))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + .WithReleases( + [ + _dataFixture.DefaultRelease(publishedVersions: 1, year: 2022), + _dataFixture.DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2021) + ]); + + var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var result = await repository.GetLatestPublishedReleaseVersion(publications[0].Id, releaseSlug: "2021-22"); + var result = await repository.GetLatestPublishedReleaseVersion(publication.Id, releaseSlug: "2021-22"); - // Expect the result to be the latest published version for the 2021-22 release of the specified publication - var expectedReleaseVersion = publications[0].ReleaseVersions - .Single(rv => rv is { Published: not null, Year: 2021, Version: 1 }); + // Expect the result to be the latest published version for the 2021-22 release + var expectedReleaseVersion = publication.Releases.Single(r => r is { Year: 2021 }).Versions[1]; Assert.NotNull(result); Assert.Equal(expectedReleaseVersion.Id, result.Id); } [Fact] - public async Task PublicationHasNoPublishedReleaseVersions_ReturnsNull() + public async Task MultiplePublications_ReturnsReleaseVersionAssociatedWithPublication() { - var publications = _dataFixture + var (publication1, publication2) = _dataFixture .DefaultPublication() - // Index 0 has an unpublished release version - // Index 1 has a published release version - .ForIndex(0, p => p.SetReleases([ - _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021) - ])) - .ForIndex(1, p => p.SetReleases([ - _dataFixture.DefaultRelease(publishedVersions: 1, year: 2021) - ])) - .GenerateList(2); + .WithReleases(_ => [_dataFixture.DefaultRelease(publishedVersions: 1, year: 2021)]) + .GenerateTuple2(); - var contextId = await AddTestData(publications); + var contextId = await AddTestData(publication1, publication2); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publications[0].Id, releaseSlug: "2021-22")); + var result = await repository.GetLatestPublishedReleaseVersion(publication1.Id, releaseSlug: "2021-22"); + + // Expect the result to be from the specified publication + var expectedReleaseVersion = publication1.Releases.Single().Versions.Single(); + + Assert.NotNull(result); + Assert.Equal(expectedReleaseVersion.Id, result.Id); } [Fact] - public async Task PublicationHasNoPublishedReleaseVersionsMatchingSlug_ReturnsNull() + public async Task PublicationHasNoPublishedReleaseVersions_ReturnsNull() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2020)]); + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021)]); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); @@ -82,34 +78,38 @@ public async Task PublicationHasNoPublishedReleaseVersionsMatchingSlug_ReturnsNu } [Fact] - public async Task PublicationHasNoReleaseVersions_ReturnsNull() + public async Task PublicationHasNoPublishedReleaseVersionsMatchingSlug_ReturnsNull() { - var publications = _dataFixture + Publication publication = _dataFixture .DefaultPublication() - // Index 0 has no release versions - // Index 1 has a published release version - .ForIndex(1, p => p.SetReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2021)])) - .GenerateList(2); + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2020)]); - var contextId = await AddTestData(publications); + var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publications[0].Id, releaseSlug: "2021-22")); + Assert.Null(await repository.GetLatestPublishedReleaseVersion(publication.Id, releaseSlug: "2021-22")); } [Fact] - public async Task PublicationDoesNotExist_ReturnsNull() + public async Task PublicationHasNoReleaseVersions_ReturnsNull() { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 1, year: 2021)]); + Publication publication = _dataFixture.DefaultPublication(); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(Guid.NewGuid(), releaseSlug: "2021-22")); + Assert.Null(await repository.GetLatestPublishedReleaseVersion(publication.Id, releaseSlug: "2021-22")); + } + + [Fact] + public async Task PublicationDoesNotExist_ReturnsNull() + { + var repository = BuildRepository(); + + Assert.Null(await repository.GetLatestPublishedReleaseVersion(publicationId: Guid.NewGuid(), + releaseSlug: "2021-22")); } } @@ -118,64 +118,68 @@ public class GetLatestReleaseVersionTests : ReleaseVersionRepositoryTests [Fact] public async Task Success() { - var publications = _dataFixture + Publication publication = _dataFixture .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2022), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2021), - _dataFixture - .DefaultRelease(publishedVersions: 2, year: 2020))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + .WithReleases( + [ + _dataFixture.DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022), + _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), + _dataFixture.DefaultRelease(publishedVersions: 2, year: 2020) + ]) + .FinishWith(p => p.ReleaseSeries = GenerateReleaseSeries(p.Releases, 2021, 2020, 2022)); + + var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var result = await repository.GetLatestReleaseVersion(publications[0].Id); + var result = await repository.GetLatestReleaseVersion(publication.Id); - // Expect the result to be the latest version taken from releases of the specified publication in - // reverse chronological order - var expectedReleaseVersion = publications[0].ReleaseVersions - .Single(rv => rv is { Year: 2022, Version: 0 }); + // Expect the result to be the latest version of the latest release in the release series + var expectedReleaseVersion = publication.Releases.Single(r => r is { Year: 2021 }).Versions[0]; Assert.NotNull(result); Assert.Equal(expectedReleaseVersion.Id, result.Id); } [Fact] - public async Task PublicationHasNoReleaseVersions_ReturnsNull() + public async Task MultiplePublications_ReturnsReleaseVersionAssociatedWithPublication() { - var publications = _dataFixture + var (publication1, publication2) = _dataFixture .DefaultPublication() - // Index 0 has no release versions - // Index 1 has a release version - .ForIndex(1, p => p.SetReleases([ - _dataFixture.DefaultRelease(publishedVersions: 1)])) - .GenerateList(2); + .WithReleases(_ => [_dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true)]) + .GenerateTuple2(); - var contextId = await AddTestData(publications); + var contextId = await AddTestData(publication1, publication2); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestReleaseVersion(publications[0].Id)); + var result = await repository.GetLatestReleaseVersion(publication1.Id); + + // Expect the result to be from the specified publication + var expectedReleaseVersion = publication1.Releases.Single().Versions.Single(); + + Assert.NotNull(result); + Assert.Equal(expectedReleaseVersion.Id, result.Id); } [Fact] - public async Task PublicationDoesNotExist_ReturnsNull() + public async Task PublicationHasNoReleaseVersions_ReturnsNull() { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + Publication publication = _dataFixture.DefaultPublication(); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestReleaseVersion(Guid.NewGuid())); + Assert.Null(await repository.GetLatestReleaseVersion(publication.Id)); + } + + [Fact] + public async Task PublicationDoesNotExist_ReturnsNull() + { + var repository = BuildRepository(); + + Assert.Null(await repository.GetLatestReleaseVersion(publicationId: Guid.NewGuid())); } } @@ -186,20 +190,18 @@ public async Task Success() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true) - .Generate(1)); + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 2, draftVersion: true)]); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var releaseVersions = publication.ReleaseVersions; + var release = publication.Releases.Single(); - // Expect only the highest published version of the release to be the latest - Assert.False(await repository.IsLatestPublishedReleaseVersion(releaseVersions[0].Id)); - Assert.True(await repository.IsLatestPublishedReleaseVersion(releaseVersions[1].Id)); - Assert.False(await repository.IsLatestPublishedReleaseVersion(releaseVersions[2].Id)); + // Expect only the latest published version of the release to be returned as the latest + Assert.False(await repository.IsLatestPublishedReleaseVersion(release.Versions[0].Id)); + Assert.True(await repository.IsLatestPublishedReleaseVersion(release.Versions[1].Id)); + Assert.False(await repository.IsLatestPublishedReleaseVersion(release.Versions[2].Id)); } [Fact] @@ -207,15 +209,13 @@ public async Task ReleaseHasNoPublishedReleaseVersions_ReturnsFalse() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true) - .Generate(1)); + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true)]); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var releaseVersion = publication.ReleaseVersions.Single(); + var releaseVersion = publication.Releases.Single().Versions.Single(); Assert.False(await repository.IsLatestPublishedReleaseVersion(releaseVersion.Id)); } @@ -223,17 +223,9 @@ public async Task ReleaseHasNoPublishedReleaseVersions_ReturnsFalse() [Fact] public async Task ReleaseVersionDoesNotExist_ReturnsFalse() { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + var repository = BuildRepository(); - var contextId = await AddTestData(publication); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); - - Assert.False(await repository.IsLatestPublishedReleaseVersion(Guid.NewGuid())); + Assert.False(await repository.IsLatestPublishedReleaseVersion(releaseVersionId: Guid.NewGuid())); } } @@ -244,36 +236,26 @@ public async Task Success() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true) - .Generate(1)); + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 2, draftVersion: true)]); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var releaseVersions = publication.ReleaseVersions; + var release = publication.Releases.Single(); - // Expect only the highest version of the release to be the latest - Assert.False(await repository.IsLatestReleaseVersion(releaseVersions[0].Id)); - Assert.False(await repository.IsLatestReleaseVersion(releaseVersions[1].Id)); - Assert.True(await repository.IsLatestReleaseVersion(releaseVersions[2].Id)); + // Expect only the latest draft version of the release to be returned as the latest + Assert.False(await repository.IsLatestReleaseVersion(release.Versions[0].Id)); + Assert.False(await repository.IsLatestReleaseVersion(release.Versions[1].Id)); + Assert.True(await repository.IsLatestReleaseVersion(release.Versions[2].Id)); } [Fact] public async Task ReleaseVersionDoesNotExist_ReturnsFalse() { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); - - var contextId = await AddTestData(publication); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); + var repository = BuildRepository(); - Assert.False(await repository.IsLatestReleaseVersion(Guid.NewGuid())); + Assert.False(await repository.IsLatestReleaseVersion(releaseVersionId: Guid.NewGuid())); } } @@ -284,29 +266,56 @@ public class AnyPublishedStateTests : ListLatestReleaseVersionIdsTests [Fact] public async Task Success() { - var publications = _dataFixture + Publication publication = _dataFixture + .DefaultPublication() + .WithReleases( + [ + _dataFixture.DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022), + _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), + _dataFixture.DefaultRelease(publishedVersions: 2, year: 2020) + ]) + .FinishWith(p => p.ReleaseSeries = GenerateReleaseSeries(p.Releases, 2021, 2020, 2022)); + + var contextId = await AddTestData(publication); + await using var contentDbContext = InMemoryContentDbContext(contextId); + var repository = BuildRepository(contentDbContext); + + var result = await ListLatestReleaseVersions(repository, publication.Id); + + // Expect the latest versions of each release, ordered by release series + Guid[] expectedReleaseVersionIds = + [ + publication.Releases.Single(r => r is { Year: 2021 }).Versions[0].Id, + publication.Releases.Single(r => r is { Year: 2020 }).Versions[1].Id, + publication.Releases.Single(r => r is { Year: 2022 }).Versions[2].Id + ]; + + Assert.Equal(expectedReleaseVersionIds, result.Select(rv => rv.Id)); + } + + [Fact] + public async Task MultiplePublications_ReturnsReleaseVersionsAssociatedWithPublication() + { + var (publication1, publication2) = _dataFixture .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + .WithReleases(_ => + [ + _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2022), + _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021) + ]) + .GenerateTuple2(); + + var contextId = await AddTestData(publication1, publication2); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var result = await repository.ListLatestReleaseVersions(publications[0].Id, publishedOnly: false); + var result = await ListLatestReleaseVersions(repository, publication1.Id); - // Expect the result to contain the highest version of each release for the specified publication + // Expect the results to be from the specified publication AssertIdsAreEqualIgnoringOrder( [ - publications[0].ReleaseVersions[0].Id, - publications[0].ReleaseVersions[3].Id, - publications[0].ReleaseVersions[5].Id + publication1.Releases.Single(r => r is { Year: 2022 }).Versions[0].Id, + publication1.Releases.Single(r => r is { Year: 2021 }).Versions[0].Id ], result); } @@ -314,68 +323,85 @@ public async Task Success() [Fact] public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() { - var publications = _dataFixture - .DefaultPublication() - // Index 0 has no release versions - // Index 1 has a published release version - .ForIndex(1, - p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + Publication publication = _dataFixture.DefaultPublication(); + + var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Empty(await repository.ListLatestReleaseVersions(publications[0].Id, publishedOnly: false)); + Assert.Empty(await ListLatestReleaseVersions(repository, publication.Id)); } [Fact] public async Task PublicationDoesNotExist_ReturnsEmpty() + { + var repository = BuildRepository(); + + Assert.Empty(await ListLatestReleaseVersions(repository, publicationId: Guid.NewGuid())); + } + + private static async Task> ListLatestReleaseVersions( + ReleaseVersionRepository repository, + Guid publicationId) + { + return await repository.ListLatestReleaseVersions(publicationId, publishedOnly: false); + } + } + + public class PublishedOnlyTests : ListLatestReleaseVersionsTests + { + [Fact] + public async Task Success() { Publication publication = _dataFixture .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + .WithReleases( + [ + _dataFixture.DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022), + _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), + _dataFixture.DefaultRelease(publishedVersions: 2, year: 2020) + ]) + .FinishWith(p => p.ReleaseSeries = GenerateReleaseSeries(p.Releases, 2021, 2020, 2022)); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Empty(await repository.ListLatestReleaseVersions(publicationId: Guid.NewGuid(), - publishedOnly: false)); + var result = await ListLatestReleaseVersions(repository, publication.Id); + + // Expect the latest published version of each release, ordered by release series + Guid[] expectedReleaseVersionIds = + [ + publication.Releases.Single(r => r is { Year: 2020 }).Versions[1].Id, + publication.Releases.Single(r => r is { Year: 2022 }).Versions[1].Id + ]; + + Assert.Equal(expectedReleaseVersionIds, result.Select(rv => rv.Id)); } - } - public class PublishedOnlyTests : ListLatestReleaseVersionsTests - { [Fact] - public async Task Success() + public async Task MultiplePublications_ReturnsReleaseVersionsAssociatedWithPublication() { - var publications = _dataFixture + var (publication1, publication2) = _dataFixture .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + .WithReleases(_ => + [ + _dataFixture.DefaultRelease(publishedVersions: 1, year: 2022), + _dataFixture.DefaultRelease(publishedVersions: 1, year: 2021) + ]) + .GenerateTuple2(); + + var contextId = await AddTestData(publication1, publication2); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var result = await repository.ListLatestReleaseVersions(publications[0].Id, publishedOnly: true); + var result = await ListLatestReleaseVersions(repository, publication1.Id); - // Expect the result to contain the highest published version of each release for the specified publication + // Expect the results to be from the specified publication AssertIdsAreEqualIgnoringOrder( [ - publications[0].ReleaseVersions[2].Id, - publications[0].ReleaseVersions[5].Id + publication1.Releases.Single(r => r is { Year: 2022 }).Versions[0].Id, + publication1.Releases.Single(r => r is { Year: 2021 }).Versions[0].Id ], result); } @@ -383,62 +409,42 @@ public async Task Success() [Fact] public async Task PublicationHasNoPublishedReleaseVersions_ReturnsEmpty() { - var publications = _dataFixture + Publication publication = _dataFixture .DefaultPublication() - // Index 0 has an unpublished release version - // Index 1 has a published release version - .ForIndex(0, - p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true) - .Generate(1))) - .ForIndex(1, - p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true)]); + + var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Empty(await repository.ListLatestReleaseVersions(publications[0].Id, publishedOnly: true)); + Assert.Empty(await ListLatestReleaseVersions(repository, publication.Id)); } [Fact] public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() { - var publications = _dataFixture - .DefaultPublication() - // Index 0 has no release versions - // Index 1 has a published release version - .ForIndex(1, - p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + Publication publication = _dataFixture.DefaultPublication(); + + var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Empty(await repository.ListLatestReleaseVersions(publications[0].Id, publishedOnly: true)); + Assert.Empty(await ListLatestReleaseVersions(repository, publication.Id)); } [Fact] public async Task PublicationDoesNotExist_ReturnsEmpty() { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + var repository = BuildRepository(); - var contextId = await AddTestData(publication); - await using var contentDbContext = InMemoryContentDbContext(contextId); - var repository = BuildRepository(contentDbContext); + Assert.Empty(await ListLatestReleaseVersions(repository, publicationId: Guid.NewGuid())); + } - Assert.Empty( - await repository.ListLatestReleaseVersions(publicationId: Guid.NewGuid(), publishedOnly: true)); + private static async Task> ListLatestReleaseVersions( + ReleaseVersionRepository repository, + Guid publicationId) + { + return await repository.ListLatestReleaseVersions(publicationId, publishedOnly: true); } } } @@ -450,163 +456,194 @@ public class AnyPublishedStateTests : ListLatestReleaseVersionIdsTests [Fact] public async Task Success() { - var publications = _dataFixture + Publication publication = _dataFixture .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + .WithReleases( + [ + _dataFixture.DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022), + _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), + _dataFixture.DefaultRelease(publishedVersions: 2, year: 2020) + ]); + + var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var result = await repository.ListLatestReleaseVersionIds(publications[0].Id, publishedOnly: false); + var result = await ListLatestReleaseVersionIds(repository, publication.Id); - // Expect the result to contain the highest version of each release for the specified publication + // Expect the latest version id's of each release AssertIdsAreEqualIgnoringOrder( [ - publications[0].ReleaseVersions[0].Id, - publications[0].ReleaseVersions[3].Id, - publications[0].ReleaseVersions[5].Id + publication.Releases.Single(r => r is { Year: 2022 }).Versions[2].Id, + publication.Releases.Single(r => r is { Year: 2021 }).Versions[0].Id, + publication.Releases.Single(r => r is { Year: 2020 }).Versions[1].Id ], result); } [Fact] - public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() + public async Task MultiplePublications_ReturnsReleaseVersionsAssociatedWithPublication() { - var publications = _dataFixture + var (publication1, publication2) = _dataFixture .DefaultPublication() - // Index 0 has no release versions - // Index 1 has a published release version - .ForIndex(1, - p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + .WithReleases(_ => + [ + _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2022), + _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021) + ]) + .GenerateTuple2(); + + var contextId = await AddTestData(publication1, publication2); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Empty(await repository.ListLatestReleaseVersionIds(publications[0].Id, publishedOnly: false)); + var result = await ListLatestReleaseVersionIds(repository, publication1.Id); + + // Expect the results to be from the specified publication + AssertIdsAreEqualIgnoringOrder( + [ + publication1.Releases.Single(r => r is { Year: 2022 }).Versions[0].Id, + publication1.Releases.Single(r => r is { Year: 2021 }).Versions[0].Id + ], + result); } [Fact] - public async Task PublicationDoesNotExist_ReturnsEmpty() + public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + Publication publication = _dataFixture.DefaultPublication(); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Empty(await repository.ListLatestReleaseVersionIds(publicationId: Guid.NewGuid(), - publishedOnly: false)); + Assert.Empty(await ListLatestReleaseVersionIds(repository, publication.Id)); + } + + [Fact] + public async Task PublicationDoesNotExist_ReturnsEmpty() + { + var repository = BuildRepository(); + + Assert.Empty(await ListLatestReleaseVersionIds(repository, publicationId: Guid.NewGuid())); + } + + private static async Task> ListLatestReleaseVersionIds( + ReleaseVersionRepository repository, + Guid publicationId) + { + return await repository.ListLatestReleaseVersionIds(publicationId, publishedOnly: false); } } - public class PublishedOnlyTrueTests : ListLatestReleaseVersionIdsTests + public class PublishedOnlyTests : ListLatestReleaseVersionIdsTests { [Fact] public async Task Success() { - var publications = _dataFixture + Publication publication = _dataFixture .DefaultPublication() - .WithReleases(_ => ListOf( - _dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2, draftVersion: true), - _dataFixture - .DefaultRelease(publishedVersions: 2))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + .WithReleases( + [ + _dataFixture.DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2022), + _dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), + _dataFixture.DefaultRelease(publishedVersions: 2, year: 2020) + ]); + + var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var result = await repository.ListLatestReleaseVersionIds(publications[0].Id, publishedOnly: true); + var result = await ListLatestReleaseVersionIds(repository, publication.Id); - // Expect the result to contain the highest published version of each release for the specified publication + // Expect the latest published version id's of each release AssertIdsAreEqualIgnoringOrder( [ - publications[0].ReleaseVersions[2].Id, - publications[0].ReleaseVersions[5].Id + publication.Releases.Single(r => r is { Year: 2022 }).Versions[1].Id, + publication.Releases.Single(r => r is { Year: 2020 }).Versions[1].Id ], result); } [Fact] - public async Task PublicationHasNoPublishedReleaseVersions_ReturnsEmpty() + public async Task MultiplePublications_ReturnsReleaseVersionsAssociatedWithPublication() { - var publications = _dataFixture + var (publication1, publication2) = _dataFixture .DefaultPublication() - // Index 0 has an unpublished release version - // Index 1 has a published release version - .ForIndex(0, - p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 0, draftVersion: true) - .Generate(1))) - .ForIndex(1, - p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); - - var contextId = await AddTestData(publications); + .WithReleases(_ => + [ + _dataFixture.DefaultRelease(publishedVersions: 1, year: 2022), + _dataFixture.DefaultRelease(publishedVersions: 1, year: 2021) + ]) + .GenerateTuple2(); + + var contextId = await AddTestData(publication1, publication2); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Empty(await repository.ListLatestReleaseVersionIds(publications[0].Id, publishedOnly: true)); + var result = await ListLatestReleaseVersionIds(repository, publication1.Id); + + // Expect the results to be from the specified publication + AssertIdsAreEqualIgnoringOrder( + [ + publication1.Releases.Single(r => r is { Year: 2022 }).Versions[0].Id, + publication1.Releases.Single(r => r is { Year: 2021 }).Versions[0].Id + ], + result); } [Fact] - public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() + public async Task PublicationHasNoPublishedReleaseVersions_ReturnsEmpty() { - var publications = _dataFixture + Publication publication = _dataFixture .DefaultPublication() - .ForIndex(1, - p => p.SetReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1))) - .GenerateList(2); + .WithReleases([_dataFixture.DefaultRelease(publishedVersions: 0, draftVersion: true)]); - var contextId = await AddTestData(publications); + var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Empty(await repository.ListLatestReleaseVersionIds(publications[0].Id, publishedOnly: true)); + Assert.Empty(await ListLatestReleaseVersionIds(repository, publication.Id)); } [Fact] - public async Task PublicationDoesNotExist_ReturnsEmpty() + public async Task PublicationHasNoReleaseVersions_ReturnsEmpty() { - Publication publication = _dataFixture - .DefaultPublication() - .WithReleases(_dataFixture - .DefaultRelease(publishedVersions: 1) - .Generate(1)); + Publication publication = _dataFixture.DefaultPublication(); var contextId = await AddTestData(publication); await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Empty( - await repository.ListLatestReleaseVersionIds(publicationId: Guid.NewGuid(), publishedOnly: true)); + Assert.Empty(await ListLatestReleaseVersionIds(repository, publication.Id)); + } + + [Fact] + public async Task PublicationDoesNotExist_ReturnsEmpty() + { + var repository = BuildRepository(); + + Assert.Empty(await ListLatestReleaseVersionIds(repository, publicationId: Guid.NewGuid())); + } + + private static async Task> ListLatestReleaseVersionIds( + ReleaseVersionRepository repository, + Guid publicationId) + { + return await repository.ListLatestReleaseVersionIds(publicationId, publishedOnly: true); } } } + private List GenerateReleaseSeries(IReadOnlyList releases, params int[] years) + { + return years.Select(year => + { + var release = releases.Single(r => r.Year == year); + return _dataFixture.DefaultReleaseSeriesItem().WithReleaseId(release.Id).Generate(); + }).ToList(); + } + private static void AssertIdsAreEqualIgnoringOrder( IReadOnlyCollection expectedIds, IReadOnlyCollection actualReleaseVersions) @@ -639,11 +676,10 @@ private static async Task AddTestData(IReadOnlyCollection p return contextId; } - private static ReleaseVersionRepository BuildRepository( - ContentDbContext contentDbContext) + private static ReleaseVersionRepository BuildRepository(ContentDbContext? contentDbContext = null) { return new ReleaseVersionRepository( - contentDbContext: contentDbContext + contentDbContext: contentDbContext ?? InMemoryContentDbContext() ); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Extensions/ReleaseSeriesItemExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Extensions/ReleaseSeriesItemExtensions.cs index 3581291c11a..a7a09eec584 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Extensions/ReleaseSeriesItemExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Extensions/ReleaseSeriesItemExtensions.cs @@ -1,4 +1,5 @@ -using System; +#nullable enable +using System; using System.Collections.Generic; using System.Linq; @@ -7,17 +8,26 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; public static class ReleaseSeriesItemExtensions { /// - /// Retrieves the release id's by the order they appear in a of type , + /// Retrieves the first release id in a of type , + /// ignoring any legacy links. + /// + /// + /// The first release id in the release series. + public static Guid? LatestReleaseId(this List releaseSeriesItems) => + SelectReleaseIds(releaseSeriesItems).FirstOrDefault(); + + /// + /// Retrieves the release id's in a of type , /// ignoring any legacy links. /// /// /// A of type containing the release id's in the order /// they appear in the release series. - public static IReadOnlyList ReleaseIds(this List releaseSeriesItems) - { - return releaseSeriesItems + public static IReadOnlyList ReleaseIds(this List releaseSeriesItems) => + SelectReleaseIds(releaseSeriesItems).ToList(); + + private static IEnumerable SelectReleaseIds(List releaseSeriesItems) => + releaseSeriesItems .Where(rsi => !rsi.IsLegacyLink) - .Select(rs => rs.ReleaseId!.Value) - .ToList(); - } + .Select(rs => rs.ReleaseId!.Value); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs index 8a01061242c..fb99630f1f0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs @@ -25,11 +25,11 @@ Task GetPublishedDate( CancellationToken cancellationToken = default); /// - /// Retrieves the latest version from all releases in reverse chronological order that are associated with a publication. + /// Retrieves the latest version of the latest release in release series order associated with a publication. /// /// The unique identifier of the publication. /// A to observe while waiting for the task to complete. - /// The latest version from all releases in reverse chronological order that are associated with a publication. + /// The latest version of the latest release in release series order associated with the publication. Task GetLatestReleaseVersion( Guid publicationId, CancellationToken cancellationToken = default); @@ -67,12 +67,12 @@ Task> ListLatestReleaseVersionIds( CancellationToken cancellationToken = default); /// - /// Retrieves the latest versions of all releases associated with a given publication in reverse chronological order. + /// Retrieves the latest versions of all releases in release series order associated with a given publication. /// /// The unique identifier of the publication. /// Flag to only include published release versions. /// A to observe while waiting for the task to complete. - /// A collection of the latest version id's of all releases associated with the publication. + /// A collection of the latest versions of all releases in release series order associated with the publication. Task> ListLatestReleaseVersions( Guid publicationId, bool publishedOnly = false, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs index f07444f4340..18ba9720d88 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; using GovUk.Education.ExploreEducationStatistics.Content.Model.Predicates; using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using Microsoft.EntityFrameworkCore; @@ -58,10 +59,16 @@ await _contentDbContext.Entry(releaseVersion) Guid publicationId, CancellationToken cancellationToken = default) { - return (await _contentDbContext.ReleaseVersions.LatestReleaseVersions(publicationId) - .ToListAsync(cancellationToken: cancellationToken)) - .OrderByReverseChronologicalOrder() - .FirstOrDefault(); + var latestReleaseId = await _contentDbContext.Publications + .Where(p => p.Id == publicationId) + .Select(p => p.ReleaseSeries.LatestReleaseId()) + .SingleOrDefaultAsync(cancellationToken: cancellationToken); + + return latestReleaseId.HasValue + ? await _contentDbContext.ReleaseVersions + .LatestReleaseVersion(releaseId: latestReleaseId.Value) + .SingleOrDefaultAsync(cancellationToken: cancellationToken) + : null; } public async Task GetLatestPublishedReleaseVersion( @@ -109,10 +116,24 @@ public async Task> ListLatestReleaseVersions( bool publishedOnly = false, CancellationToken cancellationToken = default) { + var publication = await _contentDbContext.Publications + .SingleOrDefaultAsync(p => p.Id == publicationId, cancellationToken: cancellationToken); + + if (publication == null) + { + return []; + } + + var publicationReleaseSeriesReleaseIds = publication.ReleaseSeries.ReleaseIds(); + + var releaseIdIndexMap = publicationReleaseSeriesReleaseIds + .Select((releaseId, index) => (releaseId, index)) + .ToDictionary(tuple => tuple.releaseId, tuple => tuple.index); + return (await _contentDbContext.ReleaseVersions .LatestReleaseVersions(publicationId, publishedOnly: publishedOnly) .ToListAsync(cancellationToken: cancellationToken)) - .OrderByReverseChronologicalOrder() + .OrderBy(rv => releaseIdIndexMap[rv.ReleaseId]) .ToList(); } @@ -156,13 +177,3 @@ private async Task IsLatestReleaseVersion( private record ReleaseIdVersion(Guid ReleaseId, int Version); } - -internal static class ReleaseVersionIEnumerableExtensions -{ - internal static IOrderedEnumerable OrderByReverseChronologicalOrder( - this IEnumerable query) - { - return query.OrderByDescending(releaseVersion => releaseVersion.Year) - .ThenByDescending(releaseVersion => releaseVersion.TimePeriodCoverage); - } -} From 2e744b2770a77b01b9a1fb2278f6a7fc74588c88 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Thu, 19 Dec 2024 12:15:27 +0000 Subject: [PATCH 141/144] EES-5656 Rename method GetLatestPublishedReleaseVersion to GetLatestPublishedReleaseVersionByReleaseSlug --- .../ReleaseVersionRepositoryTests.cs | 19 ++++++++++++------- .../Interfaces/IReleaseVersionRepository.cs | 2 +- .../Repository/ReleaseVersionRepository.cs | 4 ++-- .../ReleaseService.cs | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs index 88ecdddf474..54a95d1034a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model.Tests/Repository/ReleaseVersionRepositoryTests.cs @@ -16,7 +16,7 @@ public class ReleaseVersionRepositoryTests { private readonly DataFixture _dataFixture = new(); - public class GetLatestPublishedReleaseVersionTests : ReleaseVersionRepositoryTests + public class GetLatestPublishedReleaseVersionByReleaseSlugTests : ReleaseVersionRepositoryTests { [Fact] public async Task Success() @@ -33,7 +33,8 @@ public async Task Success() await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var result = await repository.GetLatestPublishedReleaseVersion(publication.Id, releaseSlug: "2021-22"); + var result = + await repository.GetLatestPublishedReleaseVersionByReleaseSlug(publication.Id, releaseSlug: "2021-22"); // Expect the result to be the latest published version for the 2021-22 release var expectedReleaseVersion = publication.Releases.Single(r => r is { Year: 2021 }).Versions[1]; @@ -54,7 +55,8 @@ public async Task MultiplePublications_ReturnsReleaseVersionAssociatedWithPublic await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - var result = await repository.GetLatestPublishedReleaseVersion(publication1.Id, releaseSlug: "2021-22"); + var result = + await repository.GetLatestPublishedReleaseVersionByReleaseSlug(publication1.Id, releaseSlug: "2021-22"); // Expect the result to be from the specified publication var expectedReleaseVersion = publication1.Releases.Single().Versions.Single(); @@ -74,7 +76,8 @@ public async Task PublicationHasNoPublishedReleaseVersions_ReturnsNull() await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publication.Id, releaseSlug: "2021-22")); + Assert.Null( + await repository.GetLatestPublishedReleaseVersionByReleaseSlug(publication.Id, releaseSlug: "2021-22")); } [Fact] @@ -88,7 +91,8 @@ public async Task PublicationHasNoPublishedReleaseVersionsMatchingSlug_ReturnsNu await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publication.Id, releaseSlug: "2021-22")); + Assert.Null( + await repository.GetLatestPublishedReleaseVersionByReleaseSlug(publication.Id, releaseSlug: "2021-22")); } [Fact] @@ -100,7 +104,8 @@ public async Task PublicationHasNoReleaseVersions_ReturnsNull() await using var contentDbContext = InMemoryContentDbContext(contextId); var repository = BuildRepository(contentDbContext); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publication.Id, releaseSlug: "2021-22")); + Assert.Null( + await repository.GetLatestPublishedReleaseVersionByReleaseSlug(publication.Id, releaseSlug: "2021-22")); } [Fact] @@ -108,7 +113,7 @@ public async Task PublicationDoesNotExist_ReturnsNull() { var repository = BuildRepository(); - Assert.Null(await repository.GetLatestPublishedReleaseVersion(publicationId: Guid.NewGuid(), + Assert.Null(await repository.GetLatestPublishedReleaseVersionByReleaseSlug(publicationId: Guid.NewGuid(), releaseSlug: "2021-22")); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs index fb99630f1f0..fb1e7400467 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/Interfaces/IReleaseVersionRepository.cs @@ -19,7 +19,7 @@ Task GetPublishedDate( /// The slug of the release. /// A to observe while waiting for the task to complete. /// The latest published version of the release associated with the publication. - Task GetLatestPublishedReleaseVersion( + Task GetLatestPublishedReleaseVersionByReleaseSlug( Guid publicationId, string releaseSlug, CancellationToken cancellationToken = default); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs index 18ba9720d88..685151bbb79 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Repository/ReleaseVersionRepository.cs @@ -71,7 +71,7 @@ await _contentDbContext.Entry(releaseVersion) : null; } - public async Task GetLatestPublishedReleaseVersion( + public async Task GetLatestPublishedReleaseVersionByReleaseSlug( Guid publicationId, string releaseSlug, CancellationToken cancellationToken = default) @@ -79,7 +79,7 @@ await _contentDbContext.Entry(releaseVersion) // There should only ever be one latest published release version with a given slug return await _contentDbContext.ReleaseVersions .LatestReleaseVersions(publicationId, releaseSlug, publishedOnly: true) - .FirstOrDefaultAsync(cancellationToken: cancellationToken); + .SingleOrDefaultAsync(cancellationToken: cancellationToken); } public async Task IsLatestPublishedReleaseVersion( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs index 68bc9328b6a..49bb95807e8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Services/ReleaseService.cs @@ -58,7 +58,7 @@ public async Task> GetRelease( // otherwise use the latest published version of the requested release var latestReleaseVersionId = releaseSlug == null ? publication.LatestPublishedReleaseVersionId - : (await _releaseVersionRepository.GetLatestPublishedReleaseVersion(publication.Id, + : (await _releaseVersionRepository.GetLatestPublishedReleaseVersionByReleaseSlug(publication.Id, releaseSlug))?.Id; return latestReleaseVersionId.HasValue From 23523e222dd4c28018d6d81586e87efc30eb4d6d Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Thu, 19 Dec 2024 17:38:58 +0000 Subject: [PATCH 142/144] EES-5656 Remove ManageContentPageViewModel.PublicationViewModel.Releases --- .../ManageContentPageServiceTests.cs | 6 ------ .../Mappings/MappingProfiles.cs | 17 ----------------- .../ManageContent/ManageContentPageViewModel.cs | 11 ----------- .../__tests__/ReleaseContentPage.test.tsx | 1 - .../__tests__/PreReleaseContentPage.test.tsx | 1 - .../src/prototypes/data/releaseContentData.ts | 7 ------- .../test/generators/releaseContentGenerators.ts | 1 - .../src/services/publicationService.ts | 5 ----- .../find-statistics/PublicationReleasePage.tsx | 6 +++++- .../__tests__/PublicationReleasePage.test.tsx | 10 ---------- .../__tests__/__data__/testReleaseData.ts | 12 ------------ .../components/__tests__/__data__/tableData.ts | 1 - 12 files changed, 5 insertions(+), 73 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs index f4dea4b73af..f5906b91e79 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/ManageContent/ManageContentPageServiceTests.cs @@ -324,12 +324,6 @@ public async Task GetManageContentPageViewModel() Assert.Equal(publication.ReleaseSeries[2].LegacyLinkUrl, contentPublicationReleaseSeries[2].LegacyLinkUrl); - var contentPublicationReleases = contentPublication.Releases; - Assert.Single(contentPublicationReleases); - Assert.Equal(otherReleaseVersion.Id, contentPublicationReleases[0].Id); - Assert.Equal(otherReleaseVersion.Slug, contentPublicationReleases[0].Slug); - Assert.Equal(otherReleaseVersion.Title, contentPublicationReleases[0].Title); - Assert.Equal(2, contentPublication.Methodologies.Count); Assert.Equal(methodology.Versions[0].Id, contentPublication.Methodologies[0].Id); Assert.Equal(methodology.Versions[0].Title, contentPublication.Methodologies[0].Title); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Mappings/MappingProfiles.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Mappings/MappingProfiles.cs index 3f802dd7f08..affee167e36 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Mappings/MappingProfiles.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Mappings/MappingProfiles.cs @@ -128,18 +128,6 @@ public MappingProfiles() Title = rv.Publication.Title, Slug = rv.Publication.Slug, Contact = rv.Publication.Contact, - Releases = rv.Publication.ReleaseVersions - .FindAll(otherReleaseVersion => rv.Id != otherReleaseVersion.Id && - IsLatestVersionOfRelease(rv.Publication.ReleaseVersions, otherReleaseVersion.Id)) - .OrderByDescending(otherReleaseVersion => otherReleaseVersion.Year) - .ThenByDescending(otherReleaseVersion => otherReleaseVersion.TimePeriodCoverage) - .Select(otherReleaseVersion => new PreviousReleaseViewModel - { - Id = otherReleaseVersion.Id, - Slug = otherReleaseVersion.Slug, - Title = otherReleaseVersion.Title, - }) - .ToList(), ReleaseSeries = new List(), // Must be hydrated after mapping ExternalMethodology = rv.Publication.ExternalMethodology != null ? new ExternalMethodology @@ -225,10 +213,5 @@ private void CreateContentBlockMap() CreateMap(); } - - private static bool IsLatestVersionOfRelease(IEnumerable releaseVersions, Guid releaseVersionId) - { - return !releaseVersions.Any(rv => rv.PreviousVersionId == releaseVersionId && rv.Id != releaseVersionId); - } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ManageContent/ManageContentPageViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ManageContent/ManageContentPageViewModel.cs index f91481a35d6..be3e5d8802d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ManageContent/ManageContentPageViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/ViewModels/ManageContent/ManageContentPageViewModel.cs @@ -80,8 +80,6 @@ public class PublicationViewModel public string Slug { get; set; } - public List Releases { get; set; } - public List ReleaseSeries { get; set; } public Contact Contact { get; set; } @@ -100,13 +98,4 @@ public class ReleaseNoteViewModel public DateTime On { get; set; } } - - public class PreviousReleaseViewModel - { - public Guid Id { get; set; } - - public string Slug { get; set; } - - public string Title { get; set; } - } } diff --git a/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx index bf84627fec8..cd8fcd19559 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/__tests__/ReleaseContentPage.test.tsx @@ -129,7 +129,6 @@ describe('ReleaseContentPage', () => { id: 'publication-id', title: 'Publication 1', slug: 'publication-1', - releases: [], releaseSeries: [], theme: { id: 'theme-1', title: 'Theme 1' }, contact: { diff --git a/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseContentPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseContentPage.test.tsx index c8c4e512df1..892b34e7b1c 100644 --- a/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseContentPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/pre-release/__tests__/PreReleaseContentPage.test.tsx @@ -91,7 +91,6 @@ describe('PreReleaseContentPage', () => { id: 'publication-id', title: 'Publication 1', slug: 'publication-1', - releases: [], releaseSeries: [], theme: { id: 'theme-1', title: 'Theme 1' }, contact: { diff --git a/src/explore-education-statistics-admin/src/prototypes/data/releaseContentData.ts b/src/explore-education-statistics-admin/src/prototypes/data/releaseContentData.ts index 6b5c0352ddc..abc3df8550b 100644 --- a/src/explore-education-statistics-admin/src/prototypes/data/releaseContentData.ts +++ b/src/explore-education-statistics-admin/src/prototypes/data/releaseContentData.ts @@ -128,13 +128,6 @@ const prototypeReleaseContent: ReleaseContent = { slug: 'methodology-slug', }, ], - releases: [ - { - id: 'previous-release-id', - slug: 'previous-release-slug', - title: 'Previous release title', - }, - ], releaseSeries: [], slug: 'publication-slug', title: 'Initial Teacher Training Census', diff --git a/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts b/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts index ab2c4eb9faa..5e71a6747bc 100644 --- a/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts +++ b/src/explore-education-statistics-admin/test/generators/releaseContentGenerators.ts @@ -92,7 +92,6 @@ const defaultPublication: Publication = { slug: 'methodology-slug', }, ], - releases: [], releaseSeries: [ { isLegacyLink: true, diff --git a/src/explore-education-statistics-common/src/services/publicationService.ts b/src/explore-education-statistics-common/src/services/publicationService.ts index d32c4c8898d..6a919192418 100644 --- a/src/explore-education-statistics-common/src/services/publicationService.ts +++ b/src/explore-education-statistics-common/src/services/publicationService.ts @@ -20,11 +20,6 @@ export interface Publication { id: string; slug: string; title: string; - releases: { - id: string; - slug: string; - title: string; - }[]; releaseSeries: ReleaseSeriesItem[]; theme: { id: string; diff --git a/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx b/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx index ac3a4d19f6b..fb89cf32d87 100644 --- a/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx +++ b/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx @@ -130,7 +130,11 @@ const PublicationReleasePage: NextPage = ({ release }) => { > View latest data:{' '} - {release.publication.releases[0].title} + { + release.publication.releaseSeries.find( + rsi => !rsi.isLegacyLink, + )?.description + } )} diff --git a/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/PublicationReleasePage.test.tsx b/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/PublicationReleasePage.test.tsx index 4dbfc470915..1522c2a7a7d 100644 --- a/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/PublicationReleasePage.test.tsx +++ b/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/PublicationReleasePage.test.tsx @@ -342,16 +342,6 @@ describe('PublicationReleasePage', () => { month: 2, year: 2022, }, - publication: { - ...testRelease.publication, - releases: [ - { - id: 'latest-release', - title: 'Latest Release Title', - slug: 'latest-release-slug', - }, - ], - }, }} />, ); diff --git a/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/__data__/testReleaseData.ts b/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/__data__/testReleaseData.ts index c7c697c1db3..b36cccdaa90 100644 --- a/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/__data__/testReleaseData.ts +++ b/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/__data__/testReleaseData.ts @@ -4,18 +4,6 @@ export const testPublication: Publication = { id: 'publication-1', title: 'Pupil absence in schools in England', slug: 'pupil-absence-in-schools-in-england', - releases: [ - { - id: 'release-2', - slug: '2018-19', - title: 'Academic year 2018/19', - }, - { - id: 'release-1', - slug: '2017-18', - title: 'Academic year 2017/18', - }, - ], releaseSeries: [ { isLegacyLink: false, diff --git a/src/explore-education-statistics-frontend/src/modules/table-tool/components/__tests__/__data__/tableData.ts b/src/explore-education-statistics-frontend/src/modules/table-tool/components/__tests__/__data__/tableData.ts index eaa670625a7..9f7823af6ae 100644 --- a/src/explore-education-statistics-frontend/src/modules/table-tool/components/__tests__/__data__/tableData.ts +++ b/src/explore-education-statistics-frontend/src/modules/table-tool/components/__tests__/__data__/tableData.ts @@ -273,7 +273,6 @@ export const testPublicationRelease: Release = { id: '', slug: '', title: '', - releases: [], releaseSeries: [], theme: { title: '', From 9412715a81c7dd64b0fd38ec21496222fe2e8fe0 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Fri, 20 Dec 2024 10:07:13 +0000 Subject: [PATCH 143/144] EES-5656 Change ContentService UpdateContent/UpdateContentStaged methods to use release series when working out the latest published release version of publication --- .../Extensions/EnumerableExtensionsTests.cs | 17 -- .../Extensions/EnumerableExtensions.cs | 26 -- .../Predicates/ReleaseVersionPredicates.cs | 16 +- .../Extensions/PublisherExtensionTests.cs | 175 ----------- .../Services/ReleaseServiceTests.cs | 287 ++++++++++++------ .../Extensions/PublisherExtensions.cs | 42 --- .../PublisherHostBuilderExtensions.cs | 1 + .../Services/ContentService.cs | 77 ++--- .../Services/Interfaces/IContentService.cs | 2 +- .../Services/Interfaces/IReleaseService.cs | 4 +- .../Services/PublishingCompletionService.cs | 26 +- .../Services/ReleaseService.cs | 86 +++--- 12 files changed, 301 insertions(+), 458 deletions(-) delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Extensions/PublisherExtensionTests.cs delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Publisher/Extensions/PublisherExtensions.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs index d70fa90b556..acdb9ab102d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs @@ -218,23 +218,6 @@ public void ToDictionaryIndexed() Assert.Equal(expected, result); } - [Fact] - public void DistinctByProperty() - { - var list = new List - { - new(1), - new(1), - new(2), - }; - - var distinct = list.DistinctByProperty(x => x.Value).ToList(); - - Assert.Equal(2, distinct.Count); - Assert.Equal(1, distinct[0].Value); - Assert.Equal(2, distinct[1].Value); - } - [Fact] public void IndexOfFirst() { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs index 5ff99b81aff..48d1210fcc7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Common.Model; using NaturalSort.Extension; @@ -229,31 +228,6 @@ public static IAsyncEnumerable WhereNotNull(this IAsyncEnumerable sour public static IEnumerable<(T item, int index)> WithIndex(this IEnumerable self) => self.Select((item, index) => (item, index)); - /// - /// Filter a list down to distinct elements based on a property of the type. - /// - /// - /// - /// As IEqualityComparers (as used in Linq's Distinct() method) compare with GetHashCode() rather than with - /// Equals(), the property being used to compare distinctions against needs to produce a reliable hash code - /// that we can use for equality. A good property type then could be a Guid Id field, as two identical Guid Ids - /// can then represent that 2 or more entities in the list are duplicates as they will have the same hash code. - /// - /// - /// Sequence of elements to filter on a distinct property - /// A supplier of a property from each entity to check for equality. The property - /// chosen must produce the same hash code for any two elements in the source list that are considered - /// duplicates. A good example would be a Guid Id. - /// - /// - public static IEnumerable DistinctByProperty( - this IEnumerable source, - Func propertyGetter) - where T : class - { - return source.Distinct(ComparerUtils.CreateComparerByProperty(propertyGetter)); - } - public static bool IsSameAsIgnoringOrder(this IEnumerable first, IEnumerable second) { var firstList = first.ToList(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Predicates/ReleaseVersionPredicates.cs b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Predicates/ReleaseVersionPredicates.cs index 57560809186..30d7023a26c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Predicates/ReleaseVersionPredicates.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Content.Model/Predicates/ReleaseVersionPredicates.cs @@ -1,5 +1,6 @@ #nullable enable using System; +using System.Collections.Generic; using System.Linq; namespace GovUk.Education.ExploreEducationStatistics.Content.Model.Predicates; @@ -55,17 +56,26 @@ public static IQueryable LatestReleaseVersions(this IQueryableThe source of type to filter. /// Unique identifier of a release to filter by. /// Flag to only include published release versions. + /// Optional list of unpublished release version ids to also consider. + /// Applicable when is true. /// An of type that contains elements from the input /// sequence filtered to only include the latest version of the release. - public static IQueryable LatestReleaseVersion(this IQueryable releaseVersions, + public static IQueryable LatestReleaseVersion( + this IQueryable releaseVersions, Guid releaseId, - bool publishedOnly = false) + bool publishedOnly = false, + IReadOnlyList? includeUnpublishedVersionIds = null) { return releaseVersions .Where(releaseVersion => releaseVersion.ReleaseId == releaseId) .Where(releaseVersion => releaseVersion.Version == releaseVersions .Where(latestVersion => latestVersion.ReleaseId == releaseId) - .Where(latestVersion => !publishedOnly || latestVersion.Published.HasValue) + .Where(latestVersion => !publishedOnly || + latestVersion.Published.HasValue || + ( + includeUnpublishedVersionIds != null && + includeUnpublishedVersionIds.Contains(latestVersion.Id) + )) .Select(latestVersion => (int?)latestVersion.Version) .Max()); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Extensions/PublisherExtensionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Extensions/PublisherExtensionTests.cs deleted file mode 100644 index 6fd5e204a10..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Extensions/PublisherExtensionTests.cs +++ /dev/null @@ -1,175 +0,0 @@ -using System; -using System.Collections.Generic; -using GovUk.Education.ExploreEducationStatistics.Content.Model; -using GovUk.Education.ExploreEducationStatistics.Publisher.Extensions; -using Xunit; - -namespace GovUk.Education.ExploreEducationStatistics.Publisher.Tests.Extensions -{ - public class PublisherExtensionTests - { - [Fact] - public void IsReleasePublished_ReleasePublished() - { - var publication = new Publication(); - - var releaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication, - Published = DateTime.UtcNow.AddSeconds(-1) - }; - - publication.ReleaseVersions = new List - { - releaseVersion - }; - - Assert.True(releaseVersion.IsReleasePublished()); - } - - [Fact] - public void IsReleasePublished_ReleaseNotPublished() - { - var publication = new Publication(); - - var releaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication, - Published = null - }; - - publication.ReleaseVersions = new List - { - releaseVersion - }; - - Assert.False(releaseVersion.IsReleasePublished()); - } - - [Fact] - public void IsReleasePublished_ReleaseNotPublishedButIncluded() - { - var publication = new Publication(); - - var releaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication, - Published = null - }; - - publication.ReleaseVersions = new List - { - releaseVersion - }; - - Assert.True(releaseVersion.IsReleasePublished(new List - { - releaseVersion.Id - })); - } - - [Fact] - public void IsReleasePublished_AmendmentReleaseNotPublished() - { - var publication = new Publication(); - - var originalReleaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication, - Version = 0, - Published = DateTime.UtcNow.AddSeconds(-1) - }; - - var amendmentReleaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication, - PreviousVersionId = originalReleaseVersion.Id, - Version = 1 - }; - - publication.ReleaseVersions = new List - { - originalReleaseVersion, - amendmentReleaseVersion - }; - - Assert.True(originalReleaseVersion.IsReleasePublished()); - Assert.False(amendmentReleaseVersion.IsReleasePublished()); - } - - [Fact] - public void IsReleasePublished_AmendmentReleasePublished() - { - var publication = new Publication(); - - var originalReleaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication, - Version = 0, - Published = DateTime.UtcNow.AddSeconds(-1) - }; - - var amendmentReleaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication, - PreviousVersionId = originalReleaseVersion.Id, - Version = 1, - Published = DateTime.UtcNow.AddSeconds(-1) - }; - - publication.ReleaseVersions = new List - { - originalReleaseVersion, - amendmentReleaseVersion - }; - - Assert.False(originalReleaseVersion.IsReleasePublished()); - Assert.True(amendmentReleaseVersion.IsReleasePublished()); - } - - [Fact] - public void IsReleasePublished_AmendmentReleaseNotPublishedButIncluded() - { - var publication = new Publication(); - - var originalReleaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication, - Version = 0, - Published = DateTime.UtcNow.AddSeconds(-1) - }; - - var amendmentReleaseVersion = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication, - PreviousVersionId = originalReleaseVersion.Id, - Version = 1 - }; - - publication.ReleaseVersions = new List - { - originalReleaseVersion, - amendmentReleaseVersion - }; - - Assert.False(originalReleaseVersion.IsReleasePublished(new List - { - amendmentReleaseVersion.Id - })); - - Assert.True(amendmentReleaseVersion.IsReleasePublished(new List - { - amendmentReleaseVersion.Id - })); - } - } -} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/ReleaseServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/ReleaseServiceTests.cs index b8b48213731..9524492242a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/ReleaseServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher.Tests/Services/ReleaseServiceTests.cs @@ -1,3 +1,7 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; @@ -7,14 +11,8 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Publisher.Services; using Microsoft.EntityFrameworkCore; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using Xunit; using static GovUk.Education.ExploreEducationStatistics.Common.Model.FileType; -using static GovUk.Education.ExploreEducationStatistics.Common.Model.TimeIdentifier; -using static GovUk.Education.ExploreEducationStatistics.Content.Model.ReleaseApprovalStatus; using static GovUk.Education.ExploreEducationStatistics.Content.Model.Tests.Utils.ContentDbUtils; namespace GovUk.Education.ExploreEducationStatistics.Publisher.Tests.Services @@ -95,108 +93,136 @@ public async Task GetFiles() } [Fact] - public async Task GetLatestRelease() + public async Task GetLatestPublishedReleaseVersion_Success() { - var publication = new Publication(); + Publication publication = _fixture.DefaultPublication() + .WithReleases(_ => + [ + _fixture.DefaultRelease(publishedVersions: 1, year: 2020), + _fixture.DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2021), + _fixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2022) + ]); - var release1V0 = new ReleaseVersion - { - Id = Guid.NewGuid(), - Publication = publication, - ReleaseName = "2017", - TimePeriodCoverage = AcademicYearQ4, - Slug = "2017-18-q4", - Published = new DateTime(2019, 4, 1), - ApprovalStatus = Approved - }; + var release2021 = publication.Releases.Single(r => r.Year == 2021); - var release2V0 = new ReleaseVersion + var contentDbContextId = Guid.NewGuid().ToString(); + await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { - Id = new Guid("e7e1aae3-a0a1-44b7-bdf3-3df4a363ce20"), - Publication = publication, - ReleaseName = "2018", - TimePeriodCoverage = AcademicYearQ1, - Slug = "2018-19-q1", - Published = new DateTime(2019, 3, 1), - ApprovalStatus = Approved, - }; + contentDbContext.Publications.Add(publication); + await contentDbContext.SaveChangesAsync(); + } - var release3V0 = new ReleaseVersion + await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { - Id = Guid.NewGuid(), - Publication = publication, - ReleaseName = "2018", - TimePeriodCoverage = AcademicYearQ2, - Slug = "2018-19-q2", - Published = new DateTime(2019, 1, 1), - ApprovalStatus = Approved, - Version = 0, - PreviousVersionId = null - }; + var service = BuildReleaseService(contentDbContext); + + var result = await service.GetLatestPublishedReleaseVersion( + publication.Id, + includeUnpublishedVersionIds: []); + + Assert.Equal(release2021.Versions[1].Id, result.Id); + } + } - var release3V1 = new ReleaseVersion + [Fact] + public async Task GetLatestPublishedReleaseVersion_NonDefaultReleaseOrder() + { + Publication publication = _fixture.DefaultPublication() + .WithReleases(_ => + [ + _fixture.DefaultRelease(publishedVersions: 1, year: 2020), + _fixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2021), + _fixture.DefaultRelease(publishedVersions: 1, year: 2022) + ]) + .FinishWith(p => + { + // Adjust the generated LatestPublishedReleaseVersion to make 2020 the latest published release + var release2020Version0 = p.Releases.Single(r => r.Year == 2020).Versions[0]; + p.LatestPublishedReleaseVersion = release2020Version0; + p.LatestPublishedReleaseVersionId = release2020Version0.Id; + + // Apply a different release series order rather than using the default + p.ReleaseSeries = + [.. GenerateReleaseSeries(p.Releases, 2021, 2020, 2022)]; + }); + + var release2020 = publication.Releases.Single(r => r.Year == 2020); + + var contentDbContextId = Guid.NewGuid().ToString(); + await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { - Id = Guid.NewGuid(), - Publication = publication, - ReleaseName = "2018", - TimePeriodCoverage = AcademicYearQ2, - Slug = "2018-19-q2", - Published = new DateTime(2019, 2, 1), - ApprovalStatus = Approved, - Version = 1, - PreviousVersionId = release3V0.Id - }; + contentDbContext.Publications.Add(publication); + await contentDbContext.SaveChangesAsync(); + } - var release3V2Deleted = new ReleaseVersion + await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { - Id = Guid.NewGuid(), - Publication = publication, - ReleaseName = "2018", - TimePeriodCoverage = AcademicYearQ2, - Slug = "2018-19-q2", - Published = null, - ApprovalStatus = Approved, - Version = 2, - PreviousVersionId = release3V1.Id, - SoftDeleted = true - }; + var service = BuildReleaseService(contentDbContext); - var release3V3NotPublished = new ReleaseVersion + var result = await service.GetLatestPublishedReleaseVersion( + publication.Id, + includeUnpublishedVersionIds: []); + + // Check the 2020 release version is considered to be the latest published release version, + // since 2020 is the first release in the release series with a published version + Assert.Equal(release2020.Versions[0].Id, result.Id); + } + } + + [Fact] + public async Task GetLatestPublishedReleaseVersion_IncludeUnpublishedReleaseVersion() + { + Publication publication = _fixture.DefaultPublication() + .WithReleases(_ => + [ + _fixture.DefaultRelease(publishedVersions: 1, year: 2020), + _fixture.DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2021), + _fixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2022) + ]); + + var release2021 = publication.Releases.Single(r => r.Year == 2021); + + var contentDbContextId = Guid.NewGuid().ToString(); + await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { - Id = Guid.NewGuid(), - Publication = publication, - ReleaseName = "2018", - TimePeriodCoverage = AcademicYearQ2, - Slug = "2018-19-q2", - Published = null, - ApprovalStatus = Approved, - Version = 3, - PreviousVersionId = release3V1.Id - }; + contentDbContext.Publications.Add(publication); + await contentDbContext.SaveChangesAsync(); + } - var release4V0 = new ReleaseVersion + await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { - Id = Guid.NewGuid(), - Publication = publication, - ReleaseName = "2018", - TimePeriodCoverage = AcademicYearQ3, - Slug = "2018-19-q3", - Published = null, - ApprovalStatus = Approved - }; + var service = BuildReleaseService(contentDbContext); - var contentDbContextId = Guid.NewGuid().ToString(); + // Include the unpublished 2021 release version id in the call + // to test the scenario where this version is about to be published + var result = await service.GetLatestPublishedReleaseVersion( + publication.Id, + includeUnpublishedVersionIds: [release2021.Versions.Single(rv => rv.Published == null).Id]); + + // Check the unpublished 2021 release version is considered to be the latest published release version, + // despite the fact that it is not published yet + Assert.Equal(release2021.Versions.Single(rv => rv.Published == null).Id, result.Id); + } + } + + [Fact] + public async Task GetLatestPublishedReleaseVersion_IncludeUnpublishedReleaseVersions() + { + Publication publication = _fixture.DefaultPublication() + .WithReleases(_ => + [ + _fixture.DefaultRelease(publishedVersions: 1, year: 2020), + _fixture.DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2021), + _fixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2022) + ]); + + var release2021 = publication.Releases.Single(r => r.Year == 2021); + var release2022 = publication.Releases.Single(r => r.Year == 2022); + var contentDbContextId = Guid.NewGuid().ToString(); await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) { - await contentDbContext.AddRangeAsync(publication); - await contentDbContext.AddRangeAsync(release1V0, - release2V0, - release3V0, - release3V1, - release3V2Deleted, - release3V3NotPublished, - release4V0); + contentDbContext.Publications.Add(publication); await contentDbContext.SaveChangesAsync(); } @@ -204,10 +230,73 @@ await contentDbContext.AddRangeAsync(release1V0, { var service = BuildReleaseService(contentDbContext); - var result = await service.GetLatestReleaseVersion(publication.Id, Enumerable.Empty()); + // Include the unpublished 2021 and 2022 release version id's in the call + // to test the scenario where both versions are about to be published together + var result = await service.GetLatestPublishedReleaseVersion( + publication.Id, + includeUnpublishedVersionIds: + [ + release2021.Versions.Single(rv => rv.Published == null).Id, + release2022.Versions.Single(rv => rv.Published == null).Id + ]); + + // Check the unpublished 2022 release version is considered to be the latest published release version, + // despite the fact that it is not published yet + Assert.Equal(release2022.Versions.Single(rv => rv.Published == null).Id, result.Id); + } + } + + [Fact] + public async Task GetLatestPublishedReleaseVersion_IgnoresIncludedUnpublishedReleaseVersions() + { + Publication publication = _fixture.DefaultPublication() + .WithReleases(_ => + [ + _fixture.DefaultRelease(publishedVersions: 1, year: 2020), + _fixture.DefaultRelease(publishedVersions: 2, draftVersion: true, year: 2021), + _fixture.DefaultRelease(publishedVersions: 0, draftVersion: true, year: 2022) + ]) + .FinishWith(p => + { + // Adjust the generated LatestPublishedReleaseVersion to make 2020 the latest published release + var release2020Version0 = p.Releases.Single(r => r.Year == 2020).Versions[0]; + p.LatestPublishedReleaseVersion = release2020Version0; + p.LatestPublishedReleaseVersionId = release2020Version0.Id; + + // Apply a different release series order rather than using the default + p.ReleaseSeries = + [.. GenerateReleaseSeries(p.Releases, 2020, 2021, 2022)]; + }); + + var release2020 = publication.Releases.Single(r => r.Year == 2020); + var release2021 = publication.Releases.Single(r => r.Year == 2021); + var release2022 = publication.Releases.Single(r => r.Year == 2022); - Assert.Equal(release3V1.Id, result.Id); - Assert.Equal("Academic year Q2 2018/19", result.Title); + var contentDbContextId = Guid.NewGuid().ToString(); + await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) + { + contentDbContext.Publications.Add(publication); + await contentDbContext.SaveChangesAsync(); + } + + await using (var contentDbContext = InMemoryContentDbContext(contentDbContextId)) + { + var service = BuildReleaseService(contentDbContext); + + // Include the unpublished 2021 and 2022 release version id's in the call + // to test the scenario where both versions are about to be published together + var result = await service.GetLatestPublishedReleaseVersion( + publication.Id, + includeUnpublishedVersionIds: + [ + release2021.Versions.Single(rv => rv.Published == null).Id, + release2022.Versions.Single(rv => rv.Published == null).Id + ]); + + // Check the 2020 release version is considered to be the latest published release version, + // despite the fact that versions for 2021 and 2022 are about to be published, + // since 2020 is the first release in the release series and has a published version + Assert.Equal(release2020.Versions[0].Id, result.Id); } } @@ -535,8 +624,16 @@ public async Task CompletePublishing_AmendedReleaseAndUpdatePublishedDateIsTrue( } } - private static ReleaseService BuildReleaseService( - ContentDbContext? contentDbContext = null) + private List GenerateReleaseSeries(IReadOnlyList releases, params int[] years) + { + return years.Select(year => + { + var release = releases.Single(r => r.Year == year); + return _fixture.DefaultReleaseSeriesItem().WithReleaseId(release.Id).Generate(); + }).ToList(); + } + + private static ReleaseService BuildReleaseService(ContentDbContext? contentDbContext = null) { contentDbContext ??= InMemoryContentDbContext(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Extensions/PublisherExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Extensions/PublisherExtensions.cs deleted file mode 100644 index cc405bd6ea2..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Extensions/PublisherExtensions.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using GovUk.Education.ExploreEducationStatistics.Content.Model; - -namespace GovUk.Education.ExploreEducationStatistics.Publisher.Extensions; - -public static class PublisherExtensions -{ - /// - /// Determines whether a release version should be published or not. - /// - /// The to test - /// Release version id's which are not published yet but are in the process of being published - /// True if the release version is the latest published version of a release or is one of the included releases - public static bool IsReleasePublished(this ReleaseVersion releaseVersion, - IEnumerable includedReleaseVersionIds = null) - { - return includedReleaseVersionIds != null && - includedReleaseVersionIds.Contains(releaseVersion.Id) || - releaseVersion.IsLatestPublishedVersionOfRelease(includedReleaseVersionIds); - } - - private static bool IsLatestPublishedVersionOfRelease(this ReleaseVersion releaseVersion, - IEnumerable includedReleaseIds) - { - if (releaseVersion.Publication?.ReleaseVersions == null || !releaseVersion.Publication.ReleaseVersions.Any()) - { - throw new ArgumentException( - "All release versions of the publication must be hydrated to test the latest published version"); - } - - return - // Release version itself must be live - releaseVersion.Live - // It must also be the latest version unless the later version is a draft not included for publishing - && !releaseVersion.Publication.ReleaseVersions.Any(rv => - (rv.Live || includedReleaseIds != null && includedReleaseIds.Contains(rv.Id)) - && rv.PreviousVersionId == releaseVersion.Id - && rv.Id != releaseVersion.Id); - } -} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs index 395fe578313..bb952026124 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/PublisherHostBuilderExtensions.cs @@ -96,6 +96,7 @@ public static IHostBuilder ConfigurePublisherHostBuilder(this IHostBuilder hostB provider.GetRequiredService>())) .AddScoped(provider => new ContentService( + contentDbContext: provider.GetRequiredService(), publicBlobStorageService: provider.GetRequiredService(), privateBlobCacheService: new BlobCacheService( provider.GetRequiredService(), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ContentService.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ContentService.cs index 72987829a04..8cddbea8b29 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ContentService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ContentService.cs @@ -4,10 +4,11 @@ using GovUk.Education.ExploreEducationStatistics.Common; using GovUk.Education.ExploreEducationStatistics.Common.Cache; using GovUk.Education.ExploreEducationStatistics.Common.Cache.Interfaces; -using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.Cache; using GovUk.Education.ExploreEducationStatistics.Publisher.Services.Interfaces; +using Microsoft.EntityFrameworkCore; using static GovUk.Education.ExploreEducationStatistics.Common.BlobContainers; using static GovUk.Education.ExploreEducationStatistics.Common.Services.FileStoragePathUtils; @@ -15,6 +16,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Publisher.Services { public class ContentService : IContentService { + private readonly ContentDbContext _contentDbContext; private readonly IBlobCacheService _privateBlobCacheService; private readonly IBlobCacheService _publicBlobCacheService; private readonly IBlobStorageService _publicBlobStorageService; @@ -24,6 +26,7 @@ public class ContentService : IContentService private readonly IPublicationCacheService _publicationCacheService; public ContentService( + ContentDbContext contentDbContext, IBlobCacheService privateBlobCacheService, IBlobCacheService publicBlobCacheService, IBlobStorageService publicBlobStorageService, @@ -32,6 +35,7 @@ public ContentService( IReleaseCacheService releaseCacheService, IPublicationCacheService publicationCacheService) { + _contentDbContext = contentDbContext; _privateBlobCacheService = privateBlobCacheService; _publicBlobCacheService = publicBlobCacheService; _publicBlobStorageService = publicBlobStorageService; @@ -94,60 +98,61 @@ await _publicBlobStorageService.DeleteBlobs( } } - public async Task UpdateContent(params Guid[] releaseVersionIds) + public async Task UpdateContent(Guid releaseVersionId) { - var releaseVersions = (await _releaseService - .List(releaseVersionIds)) - .ToList(); - - foreach (var releaseVersion in releaseVersions) - { - await _releaseCacheService.UpdateRelease( - releaseVersion.Id, - publicationSlug: releaseVersion.Publication.Slug, - releaseSlug: releaseVersion.Slug); - } - - var publications = releaseVersions - .Select(rv => rv.Publication) - .DistinctByProperty(publication => publication.Id) - .ToList(); - - foreach (var publication in publications) - { - // Cache the latest release version for the publication as a separate cache entry - var latestReleaseVersion = await _releaseService.GetLatestReleaseVersion(publication.Id, releaseVersionIds); - await _releaseCacheService.UpdateRelease( - latestReleaseVersion.Id, - publicationSlug: publication.Slug); - } + var releaseVersion = await _contentDbContext.ReleaseVersions + .Include(rv => rv.Release) + .ThenInclude(r => r.Publication) + .SingleAsync(rv => rv.Id == releaseVersionId); + + await _releaseCacheService.UpdateRelease( + releaseVersion.Id, + publicationSlug: releaseVersion.Release.Publication.Slug, + releaseSlug: releaseVersion.Release.Slug); + + var publication = releaseVersion.Release.Publication; + + // Cache the latest release version for the publication as a separate cache entry + var latestReleaseVersion = await _releaseService.GetLatestPublishedReleaseVersion( + publicationId: publication.Id, + includeUnpublishedVersionIds: [releaseVersion.Id]); + + await _releaseCacheService.UpdateRelease( + releaseVersionId: latestReleaseVersion.Id, + publicationSlug: publication.Slug); } - public async Task UpdateContentStaged(DateTime expectedPublishDate, + public async Task UpdateContentStaged( + DateTime expectedPublishDate, params Guid[] releaseVersionIds) { - var releaseVersions = (await _releaseService - .List(releaseVersionIds)) - .ToList(); + var releaseVersions = await _contentDbContext.ReleaseVersions + .Where(rv => releaseVersionIds.Contains(rv.Id)) + .Include(rv => rv.Release) + .ThenInclude(r => r.Publication) + .ToListAsync(); foreach (var releaseVersion in releaseVersions) { await _releaseCacheService.UpdateReleaseStaged( releaseVersion.Id, expectedPublishDate, - publicationSlug: releaseVersion.Publication.Slug, - releaseSlug: releaseVersion.Slug); + publicationSlug: releaseVersion.Release.Publication.Slug, + releaseSlug: releaseVersion.Release.Slug); } var publications = releaseVersions - .Select(rv => rv.Publication) - .DistinctByProperty(publication => publication.Id) + .Select(rv => rv.Release.Publication) + .DistinctBy(p => p.Id) .ToList(); foreach (var publication in publications) { // Cache the latest release version for the publication as a separate cache entry - var latestReleaseVersion = await _releaseService.GetLatestReleaseVersion(publication.Id, releaseVersionIds); + var latestReleaseVersion = await _releaseService.GetLatestPublishedReleaseVersion( + publicationId: publication.Id, + includeUnpublishedVersionIds: releaseVersionIds); + await _releaseCacheService.UpdateReleaseStaged( latestReleaseVersion.Id, expectedPublishDate, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/Interfaces/IContentService.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/Interfaces/IContentService.cs index 9e80eb61220..28792d63f3a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/Interfaces/IContentService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/Interfaces/IContentService.cs @@ -9,7 +9,7 @@ public interface IContentService Task DeletePreviousVersionsContent(params Guid[] releaseVersionIds); - Task UpdateContent(params Guid[] releaseVersionIds); + Task UpdateContent(Guid releaseVersionId); Task UpdateContentStaged(DateTime expectedPublishDate, params Guid[] releaseVersionIds); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/Interfaces/IReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/Interfaces/IReleaseService.cs index 83fc632e269..51a60318f13 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/Interfaces/IReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/Interfaces/IReleaseService.cs @@ -16,7 +16,9 @@ public interface IReleaseService Task> GetFiles(Guid releaseVersionId, params FileType[] types); - Task GetLatestReleaseVersion(Guid publicationId, IEnumerable includedReleaseVersionIds); + Task GetLatestPublishedReleaseVersion( + Guid publicationId, + IReadOnlyList? includeUnpublishedVersionIds = null); Task CompletePublishing(Guid releaseVersionId, DateTime actualPublishedDate); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs index 9bd2031ff20..bc469d819da 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/PublishingCompletionService.cs @@ -3,8 +3,6 @@ using System.Linq; using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; -using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; -using GovUk.Education.ExploreEducationStatistics.Content.Model.Predicates; using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.Cache; using GovUk.Education.ExploreEducationStatistics.Publisher.Model; using GovUk.Education.ExploreEducationStatistics.Publisher.Services.Interfaces; @@ -125,30 +123,10 @@ private async Task UpdateLatestPublishedReleaseVersionForPublication(Guid public var publication = await contentDbContext.Publications .SingleAsync(p => p.Id == publicationId); - // Get the publications release id's by the order they appear in the release series - var releaseSeriesReleaseIds = publication.ReleaseSeries.ReleaseIds(); + var latestPublishedReleaseVersion = await releaseService.GetLatestPublishedReleaseVersion(publicationId); - // Work out the publication's new latest published release version. - // This is the latest published version of the first release which has a published version - Guid? latestPublishedReleaseVersionId = null; - foreach (var releaseId in releaseSeriesReleaseIds) - { - latestPublishedReleaseVersionId = (await contentDbContext.ReleaseVersions - .LatestReleaseVersion(releaseId: releaseId, publishedOnly: true) - .SingleOrDefaultAsync())?.Id; - - if (latestPublishedReleaseVersionId != null) - { - break; - } - } - - publication.LatestPublishedReleaseVersionId = - latestPublishedReleaseVersionId ?? - throw new InvalidOperationException( - $"No latest published release version found for publication {publicationId}"); + publication.LatestPublishedReleaseVersionId = latestPublishedReleaseVersion.Id; - contentDbContext.Update(publication); await contentDbContext.SaveChangesAsync(); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ReleaseService.cs b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ReleaseService.cs index dcf55d60cef..52b3465b012 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ReleaseService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Publisher/Services/ReleaseService.cs @@ -1,39 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Extensions; +using GovUk.Education.ExploreEducationStatistics.Content.Model.Predicates; using GovUk.Education.ExploreEducationStatistics.Content.Model.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Publisher.Services.Interfaces; using Microsoft.EntityFrameworkCore; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using static GovUk.Education.ExploreEducationStatistics.Publisher.Extensions.PublisherExtensions; namespace GovUk.Education.ExploreEducationStatistics.Publisher.Services { - public class ReleaseService : IReleaseService + public class ReleaseService( + ContentDbContext contentDbContext, + IReleaseVersionRepository releaseVersionRepository + ) : IReleaseService { - private readonly ContentDbContext _contentDbContext; - private readonly IReleaseVersionRepository _releaseVersionRepository; - - public ReleaseService( - ContentDbContext contentDbContext, - IReleaseVersionRepository releaseVersionRepository) - { - _contentDbContext = contentDbContext; - _releaseVersionRepository = releaseVersionRepository; - } - public async Task Get(Guid releaseVersionId) { - return await _contentDbContext.ReleaseVersions + return await contentDbContext.ReleaseVersions .SingleAsync(releaseVersion => releaseVersion.Id == releaseVersionId); } public async Task> List(IEnumerable releaseVersionIds) { - return await _contentDbContext.ReleaseVersions + return await contentDbContext.ReleaseVersions .Where(rv => releaseVersionIds.Contains(rv.Id)) .Include(rv => rv.Publication) .Include(rv => rv.PreviousVersion) @@ -42,31 +35,48 @@ public async Task> List(IEnumerable releaseVer public async Task> GetAmendedReleases(IEnumerable releaseVersionIds) { - return await _contentDbContext.ReleaseVersions + return await contentDbContext.ReleaseVersions .Include(rv => rv.PreviousVersion) .Include(rv => rv.Publication) .Where(rv => releaseVersionIds.Contains(rv.Id) && rv.PreviousVersionId != null) .ToListAsync(); } - public async Task GetLatestReleaseVersion(Guid publicationId, - IEnumerable includedReleaseVersionIds) + public async Task GetLatestPublishedReleaseVersion( + Guid publicationId, + IReadOnlyList? includeUnpublishedVersionIds = null) { - var releases = await _contentDbContext.ReleaseVersions - .Include(rv => rv.Publication) - .Where(rv => rv.PublicationId == publicationId) - .ToListAsync(); + var publication = await contentDbContext.Publications + .SingleAsync(p => p.Id == publicationId); + + // Get the publications release id's by the order they appear in the release series + var releaseSeriesReleaseIds = publication.ReleaseSeries.ReleaseIds(); + + // Work out the publication's latest published release version. + // This is the latest published version of the first release which has either a published version + // or one of the included (about to be published) release version ids + ReleaseVersion? latestPublishedReleaseVersion = null; + foreach (var releaseId in releaseSeriesReleaseIds) + { + latestPublishedReleaseVersion = await contentDbContext.ReleaseVersions + .LatestReleaseVersion(releaseId: releaseId, + publishedOnly: true, + includeUnpublishedVersionIds: includeUnpublishedVersionIds) + .SingleOrDefaultAsync(); + + if (latestPublishedReleaseVersion != null) + { + break; + } + } - return releases - .Where(rv => rv.IsReleasePublished(includedReleaseVersionIds)) - .OrderBy(rv => rv.Year) - .ThenBy(rv => rv.TimePeriodCoverage) - .Last(); + return latestPublishedReleaseVersion ?? throw new InvalidOperationException( + $"No latest published release version found for publication {publicationId}"); } public async Task> GetFiles(Guid releaseVersionId, params FileType[] types) { - return await _contentDbContext + return await contentDbContext .ReleaseFiles .Include(rf => rf.File) .Where(rf => rf.ReleaseVersionId == releaseVersionId) @@ -77,15 +87,15 @@ public async Task> GetFiles(Guid releaseVersionId, params FileType[] public async Task CompletePublishing(Guid releaseVersionId, DateTime actualPublishedDate) { - var releaseVersion = await _contentDbContext + var releaseVersion = await contentDbContext .ReleaseVersions .Include(rv => rv.DataBlockVersions) .ThenInclude(dataBlockVersion => dataBlockVersion.DataBlockParent) .SingleAsync(rv => rv.Id == releaseVersionId); - _contentDbContext.ReleaseVersions.Update(releaseVersion); + contentDbContext.ReleaseVersions.Update(releaseVersion); - var publishedDate = await _releaseVersionRepository.GetPublishedDate(releaseVersion.Id, actualPublishedDate); + var publishedDate = await releaseVersionRepository.GetPublishedDate(releaseVersion.Id, actualPublishedDate); releaseVersion.Published = publishedDate; @@ -93,14 +103,14 @@ public async Task CompletePublishing(Guid releaseVersionId, DateTime actualPubli await UpdatePublishedDataBlockVersions(releaseVersion); - await _contentDbContext.SaveChangesAsync(); + await contentDbContext.SaveChangesAsync(); } private async Task UpdateReleaseFilePublishedDate( ReleaseVersion releaseVersion, DateTime publishedDate) { - var dataReleaseFiles = _contentDbContext.ReleaseFiles + var dataReleaseFiles = contentDbContext.ReleaseFiles .Where(releaseFile => releaseFile.ReleaseVersionId == releaseVersion.Id) .Include(rf => rf.File); @@ -143,7 +153,7 @@ private async Task UpdatePublishedDataBlockVersions(ReleaseVersion releaseVersio { var latestDataBlockParentIds = latestDataBlockParents.Select(dataBlockParent => dataBlockParent.Id); - var removedDataBlockVersions = await _contentDbContext + var removedDataBlockVersions = await contentDbContext .DataBlockVersions .Where(dataBlockVersion => dataBlockVersion.ReleaseVersionId == releaseVersion.PreviousVersionId && !latestDataBlockParentIds.Contains(dataBlockVersion.DataBlockParentId)) From 3968fb43a929712109bbf93ac464552f9b51e1c3 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Fri, 20 Dec 2024 16:20:19 +0000 Subject: [PATCH 144/144] EES-5738 More changes in response to PR comments --- .../tests/admin_and_public/bau/data_catalogue.robot | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/robot-tests/tests/admin_and_public/bau/data_catalogue.robot b/tests/robot-tests/tests/admin_and_public/bau/data_catalogue.robot index ceff983c154..659af1efdbc 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/data_catalogue.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/data_catalogue.robot @@ -219,6 +219,9 @@ Filter by geographic level user checks page contains button Local Authority District user checks testid element contains total-results 1 data set + user clicks button Show more details + user checks testid element contains Geographic levels-value Local authority district + Remove geographic level filter user clicks button Local Authority District
    Description