fixup! EES-5235 Generate LocationOptionMeta and FilterOptionMetaLink …

…PublicId Sqids based on Id sequences
dfe-analytical-services · Jun 21, 2024 · 1e9766b · 1e9766b
1 parent 6f4ce57
commit 1e9766b
Show file tree

Hide file tree

Showing 14 changed files with 90 additions and 59 deletions.
diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Fixture/TestApplicationFactory.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Fixture/TestApplicationFactory.cs
@@ -35,7 +35,7 @@ public async Task ClearTestData<TDbContext>() where TDbContext : DbContext
         var tables = context.Model.GetEntityTypes()
             .Select(type => type.GetTableName())
             .Distinct()
-            .Cast<string>()
+            .OfType<string>()
             .ToList();
 
         foreach (var table in tables)

diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/FunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/FunctionsIntegrationTest.cs
@@ -53,7 +53,7 @@ protected void ClearTestData<TDbContext>() where TDbContext : DbContext
         var tables = context.Model.GetEntityTypes()
             .Select(type => type.GetTableName())
             .Distinct()
-            .Cast<string>()
+            .OfType<string>()
             .ToList();
 
         foreach (var table in tables)

diff --git a/...cation.ExploreEducationStatistics.Public.Data.Api.Tests/Fixture/TestApplicationFactory.cs b/...cation.ExploreEducationStatistics.Public.Data.Api.Tests/Fixture/TestApplicationFactory.cs
@@ -32,7 +32,7 @@ public async Task ClearTestData<TDbContext>() where TDbContext : DbContext
         var tables = context.Model.GetEntityTypes()
             .Select(type => type.GetTableName())
             .Distinct()
-            .Cast<string>()
+            .OfType<string>()
             .ToList();
 
         foreach (var table in tables)

diff --git a/...Uk.Education.ExploreEducationStatistics.Public.Data.Model/Database/PublicDataDbContext.cs b/...Uk.Education.ExploreEducationStatistics.Public.Data.Model/Database/PublicDataDbContext.cs
@@ -1,11 +1,16 @@
+using System.Diagnostics.CodeAnalysis;
 using GovUk.Education.ExploreEducationStatistics.Common.Utils;
 using Microsoft.EntityFrameworkCore;
 
 namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database;
 
 public class PublicDataDbContext : DbContext
 {
-    public PublicDataDbContext(DbContextOptions<PublicDataDbContext> options, bool updateTimestamps = true) : base(options)
+    public const string FilterOptionMetaLinkSequence = "FilterOptionMetaLink_seq";
+    public const string LocationOptionMetasIdSequence = "LocationOptionMetas_Id_seq";
+
+    public PublicDataDbContext(
+        DbContextOptions<PublicDataDbContext> options, bool updateTimestamps = true) : base(options)
     {
         Configure(updateTimestamps: updateTimestamps);
     }
@@ -23,7 +28,25 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
     {
         modelBuilder.ApplyConfigurationsFromAssembly(typeof(PublicDataDbContext).Assembly);
 
-        modelBuilder.HasSequence<int>("FilterOptionMetaLink_Id_seq");
+        modelBuilder.HasSequence<int>(FilterOptionMetaLinkSequence);
+    }
+
+    [SuppressMessage("Security", "EF1002:Risk of vulnerability to SQL injection.")]
+    public Task<int> NextSequenceValue(string sequenceName, CancellationToken cancellationToken = default)
+    {
+        return Database.SqlQueryRaw<int>($"""
+                                          SELECT nextval('public."{sequenceName}"') AS "Value"
+                                          """)
+            .FirstAsync(cancellationToken);
+    }
+
+    [SuppressMessage("Security", "EF1002:Risk of vulnerability to SQL injection.")]
+    public Task<int> SetSequenceValue(string sequenceName, long value, CancellationToken cancellationToken = default)
+    {
+        return Database.SqlQueryRaw<int>($"""
+                                          SELECT setval('public."{sequenceName}"', {value}) AS "Value"
+                                          """)
+            .FirstAsync(cancellationToken);
     }
 
     public DbSet<DataSet> DataSets { get; init; } = null!;

diff --git a/...ata.Model/Migrations/20240619085713_EES5235_AddFilterOptionMetaLinkIdSequence.Designer.cs b/...ata.Model/Migrations/20240619085713_EES5235_AddFilterOptionMetaLinkIdSequence.Designer.cs
diff --git a/....Public.Data.Model/Migrations/20240619085713_EES5235_AddFilterOptionMetaLinkIdSequence.cs b/....Public.Data.Model/Migrations/20240619085713_EES5235_AddFilterOptionMetaLinkIdSequence.cs
@@ -13,13 +13,13 @@ public partial class EES5235_AddFilterOptionMetaLinkIdSequence : Migration
     protected override void Up(MigrationBuilder migrationBuilder)
     {
         migrationBuilder.CreateSequence<int>(
-            name: "FilterOptionMetaLink_Id_seq");
+            name: "FilterOptionMetaLink_seq");
     }
 
     /// <inheritdoc />
     protected override void Down(MigrationBuilder migrationBuilder)
     {
         migrationBuilder.DropSequence(
-            name: "FilterOptionMetaLink_Id_seq");
+            name: "FilterOptionMetaLink_seq");
     }
 }
diff --git a/...ploreEducationStatistics.Public.Data.Model/Migrations/PublicDataDbContextModelSnapshot.cs b/...ploreEducationStatistics.Public.Data.Model/Migrations/PublicDataDbContextModelSnapshot.cs
@@ -23,7 +23,7 @@ protected override void BuildModel(ModelBuilder modelBuilder)
 
             NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
 
-            modelBuilder.HasSequence<int>("FilterOptionMetaLink_Id_seq");
+            modelBuilder.HasSequence<int>("FilterOptionMetaLink_seq");
 
             modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.ChangeSetFilterOptions", b =>
                 {

diff --git a/...eEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs b/...eEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs
@@ -114,11 +114,11 @@ public async Task DataSetVersionMeta_CorrectGeographicLevels(ProcessorTestData t
 
         [Theory]
         [MemberData(nameof(TestDataFilesWithInsertBatchSize))]
-        public async Task DataSetVersionMeta_CorrectLocations(ProcessorTestData testData, int insertBatchSize)
+        public async Task DataSetVersionMeta_CorrectLocations(ProcessorTestData testData, int metaInsertBatchSize)
         {
             var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage());
 
-            await ImportMetadata(testData, dataSetVersion, instanceId, insertBatchSize);
+            await ImportMetadata(testData, dataSetVersion, instanceId, metaInsertBatchSize);
 
             await using var publicDataDbContext = GetDbContext<PublicDataDbContext>();
 
@@ -189,11 +189,11 @@ public async Task DataSetVersionMeta_CorrectTimePeriods(ProcessorTestData testDa
 
         [Theory]
         [MemberData(nameof(TestDataFilesWithInsertBatchSize))]
-        public async Task DataSetVersionMeta_CorrectFilters(ProcessorTestData testData, int insertBatchSize)
+        public async Task DataSetVersionMeta_CorrectFilters(ProcessorTestData testData, int metaInsertBatchSize)
         {
             var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage());
 
-            await ImportMetadata(testData, dataSetVersion, instanceId, insertBatchSize);
+            await ImportMetadata(testData, dataSetVersion, instanceId, metaInsertBatchSize);
 
             await using var publicDataDbContext = GetDbContext<PublicDataDbContext>();
 
@@ -422,15 +422,15 @@ private async Task ImportMetadata(
             ProcessorTestData testData,
             DataSetVersion dataSetVersion,
             Guid instanceId,
-            int? insertBatchSize = null)
+            int? metaInsertBatchSize = null)
         {
             SetupCsvDataFilesForDataSetVersion(testData, dataSetVersion);
 
             // Override default app configuration if provided
-            if (insertBatchSize.HasValue)
+            if (metaInsertBatchSize.HasValue)
             {
-                var processorOptions = GetRequiredService<IOptions<ProcessorOptions>>();
-                processorOptions.Value.InsertBatchSize = insertBatchSize.Value;
+                var appSettingOptions = GetRequiredService<IOptions<AppSettingOptions>>();
+                appSettingOptions.Value.MetaInsertBatchSize = metaInsertBatchSize.Value;
             }
 
             var function = GetRequiredService<ImportMetadataFunction>();

diff --git a/...ata.Processor/Options/ProcessorOptions.cs → ...ta.Processor/Options/AppSettingOptions.cs b/...ata.Processor/Options/ProcessorOptions.cs → ...ta.Processor/Options/AppSettingOptions.cs
@@ -1,12 +1,12 @@
 namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Options;
 
-public class ProcessorOptions
+public class AppSettingOptions
 {
-    public static readonly string Section = "Processor";
+    public static readonly string Section = "AppSettings";
 
     /// <summary>
     /// Batch size to use when inserting location option meta rows, location option meta link rows,
     /// and filter option meta link rows into the public data db.
     /// </summary>
-    public required int InsertBatchSize { get; set; }
+    public required int MetaInsertBatchSize { get; set; }
 }
diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs
@@ -86,10 +86,10 @@ public static IHostBuilder ConfigureProcessorHostBuilder(this IHostBuilder hostB
                     .AddScoped<IPrivateBlobStorageService, PrivateBlobStorageService>()
                     .AddScoped<IValidator<DataSetCreateRequest>,
                         DataSetCreateRequest.Validator>()
+                    .Configure<AppSettingOptions>(
+                        hostBuilderContext.Configuration.GetSection(AppSettingOptions.Section))
                     .Configure<DataFilesOptions>(
-                        hostBuilderContext.Configuration.GetSection(DataFilesOptions.Section))
-                    .Configure<ProcessorOptions>(
-                        hostBuilderContext.Configuration.GetSection(ProcessorOptions.Section));
+                        hostBuilderContext.Configuration.GetSection(DataFilesOptions.Section));
             });
     }
 }
diff --git a/...ation.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs b/...ation.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs
@@ -16,9 +16,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repos
 
 public class FilterMetaRepository(
     PublicDataDbContext publicDataDbContext,
-    IOptions<ProcessorOptions> processorOptions,
+    IOptions<AppSettingOptions> appSettingOptions,
     IDataSetVersionPathResolver dataSetVersionPathResolver) : IFilterMetaRepository
 {
+    private readonly AppSettingOptions _appSettingOptions = appSettingOptions.Value;
+
     public async Task CreateFilterMetas(
         IDuckDbConnection duckDbConnection,
         DataSetVersion dataSetVersion,
@@ -82,10 +84,9 @@ await optionTable
                 .InsertWhenNotMatched()
                 .MergeAsync(cancellationToken);
 
-            var startIndex = (await publicDataDbContext.Database
-                    .SqlQuery<int>($"SELECT nextval('public.\"FilterOptionMetaLink_Id_seq\"')")
-                    .ToListAsync(cancellationToken: cancellationToken))
-                .First();
+            var startIndex = await publicDataDbContext.NextSequenceValue(
+                PublicDataDbContext.FilterOptionMetaLinkSequence,
+                cancellationToken);
 
             var current = 0;
 
@@ -94,7 +95,7 @@ await optionTable
                 var batchStartIndex = startIndex + current;
                 var batch = options
                     .Skip(current)
-                    .Take(processorOptions.Value.InsertBatchSize)
+                    .Take(_appSettingOptions.MetaInsertBatchSize)
                     .ToList();
 
                 // Although not necessary for filter options, we've adopted the 'row key'
@@ -119,7 +120,7 @@ await optionTable
                 publicDataDbContext.FilterOptionMetaLinks.AddRange(links);
                 await publicDataDbContext.SaveChangesAsync(cancellationToken);
 
-                current += processorOptions.Value.InsertBatchSize;
+                current += _appSettingOptions.MetaInsertBatchSize;
             }
 
             var insertedLinks = await publicDataDbContext.FilterOptionMetaLinks
@@ -133,8 +134,9 @@ await optionTable
                     $"Inserted: {insertedLinks}, expected: {options.Count}");
             }
 
-            await publicDataDbContext.Database.ExecuteSqlAsync(
-                $"SELECT setval('public.\"FilterOptionMetaLink_Id_seq\"', {startIndex + insertedLinks - 1})",
+            await publicDataDbContext.SetSequenceValue(
+                PublicDataDbContext.FilterOptionMetaLinkSequence,
+                startIndex + insertedLinks - 1,
                 cancellationToken);
         }
     }

diff --git a/...ion.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationMetaRepository.cs b/...ion.ExploreEducationStatistics.Public.Data.Processor/Repository/LocationMetaRepository.cs
@@ -22,9 +22,11 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repos
 
 public class LocationMetaRepository(
     PublicDataDbContext publicDataDbContext,
-    IOptions<ProcessorOptions> processorOptions,
+    IOptions<AppSettingOptions> appSettingOptions,
     IDataSetVersionPathResolver dataSetVersionPathResolver) : ILocationMetaRepository
 {
+    private readonly AppSettingOptions _appSettingOptions = appSettingOptions.Value;
+
     public async Task CreateLocationMetas(
         IDuckDbConnection duckDbConnection,
         DataSetVersion dataSetVersion,
@@ -78,7 +80,7 @@ FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', AL
             {
                 var batch = options
                     .Skip(current)
-                    .Take(processorOptions.Value.InsertBatchSize)
+                    .Take(_appSettingOptions.MetaInsertBatchSize)
                     .ToList();
 
                 // We create a 'row key' for each option that allows us to quickly
@@ -114,10 +116,9 @@ FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', AL
                         .Where(o => !existingRowKeys.Contains(o.GetRowKey()))
                         .ToList();
 
-                    var startIndex = (await publicDataDbContext.Database
-                            .SqlQuery<int>($"SELECT nextval('public.\"LocationOptionMetas_Id_seq\"')")
-                            .ToListAsync(cancellationToken: cancellationToken))
-                        .First();
+                    var startIndex = await publicDataDbContext.NextSequenceValue(
+                        PublicDataDbContext.LocationOptionMetasIdSequence,
+                        cancellationToken);
 
                     foreach (var option in newOptions)
                     {
@@ -129,8 +130,9 @@ await optionTable.BulkCopyAsync(new BulkCopyOptions { KeepIdentity = true },
                         newOptions,
                         cancellationToken);
 
-                    await publicDataDbContext.Database.ExecuteSqlAsync(
-                        $"SELECT setval('public.\"LocationOptionMetas_Id_seq\"', {startIndex - 1})",
+                    await publicDataDbContext.SetSequenceValue(
+                        PublicDataDbContext.LocationOptionMetasIdSequence,
+                        startIndex - 1,
                         cancellationToken);
                 }
 
@@ -146,7 +148,7 @@ await publicDataDbContext.Database.ExecuteSqlAsync(
                 publicDataDbContext.LocationOptionMetaLinks.AddRange(links);
                 await publicDataDbContext.SaveChangesAsync(cancellationToken);
 
-                current += processorOptions.Value.InsertBatchSize;
+                current += _appSettingOptions.MetaInsertBatchSize;
             }
 
             var insertedLinks = await publicDataDbContext.LocationOptionMetaLinks

diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/appsettings.json b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/appsettings.json
@@ -6,10 +6,10 @@
     }
   },
   "CoreStorage": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://data-storage:10000/devstoreaccount1;QueueEndpoint=http://data-storage:10001/devstoreaccount1;TableEndpoint=http://data-storage:10002/devstoreaccount1",
+  "AppSettings": {
+    "MetaInsertBatchSize": 1000
+  },
   "DataFiles": {
     "BasePath": "data/public-api-data"
-  },
-  "Processor": {
-    "InsertBatchSize": 1000
   }
 }
diff --git a/...ovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs b/...ovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs
@@ -111,7 +111,7 @@ private static async Task<PublicDataDbContext> SetUpPublicDataDbContext(Cancella
         var tables = dbContext.Model.GetEntityTypes()
             .Select(type => type.GetTableName())
             .Distinct()
-            .Cast<string>()
+            .OfType<string>()
             .ToList();
 
         // Clear any tables in case we're re-running the command
@@ -450,10 +450,10 @@ await optionTable
                     .InsertWhenNotMatched()
                     .MergeAsync(_cancellationToken);
 
-                var startIndex = (await _dbContext.Database
-                        .SqlQuery<int>($"SELECT nextval('public.\"FilterOptionMetaLink_Id_seq\"')")
-                        .ToListAsync(cancellationToken: _cancellationToken))
-                    .First();
+                var startIndex = await _dbContext.NextSequenceValue(
+                    PublicDataDbContext.FilterOptionMetaLinkSequence,
+                    _cancellationToken
+                );
 
                 var current = 0;
                 const int batchSize = 1000;
@@ -502,9 +502,11 @@ await optionTable
                         $"Inserted: {insertedLinks}, expected: {options.Count}");
                 }
 
-                await _dbContext.Database.ExecuteSqlAsync(
-                    $"SELECT setval('public.\"FilterOptionMetaLink_Id_seq\"', {startIndex + insertedLinks - 1})",
-                    _cancellationToken);
+                await _dbContext.SetSequenceValue(
+                    PublicDataDbContext.FilterOptionMetaLinkSequence,
+                    startIndex + insertedLinks - 1,
+                    _cancellationToken
+                );
             }
         }
 
@@ -594,10 +596,10 @@ FROM read_csv('{_dataFilePath:raw}', ALL_VARCHAR = true)
                             .Where(o => !existingRowKeys.Contains(o.GetRowKey()))
                             .ToList();
 
-                        var startIndex = (await _dbContext.Database
-                                .SqlQuery<int>($"SELECT nextval('public.\"LocationOptionMetas_Id_seq\"')")
-                                .ToListAsync(cancellationToken: _cancellationToken))
-                            .First();
+                        var startIndex = await _dbContext.NextSequenceValue(
+                            PublicDataDbContext.LocationOptionMetasIdSequence,
+                            _cancellationToken
+                        );
 
                         foreach (var option in newOptions)
                         {
@@ -609,9 +611,11 @@ await optionTable.BulkCopyAsync(new BulkCopyOptions { KeepIdentity = true },
                             newOptions,
                             cancellationToken: _cancellationToken);
 
-                        await _dbContext.Database.ExecuteSqlAsync(
-                            $"SELECT setval('public.\"LocationOptionMetas_Id_seq\"', {startIndex - 1})",
-                            _cancellationToken);
+                        await _dbContext.SetSequenceValue(
+                            PublicDataDbContext.LocationOptionMetasIdSequence,
+                            startIndex - 1,
+                            _cancellationToken
+                        );
                     }
 
                     var links = await optionTable