Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EES-4932 Bulk import: allow users to import multiple data sets from a single zip file #5030

Merged
merged 32 commits into from
Jul 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
c2bac0c
EES-4932 Add Bulk ZIP upload option to frontend
mmyoungman May 30, 2024
17d27bb
EES-4932 Add bulk uploads to backend
mmyoungman May 30, 2024
ed5d9c3
EES-4932 Fix deleting data set when other files have same SourceId
mmyoungman Jun 3, 2024
ca7ec34
ES-4932 Improve error messages
mmyoungman Jun 3, 2024
235e53e
EES-4932 Add additional bulk zip validation
mmyoungman Jun 6, 2024
db931bd
EES-4932 Refactor IsZipFile into IsValidZipFile
mmyoungman Jun 6, 2024
4243164
EES-4932 More import validation error refactoring
mmyoungman Jun 6, 2024
8e847fd
EES-4932 Refactor ValidateSubjectName
mmyoungman Jun 6, 2024
9e140d2
EES-4932 More refactoring around subject name validation
mmyoungman Jun 7, 2024
353a301
EES-4932 Switch to using ArchiveDataSetFile
mmyoungman Jun 7, 2024
ede4b20
EES-4932 Take Streams rather than Func<Task<Stream>>>s to fix unit tests
mmyoungman Jun 17, 2024
9de0d18
EES-4932 Refactor Upload methods
mmyoungman Jun 17, 2024
fec3343
EES-4932 Various renaming and cleanup
mmyoungman Jun 18, 2024
86b47bd
EES-4932 DeflateStreams don't support .Length
mmyoungman Jun 18, 2024
f3b103b
EES-4932 Fix Magic mime type / encoding guessing
mmyoungman Jun 20, 2024
7983deb
EES-4932 Fix replacement dup data set name error
mmyoungman Jun 21, 2024
bd99b2b
EES-4932 Fix frontend validation
mmyoungman Jun 21, 2024
f831bf2
EES-4932 Outline what unit tests need writing
mmyoungman Jun 21, 2024
960b09b
EES-4932 Update backend unit tests
mmyoungman Jun 21, 2024
84255ed
EES-4932 More updates to backend unit tests
mmyoungman Jun 24, 2024
c80c23f
EES-4932 Even more backend unit test updates
mmyoungman Jun 25, 2024
7a2d93b
EES-4932 Yet more backend unit test updates
mmyoungman Jun 25, 2024
532cea3
EES-4932 fix updating upload statuses
amyb-hiveit Jun 24, 2024
7fc0de5
EES-4932 open all bulk uploaded file details
amyb-hiveit Jun 24, 2024
7c50414
EES-4932 use server error messages on data upload form
amyb-hiveit Jun 25, 2024
4bdf992
EES-4932 tidying and updating tests
amyb-hiveit Jun 25, 2024
92a3c74
EES-4932 Backend unit test updates
mmyoungman Jun 27, 2024
cdff09c
EES-4932 Cleanup and renaming
mmyoungman Jun 28, 2024
ef0b3d0
EES-4932 Trim whitespace from archive data set titles
mmyoungman Jul 5, 2024
4545a5c
EES-4932 Changes in response to PR comments
mmyoungman Jul 11, 2024
7640d6a
EES-4932 More changes in response to PR comments
mmyoungman Jul 15, 2024
dcb83c2
Merge branch 'dev' into EES-4932
mmyoungman Jul 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using GovUk.Education.ExploreEducationStatistics.Admin.Controllers.Api;
using GovUk.Education.ExploreEducationStatistics.Admin.Models;
using GovUk.Education.ExploreEducationStatistics.Admin.Requests;
using GovUk.Education.ExploreEducationStatistics.Admin.Services;
using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces;
using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels;
using GovUk.Education.ExploreEducationStatistics.Common.Extensions;
Expand Down Expand Up @@ -52,15 +51,15 @@ public async Task Create_Release_Returns_Ok()
}

[Fact]
public async Task AddDataFilesAsync_UploadsTheFiles_Returns_Ok()
public async Task UploadDataSet_Success()
{
var dataFile = MockFile("datafile.csv");
var metaFile = MockFile("metafile.csv");

var dataFileInfo = new DataFileInfo
{
Name = "Subject name",
FileName = "subject-name.csv",
Name = "Data set title",
FileName = "data-set.csv",
Size = "1 Kb",
};

Expand All @@ -69,27 +68,27 @@ public async Task AddDataFilesAsync_UploadsTheFiles_Returns_Ok()
.Setup(service => service.Upload(_releaseVersionId,
dataFile,
metaFile,
null,
"Subject name"))
"Data set title",
null))
.ReturnsAsync(dataFileInfo);

// Call the method under test
var controller = BuildController(releaseDataFileService: releaseDataFileService.Object);

var result = await controller.AddDataFilesAsync(releaseVersionId: _releaseVersionId,
var result = await controller.UploadDataSet(releaseVersionId: _releaseVersionId,
replacingFileId: null,
subjectName: "Subject name",
title: "Data set title",
file: dataFile,
metaFile: metaFile);

VerifyAllMocks(releaseDataFileService);

var dataFileInfoResult = result.AssertOkResult();
Assert.Equal("Subject name", dataFileInfoResult.Name);
Assert.Equal("Data set title", dataFileInfoResult.Name);
}

[Fact]
public async Task AddDataFilesAsync_UploadsTheFiles_Returns_ValidationProblem()
public async Task UploadDataSet_Fail_ValidationProblem()
{
var dataFile = MockFile("datafile.csv");
var metaFile = MockFile("metafile.csv");
Expand All @@ -99,16 +98,16 @@ public async Task AddDataFilesAsync_UploadsTheFiles_Returns_ValidationProblem()
.Setup(service => service.Upload(_releaseVersionId,
dataFile,
metaFile,
null,
"Subject name"))
"Data set title",
null))
.ReturnsAsync(ValidationActionResult(CannotOverwriteFile));

var controller = BuildController(releaseDataFileService: releaseDataFileService.Object);

// Call the method under test
var result = await controller.AddDataFilesAsync(releaseVersionId: _releaseVersionId,
var result = await controller.UploadDataSet(releaseVersionId: _releaseVersionId,
replacingFileId: null,
subjectName: "Subject name",
title: "Data set title",
file: dataFile,
metaFile: metaFile);

Expand Down Expand Up @@ -154,7 +153,7 @@ public async Task GetDataFileInfo_Returns_A_List_Of_Files()
}

[Fact]
public async Task DeleteDataFilesAsync_Returns_OK()
public async Task DeleteDataFiles_Success()
{
var releaseService = new Mock<IReleaseService>(Strict);

Expand All @@ -174,7 +173,7 @@ public async Task DeleteDataFilesAsync_Returns_OK()
}

[Fact]
public async Task DeleteDataFilesAsync_Returns_ValidationProblem()
public async Task DeleteDataFiles_Fail_UnableToFindMetaFileToDelete()
{
var releaseService = new Mock<IReleaseService>(Strict);

Expand Down Expand Up @@ -257,7 +256,7 @@ public async Task CancelFileImport()
}

[Fact]
public async Task CancelFileImportButNotAllowed()
public async Task CancelFileImport_Fail_Forbidden()
{
var importService = new Mock<IDataImportService>(Strict);

Expand Down
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
time_period,time_identifier,geographic_level,country_code,country_name,old_la_code,new_la_code,la_name,region_code,region_name,lad_code,lad_name,local_enterprise_partnership_code,local_enterprise_partnership_name,rsc_region_lead_name,opportunity_area_code,opportunity_area_name,pcon_code,pcon_name,ward_code,ward_name,admission_numbers
2018,Calendar year,Local authority,E92000001,England,330,E08000025,Birmingham,,,,,,,,,,,,,,3962
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading