From a2983b086ad825e268b639287c6e50118626ac6e Mon Sep 17 00:00:00 2001 From: JoanaCMS <70145179+JoanaCMS@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:57:33 +0200 Subject: [PATCH] Feat/MET-5305 Accept .tar and .tar.gz uploads harvesting (#130) * MET-5305 Updating code to accept other file harvesting * MET-5305 Fixed unit tests * MET-5305 Changed Step enum and fixed unit tests * MET-5305 Fixed issue of harvesting other files * MET-5305 Fixed http harvesting to accept .tar and .tar.gz files * MET-5303 Code refactoring * MET-5305 Created unit tests and fixed code review comments * MET-5305 Changed unit tests --- pom.xml | 2 +- .../europeana/metis/sandbox/common/Step.java | 2 +- .../InvalidCompressedFileException.java | 14 ++ .../exception/InvalidZipFileException.java | 11 -- .../sandbox/controller/DatasetController.java | 133 +++++++++++++----- .../advice/ControllerErrorHandler.java | 6 +- .../workflow/HarvestPublishService.java | 19 +-- .../workflow/HarvestPublishServiceImpl.java | 40 +++--- .../service/workflow/HarvestService.java | 45 +++--- .../service/workflow/HarvestServiceImpl.java | 13 +- src/main/resources/sample.application.yml | 4 + .../metis/sandbox/common/StepTest.java | 4 +- .../controller/DatasetControllerTest.java | 93 ++++++++---- .../dto/report/ProgressInfoDtoTest.java | 4 +- .../executor/EventRecordLogConsumerTest.java | 6 +- .../executor/workflow/CloseExecutorTest.java | 8 +- .../ExternalValidationExecutorTest.java | 8 +- .../workflow/NormalizationExecutorTest.java | 8 +- .../workflow/PublishExecutorTest.java | 8 +- .../executor/workflow/StepExecutorTest.java | 22 +-- ...ansformationToEdmExternalExecutorTest.java | 6 +- .../dataset/DatasetReportServiceImplTest.java | 18 +-- .../dataset/RecordPublishServiceImplTest.java | 10 +- .../metrics/MetricsServiceImplTest.java | 10 +- .../PatternAnalysisServiceImplIT.java | 2 +- .../record/RecordLogServiceImplTest.java | 18 +-- .../HarvestPublishServiceImplTest.java | 20 +-- .../workflow/HarvestServiceImplTest.java | 48 ++++--- .../response_body_dataset_valid_small.txt | 2 +- src/test/resources/zip/records_to_test.tar | Bin 0 -> 51200 bytes src/test/resources/zip/sandbox.tar.gz | Bin 0 -> 7300 bytes 31 files changed, 354 insertions(+), 230 deletions(-) create mode 100644 src/main/java/eu/europeana/metis/sandbox/common/exception/InvalidCompressedFileException.java delete mode 100644 src/main/java/eu/europeana/metis/sandbox/common/exception/InvalidZipFileException.java create mode 100644 src/test/resources/zip/records_to_test.tar create mode 100644 src/test/resources/zip/sandbox.tar.gz diff --git a/pom.xml b/pom.xml index 8d0d1cb1..f286a089 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ 11 1.7.0 - 10 + 11-SNAPSHOT 2.15.3 42.6.0 1.12.261 diff --git a/src/main/java/eu/europeana/metis/sandbox/common/Step.java b/src/main/java/eu/europeana/metis/sandbox/common/Step.java index 75d6bd24..0feb59df 100644 --- a/src/main/java/eu/europeana/metis/sandbox/common/Step.java +++ b/src/main/java/eu/europeana/metis/sandbox/common/Step.java @@ -5,7 +5,7 @@ */ public enum Step { HARVEST_OAI_PMH("harvest OAI-PMH",1), - HARVEST_ZIP("harvest zip", 2), + HARVEST_FILE("harvest file", 2), TRANSFORM_TO_EDM_EXTERNAL("transform to EDM external", 3), VALIDATE_EXTERNAL("validate (edm external)", 4), TRANSFORM("transform", 5), diff --git a/src/main/java/eu/europeana/metis/sandbox/common/exception/InvalidCompressedFileException.java b/src/main/java/eu/europeana/metis/sandbox/common/exception/InvalidCompressedFileException.java new file mode 100644 index 00000000..3fb6a472 --- /dev/null +++ b/src/main/java/eu/europeana/metis/sandbox/common/exception/InvalidCompressedFileException.java @@ -0,0 +1,14 @@ +package eu.europeana.metis.sandbox.common.exception; + +/** + * Exception class for when an invalid type of compressed file is detected + */ +public class InvalidCompressedFileException extends RuntimeException { + + private static final long serialVersionUID = -2555540887797325483L; + + public InvalidCompressedFileException(Throwable cause) { + super("File provided is not valid compressed file. ", cause); + } + +} diff --git a/src/main/java/eu/europeana/metis/sandbox/common/exception/InvalidZipFileException.java b/src/main/java/eu/europeana/metis/sandbox/common/exception/InvalidZipFileException.java deleted file mode 100644 index 79a8ed52..00000000 --- a/src/main/java/eu/europeana/metis/sandbox/common/exception/InvalidZipFileException.java +++ /dev/null @@ -1,11 +0,0 @@ -package eu.europeana.metis.sandbox.common.exception; - -public class InvalidZipFileException extends RuntimeException { - - private static final long serialVersionUID = -2555540887797325483L; - - public InvalidZipFileException(Throwable cause) { - super("File provided is not valid zip. ", cause); - } - -} diff --git a/src/main/java/eu/europeana/metis/sandbox/controller/DatasetController.java b/src/main/java/eu/europeana/metis/sandbox/controller/DatasetController.java index 16ef65bd..130998e7 100644 --- a/src/main/java/eu/europeana/metis/sandbox/controller/DatasetController.java +++ b/src/main/java/eu/europeana/metis/sandbox/controller/DatasetController.java @@ -4,6 +4,7 @@ import eu.europeana.indexing.tiers.view.RecordTierCalculationView; import eu.europeana.metis.sandbox.common.OaiHarvestData; import eu.europeana.metis.sandbox.common.Step; +import eu.europeana.metis.sandbox.common.exception.InvalidCompressedFileException; import eu.europeana.metis.sandbox.common.exception.NoRecordFoundException; import eu.europeana.metis.sandbox.common.exception.XsltProcessingException; import eu.europeana.metis.sandbox.common.locale.Country; @@ -18,6 +19,7 @@ import eu.europeana.metis.sandbox.service.record.RecordLogService; import eu.europeana.metis.sandbox.service.record.RecordTierCalculationService; import eu.europeana.metis.sandbox.service.workflow.HarvestPublishService; +import eu.europeana.metis.utils.CompressedFileExtension; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; @@ -25,6 +27,7 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.validator.routines.UrlValidator; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.GetMapping; @@ -40,6 +43,10 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Set; import java.util.regex.Pattern; @@ -116,13 +123,13 @@ public DatasetController(DatasetService datasetService, DatasetLogService datase } /** - * POST API calls for harvesting and processing the records given a zip file + * POST API calls for harvesting and processing the records given a zip, tar or tar.gz file * * @param datasetName the given name of the dataset to be processed * @param country the given country from which the records refer to * @param language the given language that the records contain * @param stepsize the stepsize - * @param dataset the given dataset itself to be processed as a zip file + * @param dataset the given dataset itself to be processed as a compressed file * @param xsltFile the xslt file used for transformation to edm external * @return 202 if it's processed correctly, 4xx or 500 otherwise */ @@ -137,9 +144,10 @@ public DatasetIdDto harvestDatasetFromFile( @Parameter(description = "country of the dataset", required = true) @RequestParam Country country, @Parameter(description = "language of the dataset", required = true) @RequestParam Language language, @Parameter(description = "step size to apply in record selection", schema = @Schema(description = "step size", defaultValue = "1")) @RequestParam(required = false) Integer stepsize, - @Parameter(description = "dataset records uploaded in a zip file", required = true) @RequestParam MultipartFile dataset, + @Parameter(description = "dataset records uploaded in a zip, tar or tar.gz file", required = true) @RequestParam MultipartFile dataset, @Parameter(description = "xslt file to transform to EDM external") @RequestParam(required = false) MultipartFile xsltFile) { checkArgument(NAME_PATTERN.matcher(datasetName).matches(), MESSAGE_FOR_DATASET_VALID_NAME); + CompressedFileExtension compressedFileExtension = getCompressedFileExtensionTypeFromUploadedFile(dataset); if (stepsize != null) { checkArgument(stepsize > 0, MESSAGE_FOR_STEP_SIZE_VALID_VALUE); } @@ -150,13 +158,13 @@ public DatasetIdDto harvestDatasetFromFile( DatasetMetadata datasetMetadata = DatasetMetadata.builder().withDatasetId(createdDatasetId) .withDatasetName(datasetName).withCountry(country).withLanguage(language) .withStepSize(stepsize).build(); - harvestPublishService.runHarvestZipAsync(dataset, datasetMetadata) + harvestPublishService.runHarvestFileAsync(dataset, datasetMetadata, compressedFileExtension) .exceptionally(e -> datasetLogService.logException(createdDatasetId, e)); return new DatasetIdDto(createdDatasetId); } /** - * POST API calls for harvesting and processing the records given a URL of a zip file + * POST API calls for harvesting and processing the records given a URL of a compressed file * * @param datasetName the given name of the dataset to be processed * @param country the given country from which the records refer to @@ -182,6 +190,7 @@ public DatasetIdDto harvestDatasetFromURL( @Parameter(description = "xslt file to transform to EDM external") @RequestParam(required = false) MultipartFile xsltFile) { checkArgument(NAME_PATTERN.matcher(datasetName).matches(), MESSAGE_FOR_DATASET_VALID_NAME); + CompressedFileExtension compressedFileExtension = getCompressedFileExtensionTypeFromUrl(url); if (stepsize != null) { checkArgument(stepsize > 0, MESSAGE_FOR_STEP_SIZE_VALID_VALUE); } @@ -194,7 +203,7 @@ public DatasetIdDto harvestDatasetFromURL( DatasetMetadata datasetMetadata = DatasetMetadata.builder().withDatasetId(createdDatasetId) .withDatasetName(datasetName).withCountry(country).withLanguage(language) .withStepSize(stepsize).build(); - harvestPublishService.runHarvestHttpZipAsync(url, datasetMetadata) + harvestPublishService.runHarvestHttpFileAsync(url, datasetMetadata, compressedFileExtension) .exceptionally(e -> datasetLogService.logException(createdDatasetId, e)); return new DatasetIdDto(createdDatasetId); } @@ -307,7 +316,7 @@ public String getRecord(@PathVariable("id") String datasetId, @RequestParam Stri private Set getSetFromStep(String step) { Set steps; if (step == null || step.isBlank() || step.equals("HARVEST")) { - steps = Set.of(Step.HARVEST_ZIP, Step.HARVEST_OAI_PMH); + steps = Set.of(Step.HARVEST_FILE, Step.HARVEST_OAI_PMH); } else { try { steps = Set.of(Step.valueOf(step)); @@ -372,39 +381,87 @@ private InputStream createXsltAsInputStreamIfPresent(MultipartFile xslt) { return new ByteArrayInputStream(new byte[0]); } - private static class CountryView { - - @JsonProperty("name") - private final String name; - @JsonProperty("xmlValue") - private final String xmlValue; - - /** - * Instantiates a new Country view. - * - * @param country the country - */ - CountryView(Country country) { - this.name = country.name(); - this.xmlValue = country.xmlValue(); + private CompressedFileExtension getCompressedFileExtensionTypeFromUrl(String url) { + + try { + if (url.startsWith("file:/")) { + Path path = Path.of(url); + String fileContentType = Files.probeContentType(path); + + return getCompressedFileExtensionType(fileContentType); + } else { + + URLConnection urlConnection = new URL(url).openConnection(); + String fileContentType = urlConnection.getContentType(); + + if (StringUtils.isEmpty(fileContentType)) { + throw new InvalidCompressedFileException(new Exception("There was an issue inspecting file's content type")); + } + + return getCompressedFileExtensionType(fileContentType); + + + } + } catch (IOException e) { + throw new InvalidCompressedFileException(e); + } } - private static class LanguageView { - - @JsonProperty("name") - private final String name; - @JsonProperty("xmlValue") - private final String xmlValue; - - /** - * Instantiates a new Language view. - * - * @param language the language - */ - LanguageView(Language language) { - this.name = language.name(); - this.xmlValue = language.xmlValue(); + private CompressedFileExtension getCompressedFileExtensionTypeFromUploadedFile(MultipartFile uploadedFile){ + String fileContentType = uploadedFile.getContentType(); + if (StringUtils.isEmpty(fileContentType)) { + throw new InvalidCompressedFileException(new Exception("There was an issue inspecting file's content type")); + } + + return getCompressedFileExtensionType(fileContentType); + } + + private CompressedFileExtension getCompressedFileExtensionType(String fileContentType){ + if (fileContentType.contains("gzip")) { + return CompressedFileExtension.TAR_GZ; + } else if (fileContentType.contains("zip")) { + return CompressedFileExtension.ZIP; + } else if (fileContentType.contains("x-tar")) { + return CompressedFileExtension.TAR; + } else { + throw new InvalidCompressedFileException(new Exception("The compressed file type is invalid")); + } + } + + private static class CountryView { + + @JsonProperty("name") + private final String name; + @JsonProperty("xmlValue") + private final String xmlValue; + + /** + * Instantiates a new Country view. + * + * @param country the country + */ + CountryView(Country country) { + this.name = country.name(); + this.xmlValue = country.xmlValue(); + } + } + + private static class LanguageView { + + @JsonProperty("name") + private final String name; + @JsonProperty("xmlValue") + private final String xmlValue; + + /** + * Instantiates a new Language view. + * + * @param language the language + */ + LanguageView(Language language) { + this.name = language.name(); + this.xmlValue = language.xmlValue(); + } } } -} diff --git a/src/main/java/eu/europeana/metis/sandbox/controller/advice/ControllerErrorHandler.java b/src/main/java/eu/europeana/metis/sandbox/controller/advice/ControllerErrorHandler.java index bdc551f7..6869cf93 100644 --- a/src/main/java/eu/europeana/metis/sandbox/controller/advice/ControllerErrorHandler.java +++ b/src/main/java/eu/europeana/metis/sandbox/controller/advice/ControllerErrorHandler.java @@ -3,7 +3,7 @@ import static java.lang.String.format; import eu.europeana.metis.sandbox.common.exception.InvalidDatasetException; -import eu.europeana.metis.sandbox.common.exception.InvalidZipFileException; +import eu.europeana.metis.sandbox.common.exception.InvalidCompressedFileException; import eu.europeana.metis.sandbox.common.exception.NoRecordFoundException; import eu.europeana.metis.sandbox.common.exception.RecordParsingException; import eu.europeana.metis.sandbox.common.exception.ServiceException; @@ -48,8 +48,8 @@ public ResponseEntity handleIllegalArgumentException(IllegalArgumentExce return new ResponseEntity<>(exceptionModel, exceptionModel.getStatus()); } - @ExceptionHandler(InvalidZipFileException.class) - public ResponseEntity handleIInvalidZipFileException(InvalidZipFileException ex) { + @ExceptionHandler(InvalidCompressedFileException.class) + public ResponseEntity handleInvalidCompressedFileException(InvalidCompressedFileException ex) { var exceptionModel = new ExceptionModelDto(HttpStatus.BAD_REQUEST.value(), HttpStatus.BAD_REQUEST, ex.getMessage()); LOGGER.error(ex.getMessage(), ex); diff --git a/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishService.java b/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishService.java index a2467ccb..71aeebcd 100644 --- a/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishService.java +++ b/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishService.java @@ -3,6 +3,7 @@ import eu.europeana.metis.sandbox.common.OaiHarvestData; import eu.europeana.metis.sandbox.common.exception.ServiceException; import eu.europeana.metis.sandbox.domain.DatasetMetadata; +import eu.europeana.metis.utils.CompressedFileExtension; import org.springframework.web.multipart.MultipartFile; import java.util.concurrent.CompletableFuture; @@ -10,33 +11,35 @@ public interface HarvestPublishService { /** - * Start the harvest of a zip asynchronously on the given file {@link MultipartFile} + * Start the harvest of a compressed file asynchronously on the given file {@link MultipartFile} * - * @param file zip file containing one or more records - * @param datasetMetadata object that encapsulates all data related to the dataset + * @param file compressed file containing one or more records + * @param datasetMetadata object that encapsulates all data related to the dataset + * @param compressedFileExtension the content type of file being uploaded * @return A HarvestContent object containing the content of the harvest and a bollean indicating * if it reached the max number of records * @throws ServiceException if file is not valid, error reading file, if records are empty */ - CompletableFuture runHarvestZipAsync(MultipartFile file, DatasetMetadata datasetMetadata); + CompletableFuture runHarvestFileAsync(MultipartFile file, DatasetMetadata datasetMetadata, CompressedFileExtension compressedFileExtension); /** * Start the harvest of an url asynchronously on the given URL {@link String} * - * @param url URL for zip file containing one or more records - * @param datasetMetadata the object that encapsulates all data related to the dataset + * @param url URL for compressed file containing one or more records + * @param datasetMetadata the object that encapsulates all data related to the dataset + * @param compressedFileExtension the content type of the file being uploaded * @return A HarvestContent object containing the content of the harvest and a boolean indicating * if it reached the max number of records * @throws ServiceException if error processing URL, if URL timeout, if records are empty */ - CompletableFuture runHarvestHttpZipAsync(String url, DatasetMetadata datasetMetadata); + CompletableFuture runHarvestHttpFileAsync(String url, DatasetMetadata datasetMetadata, CompressedFileExtension compressedFileExtension); /** * Async publish to message broker for further processing. This will send messages to 'harvestOai` * queue * * @param datasetMetadata the objevct that encapsulates all dataset related to the dataset - * @param oaiHarvestData And object that encapsulates the data necessary for OAI-PMH harvesting + * @param oaiHarvestData And object that encapsulates the data necessary for OAI-PMH harvesting * @return {@link CompletableFuture} of the process */ CompletableFuture runHarvestOaiPmhAsync(DatasetMetadata datasetMetadata, OaiHarvestData oaiHarvestData); diff --git a/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishServiceImpl.java b/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishServiceImpl.java index 9c1424cf..1c7fe126 100644 --- a/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishServiceImpl.java +++ b/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishServiceImpl.java @@ -5,12 +5,15 @@ import eu.europeana.metis.sandbox.common.exception.ServiceException; import eu.europeana.metis.sandbox.domain.DatasetMetadata; import eu.europeana.metis.sandbox.domain.Record; + import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.UnknownHostException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; + +import eu.europeana.metis.utils.CompressedFileExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -32,46 +35,49 @@ public HarvestPublishServiceImpl(HarvestService harvestService, } @Override - public CompletableFuture runHarvestZipAsync(MultipartFile file, DatasetMetadata datasetMetadata){ + public CompletableFuture runHarvestFileAsync(MultipartFile file, DatasetMetadata datasetMetadata, + CompressedFileExtension compressedFileExtension) { try { Record.RecordBuilder recordDataEncapsulated = Record.builder() - .datasetId(datasetMetadata.getDatasetId()) - .datasetName(datasetMetadata.getDatasetName()) - .country(datasetMetadata.getCountry()) - .language(datasetMetadata.getLanguage()); - return runHarvestZipAsync(file.getInputStream(), recordDataEncapsulated, datasetMetadata); + .datasetId(datasetMetadata.getDatasetId()) + .datasetName(datasetMetadata.getDatasetName()) + .country(datasetMetadata.getCountry()) + .language(datasetMetadata.getLanguage()); + return runHarvestFileAsync(file.getInputStream(), recordDataEncapsulated, datasetMetadata, compressedFileExtension); } catch (IOException e) { throw new ServiceException("Error harvesting records from file " + file.getName(), e); } } @Override - public CompletableFuture runHarvestHttpZipAsync(String url, DatasetMetadata datasetMetadata){ + public CompletableFuture runHarvestHttpFileAsync(String url, DatasetMetadata datasetMetadata, + CompressedFileExtension compressedFileExtension) { Record.RecordBuilder recordDataEncapsulated = Record.builder() - .datasetId(datasetMetadata.getDatasetId()) - .datasetName(datasetMetadata.getDatasetName()) - .country(datasetMetadata.getCountry()) - .language(datasetMetadata.getLanguage()); + .datasetId(datasetMetadata.getDatasetId()) + .datasetName(datasetMetadata.getDatasetName()) + .country(datasetMetadata.getCountry()) + .language(datasetMetadata.getLanguage()); return CompletableFuture.runAsync(() -> { try (InputStream input = new URL(url).openStream()) { harvestService.harvest(input, datasetMetadata.getDatasetId(), recordDataEncapsulated, - datasetMetadata.getStepSize()); + datasetMetadata.getStepSize(), compressedFileExtension); } catch (UnknownHostException e) { throw new ServiceException(HARVESTING_ERROR_MESSAGE + datasetMetadata.getDatasetId() - + " - unknown host: " + e.getMessage()); + + " - unknown host: " + e.getMessage()); } catch (IOException | HarvesterException e) { throw new ServiceException(HARVESTING_ERROR_MESSAGE + datasetMetadata.getDatasetId(), e); } }, asyncServiceTaskExecutor); } - private CompletableFuture runHarvestZipAsync(InputStream inputStreamToHarvest, - Record.RecordBuilder recordDataEncapsulated, - DatasetMetadata datasetMetadata) { + private CompletableFuture runHarvestFileAsync(InputStream inputStreamToHarvest, + Record.RecordBuilder recordDataEncapsulated, + DatasetMetadata datasetMetadata, + CompressedFileExtension compressedFileExtension) { return CompletableFuture.runAsync(() -> { try { harvestService.harvest(inputStreamToHarvest, datasetMetadata.getDatasetId(), recordDataEncapsulated, - datasetMetadata.getStepSize()); + datasetMetadata.getStepSize(), compressedFileExtension); } catch (HarvesterException e) { throw new ServiceException(HARVESTING_ERROR_MESSAGE + datasetMetadata.getDatasetId(), e); } diff --git a/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestService.java b/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestService.java index fa4b740e..a3bb9695 100644 --- a/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestService.java +++ b/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestService.java @@ -3,30 +3,35 @@ import eu.europeana.metis.harvesting.HarvesterException; import eu.europeana.metis.sandbox.common.OaiHarvestData; import eu.europeana.metis.sandbox.domain.Record; +import eu.europeana.metis.utils.CompressedFileExtension; + import java.io.InputStream; public interface HarvestService { - /** - * Harvest the given OAI endpoint with the given datasetId, data for the records and OAI-PMH data - * - * @param datasetId The id of the dataset to be harvested - * @param recordDataEncapsulated The encapsulation of data to be used to harvest each record - * @param oaiHarvestData The object that encapsulate the necessary data for harvesting - * @param stepSize - */ - void harvestOaiPmh(String datasetId, Record.RecordBuilder recordDataEncapsulated, OaiHarvestData oaiHarvestData, - Integer stepSize); + /** + * Harvest the given OAI endpoint with the given datasetId, data for the records and OAI-PMH data + * + * @param datasetId The id of the dataset to be harvested + * @param recordDataEncapsulated The encapsulation of data to be used to harvest each record + * @param oaiHarvestData The object that encapsulate the necessary data for harvesting + * @param stepSize The step size to apply in the record selection + */ + void harvestOaiPmh(String datasetId, Record.RecordBuilder recordDataEncapsulated, OaiHarvestData oaiHarvestData, + Integer stepSize); + + /** + * Harvest the input stream {@link InputStream} with the given datasetId and data of the records + * + * @param inputStream The input stream to harvest from + * @param datasetId The id of the dataset to be harvested + * @param recordDataEncapsulated The encapsulation of data to be used to harvest each record + * @param stepSize The step size to apply in the record selection + * @param compressedFileExtension The content type of the file being uploaded + * @throws HarvesterException In case an issue occurs while harvesting + */ + void harvest(InputStream inputStream, String datasetId, Record.RecordBuilder recordDataEncapsulated, Integer stepSize, + CompressedFileExtension compressedFileExtension) throws HarvesterException; - /** - * Harvest the input stream {@link InputStream} with the given datasetId and data of the records - * - * @param inputStream The input stream to harvest from - * @param datasetId The id of the dataset to be harvested - * @param recordDataEncapsulated The encapsulation of data to be used to harvest each record - * @param stepSize The step size to apply in the record selection - * @throws HarvesterException In case an issue occurs while harvesting - */ - void harvest(InputStream inputStream, String datasetId, Record.RecordBuilder recordDataEncapsulated, Integer stepSize) throws HarvesterException; } diff --git a/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestServiceImpl.java b/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestServiceImpl.java index c44af833..8196fe58 100644 --- a/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestServiceImpl.java +++ b/src/main/java/eu/europeana/metis/sandbox/service/workflow/HarvestServiceImpl.java @@ -198,16 +198,17 @@ private RecordInfo harvestOaiRecords(String datasetId, OaiHarvestData oaiHarvest } @Override - public void harvest(InputStream inputStream, String datasetId, RecordBuilder recordDataEncapsulated, Integer stepSize) + public void harvest(InputStream inputStream, String datasetId, RecordBuilder recordDataEncapsulated, Integer stepSize, + CompressedFileExtension compressedFileExtension) throws ServiceException { - publishHarvestedRecords(harvestInputStreamIdentifiers(inputStream, datasetId, recordDataEncapsulated, stepSize), + publishHarvestedRecords(harvestInputStreamIdentifiers(inputStream, datasetId, recordDataEncapsulated, stepSize, compressedFileExtension), datasetId, "Error harvesting file records", - Step.HARVEST_ZIP); + Step.HARVEST_FILE); } private List harvestInputStreamIdentifiers(InputStream inputStream, String datasetId, - Record.RecordBuilder recordDataEncapsulated, Integer stepSize) { + Record.RecordBuilder recordDataEncapsulated, Integer stepSize, CompressedFileExtension compressedFileExtension) { List> exception = new ArrayList<>(1); List recordInfoList = new ArrayList<>(); final int numberOfRecordsToStepInto = stepSize == null ? DEFAULT_STEP_SIZE : stepSize; @@ -218,7 +219,7 @@ private List harvestInputStreamIdentifiers(InputStream inputStream, AtomicInteger nextIndexToSelect = new AtomicInteger(numberOfRecordsToStepInto - 1); final HttpRecordIterator iterator = httpHarvester.createTemporaryHttpHarvestIterator(inputStream, - CompressedFileExtension.ZIP); + compressedFileExtension); final String extractedDirectoryFromIterator = iterator.getExtractedDirectory(); iterator.forEach(path -> { try (InputStream content = Files.newInputStream(path)) { @@ -293,7 +294,7 @@ private RecordInfo harvestInputStream(InputStream inputStream, String datasetId, return recordInfo; } catch (RuntimeException | IOException e) { LOGGER.error("Error harvesting file records: {} with exception {}", recordEntity.getId(), e); - saveErrorWhileHarvesting(recordToHarvest, tmpProviderId, Step.HARVEST_ZIP, new RuntimeException(e)); + saveErrorWhileHarvesting(recordToHarvest, tmpProviderId, Step.HARVEST_FILE, new RuntimeException(e)); return null; } } diff --git a/src/main/resources/sample.application.yml b/src/main/resources/sample.application.yml index 1befb1a3..0be5fa3c 100644 --- a/src/main/resources/sample.application.yml +++ b/src/main/resources/sample.application.yml @@ -4,6 +4,10 @@ info: description: Sandbox to process Datasets version: 1.0.0 repository: https://github.com/europeana/metis-sandbox + contact: + name: Europeana Foundation + email: info@europeana.eu + url: www.europeana.eu spring: profiles: diff --git a/src/test/java/eu/europeana/metis/sandbox/common/StepTest.java b/src/test/java/eu/europeana/metis/sandbox/common/StepTest.java index 6a16ff38..429fff25 100644 --- a/src/test/java/eu/europeana/metis/sandbox/common/StepTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/common/StepTest.java @@ -15,7 +15,7 @@ class StepTest { private static Stream provideExpectedStepsAndValues() { return Stream.of( Arguments.of(Step.HARVEST_OAI_PMH, "harvest OAI-PMH", 1), - Arguments.of(Step.HARVEST_ZIP, "harvest zip", 2), + Arguments.of(Step.HARVEST_FILE, "harvest file", 2), Arguments.of(Step.TRANSFORM_TO_EDM_EXTERNAL, "transform to EDM external", 3), Arguments.of(Step.VALIDATE_EXTERNAL, "validate (edm external)", 4), Arguments.of(Step.TRANSFORM, "transform", 5), @@ -34,4 +34,4 @@ void valueOf(Step expectedStep, String value, Integer precedence) { assertEquals(expectedStep.value(), value); assertEquals(expectedStep.precedence(), precedence); } -} \ No newline at end of file +} diff --git a/src/test/java/eu/europeana/metis/sandbox/controller/DatasetControllerTest.java b/src/test/java/eu/europeana/metis/sandbox/controller/DatasetControllerTest.java index 34b7a903..5f66ca85 100644 --- a/src/test/java/eu/europeana/metis/sandbox/controller/DatasetControllerTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/controller/DatasetControllerTest.java @@ -56,6 +56,8 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; + +import eu.europeana.metis.utils.CompressedFileExtension; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -70,6 +72,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.ResultMatcher; +import org.springframework.web.multipart.MultipartFile; @ExtendWith(SpringExtension.class) @@ -100,13 +103,31 @@ class DatasetControllerTest { @Mock private CompletableFuture asyncResult; + private static Stream provideDifferentCompressedFiles() { + return Stream.of( + new MockMultipartFile("dataset", "dataset.txt", "application/zip", + "".getBytes()), + new MockMultipartFile("dataset", "dataset.txt", "application/x-tar", + "".getBytes()), + new MockMultipartFile("dataset", "dataset.txt", "application/gzip", + "".getBytes()) + ); + } + + private static Stream provideDifferentUrlsOfCompressedFiles() { + return Stream.of( + Paths.get("zip", "dataset-valid.zip").toUri().toString(), + Paths.get("zip", "sandbox.tar.gz").toUri().toString(), + Paths.get("zip", "records_to_test.tar").toUri().toString() + ); + } private static Stream steps() { return Stream.of( - arguments(null, Set.of(Step.HARVEST_ZIP, Step.HARVEST_OAI_PMH), status().isOk(), + arguments(null, Set.of(Step.HARVEST_FILE, Step.HARVEST_OAI_PMH), status().isOk(), content().string("exampleString")), - arguments("", Set.of(Step.HARVEST_ZIP, Step.HARVEST_OAI_PMH), status().isOk(), + arguments("", Set.of(Step.HARVEST_FILE, Step.HARVEST_OAI_PMH), status().isOk(), content().string("exampleString")), - arguments("HARVEST", Set.of(Step.HARVEST_ZIP, Step.HARVEST_OAI_PMH), status().isOk(), + arguments("HARVEST", Set.of(Step.HARVEST_FILE, Step.HARVEST_OAI_PMH), status().isOk(), content().string("exampleString")), arguments("TRANSFORM_TO_EDM_EXTERNAL", Set.of(Step.TRANSFORM_TO_EDM_EXTERNAL), status().isOk(), content().string("exampleString")), @@ -131,16 +152,14 @@ private static Stream steps() { @BeforeEach public void setup() { - when(harvestPublishService.runHarvestZipAsync(any(), any())).thenReturn(asyncResult); - when(harvestPublishService.runHarvestHttpZipAsync(any(), any())).thenReturn(asyncResult); + when(harvestPublishService.runHarvestFileAsync(any(), any(), any())).thenReturn(asyncResult); + when(harvestPublishService.runHarvestHttpFileAsync(any(), any(), any())).thenReturn(asyncResult); when(harvestPublishService.runHarvestOaiPmhAsync(any(), any())).thenReturn(asyncResult); } - @Test - void processDatasetFromZipFile_withoutXsltFile_expectSuccess() throws Exception { - - MockMultipartFile mockMultipart = new MockMultipartFile("dataset", "dataset.txt", "text/plain", - "".getBytes()); + @ParameterizedTest + @MethodSource("provideDifferentCompressedFiles") + void processDatasetFromZipFile_withoutXsltFile_expectSuccess(MockMultipartFile mockMultipart) throws Exception { when(datasetService.createEmptyDataset(eq("my-data-set"), eq(ITALY), eq(IT), any(ByteArrayInputStream.class))) @@ -157,11 +176,9 @@ void processDatasetFromZipFile_withoutXsltFile_expectSuccess() throws Exception verify(datasetLogService, never()).logException(any(), any()); } - @Test - void processDatasetFromZipFile_withXsltFile_expectSuccess() throws Exception { - - MockMultipartFile mockMultipart = new MockMultipartFile("dataset", "dataset.txt", "text/plain", - "".getBytes()); + @ParameterizedTest + @MethodSource("provideDifferentCompressedFiles") + void processDatasetFromZipFile_withXsltFile_expectSuccess(MockMultipartFile mockMultipart) throws Exception { MockMultipartFile xsltMock = new MockMultipartFile("xsltFile", "xslt.xsl", "application/xslt+xml", @@ -183,10 +200,9 @@ void processDatasetFromZipFile_withXsltFile_expectSuccess() throws Exception { verify(datasetLogService, never()).logException(any(), any()); } - @Test - void processDatasetFromURL_withoutXsltFile_expectSuccess() throws Exception { - - String url = Paths.get("zip", "dataset-valid.zip").toUri().toString(); + @ParameterizedTest + @MethodSource("provideDifferentUrlsOfCompressedFiles") + void processDatasetFromURL_withoutXsltFile_expectSuccess(String url) throws Exception { when(datasetService.createEmptyDataset(eq("my-data-set"), eq(ITALY), eq(IT), any(ByteArrayInputStream.class))) @@ -203,10 +219,9 @@ void processDatasetFromURL_withoutXsltFile_expectSuccess() throws Exception { verify(datasetLogService, never()).logException(any(), any()); } - @Test - void processDatasetFromURL_withXsltFile_expectSuccess() throws Exception { - - final String url = Paths.get("zip", "dataset-valid.zip").toUri().toString(); + @ParameterizedTest + @MethodSource("provideDifferentUrlsOfCompressedFiles") + void processDatasetFromURL_withXsltFile_expectSuccess(String url) throws Exception { MockMultipartFile xsltMock = new MockMultipartFile("xsltFile", "xslt.xsl", "application/xslt+xml", @@ -280,7 +295,7 @@ void processDatasetFromOAIWithXsltFile_expectSuccess() throws Exception { @Test void processDatasetFromFile_invalidName_expectFail() throws Exception { - var dataset = new MockMultipartFile("dataset", "dataset.txt", "text/plain", + var dataset = new MockMultipartFile("dataset", "dataset.txt", "application/zip", "".getBytes()); mvc.perform(multipart("/dataset/{name}/harvestByFile", "my-data=set") @@ -295,7 +310,7 @@ void processDatasetFromFile_invalidName_expectFail() throws Exception { @Test void processDatasetFromFile_invalidStepSize_expectFail() throws Exception { - var dataset = new MockMultipartFile("dataset", "dataset.txt", "text/plain", + var dataset = new MockMultipartFile("dataset", "dataset.txt", "application/zip", "".getBytes()); mvc.perform(multipart("/dataset/{name}/harvestByFile", "my-data-set") @@ -308,6 +323,22 @@ void processDatasetFromFile_invalidStepSize_expectFail() throws Exception { is("Step size must be a number higher than zero"))); } + @Test + void processDatasetFromFile_invalidFileType_expectFail() throws Exception { + + var dataset = new MockMultipartFile("dataset", "dataset.txt", "text/plain", + "".getBytes()); + + mvc.perform(multipart("/dataset/{name}/harvestByFile", "my-data-set") + .file(dataset) + .param("country", ITALY.name()) + .param("language", IT.name()) + .param("stepsize", "-1")) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.message", + is("File provided is not valid compressed file. "))); + } + @Test void processDatasetFromURL_invalidName_expectFail() throws Exception { @@ -326,7 +357,7 @@ void processDatasetFromURL_invalidName_expectFail() throws Exception { @Test void processDatasetFromURL_invalidStepSize_expectFail() throws Exception { - final String url = "zip" + File.separator + "dataset-valid.zip"; + final String url = Paths.get("zip" + File.separator + "dataset-valid.zip").toUri().toString(); mvc.perform(post("/dataset/{name}/harvestByUrl", "my-data-set") .param("name", "invalidDatasetName") @@ -445,7 +476,7 @@ void retrieveDataset_expectSuccess() throws Exception { var error1 = new ErrorInfoDto(message1, Status.FAIL, List.of("1", "2")); var error2 = new ErrorInfoDto(message2, Status.FAIL, List.of("3", "4")); var errors = List.of(error1, error2); - var createProgress = new ProgressByStepDto(Step.HARVEST_ZIP, 10, 0, 0, List.of()); + var createProgress = new ProgressByStepDto(Step.HARVEST_FILE, 10, 0, 0, List.of()); var externalProgress = new ProgressByStepDto(Step.VALIDATE_EXTERNAL, 7, 3, 0, errors); var datasetInfoDto = new DatasetInfoDto("12345", "Test", LocalDateTime.MIN, Language.NL, Country.NETHERLANDS, false, false); @@ -557,7 +588,7 @@ void getRecord_stepOptional_expectSuccess() throws Exception { final String recordId = "europeanaId"; final String returnString = "exampleString"; when(recordLogService.getProviderRecordString(recordId, datasetId, - Set.of(Step.HARVEST_ZIP, Step.HARVEST_OAI_PMH))) + Set.of(Step.HARVEST_FILE, Step.HARVEST_OAI_PMH))) .thenReturn(returnString); mvc.perform(get("/dataset/{id}/record", datasetId) @@ -586,13 +617,13 @@ void getRecord_NoRecordFoundException() throws Exception { @Test void processDatasetFromZipFile_AsyncExecutionException_expectLogging() throws Exception { - MockMultipartFile mockMultipart = new MockMultipartFile("dataset", "dataset.txt", "text/plain", + MockMultipartFile mockMultipart = new MockMultipartFile("dataset", "dataset.txt", "application/zip", "".getBytes()); when(datasetService.createEmptyDataset(eq("my-data-set"), eq(ITALY), eq(IT), any(ByteArrayInputStream.class))) .thenReturn("12345"); ServiceException exception = new ServiceException("Test error"); - when(harvestPublishService.runHarvestZipAsync(any(), any())).thenReturn( + when(harvestPublishService.runHarvestFileAsync(any(), any(), eq(CompressedFileExtension.ZIP))).thenReturn( CompletableFuture.failedFuture(exception)); mvc.perform(multipart("/dataset/{name}/harvestByFile", "my-data-set") @@ -607,7 +638,7 @@ void processDatasetFromZipFile_AsyncExecutionException_expectLogging() throws Ex @Test void processDatasetFromURL_AsyncExecutionException_expectLogging() throws Exception { ServiceException exception = new ServiceException("Test error"); - when(harvestPublishService.runHarvestHttpZipAsync(any(), any())).thenReturn( + when(harvestPublishService.runHarvestHttpFileAsync(any(), any(), any())).thenReturn( CompletableFuture.failedFuture(exception)); String url = Paths.get("zip", "dataset-valid.zip").toUri().toString(); when(datasetService.createEmptyDataset(eq("my-data-set"), eq(ITALY), eq(IT), diff --git a/src/test/java/eu/europeana/metis/sandbox/dto/report/ProgressInfoDtoTest.java b/src/test/java/eu/europeana/metis/sandbox/dto/report/ProgressInfoDtoTest.java index c1349458..9d66d154 100644 --- a/src/test/java/eu/europeana/metis/sandbox/dto/report/ProgressInfoDtoTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/dto/report/ProgressInfoDtoTest.java @@ -70,7 +70,7 @@ private static ProgressInfoDto getTestProgressInfoDto() { @NotNull private static List getProgressByStepDtoList(int mediaProcessed, int published) { - return List.of(new ProgressByStepDto(Step.HARVEST_ZIP, 5, 0, 0, List.of()), + return List.of(new ProgressByStepDto(Step.HARVEST_FILE, 5, 0, 0, List.of()), new ProgressByStepDto(Step.TRANSFORM_TO_EDM_EXTERNAL, 5, 0, 0, List.of()), new ProgressByStepDto(Step.VALIDATE_EXTERNAL, 5, 0, 0, List.of()), new ProgressByStepDto(Step.TRANSFORM, 5, 0, 0, List.of()), @@ -86,7 +86,7 @@ private static ProgressInfoDto getTestHarvestingIdsInfoDto() { return new ProgressInfoDto("http://metis-sandbox", null, 0L, - List.of(new ProgressByStepDto(Step.HARVEST_ZIP, 0, 0, 0, List.of())), + List.of(new ProgressByStepDto(Step.HARVEST_FILE, 0, 0, 0, List.of())), new DatasetInfoDto("datasetId", "datasetName", LocalDateTime.parse("2022-03-14T22:50:22"), diff --git a/src/test/java/eu/europeana/metis/sandbox/executor/EventRecordLogConsumerTest.java b/src/test/java/eu/europeana/metis/sandbox/executor/EventRecordLogConsumerTest.java index 58b19a21..d48e5112 100644 --- a/src/test/java/eu/europeana/metis/sandbox/executor/EventRecordLogConsumerTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/executor/EventRecordLogConsumerTest.java @@ -35,7 +35,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); consumer.logRecord(recordEvent); @@ -48,7 +48,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); doThrow(new RecordProcessingException("1", new Exception())).when(recordLogService) .logRecordEvent(any(RecordProcessEvent.class)); @@ -56,4 +56,4 @@ var record = Record.builder() verify(recordLogService).logRecordEvent(any(RecordProcessEvent.class)); } -} \ No newline at end of file +} diff --git a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/CloseExecutorTest.java b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/CloseExecutorTest.java index fc01e7a8..598254b1 100644 --- a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/CloseExecutorTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/CloseExecutorTest.java @@ -39,7 +39,7 @@ class CloseExecutorTest { @Test void close_expectSuccess() { var record = getTestRecord(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); consumer.close(recordEvent); verify(amqpTemplate).convertAndSend(any(), captor.capture()); @@ -49,7 +49,7 @@ var record = getTestRecord(); @Test void close_inputMessageWithFailStatus_expectNoInteractions() { var record = getTestRecord(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.FAIL); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.FAIL); consumer.close(recordEvent); verify(amqpTemplate, never()).convertAndSend(any(), any(RecordProcessEvent.class)); @@ -58,7 +58,7 @@ var record = getTestRecord(); @Test void close_exception_expectLogError() { var record = getTestRecord(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); final RuntimeException runtimeException = new AmqpException("Queue Failure"); doThrow(runtimeException).when(amqpTemplate).convertAndSend(any(), any(Object.class)); @@ -69,4 +69,4 @@ private Record getTestRecord() { return Record.builder().datasetId("").datasetName("").country(Country.ITALY).language(Language.IT).content("".getBytes()) .recordId(1L).build(); } -} \ No newline at end of file +} diff --git a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/ExternalValidationExecutorTest.java b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/ExternalValidationExecutorTest.java index 398db757..9c360ef8 100644 --- a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/ExternalValidationExecutorTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/ExternalValidationExecutorTest.java @@ -45,7 +45,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); when(service.validate(record)).thenReturn(new RecordInfo(record)); consumer.validateExternal(recordEvent); @@ -62,7 +62,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.FAIL); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.FAIL); consumer.validateExternal(recordEvent); @@ -76,7 +76,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); when(service.validate(record)).thenThrow(new RecordProcessingException("1", new Exception())); @@ -87,4 +87,4 @@ var record = Record.builder() assertEquals(Status.FAIL, captor.getValue().getStatus()); } -} \ No newline at end of file +} diff --git a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/NormalizationExecutorTest.java b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/NormalizationExecutorTest.java index b0faaaec..da392c0d 100644 --- a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/NormalizationExecutorTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/NormalizationExecutorTest.java @@ -45,7 +45,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); when(service.normalize(record)).thenReturn(new RecordInfo(record)); consumer.normalize(recordEvent); @@ -62,7 +62,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.FAIL); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.FAIL); consumer.normalize(recordEvent); @@ -76,7 +76,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); when(service.normalize(record)).thenThrow(new RecordProcessingException("1", new Exception())); @@ -87,4 +87,4 @@ var record = Record.builder() assertEquals(Status.FAIL, captor.getValue().getStatus()); } -} \ No newline at end of file +} diff --git a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/PublishExecutorTest.java b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/PublishExecutorTest.java index 85fd39b9..2418957b 100644 --- a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/PublishExecutorTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/PublishExecutorTest.java @@ -45,7 +45,7 @@ var record = Record.builder() .datasetId("").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); when(service.index(record)).thenReturn(new RecordInfo(record)); consumer.publish(recordEvent); @@ -62,7 +62,7 @@ var record = Record.builder() .datasetId("").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record),Step.HARVEST_ZIP, Status.FAIL); + var recordEvent = new RecordProcessEvent(new RecordInfo(record),Step.HARVEST_FILE, Status.FAIL); consumer.publish(recordEvent); @@ -76,7 +76,7 @@ var record = Record.builder() .datasetId("").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); when(service.index(record)) .thenThrow(new RecordProcessingException("1", new Exception())); @@ -88,4 +88,4 @@ var record = Record.builder() assertEquals(Status.FAIL, captor.getValue().getStatus()); } -} \ No newline at end of file +} diff --git a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/StepExecutorTest.java b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/StepExecutorTest.java index 0edc12e8..7fee06d9 100644 --- a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/StepExecutorTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/StepExecutorTest.java @@ -51,9 +51,9 @@ class StepExecutorTest { void consumeEventStatusSuccess_expectEventSuccess() { final String routingKey = "routingKey"; final Record myTestRecord = getTestRecord(1L); - final RecordProcessEvent myEvent = new RecordProcessEvent(new RecordInfo(myTestRecord), Step.HARVEST_ZIP, Status.SUCCESS); + final RecordProcessEvent myEvent = new RecordProcessEvent(new RecordInfo(myTestRecord), Step.HARVEST_FILE, Status.SUCCESS); - stepExecutor.consume(routingKey, myEvent, Step.HARVEST_ZIP, this::getRecordInfo); + stepExecutor.consume(routingKey, myEvent, Step.HARVEST_FILE, this::getRecordInfo); verify(amqpTemplate).convertAndSend(eq(routingKey), captor.capture()); @@ -64,9 +64,9 @@ void consumeEventStatusSuccess_expectEventSuccess() { void consumeEventStatusFailQueueNeverCalled_expectException() { final String routingKey = "routingKey"; final Record myTestRecord = getTestRecord(1L); - final RecordProcessEvent myEvent = new RecordProcessEvent(new RecordInfo(myTestRecord), Step.HARVEST_ZIP, Status.FAIL); + final RecordProcessEvent myEvent = new RecordProcessEvent(new RecordInfo(myTestRecord), Step.HARVEST_FILE, Status.FAIL); - stepExecutor.consume(routingKey, myEvent, Step.HARVEST_ZIP, this::getRecordInfo); + stepExecutor.consume(routingKey, myEvent, Step.HARVEST_FILE, this::getRecordInfo); verify(amqpTemplate, never()).convertAndSend(eq(routingKey), captor.capture()); @@ -78,9 +78,9 @@ void consumeEventStatusSuccessThrowsRuntimeException_expectEventFail() { final String routingKey = "routingKey"; final long expectedRecordId = 1L; final Record myTestRecord = getTestRecord(expectedRecordId); - final RecordProcessEvent myEvent = new RecordProcessEvent(new RecordInfo(myTestRecord), Step.HARVEST_ZIP, Status.SUCCESS); + final RecordProcessEvent myEvent = new RecordProcessEvent(new RecordInfo(myTestRecord), Step.HARVEST_FILE, Status.SUCCESS); - stepExecutor.consume(routingKey, myEvent, Step.HARVEST_ZIP, () -> + stepExecutor.consume(routingKey, myEvent, Step.HARVEST_FILE, () -> { throw new RuntimeException("General Failure"); }); @@ -95,9 +95,9 @@ void consumeEventStatusSuccessThrowsRecordProcessingException_expectEventFail() final String routingKey = "routingKey"; final long expectedRecordId = 2L; final Record myTestRecord = getTestRecord(expectedRecordId); - final RecordProcessEvent myEvent = new RecordProcessEvent(new RecordInfo(myTestRecord), Step.HARVEST_ZIP, Status.SUCCESS); + final RecordProcessEvent myEvent = new RecordProcessEvent(new RecordInfo(myTestRecord), Step.HARVEST_FILE, Status.SUCCESS); - stepExecutor.consume(routingKey, myEvent, Step.HARVEST_ZIP, () -> + stepExecutor.consume(routingKey, myEvent, Step.HARVEST_FILE, () -> { throw new RecordProcessingException("2", new Throwable("Record failure")); }); @@ -112,18 +112,18 @@ void consumeEventStatusSuccessThrowsRabbitMQException_expectEventSuccess() { final String routingKey = "routingKey"; final long expectedRecordId = 2L; final Record myTestRecord = getTestRecord(expectedRecordId); - final RecordProcessEvent myEvent = new RecordProcessEvent(new RecordInfo(myTestRecord), Step.HARVEST_ZIP, Status.SUCCESS); + final RecordProcessEvent myEvent = new RecordProcessEvent(new RecordInfo(myTestRecord), Step.HARVEST_FILE, Status.SUCCESS); final RuntimeException runtimeException = new AmqpException("Queue Failure"); doThrow(runtimeException) .when(amqpTemplate) .convertAndSend(eq(routingKey), any(Object.class)); - assertDoesNotThrow(() -> stepExecutor.consume(routingKey, myEvent, Step.HARVEST_ZIP, this::getRecordInfo)); + assertDoesNotThrow(() -> stepExecutor.consume(routingKey, myEvent, Step.HARVEST_FILE, this::getRecordInfo)); } private void assertRecordId_CreatedStepAndStatus(final Long RecordId, final Status status) { assertEquals(RecordId, captor.getValue().getRecord().getRecordId()); - assertEquals(Step.HARVEST_ZIP, captor.getValue().getStep()); + assertEquals(Step.HARVEST_FILE, captor.getValue().getStep()); assertEquals(status, captor.getValue().getStatus()); } diff --git a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/TransformationToEdmExternalExecutorTest.java b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/TransformationToEdmExternalExecutorTest.java index 5bd73379..05483432 100644 --- a/src/test/java/eu/europeana/metis/sandbox/executor/workflow/TransformationToEdmExternalExecutorTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/executor/workflow/TransformationToEdmExternalExecutorTest.java @@ -45,7 +45,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); when(service.transform(record)).thenReturn(new RecordInfo(record)); consumer.transformationToEdmExternal(recordEvent); @@ -62,7 +62,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.FAIL); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.FAIL); consumer.transformationToEdmExternal(recordEvent); @@ -76,7 +76,7 @@ var record = Record.builder() .datasetId("1").datasetName("").country(Country.ITALY).language(Language.IT) .content("".getBytes()) .recordId(1L).build(); - var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var recordEvent = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); when(service.transform(record)).thenThrow(new RecordProcessingException("1", new Exception())); diff --git a/src/test/java/eu/europeana/metis/sandbox/service/dataset/DatasetReportServiceImplTest.java b/src/test/java/eu/europeana/metis/sandbox/service/dataset/DatasetReportServiceImplTest.java index 608a026f..e9e9b21e 100644 --- a/src/test/java/eu/europeana/metis/sandbox/service/dataset/DatasetReportServiceImplTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/service/dataset/DatasetReportServiceImplTest.java @@ -107,7 +107,7 @@ void getReportWithRecordErrors_expectSuccess() { var error1 = new ErrorInfoDto(message1, Status.FAIL, List.of("europeanaId1 | providerId1", "europeanaId2 | providerId2")); var error2 = new ErrorInfoDto(message2, Status.FAIL, List.of("europeanaId3 | providerId3", "europeanaId4 | providerId4")); var errors = List.of(error1, error2); - var createProgress = new ProgressByStepDto(Step.HARVEST_ZIP, 5, 0, 0, List.of()); + var createProgress = new ProgressByStepDto(Step.HARVEST_FILE, 5, 0, 0, List.of()); var externalProgress = new ProgressByStepDto(Step.VALIDATE_EXTERNAL, 1, 4, 0, errors); var report = new ProgressInfoDto( "A review URL will be generated when the dataset has finished processing.", @@ -116,7 +116,7 @@ void getReportWithRecordErrors_expectSuccess() { new DatasetInfoDto("", "", LocalDateTime.now(), Language.NL, Country.NETHERLANDS, false, false), "", emptyList(), null); - var recordViewCreate = new StepStatistic(Step.HARVEST_ZIP, Status.SUCCESS, 5L); + var recordViewCreate = new StepStatistic(Step.HARVEST_FILE, Status.SUCCESS, 5L); var recordViewExternal1 = new StepStatistic(Step.VALIDATE_EXTERNAL, Status.SUCCESS, 1L); var recordViewExternal2 = new StepStatistic(Step.VALIDATE_EXTERNAL, Status.FAIL, 4L); var errorView1 = new ErrorLogViewImpl(1L, getTestRecordEntity(1L), Step.VALIDATE_EXTERNAL, Status.FAIL, @@ -142,7 +142,7 @@ void getReportWithRecordErrors_expectSuccess() { @Test void getReportWithoutErrors_expectSuccess() { var dataset = createDataset(5L); - var createProgress = new ProgressByStepDto(Step.HARVEST_ZIP, 5, 0, 0, List.of()); + var createProgress = new ProgressByStepDto(Step.HARVEST_FILE, 5, 0, 0, List.of()); var externalProgress = new ProgressByStepDto(Step.VALIDATE_EXTERNAL, 5, 0, 0, List.of()); var report = new ProgressInfoDto( "A review URL will be generated when the dataset has finished processing.", @@ -150,7 +150,7 @@ void getReportWithoutErrors_expectSuccess() { List.of(createProgress, externalProgress), new DatasetInfoDto("", "", LocalDateTime.now(), null, null, false, false), "", emptyList(), null); - var recordViewCreate = new StepStatistic(Step.HARVEST_ZIP, Status.SUCCESS, 5L); + var recordViewCreate = new StepStatistic(Step.HARVEST_FILE, Status.SUCCESS, 5L); var recordViewExternal = new StepStatistic(Step.VALIDATE_EXTERNAL, Status.SUCCESS, 5L); when(datasetRepository.findById(1)).thenReturn(Optional.of(dataset)); @@ -167,7 +167,7 @@ void getReportWithoutErrors_expectSuccess() { @Test void getReportCompleted_expectSuccess() { var dataset = createDataset(5L); - var createProgress = new ProgressByStepDto(Step.HARVEST_ZIP, 5, 0, 0, List.of()); + var createProgress = new ProgressByStepDto(Step.HARVEST_FILE, 5, 0, 0, List.of()); var externalProgress = new ProgressByStepDto(Step.VALIDATE_EXTERNAL, 5, 0, 0, List.of()); var publishProgress = new ProgressByStepDto(Step.PUBLISH, 5, 0, 0, List.of()); var tiersZeroInfo = new TiersZeroInfo(new TierStatistics(2, List.of("europeanaId1", "europeanaId2")), @@ -178,7 +178,7 @@ void getReportCompleted_expectSuccess() { new DatasetInfoDto("", "", LocalDateTime.now(), Language.NL, Country.NETHERLANDS, false, false), "", emptyList(), tiersZeroInfo); - var recordViewCreate = new StepStatistic(Step.HARVEST_ZIP, Status.SUCCESS, 5L); + var recordViewCreate = new StepStatistic(Step.HARVEST_FILE, Status.SUCCESS, 5L); var recordViewExternal = new StepStatistic(Step.VALIDATE_EXTERNAL, Status.SUCCESS, 5L); var recordViewPublish = new StepStatistic(Step.PUBLISH, Status.SUCCESS, 5L); var recordViewClose = new StepStatistic(Step.CLOSE, Status.SUCCESS, 5L); @@ -213,7 +213,7 @@ void getReportCompletedAllErrors_expectSuccess() { var error1 = new ErrorInfoDto(message1, Status.FAIL, List.of("europeanaId1 | providerId1", "europeanaId2 | providerId2")); var error2 = new ErrorInfoDto(message2, Status.FAIL, List.of("europeanaId3 | providerId3", "europeanaId4 | providerId4", "europeanaId5 | providerId5")); var errors = List.of(error1, error2); - var createProgress = new ProgressByStepDto(Step.HARVEST_ZIP, 5, 0, 0, List.of()); + var createProgress = new ProgressByStepDto(Step.HARVEST_FILE, 5, 0, 0, List.of()); var externalProgress = new ProgressByStepDto(Step.VALIDATE_EXTERNAL, 0, 5, 0, errors); var report = new ProgressInfoDto( @@ -222,7 +222,7 @@ void getReportCompletedAllErrors_expectSuccess() { new DatasetInfoDto("", "", LocalDateTime.now(), Language.NL, Country.NETHERLANDS, false, false), "", emptyList(), null); - var recordViewCreate = new StepStatistic(Step.HARVEST_ZIP, Status.SUCCESS, 5L); + var recordViewCreate = new StepStatistic(Step.HARVEST_FILE, Status.SUCCESS, 5L); var recordViewExternal = new StepStatistic(Step.VALIDATE_EXTERNAL, Status.FAIL, 5L); var errorView1 = new ErrorLogViewImpl(1L, getTestRecordEntity(1L), Step.VALIDATE_EXTERNAL, Status.FAIL, @@ -265,7 +265,7 @@ void getReport_retrieveEmptyDataset_expectSuccess() { @Test void getReport_WithDatasetWarnings_ShouldNotFail() { var dataset = createDataset(5L); - var recordViewCreate = new StepStatistic(Step.HARVEST_ZIP, Status.SUCCESS, 5L); + var recordViewCreate = new StepStatistic(Step.HARVEST_FILE, Status.SUCCESS, 5L); var recordViewExternal = new StepStatistic(Step.VALIDATE_EXTERNAL, Status.SUCCESS, 5L); when(datasetRepository.findById(1)).thenReturn(Optional.of(dataset)); when(recordLogRepository.getStepStatistics("1")).thenReturn( diff --git a/src/test/java/eu/europeana/metis/sandbox/service/dataset/RecordPublishServiceImplTest.java b/src/test/java/eu/europeana/metis/sandbox/service/dataset/RecordPublishServiceImplTest.java index 5e4f4f53..5f3fd5ac 100644 --- a/src/test/java/eu/europeana/metis/sandbox/service/dataset/RecordPublishServiceImplTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/service/dataset/RecordPublishServiceImplTest.java @@ -51,7 +51,7 @@ void publishWithoutXslt_expectSuccess() { Dataset dataset = new Dataset("1234", Set.of(record1, record2), 0); - dataset.getRecords().forEach(record -> service.publishToHarvestQueue(new RecordInfo(record), Step.HARVEST_ZIP)); + dataset.getRecords().forEach(record -> service.publishToHarvestQueue(new RecordInfo(record), Step.HARVEST_FILE)); verify(amqpTemplate, times(2)).convertAndSend(eq("createdQueue"), any(RecordProcessEvent.class)); } @@ -69,7 +69,7 @@ void publishWithoutXslt_asyncFail_expectNoFail() { doThrow(new AmqpException("Issue publishing this record")).when(amqpTemplate) .convertAndSend(anyString(), any(RecordProcessEvent.class)); - dataset.getRecords().forEach(record -> service.publishToHarvestQueue(new RecordInfo(record), Step.HARVEST_ZIP)); + dataset.getRecords().forEach(record -> service.publishToHarvestQueue(new RecordInfo(record), Step.HARVEST_FILE)); verify(amqpTemplate, times(2)).convertAndSend(eq("createdQueue"), any(RecordProcessEvent.class)); } @@ -84,7 +84,7 @@ void publishWithXslt_expectSuccess() { Dataset dataset = new Dataset("1234", Set.of(record1, record2), 0); - dataset.getRecords().forEach(record -> service.publishToTransformationToEdmExternalQueue(new RecordInfo(record), Step.HARVEST_ZIP)); + dataset.getRecords().forEach(record -> service.publishToTransformationToEdmExternalQueue(new RecordInfo(record), Step.HARVEST_FILE)); verify(amqpTemplate, times(2)).convertAndSend(eq("transformationEdmExternalQueue"), any(RecordProcessEvent.class)); } @@ -102,9 +102,9 @@ void publishWithXslt_asyncFail_expectNoFail() { doThrow(new AmqpException("Issue publishing this record")).when(amqpTemplate) .convertAndSend(anyString(), any(RecordProcessEvent.class)); - dataset.getRecords().forEach(record -> service.publishToTransformationToEdmExternalQueue(new RecordInfo(record), Step.HARVEST_ZIP)); + dataset.getRecords().forEach(record -> service.publishToTransformationToEdmExternalQueue(new RecordInfo(record), Step.HARVEST_FILE)); verify(amqpTemplate, times(2)).convertAndSend(eq("transformationEdmExternalQueue"), any(RecordProcessEvent.class)); } -} \ No newline at end of file +} diff --git a/src/test/java/eu/europeana/metis/sandbox/service/metrics/MetricsServiceImplTest.java b/src/test/java/eu/europeana/metis/sandbox/service/metrics/MetricsServiceImplTest.java index 9fbb88a1..9790d943 100644 --- a/src/test/java/eu/europeana/metis/sandbox/service/metrics/MetricsServiceImplTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/service/metrics/MetricsServiceImplTest.java @@ -60,7 +60,7 @@ private void assertRepositoriesAndMeterRegistry() { void processMetrics() { when(recordRepository.getMetricDatasetStatistics()).thenReturn(List.of(new DatasetStatistic("1", 10L))); when(recordLogRepository.getMetricStepStatistics()).thenReturn( - List.of(new StepStatistic(Step.HARVEST_ZIP, Status.SUCCESS, 10L))); + List.of(new StepStatistic(Step.HARVEST_FILE, Status.SUCCESS, 10L))); when(problemPatternRepository.getMetricProblemPatternStatistics()).thenReturn(List.of( new DatasetProblemPatternStatistic(ProblemPatternId.P2.name(), 5L))); when(amqpConfiguration.getAllQueuesNames()).thenReturn(List.of("sandbox.record.created")); @@ -68,7 +68,7 @@ void processMetrics() { metricsService.processMetrics(); assertEquals(1L, meterRegistry.get("sandbox.metrics.dataset.count").gauge().value()); - assertEquals(10L, meterRegistry.get("sandbox.metrics.dataset.harvest_zip.success").gauge().value()); + assertEquals(10L, meterRegistry.get("sandbox.metrics.dataset.harvest_file.success").gauge().value()); assertEquals(5L, meterRegistry.get("sandbox.metrics.dataset.p2").gauge().value()); assertEquals(10L, meterRegistry.get("sandbox.metrics.queue.created").gauge().value()); assertRepositoriesAndMeterRegistry(); @@ -88,7 +88,7 @@ void processMetrics_whenStepNull() { when(recordLogRepository.getMetricStepStatistics()).thenReturn(null); metricsService.processMetrics(); - assertEquals(0L, meterRegistry.get("sandbox.metrics.dataset.harvest_zip.success").gauge().value()); + assertEquals(0L, meterRegistry.get("sandbox.metrics.dataset.harvest_file.success").gauge().value()); assertRepositoriesAndMeterRegistry(); } @@ -106,7 +106,7 @@ void processMetrics_whenDataAvailable() { when(recordRepository.getMetricDatasetStatistics()).thenReturn(List.of(new DatasetStatistic("1", 5L), new DatasetStatistic("2", 5L))); when(recordLogRepository.getMetricStepStatistics()).thenReturn( - List.of(new StepStatistic(Step.HARVEST_ZIP, Status.SUCCESS, 10L))); + List.of(new StepStatistic(Step.HARVEST_FILE, Status.SUCCESS, 10L))); when(problemPatternRepository.getMetricProblemPatternStatistics()).thenReturn(List.of( new DatasetProblemPatternStatistic(ProblemPatternId.P2.name(), 5L))); when(amqpConfiguration.getAllQueuesNames()).thenReturn(List.of("sandbox.record.created")); @@ -114,7 +114,7 @@ void processMetrics_whenDataAvailable() { metricsService.processMetrics(); assertEquals(2L, meterRegistry.get("sandbox.metrics.dataset.count").gauge().value()); - assertEquals(10L, meterRegistry.get("sandbox.metrics.dataset.harvest_zip.success").gauge().value()); + assertEquals(10L, meterRegistry.get("sandbox.metrics.dataset.harvest_file.success").gauge().value()); assertEquals(5L, meterRegistry.get("sandbox.metrics.dataset.p2").gauge().value()); assertEquals(10L, meterRegistry.get("sandbox.metrics.queue.created").gauge().value()); assertRepositoriesAndMeterRegistry(); diff --git a/src/test/java/eu/europeana/metis/sandbox/service/problempatterns/PatternAnalysisServiceImplIT.java b/src/test/java/eu/europeana/metis/sandbox/service/problempatterns/PatternAnalysisServiceImplIT.java index 9deae7f1..0f3a7108 100644 --- a/src/test/java/eu/europeana/metis/sandbox/service/problempatterns/PatternAnalysisServiceImplIT.java +++ b/src/test/java/eu/europeana/metis/sandbox/service/problempatterns/PatternAnalysisServiceImplIT.java @@ -202,7 +202,7 @@ void generateRecordPatternAnalysisTest() { .getMessageReport())); //Empty result - assertTrue(patternAnalysisServiceImpl.getDatasetPatternAnalysis("1", Step.HARVEST_ZIP, nowP6) + assertTrue(patternAnalysisServiceImpl.getDatasetPatternAnalysis("1", Step.HARVEST_FILE, nowP6) .isEmpty()); } diff --git a/src/test/java/eu/europeana/metis/sandbox/service/record/RecordLogServiceImplTest.java b/src/test/java/eu/europeana/metis/sandbox/service/record/RecordLogServiceImplTest.java index 1528190d..e048df6b 100644 --- a/src/test/java/eu/europeana/metis/sandbox/service/record/RecordLogServiceImplTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/service/record/RecordLogServiceImplTest.java @@ -58,7 +58,7 @@ var record = Record.builder().recordId(1L).content("".getBytes()).datasetId("1") .language(Language.IT).country(Country.ITALY).datasetName("").build(); var recordError = new RecordError("message", "stack"); - var event = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var event = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); service.logRecordEvent(event); @@ -76,7 +76,7 @@ void logRecord_unableToSaveRecord_expectFail() { var record = Record.builder().recordId(1L).content("".getBytes()).datasetId("1") .language(Language.IT).country(Country.ITALY).datasetName("").build(); - var event = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var event = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); when(recordLogRepository.save(any(RecordLogEntity.class))) .thenThrow(new RuntimeException("Exception saving")); @@ -89,7 +89,7 @@ void logRecord_unableToSaveRecordErrors_expectFail() { var record = Record.builder().recordId(1L).content("".getBytes()).datasetId("1") .language(Language.IT).country(Country.ITALY).datasetName("").build(); - var event = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_ZIP, Status.SUCCESS); + var event = new RecordProcessEvent(new RecordInfo(record), Step.HARVEST_FILE, Status.SUCCESS); when(errorLogRepository.saveAll(anyList())) .thenThrow(new RuntimeException("Exception saving")); @@ -121,9 +121,9 @@ void getProviderRecordStringByStep_expectSuccess() throws Exception { final RecordLogEntity recordLogEntity = new RecordLogEntity(); recordLogEntity.setContent("content"); when(recordLogRepository.findRecordLogByRecordIdDatasetIdAndStepIn("recordId", "datasetId", - Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_ZIP))).thenReturn(Set.of(recordLogEntity)); + Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_FILE))).thenReturn(Set.of(recordLogEntity)); final String providerRecord = service.getProviderRecordString("recordId", "datasetId", - Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_ZIP)); + Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_FILE)); assertNotNull(providerRecord); assertEquals("content", providerRecord); } @@ -132,15 +132,15 @@ void getProviderRecordStringByStep_expectSuccess() throws Exception { void getProviderRecordString_expectFail() { //Case null entity when(recordLogRepository.findRecordLogByRecordIdDatasetIdAndStepIn("recordId", "datasetId", - Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_ZIP))).thenReturn(Collections.emptySet()); + Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_FILE))).thenReturn(Collections.emptySet()); assertThrows(NoRecordFoundException.class, - () -> service.getProviderRecordString("recordId", "datasetId", Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_ZIP))); + () -> service.getProviderRecordString("recordId", "datasetId", Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_FILE))); //Case null content when(recordLogRepository.findRecordLogByRecordIdDatasetIdAndStepIn("recordId", "datasetId", - Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_ZIP))).thenReturn(Set.of(new RecordLogEntity())); + Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_FILE))).thenReturn(Set.of(new RecordLogEntity())); assertThrows(NoRecordFoundException.class, - () -> service.getProviderRecordString("recordId", "datasetId", Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_ZIP))); + () -> service.getProviderRecordString("recordId", "datasetId", Set.of(Step.HARVEST_OAI_PMH, Step.HARVEST_FILE))); } @Test diff --git a/src/test/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishServiceImplTest.java b/src/test/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishServiceImplTest.java index 380a8a7e..814e52bd 100644 --- a/src/test/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishServiceImplTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/service/workflow/HarvestPublishServiceImplTest.java @@ -23,6 +23,8 @@ import java.nio.file.Paths; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; + +import eu.europeana.metis.utils.CompressedFileExtension; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -58,9 +60,10 @@ void runZipHarvestAsync_expectSuccess() throws IOException, HarvesterException { .withLanguage(Language.NL) .withStepSize(5) .build(); - asyncHarvestPublishService.runHarvestZipAsync(datasetFile, datasetMetadata); + asyncHarvestPublishService.runHarvestFileAsync(datasetFile, datasetMetadata, CompressedFileExtension.ZIP); - verify(harvestService, times(1)).harvest(any(InputStream.class), eq("datasetId"), any(Record.RecordBuilder.class), eq(5)); + verify(harvestService, times(1)).harvest(any(InputStream.class), eq("datasetId"), any(Record.RecordBuilder.class), eq(5), + eq(CompressedFileExtension.ZIP)); } @@ -80,7 +83,7 @@ void runZipHarvestAsync_expectFail() throws IOException { when(datasetFile.getInputStream()).thenThrow(new IOException("error test")); assertThrows(ServiceException.class, () -> - asyncHarvestPublishService.runHarvestZipAsync(datasetFile, datasetMetadata)); + asyncHarvestPublishService.runHarvestFileAsync(datasetFile, datasetMetadata, CompressedFileExtension.ZIP)); } @@ -94,11 +97,12 @@ void runHttpHarvestAsync_expectSuccess() throws HarvesterException { .withStepSize(5) .build(); - CompletableFuture future = asyncHarvestPublishService.runHarvestHttpZipAsync( - "http://ftp.eanadev.org/uploads/Hauenstein-0.zip", datasetMetadata); + CompletableFuture future = asyncHarvestPublishService.runHarvestHttpFileAsync( + "http://ftp.eanadev.org/uploads/Hauenstein-0.zip", datasetMetadata, CompressedFileExtension.ZIP); assertTrue(!future.isCompletedExceptionally() || future.isCancelled()); - verify(harvestService, times(1)).harvest(any(InputStream.class), eq("datasetId"), any(Record.RecordBuilder.class), eq(5)); + verify(harvestService, times(1)).harvest(any(InputStream.class), eq("datasetId"), any(Record.RecordBuilder.class), eq(5), + eq(CompressedFileExtension.ZIP)); } @Test @@ -111,8 +115,8 @@ void runHttpHarvestAsync_expectFail() { .withStepSize(5) .build(); - CompletableFuture future = asyncHarvestPublishService.runHarvestHttpZipAsync("http://myfake-test-url.com", - datasetMetadata); + CompletableFuture future = asyncHarvestPublishService.runHarvestHttpFileAsync("http://myfake-test-url.com", + datasetMetadata,null); assertTrue(future.isCompletedExceptionally() && !future.isCancelled()); } diff --git a/src/test/java/eu/europeana/metis/sandbox/service/workflow/HarvestServiceImplTest.java b/src/test/java/eu/europeana/metis/sandbox/service/workflow/HarvestServiceImplTest.java index bd90a461..38283631 100644 --- a/src/test/java/eu/europeana/metis/sandbox/service/workflow/HarvestServiceImplTest.java +++ b/src/test/java/eu/europeana/metis/sandbox/service/workflow/HarvestServiceImplTest.java @@ -98,9 +98,10 @@ void harvest_notExceedingRecordLimitWithoutXslt_expectSuccess() throws Harvester when(recordRepository.save(any(RecordEntity.class))).thenReturn(recordEntity1) .thenReturn(recordEntity2); - harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null); + harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null, + CompressedFileExtension.ZIP); - assertHarvestProcessWithoutXslt(recordPublishService, 2, Step.HARVEST_ZIP, 2L); + assertHarvestProcessWithoutXslt(recordPublishService, 2, Step.HARVEST_FILE, 2L); } @Test @@ -118,9 +119,10 @@ void harvest_exceedingRecordLimitWithoutXslt_expectSuccess() throws HarvesterExc when(datasetService.isXsltPresent("datasetId")).thenReturn(false); when(recordRepository.save(any(RecordEntity.class))).thenReturn(recordEntity1); - harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null); + harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null, + CompressedFileExtension.ZIP); - assertHarvestProcessWithoutXslt(recordPublishService, 1, Step.HARVEST_ZIP, 1L); + assertHarvestProcessWithoutXslt(recordPublishService, 1, Step.HARVEST_FILE, 1L); } @Test @@ -144,9 +146,10 @@ void harvest_withStepSize_expectSuccess() throws HarvesterException { .thenReturn(recordEntity3) .thenReturn(recordEntity4); - harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), 2); + harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), 2, + CompressedFileExtension.ZIP); - assertHarvestProcessWithoutXslt(recordPublishService, 2, Step.HARVEST_ZIP, 2L); + assertHarvestProcessWithoutXslt(recordPublishService, 2, Step.HARVEST_FILE, 2L); } @Test @@ -171,7 +174,8 @@ void harvest_withStepSize_expectFail() throws HarvesterException { .thenReturn(recordEntity4); assertThrows(StepIsTooBigException.class, ()-> - harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), 10)); + harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), 10, + CompressedFileExtension.ZIP)); } @Test @@ -195,9 +199,10 @@ void harvest_notExceedingRecordLimitWithXslt_expectSuccess() throws HarvesterExc .thenReturn(recordEntity3) .thenReturn(recordEntity4); - harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null); + harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null, + CompressedFileExtension.ZIP); - assertHarvestProcessWithXslt(recordPublishService, 2, Step.HARVEST_ZIP, 2L); + assertHarvestProcessWithXslt(recordPublishService, 2, Step.HARVEST_FILE, 2L); } @Test @@ -221,9 +226,10 @@ void harvest_withStepSizeAndWithXslt_expectSuccess() throws HarvesterException { .thenReturn(recordEntity3) .thenReturn(recordEntity4); - harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), 2); + harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), 2, + CompressedFileExtension.ZIP); - assertHarvestProcessWithXslt(recordPublishService, 2, Step.HARVEST_ZIP, 2L); + assertHarvestProcessWithXslt(recordPublishService, 2, Step.HARVEST_FILE, 2L); } @Test @@ -241,9 +247,10 @@ void harvest_exceedingRecordLimitWithXslt_expectSuccess() throws HarvesterExcept when(datasetService.isXsltPresent("datasetId")).thenReturn(true); when(recordRepository.save(any(RecordEntity.class))).thenReturn(recordEntity1); - harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null); + harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null, + CompressedFileExtension.ZIP); - assertHarvestProcessWithXslt(recordPublishService, 1, Step.HARVEST_ZIP, 1L); + assertHarvestProcessWithXslt(recordPublishService, 1, Step.HARVEST_FILE, 1L); } @Test @@ -270,7 +277,7 @@ void harvest_failsWithErrorMessage_expectSuccess() throws HarvesterException { .language(Language.NL).country(Country.NETHERLANDS).build()); harvestService.harvest(new ByteArrayInputStream("inputStream".getBytes(StandardCharsets.UTF_8)), "datasetId", - recordBuilderToTest, null); + recordBuilderToTest, null, CompressedFileExtension.ZIP); verify(recordPublishService, times(0)).publishToHarvestQueue(captorRecordInfo.capture(), any(Step.class)); verify(recordRepository, times(2)).save(any(RecordEntity.class)); } @@ -281,7 +288,8 @@ void harvest_expectFail() throws HarvesterException { HarvesterException.class); assertThrows(ServiceException.class, - () -> harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null)); + () -> harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null, + CompressedFileExtension.ZIP)); } @Test @@ -304,9 +312,10 @@ void harvest_duplicatedById_expectSuccess() throws HarvesterException { .thenReturn(null) .thenReturn(recordEntity2); - harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null); + harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null, + CompressedFileExtension.ZIP); - assertHarvestProcessWithoutXslt(recordPublishService, 3, Step.HARVEST_ZIP, 3L); + assertHarvestProcessWithoutXslt(recordPublishService, 3, Step.HARVEST_FILE, 3L); } @Test @@ -330,9 +339,10 @@ void harvest_duplicatedByContent_expectSuccess() throws HarvesterException { .thenReturn(null) .thenReturn(recordEntity2); - harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null); + harvestService.harvest(new ByteArrayInputStream(new byte[0]), "datasetId", createMockEncapsulatedRecord(), null, + CompressedFileExtension.ZIP); - assertHarvestProcessWithoutXslt(recordPublishService, 3, Step.HARVEST_ZIP, 3L); + assertHarvestProcessWithoutXslt(recordPublishService, 3, Step.HARVEST_FILE, 3L); } @Test diff --git a/src/test/resources/response_body_dataset_valid_small.txt b/src/test/resources/response_body_dataset_valid_small.txt index 46475a87..e173b45d 100644 --- a/src/test/resources/response_body_dataset_valid_small.txt +++ b/src/test/resources/response_body_dataset_valid_small.txt @@ -1 +1 @@ -{"dataset-info":{"country":"Italy","transformed-to-edm-external":false,"dataset-id":"1","language":"Italian","dataset-name":"testDataset","record-limit-exceeded":false},"processed-records":2,"portal-publish":"http://metis-test1_testDataset*","total-records":2,"progress-by-step":[{"fail":0,"warn":0,"total":2,"success":2,"step":"HARVEST_ZIP"},{"fail":0,"warn":0,"total":2,"success":2,"step":"VALIDATE_EXTERNAL"},{"fail":0,"warn":0,"total":2,"success":2,"step":"TRANSFORM"},{"fail":0,"warn":0,"total":2,"success":2,"step":"VALIDATE_INTERNAL"},{"fail":0,"warn":0,"total":2,"success":2,"step":"NORMALIZE"},{"fail":0,"warn":0,"total":2,"success":2,"step":"ENRICH"},{"fail":0,"warn":0,"total":2,"success":2,"step":"MEDIA_PROCESS"},{"fail":0,"warn":0,"total":2,"success":2,"step":"PUBLISH"}],"status":"COMPLETED","dataset-logs":[],"records-published-successfully":true} +{"dataset-info":{"country":"Italy","transformed-to-edm-external":false,"dataset-id":"1","language":"Italian","dataset-name":"testDataset","record-limit-exceeded":false},"processed-records":2,"portal-publish":"http://metis-test1_testDataset*","total-records":2,"progress-by-step":[{"fail":0,"warn":0,"total":2,"success":2,"step":"HARVEST_FILE"},{"fail":0,"warn":0,"total":2,"success":2,"step":"VALIDATE_EXTERNAL"},{"fail":0,"warn":0,"total":2,"success":2,"step":"TRANSFORM"},{"fail":0,"warn":0,"total":2,"success":2,"step":"VALIDATE_INTERNAL"},{"fail":0,"warn":0,"total":2,"success":2,"step":"NORMALIZE"},{"fail":0,"warn":0,"total":2,"success":2,"step":"ENRICH"},{"fail":0,"warn":0,"total":2,"success":2,"step":"MEDIA_PROCESS"},{"fail":0,"warn":0,"total":2,"success":2,"step":"PUBLISH"}],"status":"COMPLETED","dataset-logs":[],"records-published-successfully":true} diff --git a/src/test/resources/zip/records_to_test.tar b/src/test/resources/zip/records_to_test.tar new file mode 100644 index 0000000000000000000000000000000000000000..48018ab3d35234d275e9c920e0dd2e1aa627e308 GIT binary patch literal 51200 zcmeI5TXWks8isS7Ux9IEHtmcp>gdy^-gTX%>84IHN!ng|AQF;L@t}gB9Obv{OfGw! zn_lJ4?*G~s;6WgDup3K`wShAeTO^JjfREtAPcM7iXYpAt5@(Xd(z&;epE`c__B!|Q ztFhOqo4?g5^>eS$Zns*yyUlhJ=Nqkht8XZ3@#AU?w&)P5WFYwrU8ru|U^yt~^r$3341IMZl0njMUPqtoiNRZ5Ki zZmYR>kJN8k@pbZl<>UY8Ul?FAW+LWM_}x~cR^K8lbR&<4{qMGZe*1K1f9qdI_a6zb zcli48Q-aiCd`P_@{%&g^<>=6H&d<+l=dD^K`c4=->ePd+`y~HKGq`S%+!c&UK4xwd z1W_1g`Q~I1L6JvQ;c_o>cU%$Z9B5DM=tE~XbT;nP>b2$>3;RBg2WLL*GQXzM%$Dm- zn=(p-uS@e>hxsgEp^TkIt>MhH-*c%fwPxC3b!uKMk_p2|W>xykoH`;pD(XmL)=PXI zx;5BI+110@*=krfaU;QsY|2nLR&xhVz$Epkq)rsd$dCG*#ZHzy|2?NXuWWRdh-k#% z8EY(YVt2p-ic-8_w)MS;_KIfcLDwx;hn@O7lSZEKiwxyCq)s`_**dB{Mbknm+yQ1& zI**XRSzz7hd~R*cdcEn?_Z*y_wTAHM529H#Qh4LYrQI42dy(@bn=eQ(+fdBBBA3y% z#=*gXgZi4y9i-TaC!wV8cEUN6R88G^C7`)=r`~XMc0C-kt&WG$qP%TYyE^ZvSu-c= z*qvroZP2R&uEBL;CdS-lbJ}{|kN4YWqlJ~WRD=60a*BixwvLp8c!V|_9{2l#_2Chs zkf6px+Kmz{wFh*1{Ot7E$=S@6DJ?@4gZX%vCuiQtvzNLc!Qv&_Gg{fupD$OH~o<|pdkc8FQ-wdMj@cT)nh02r0aHXynG#m9!8%rmY@J1e-o$I@)kK%nUsqfGccVc*F zpP?~JdF_f6gcEq(+WWHA9>9N`p~U7pY4<7I2rK6DVcC-GS~b8W*S%J(7eT zDwP9E*qd8}utGPC3 z@8QF%!ECY9o$Q3}PKdcxt2^1;dgN4mRO+}?J(+_j{7vq-vKgk1FK{-6uR#akmO{yU zoQb0u52t%*B*=S-Y`P6 z%Zq$oJZ3#*jzxLl+SpO!pxM$1^rvab`qG=Wjd?|?^3iGPLyyI-;3Mt&v12&&soHb0 zaBF$D9>Qc~K*!X*{ODhP9Qs2_xZmeKC(o1dIJLzyoaYl50~YErDfwR?e17?XKMWa} z(D%qmLu`9VmkV7M!HrN9CuEq2#K$HF=HwTDVD0spU_%t0T>L?YJ{6ZALlRS|eo@pO zTz;JRMg2{zThC)Y9};Xp`;0t}+(d0Fw~IB}o#^Fpy2SSR0{bzHNAP-)fX&qy9iy#S zI=Bd{#S)AJkFi8?qKQ>2xr;k_f~%-nn{ZL?fW|r-<-!%TCrQYVHKrJFtU8?xxBRrL zcFd}csw5Z%$w|*=Or|3|9V2tvrps2{4H%>L+lcjbru3|tUQR}E7`VLW%emOxq+6;! zhG?Rew?nCPTJ<_EW}SM+@bH+*x{z{}lD(7U`;w`33slm)ev(bO{P!czO{Zi$!lZ{Q zEEoAU5kod1--LQad`sS5d=f(}D^+eO-_cA#*XrO%U9V+7bQ*7 zWg=9&8#?BC<`WNX#%JYDo>1XW=G6Y0jS~0|7jEMfHcOGWO;GcH&>-byil!6?RBQBb zXk+CIRE?tY{RvjaQ6#BOQ5={qgmm@s_!Yd}%U;EuR5$s?;HaxLk6*x5{20DrzbD`u zRcBPqUAo<9%-KzA2lM0U@efbZeP!Wr(vQaGK=)Gb{(oiN*V#>pZhW?5SkLC)H16UH zZA07IIB#aN`(nEax> z^B?g4UCaM(V}e-zubx|2{+~VAfmbi?%=EJa%l|F^zia$Iy)vza|7T^DQF)2qZxa7+ ze<}XItm1X?e}kOp+p7$`TK=y<-VAfM{Qt}I|El8x_b~9j!T-~Q`rh0~FeM@C2_riS zYhEqAe^;N(vr}kW0a6)~(NqYJ3;?i0kCGwnCO)1HAr>eEOqO&qQO~@(7k~N}pDsUK zd{U%b9R&*~0UJuz^3I z{fj?i;2l)2fOJIO;(tbBJS@!-6b+&zW7HYX-X^_-r4Oa{;In zJj_wvbk3B>c4g97B5sNJ)lUpoOvF#FL8QqHe@>^mBJrj_ELig0gUj{Z&4A104W4OS z10GlWhK0*^)Af0qE(@1$gZ-&Gmhj*G+v2~Cy~dRP))6NT{@ZF;{(Bp9!17MhYS?e-o1|*5vbMNid=>i_h(}buvbdTr{_XDX zE}_2bRUFyfFW*p#dM#WbvsS|t|A6gy&_ek5CijoLAg7OEVCLxSek|eomSaL~{lL5l1 z0~*EyK2js7;QJ4FjvyeT@kU|>RFp|tQ)Lnf5hJBoXY#pU-8IX zXNx=;@}R6~a-~DMLfT|bYqZ$OY-qV|*-OYVb)kq>w5T|+k0@g_5msh+V z8jm_Hjko&WsdRe|nw2M5aSYQ+X?(?TEhh3h;SwspTKx|tWF4k4+){~M_i`-|Q}kLV zq5=LXacqdgCIJGoDNx8X9y<*wS6sVG2SCudl86~cp2}()Ri>I{3DWlQS3OxKynQc4YALmvkozBKYE`-Qr_;GUe9ob77bf8Jp0-Si-uEhxP<>|CvY45x7{%G zpX*Jn|A8=YV&DIGL%4g5?3Vvp{%iTK<-Z%nf79)0vERE9|DCth-B|v+_oevnvWnNm ze{D3jeg9+jilR;7zZJ)|nE&d8o6LVL-!~~s`oU%(-15fJ^m>LfvTzU$bZv>Yv#XJ|9SeF#Nt|FajV6xm;g{pgZLx6c$m&1mmazKI&rZ(F z7q-rOjmVu7Q`kEw@6$}LXK`zY3!=2|PL}Z;nL_&-zt2Z(cRJ@{{@ZBZ()<6K&3b35 z|Dg4s@&3QMjsLxkiDUV%<-eByTK>C9{5QQatw;k}U*#$!aYOm;(wP{ydM6G zIxC}Q)tIds^BvcJNS%#&+ikqw&%g<(hg-sbb2;VHi%%ZyQMxmMCc`Oo36%WsFAVSO zHEu%ffAQEO`yxqx{xJ)41ouxM3WyvhIpb9D3N6( znH9Gn-HT81;tvQjE18Q(c*%s4LYH|&C%zJ7?yCWiG=N;`3Dh4Fp9(^|Q6ja5Lp-1Y zq7sF+F=QmPPJ{?1>J5kL?{4DtnWhm&6Q+{Q(cc=Y?xYy5T*&z`!&@3<0OElcBlgHo zQ6EFiT_`R?2;) zT*(Cb?U~ESwB)zv`>D)n>-)9rKhtw2(K`w9fr?>G!W8gFSbxhIc->}m2(PJmycAlU59*|*G=_MCZTc_D=p7zG1&74E??$ zxA`of(b|O2>*}7}G_HLXV4nr7T1QO>&H)z?%xLJv~^eu kSPNJSSPNJSSPNJSSPNJSSPNJSSPNJSSPNJSe5Drn9~DDH{r~^~ literal 0 HcmV?d00001 diff --git a/src/test/resources/zip/sandbox.tar.gz b/src/test/resources/zip/sandbox.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..cacda43b2d87d6ae0fd167cc80c398ee92cab3dc GIT binary patch literal 7300 zcmV-~9DCy*iwFP!000001MEH9jvGgI`hoq2!WfCxh{(RMNj7JtF=j@RE$eEHyczpN zD0UawHS7!B7tX~mmSV5%jgtiyy9t795(IH#2Z$X>yGmqxtpzgPm&HIHy-APGm*kvo zUYfUIHA#)VF*(o-*;VI$>YO^Ko27udX4Tud+4Md0rdp7Ts;m~I6LWpZ@G6&!_*W?x z<@Ddq>x80dg+f8oG+8?#E1FVNPLSe!*%zt0TM zH}PVNo6OAnW^D!W%X`ed&dj9}sdHd|>(d+4BVEyTLMiIrTBFcKgh>CW7)2fB8%=>?;N0QW0MvLe-)Cv_)wB?ua0 zp)?d7$0b-ktP26}`v5>-m->dqTP%n{VM2#LR5dvb3Ru3@UVQ5>U2Wspq0O5 zewOFXACAlO+0MZi!pbbo8Ipm?ukk+zIaXHkPzqGPJ9cozMs?_Hv15)>F zlevt%MD3uu|DkI#|FqPBO(BbAT&6bsb!iBuUs?U;vro3ay7dO3-#qwP4*pS#RuOi1 zYKkZ1B%*b;Qg0LLUPw(cJU8@tH3A#N;`=P{BHw7cVF21Pb8a=y-I^!WT~kt&QVHBw z-CUCT(D;msqo@?*Ql&6}q8U|f9$3s@sWML1sc(~)7%prSG3k0t+WHjV;046sO z2Mqe8l?QV>4g^gIST0$Of;z1-((+v5vIwB}e+!TS3->}2xh8RVNE$v5c`_d_u^m=t zoeQfj;A%u}9o|=2-DW&w;Ycr9hKS&Xyv9LOtLus^SJa}e$5T@yX(@M`X(3IvwDM!m zws9FH1YkTpKZvSZ%m@cSR()DUUSp89PU85;5D(Nbo|? zaY12_bD_l>*+4GS9sX=o(OD1Is2^Uft-QW^=|*~)v@fTK)jZ#!XjjnO#`SNwK%wIJXHjx*tb7=Wt{l>7)ssRI2Ns>yrF)R=wU1J&QCrr!?4} zYSNJAJqY5|$kR~bPQ4+mz7BhJ*pZh@OMRA4lsO_3Ajx9pyw94T5rYMR z6k3r}bt$*e8*Y9FyaB{mUai-CR*w_X!W-u74Lc@n<|z+BZRiRP6x$rimYsalYQBI; z|BT6lrcHPMlse5Hd6aYW%y2r72hHjkx!*}>+i*$U+V3j@)tA(VtlfX|S1v|&82L@l zvvGr+xKDR54T16PT4!MYZvs%R&tMZ3Z?wsk*rBLh`VbeH?Ul3w}2X{p^#TKTpcVBhoC z=>Bi#YP%2Ixt@o}GX)Ld{;yn?HLZyK-;%cI{~pV0(f?iaf4@Ilphf?8(f>WtxQqVp zqW?PyWYPa!^nd@j)8L~2yD<5p|NH$g`lA24@PA^S^~+0sFtITwL^jkK-lQ|D99w_u2GvGq;qzZF@`uSC!GL1+PS*T&CtqUUi`8p?kU0(g0|JirV2A zOsmx>chb(iLJ!^!xHA!P7tsJ-Q8WYeCt#funipZ(!P`EyZTMpei=2(fkY=Z(IwP6U zc=7G!2u7&mP*Z@+lvrm}nGhNLxQg?Tq@8=|;M47x)jbDr2#7WxBHw~@%vle|!NRxb z5Bxa%fq(b?L0AvR!NSzxe<}?(4(?e8_k#F$FWr&gOMesRztu7 z=|?_e!#WaGWxjq^glu+lydb?n6BhLdG`Va~*?&|VZ_4{nLGn~<91_OBlN@+fb- z+glN$JdP`OI?MW`sJfDc8wIDj0pj3x3Kx{5l@yRPqrCz5@<0O`S8m%ZX;%_ReYJ)lU^-f%q|d@Lka=@dKpdH_1$_C<{somn)sJZ6ZhtHT zqZT2Ck%OOz2JA7YhZx7|f~IPV$;)_vk1<-rxBbzP%r80$b8(!QyMndFnoQ{r562aC z9O>W=hV&7cw9{CvdLFXTP1BC&^$&=`=Ve;#0=3X+->%R@|{z<5*dIrlIi}TQ*I05@hm3FFrZKuGZGLA6hGlTrT3b zABG0{C!b{$Co7sZFgY1G?QPr30d?5wnCINgYLl6q#t)}-2Bp=$?HoSjHVI9O)8uPV z_(rP8a#<8%ge*^-?Zts#+V z!pz&Ak*|9#$=~)QT8|$ycXdSu97eL>6_EQ7WI;LE!ul=(x2slI*0L9;1M+ zW&%Q(D98l#GP7_Cj>X$Flwg5-RUQS?=z~?KRppFi6BAB{B_%koPQ}30Fr3svxOVBa z*ESA5xPFSfhUU4A=9%n2*Dk&MasrhyYBsmaXYd`o^8)}G@n>5FecGPR-ieimsg6ov zx~$}6HK!`#iA>jZ1X1KUrn3(3H@m)ivnyHlHkiDIqrS|bVgK1#!oRK5GirV)bkr4O@ zfDcKj0OjH1N@8ZKtj^t-a1Px~Nu)`x#uJ#E91E0F)LfxGjMoxTT<@qycT>vijNw;S z!kTpM?2Yp|o3)s&^oiZU*VbxBmE#pFDE(Ny;~6kYB~MF`j`-h)r0BnX!D;xX&ZbY_606_-GDt?KilAWR*CahBD_YZq9 zPXyBP6V(QxXK%5u-ep;+wi4JnH3RO1S;*G>nBi~kzfS_b|FL0bp=(8Lp7-|ud5@YI zAcIC%THpKXuXq-!f!)&-3%9a>)+vJSe@-mw@hoaG0B{3O!OG}IL96ZZA+=eSwZdW{ zAN}s@+vL&RN1uKD*N?t@bocAqSunVWA+(4Au{QE|Gl2587YEGnY%dGV0Mx|mnU@<2 z+a{%3)Xm~YMv(S}{ZF#cOrlDjI1r#2%@S7ZL_OOa4{HzxGik!>{=det(`RND*Y!R0 zK(I$-6KbP=AtPkr+KoMm5yqnJ3Dwh}16P(=muqUW3ETcr&;e`MS>f8tW-1^Sk{_lx zetv2sz&NwT!To4BZ%l^jEY)29QhmmNSGdbU27rShLX>WPDmYv@>lxuNAYtu=^d=Nw z15irkf9~|hF)_G!sini17#Iq-k)0Z>OGw<_G>o3*2B)s;wWh{!QpfLHK2uB7{3Q1D zZ!yH+K&9CM<5=5X##sUn>*NzmV!}|kJILZGz1@V#T`!sfPN@t5rR$TpI2l`?R6sAO z;kZ6Yj_LADN=2}}Z023t$-CT&(E$QWIxB0Hl3-^R6;tyWtI{AjKg^M2y;^XD zfa?GuHR{4Ji6y-tUv*g?MD2DkHaE3`R;cJgwoFa~sk1QL%`-ESC=I28ENik>6li4w zf+{*Mh%!oJJsj1s?=oQEDx>@(Y!J9)3>XH6!By**L zCM%ulnC(y_yi7yG%IM@3trYv^WGutYa-d%<$!e}3hCs->HE{5fFuquvs#^m;m8joX zmK0UdbjheHN~K&Z>4~R{vgVpLu$JFy#>`3Z(WR0s7ae4!|3agfn5 zrKlH5s<@+#2`q9^Q!C=!jKxx{6f)=q1At9~KsgH=7|Wac&_^B|;(#PJ4;mq=I|l@`ttDp1R-*D;v;TN%>4^Vzo!)=`UE0}ou@J=rN z^8l~^h#dX{V0-KExAB|@G4?y8H-P}w!+(Ma4*=uC!~aSRKYsu4&P499oLvL$p{8o2 z*>SX85?VAQ!EO*ThlYHQ1>kwC1{-a7LtT^j@quFanY0&PfEeO~fZ_?;z6X8?V`gMb zHUSo8*QtXRux;9Nt;r7T$1BH(R}Sk*z@Y%ZMHI$weh90Ud+zgug~W{yVCOBk>hFIH zrAY8RSlbOqWu ztk{e z3}a`9!8%DDPdqP^ zP#ADCuMNCObB<-S6W*Z7YFjpi-!)?&+RX%mDIlY1@h%K1`>Vgmc;^MpI_hqbl#h;8BbxJzE7f- z%UU5PF4E*aNl7ncC_KmaNeTr;*HrzDOxorqcnsnK!PFA4q5;Qq>9jyyF4&G5-)Siq zGt_t^IxPj!9>us;OU>Ta1qJ8 z7hk<2n-W^NB+JjDDN)e$Id>&WWx?PwQd?Adtx_!N%25r8%btt>n*;2vlmF-8IXiLh@?$-G2a4PNp}3Cz0yeO}#sXQehk+N7 zRhPMhR()zwDCTJVF#@RGy}qNH39D_;5G$(xYwzfK;x>YRW!0CwkPuKKXjNG{p`<-E zq-`2SQD3UP^Wt0Zt*JjQ;hsuqQne3_`qoN$0HrTTNh3rMq@m@(@aj{40{^5lySC3Z zN6vd8dD@W>=k?6&*UawB-0qEEx5sFytKo^Cdf>$2>-c#Pf+rLSo7$ADdOlgCUN_zb z2b2WycAMO7HVGrR3Lt{;0>VVValwYh2E_Krg69LjdLf8Stud1Z-jn1g`3L?ElD)w% zgN?zD%G@K#!C*bvA8f$GUh*Q@nIg$PNsb2J11>z05@`0BRlU*p=!N;BC6}|#SFM<0r;JzOt7%vh$B1>i8Dan82vOKkCq+O`wMhop={!jRP|$ z3W&%?q(<}65mOREhXcIU?gNiLJ^8^qIrQM;eW`KxuZ(4fKH6lv3c6~lA%zv-cq3f` z;6wbEP&%(p)Io#rL>ezk@yU8m$w}h@Gqqlw9Z3socS3h5tR) zs?g0Uq`D`yb;krZKA<^X@9G}T)~yM5g=Wx)MP7#r7aiT2Fl?twgG*IIkg(}#5~5~l zH?XHqBTc(Mz`lBoMwA9&7iH<>P_s6n-$EWv(5qkqnhEM@t9tZgk45?wtxh0&_5>n? zn}{xrIKOR`XkRs}WHB`l9hOgLX$5QN>#@e>J=9J8Q%px#H?#;YI2F9F`Y`k1{Os(t z^j;b;o}Zojz#5V2@B&g2UQjN=cbbGyBbVd_=8rj(^iYZ*jQxYPlJKo!$#g!M8ge(h zHy1?UdX(Fn-Rqk!V_B6v-Qz2&=IurXsC*H1m}TP#77L2`%r&Fp>5;!;p7XchCWG`P zUh@2HhAQX53Vr;NZo~_TyvC@XUIEM>}`7FsJ z7!@m`m}d*qY)LAGSyivb!pzmHAI)ByAE`!#M@_wRn15F9vUHuk)nc41P~RP1@4n62 za+fTj71gWkyU1Tg1JCDNZ7idID!eYMi_3s7GzpxpnHjtdQH$!Uh`DavDpHG}7H)d7 zq89EVeZWWbGvuLL4M`V&bo5dH{-X;s$;fBa4O;P^c$-(Uk)-D`5x@d`J6p6M{pFdG(k)R@b`%Y`b0Nt220wC~&|6B0Vs2V1ond_z)-q7WP9D30oe=Tfr~g z6h0QQ2h3qvK|=tql}I@7xrH|%c!;+;%B{tOfaS*C^;{PdR4YuG4K3SxD7^56Jj!G> zPG3F0f4u%3p{ECrFT)@0(AIF5Dx&7}*i{i)_nOTqGCxPYzB0s5U!A#%Zhh0Zd1uts z^2&ComiOu&Ibugxf|)yKuXw*(!X%3~wpvcfo$L4Rs7|OEDz55SdOhn}9aDJ{o8_I_ z%9Ya{S@XyZ55L<8Wh85~5xlC)qHHlDLr}0=w972)z_-xuVO80iG>2IPTh*EcY6?3R z%~Fu5!n7@{H`*Y-2s=36F}jhfDj)74SvEA6fl>iKtC00AFH=RM1jYIUQ_p_YwAz<>b*1`HT5 eV8DO@0|pEjFkrxd0Rsl!H~bGw-j-wl$N&I+iZ$*4 literal 0 HcmV?d00001