diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..be23e7b
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+.vscode
+target/
\ No newline at end of file
diff --git a/.java-version b/.java-version
new file mode 100644
index 0000000..b4de394
--- /dev/null
+++ b/.java-version
@@ -0,0 +1 @@
+11
diff --git a/README.md b/README.md
index 38cf230..cb9ae07 100644
--- a/README.md
+++ b/README.md
@@ -70,7 +70,7 @@ An example execution is
amt-to-flat-file
NCTS_SCT_RF2_DISTRIBUTION_32506021000036107-XXX-SNAPSHOT.zip
- amt-flat-file.csv
+ amt-flat-file
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
new file mode 100644
index 0000000..a67cb7b
--- /dev/null
+++ b/azure-pipelines.yml
@@ -0,0 +1,26 @@
+trigger:
+ branches:
+ include:
+ - '*'
+pool:
+ vmImage: ubuntu-latest
+
+steps:
+
+- task: MavenAuthenticate@0
+ inputs:
+ artifactsFeeds: 'amtv4'
+
+- task: Maven@3
+ displayName: Build and Deploy on NCTS Devops
+ inputs:
+ mavenPomFile: 'pom.xml'
+ mavenOptions: '-Xmx3072m'
+ options: '-B'
+ javaHomeOption: 'JDKVersion'
+ jdkVersionOption: '1.11'
+ jdkArchitectureOption: 'x64'
+ publishJUnitResults: false
+ testResultsFiles: '**/surefire-reports/TEST-*.xml'
+ mavenAuthenticateFeed: true
+ goals: 'deploy'
diff --git a/pom.xml b/pom.xml
index 937d2fa..53cb831 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
au.gov.digitalhealth.terminology
amt-to-flat-file
- master-SNAPSHOT
+ amt-v4-SNAPSHOT
AMT flat file generator
Generates an "AMT Flat File" from a Snapshot SNOMED CT-AU release
@@ -17,53 +17,81 @@
https://github.com/AuDigitalHealth/amt-flat-file-generator.git
+
+ 11
+ amtv4
+ https://pkgs.dev.azure.com/OD225632-NCTS-ContentAndTooling/OD225632-NCTS-ContentAndTooling/_packaging/amtv4/maven/v1
+
+
- org.openmbee.junit
- junit-xml-parser
- 1.0.0
+ jakarta.xml.bind
+ jakarta.xml.bind-api
+ 4.0.0
+
+
+ com.sun.xml.bind
+ jaxb-impl
+ 4.0.0
+ runtime
+
+
+ org.projectlombok
+ lombok
+ 1.18.28
+ provided
+
+
+ junit
+ junit
+ 4.13.2
+
+
+ commons-io
+ commons-io
+ 2.13.0
org.apache.maven
maven-plugin-api
- 3.0
+ 3.9.0
org.apache.maven.plugin-tools
maven-plugin-annotations
- 3.4
+ 3.8.1
provided
org.jgrapht
jgrapht-core
- 1.3.0
+ 1.5.1
commons-cli
commons-cli
- 1.4
+ 1.5.0
org.apache.commons
commons-lang3
- 3.5
+ 3.12.0
org.testng
testng
- 6.9.8
+ 7.7.0
test
org.apache.maven.wagon
wagon-http
- 3.2.0
+ 3.5.3
org.apache.tika
tika-core
- 1.22
+ 2.7.0
@@ -76,8 +104,8 @@
maven-compiler-plugin
3.7.0
- 1.8
- 1.8
+ ${java.version}
+ ${java.version}
@@ -95,7 +123,7 @@
org.apache.maven.plugins
maven-plugin-plugin
- 3.3
+ 3.8.1
amt-to-flat-file
true
@@ -149,4 +177,50 @@
+
+
+
+ spring-releases
+ https://repo.spring.io/release
+
+
+ ${deployment.repo.id}
+ ${deployment.repo.url}
+
+ true
+
+
+ true
+
+
+
+
+
+ ${deployment.repo.id}
+ ${deployment.repo.url}
+
+ true
+
+
+ true
+
+
+
+
+
+ ${deployment.repo.id}
+ ${deployment.repo.url}
+ Maven Release Feed
+ default
+ true
+
+
+ ${deployment.repo.id}
+ ${deployment.repo.url}
+ Maven Snapshot Release Feed
+ default
+ true
+
+
+
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Amt2FlatFile.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Amt2FlatFile.java
index c569ab4..bd9cf7d 100644
--- a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Amt2FlatFile.java
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Amt2FlatFile.java
@@ -5,7 +5,6 @@
import java.io.FileWriter;
import java.io.IOException;
import java.net.URI;
-import java.net.URISyntaxException;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
@@ -17,10 +16,14 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
+import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
@@ -28,400 +31,527 @@
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.commons.lang3.tuple.Triple;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.tika.Tika;
+import au.gov.digitalhealth.terminology.amtflatfile.Junit.JUnitTestSuite;
/**
- * This is both a Java CLI class compiled into a runnable JAR, and a Maven Mojo to transform a ZIP file of SNOMED CT-AU
- * release files into an AMT flat file. Note it only really needs the snapshot files, and expects file names to match
- * the release filename convention.
+ * This is both a Java CLI class compiled into a runnable JAR, and a Maven Mojo to transform a ZIP
+ * file of SNOMED CT-AU release files into an AMT flat file. Note it only really needs the snapshot
+ * files, and expects file names to match the release filename convention.
*/
@Mojo(name = "amt-to-flat-file")
public class Amt2FlatFile extends AbstractMojo {
- private static final int MAX_ZIP_FILE_SIZE = 600000000;
-
- private static final String INPUT_FILE_OPTION = "i";
-
- private static final String OUTPUT_FILE_OPTION = "o";
-
- private static final String EXIT_ON_ERROR_OPTION = "e";
-
- private static final String REPLACEMENT_FILE_PATH = "r";
-
- private static final String JUNIT_FILE_PATH = "j";
-
- private static final Logger logger = Logger.getLogger(Amt2FlatFile.class.getCanonicalName());
-
- private JUnitTestSuite_EXT testSuite;
-
- @Parameter(property = "inputZipFilePath", required = true)
- private String inputZipFilePath;
-
- @Parameter(property = "outputFilePath", required = true)
- private String outputFilePath;
-
- @Parameter(property = "replacementsOutputFilePath", required = false)
- private String replacementsOutputFilePath;
-
- @Parameter(property = "junitFilePath", required = false, defaultValue = "target/ValidationErrors.xml")
- private String junitFilePath;
-
- @Parameter(property = "exitOnError", required = false, defaultValue = "false")
- private boolean exitOnError;
-
- private AmtCache conceptCache;
-
- private Tika tika = new Tika();
-
- public static void main(String args[]) throws IOException, URISyntaxException {
- long start = System.currentTimeMillis();
- Options options = new Options();
-
- options.addOption(Option.builder(INPUT_FILE_OPTION)
- .longOpt("inputFile")
- .argName("AMT_ZIP_FILE_PATH")
- .hasArg()
- .desc("Input AMT release ZIP file")
- .required(true)
- .build());
- options.addOption(Option.builder(OUTPUT_FILE_OPTION)
- .longOpt("outputFile")
- .argName("OUTPUT_FILE")
- .hasArg()
- .desc("Output file path to write out the flat file")
- .required()
- .build());
- options.addOption(Option.builder(EXIT_ON_ERROR_OPTION)
- .longOpt("exit-on-error")
- .argName("EXIT_ON_ERROR")
- .desc("Flag dictating whether the program will exit on an error or keep processing")
- .build());
- options.addOption(Option.builder(JUNIT_FILE_PATH)
- .longOpt("junitFile")
- .argName("JUNIT_FILE")
- .hasArg()
- .desc("Output file path to write out the junit result file")
- .build());
- options.addOption(Option.builder(REPLACEMENT_FILE_PATH)
- .longOpt("replacementsOutputFile")
- .argName("REPLACEMENTS_FILE_PATH")
- .hasArg()
- .desc("Output file path to write out the file listing inactive AMT concepts and their replacement active concepts")
- .build());
-
- CommandLineParser parser = new DefaultParser();
- try {
- CommandLine line = parser.parse(options, args);
-
- Amt2FlatFile amt2FlatFile = new Amt2FlatFile();
- amt2FlatFile.setInputZipFilePath(line.getOptionValue(INPUT_FILE_OPTION));
- amt2FlatFile.setOutputFilePath(line.getOptionValue(OUTPUT_FILE_OPTION));
- amt2FlatFile.setExitOnError(line.hasOption(EXIT_ON_ERROR_OPTION));
- amt2FlatFile.setJunitFilePath(line.getOptionValue(JUNIT_FILE_PATH));
- amt2FlatFile.setReplacementsFilePath(line.getOptionValue(REPLACEMENT_FILE_PATH));
- amt2FlatFile.execute();
-
- } catch (ParseException exp) {
- logger.severe("Parsing failed. Reason: " + exp.getMessage());
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("Amt2FlatFile", options);
- } catch (MojoExecutionException | MojoFailureException e) {
- logger.severe("Failed due to execution exception");
- throw new RuntimeException(e);
- }
- logger.info("Done in " + (System.currentTimeMillis() - start) + " milliseconds");
- }
-
- @Override
- public void execute() throws MojoExecutionException, MojoFailureException {
- logger.info("Input file is " + inputZipFilePath);
- logger.info("Output will be written to " + outputFilePath);
-
- validateInputZipFile(inputZipFilePath);
-
- validateOutputPath(outputFilePath, "text/csv");
-
- if (replacementsOutputFilePath == null || replacementsOutputFilePath.isEmpty()) {
- logger.info("Replacement file was not requested and will not be written");
- } else {
- validateOutputPath(replacementsOutputFilePath, "text/csv");
- logger.info("Replacement file will be written to " + replacementsOutputFilePath);
- }
-
- if (junitFilePath == null || junitFilePath.isEmpty()) {
- logger.info("JUnit file was not requested and will not be written");
- } else {
- validateOutputPath(junitFilePath, "application/xml");
- logger.info("JUnit file will be written to " + junitFilePath);
- }
-
- if (exitOnError) {
- logger.info("AMT flat file generation will be aborted if any errors are detected");
- } else {
- logger.warning("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
- logger.warning("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
- logger.warning(
- "Configured to continue regardless of detected errors. This is useful for testing pre-release AMT content, "
- + "but is NOT recommended for any other use! Resultant AMT flat file may be unreliable. "
- + "Consider rerunning with the -e or --exit-on-error flag set!!!");
- logger.warning("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
- logger.warning("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
-
- }
-
- //initialise test suite
- this.testSuite = new JUnitTestSuite_EXT();
- try (FileSystem zipFileSystem = FileSystems.newFileSystem(URI.create(
- "jar:file:" + FileSystems.getDefault().getPath(inputZipFilePath).toAbsolutePath().toString()),
- new HashMap<>());) {
-
- conceptCache = new AmtCache(zipFileSystem, this.testSuite, exitOnError);
- writeFlatFile(FileSystems.getDefault().getPath(outputFilePath));
- if (replacementsOutputFilePath != null && !replacementsOutputFilePath.isEmpty()) {
- writeReplacementsFile(FileSystems.getDefault().getPath(replacementsOutputFilePath));
- }
- if (junitFilePath == null || junitFilePath.trim().isEmpty()) {
- junitFilePath = "target/ValidationErrors.xml";
- }
- BufferedWriter outputJunitXml = new BufferedWriter(new FileWriter(junitFilePath));
- testSuite.writeToFile(outputJunitXml);
- logger.info("Output junit results to: " + new File(junitFilePath).getAbsolutePath());
- } catch (IOException e) {
- throw new MojoExecutionException("Failed due to IO error executing transformation", e);
- }
- }
-
- private void validateOutputPath(String outputPath, String expectedMimeType) {
- try {
- Path path = Paths.get(outputPath);
-
- if (Files.isSymbolicLink(path)) {
- throw new SecurityException("The output " + outputPath + " file must not be a symlink for security reasons");
- }
+ private static final int MAX_ZIP_FILE_SIZE = 1500000000;
+
+ private static final String INPUT_FILE_OPTION = "i";
+
+ private static final String OUTPUT_FILE_OPTION = "o";
+
+ private static final String EXIT_ON_ERROR_OPTION = "e";
+
+ private static final String REPLACEMENT_FILE_PATH = "r";
+
+ private static final String JUNIT_FILE_PATH = "j";
+
+ private static final Logger logger = Logger.getLogger(Amt2FlatFile.class.getCanonicalName());
+ public static final String WARNING_MESSAGE = "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!";
+
+ private JUnitTestSuite testSuite;
+
+ @Setter
+ @Getter
+ @Parameter(property = "inputZipFilePath", required = true)
+ private String inputZipFilePath;
+
+ @Getter
+ @Setter
+ @Parameter(property = "outputFilePath", required = true)
+ private String outputFilePath;
+
+ @Parameter(property = "replacementsOutputFilePath")
+ private String replacementsOutputFilePath;
+
+ @Setter
+ @Parameter(property = "junitFilePath", defaultValue = "target/ValidationErrors.xml")
+ private String junitFilePath;
+
+ @Setter
+ @Parameter(property = "exitOnError", defaultValue = "false")
+ private boolean exitOnError;
+
+ private AmtCache conceptCache;
+
+ private final Tika tika = new Tika();
+
+ public static void main(String[] args) {
+ long start = System.currentTimeMillis();
+ Options options = new Options();
+
+ options.addOption(Option.builder(INPUT_FILE_OPTION)
+ .longOpt("inputFile")
+ .argName("AMT_ZIP_FILE_PATH")
+ .hasArg()
+ .desc("Input AMT release ZIP file")
+ .required(true)
+ .build());
+ options.addOption(Option.builder(OUTPUT_FILE_OPTION)
+ .longOpt("outputFile")
+ .argName("OUTPUT_FILE")
+ .hasArg()
+ .desc("Output file path to write out the flat file")
+ .required()
+ .build());
+ options.addOption(Option.builder(EXIT_ON_ERROR_OPTION)
+ .longOpt("exit-on-error")
+ .argName("EXIT_ON_ERROR")
+ .desc("Flag dictating whether the program will exit on an error or keep processing")
+ .build());
+ options.addOption(Option.builder(JUNIT_FILE_PATH)
+ .longOpt("junitFile")
+ .argName("JUNIT_FILE")
+ .hasArg()
+ .desc("Output file path to write out the junit result file")
+ .build());
+ options.addOption(Option.builder(REPLACEMENT_FILE_PATH)
+ .longOpt("replacementsOutputFile")
+ .argName("REPLACEMENTS_FILE_PATH")
+ .hasArg()
+ .desc(
+ "Output file path to write out the file listing inactive AMT concepts and their replacement active concepts")
+ .build());
+
+ CommandLineParser parser = new DefaultParser();
+ try {
+ CommandLine line = parser.parse(options, args);
+
+ Amt2FlatFile amt2FlatFile = new Amt2FlatFile();
+ amt2FlatFile.setInputZipFilePath(line.getOptionValue(INPUT_FILE_OPTION));
+ amt2FlatFile.setOutputFilePath(line.getOptionValue(OUTPUT_FILE_OPTION));
+ amt2FlatFile.setExitOnError(line.hasOption(EXIT_ON_ERROR_OPTION));
+ amt2FlatFile.setJunitFilePath(line.getOptionValue(JUNIT_FILE_PATH));
+ amt2FlatFile.setReplacementsFilePath(line.getOptionValue(REPLACEMENT_FILE_PATH));
+ amt2FlatFile.execute();
+
+ } catch (ParseException exp) {
+ logger.severe("Parsing failed. Reason: " + exp.getMessage());
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("Amt2FlatFile", options);
+ } catch (MojoExecutionException | MojoFailureException e) {
+ logger.severe("Failed due to execution exception");
+ throw new RuntimeException(e);
+ }
+ logger.info("Done in " + (System.currentTimeMillis() - start) + " milliseconds");
+ }
+
+ @Override
+ public void execute() throws MojoExecutionException, MojoFailureException {
+ logger.log(Level.INFO, "Input file is {0}", inputZipFilePath);
+ logger.log(Level.INFO, "CSV Output will be written to {0}",
+ FileFormat.CSV.getFilePath(outputFilePath));
+ logger.log(Level.INFO, "TSV Output will be written to {0}",
+ FileFormat.TSV.getFilePath(outputFilePath));
+
+ validateInputZipFile(inputZipFilePath);
+
+ validateOutputPath(FileFormat.TSV.getFilePath(outputFilePath),
+ FileFormat.TSV.getMimeType());
+ validateOutputPath(FileFormat.CSV.getFilePath(outputFilePath),
+ FileFormat.CSV.getMimeType());
+
+ if (replacementsOutputFilePath == null || replacementsOutputFilePath.isEmpty()) {
+ logger.info("Replacement file was not requested and will not be written");
+ } else {
+ validateOutputPath(FileFormat.TSV.getFilePath(replacementsOutputFilePath),
+ FileFormat.TSV.getMimeType());
+ validateOutputPath(FileFormat.CSV.getFilePath(replacementsOutputFilePath),
+ FileFormat.CSV.getMimeType());
+ logger.log(Level.INFO, "CSV Replacement file will be written to {0}",
+ replacementsOutputFilePath);
+ logger.log(Level.INFO, "TSV Replacement file will be written to {0}",
+ replacementsOutputFilePath);
+ }
- if (Files.exists(path, LinkOption.NOFOLLOW_LINKS)) {
- BasicFileAttributes attr = Files.readAttributes(
- path, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
-
- if (!attr.isRegularFile()) {
- throw new SecurityException(
- "The specified output file " + outputPath + " exists, but is not a regular file. Cannot be overwritten.");
- } else if (!tika.detect(path).equals(expectedMimeType)) {
- throw new SecurityException(
- "The specified output file " + outputPath + " exists, but is not a " + expectedMimeType
- + " file as expected, detected type was " + tika.detect(path) + ". Cannot be overwritten");
- }
- }
- } catch (IOException e) {
- throw new IllegalArgumentException("Could not validate output file path " + outputPath, e);
- }
+ if (junitFilePath == null || junitFilePath.isEmpty()) {
+ logger.info("JUnit file was not requested and will not be written");
+ } else {
+ validateOutputPath(junitFilePath, "application/xml");
+ logger.info("JUnit file will be written to " + junitFilePath);
}
- private void validateInputZipFile(String inputZipFilePath) {
- try {
- Path path = Paths.get(inputZipFilePath);
+ if (exitOnError) {
+ logger.info("AMT flat file generation will be aborted if any errors are detected");
+ } else {
+ logger.warning(
+ WARNING_MESSAGE);
+ logger.warning(
+ WARNING_MESSAGE);
+ logger.warning(
+ "Configured to continue regardless of detected errors. This is useful for testing pre-release AMT content, "
+ + "but is NOT recommended for any other use! Resultant AMT flat file may be unreliable. "
+ + "Consider rerunning with the -e or --exit-on-error flag set!!!");
+ logger.warning(
+ WARNING_MESSAGE);
+ logger.warning(
+ WARNING_MESSAGE);
- if (Files.isSymbolicLink(path)) {
- throw new SecurityException("The input ZIP file must not be a symlink for security reasons");
- } else if (!Files.exists(path, LinkOption.NOFOLLOW_LINKS)) {
- throw new IllegalArgumentException("Specified input ZIP file " + inputZipFilePath + " does not exist");
- }
+ }
- BasicFileAttributes attr = Files.readAttributes(
- path, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
-
- if (!attr.isRegularFile()) {
- throw new SecurityException("The input ZIP file must be a regular file");
- } else if (attr.size() > MAX_ZIP_FILE_SIZE) {
- throw new SecurityException("For security, input ZIP files over 600M are not accepted. "
- + "This should permit RF2 ALL or SNAPSHOT bundles requiring the required files");
- } else if (!tika.detect(path).equals("application/zip")) {
- throw new SecurityException(
- "TThe input ZIP file " + inputZipFilePath + " is not a zip file as expected, detected type was "
- + tika.detect(path));
- }
- } catch (IOException e) {
- throw new IllegalArgumentException("Could not validate input ZIP file path " + inputZipFilePath, e);
+ //initialise test suite
+ this.testSuite = new JUnitTestSuite();
+ try (FileSystem zipFileSystem = FileSystems.newFileSystem(URI.create(
+ "jar:file:" + FileSystems.getDefault().getPath(inputZipFilePath).toAbsolutePath()),
+ new HashMap<>())) {
+
+ conceptCache = new AmtCache(zipFileSystem, exitOnError, testSuite);
+ writeFlatFile(
+ FileSystems.getDefault().getPath(FileFormat.CSV.getFilePath(outputFilePath)),
+ FileFormat.CSV);
+ writeFlatFile(
+ FileSystems.getDefault().getPath(FileFormat.TSV.getFilePath(outputFilePath)),
+ FileFormat.TSV);
+ if (replacementsOutputFilePath != null && !replacementsOutputFilePath.isEmpty()) {
+ writeReplacementsFile(
+ FileSystems.getDefault()
+ .getPath(FileFormat.CSV.getFilePath(replacementsOutputFilePath)),
+ FileFormat.CSV);
+ writeReplacementsFile(
+ FileSystems.getDefault()
+ .getPath(FileFormat.TSV.getFilePath(replacementsOutputFilePath)),
+ FileFormat.TSV);
+ }
+ if (junitFilePath == null || junitFilePath.trim().isEmpty()) {
+ junitFilePath = "target/ValidationErrors.xml";
+ }
+ File junitFile = new File(junitFilePath);
+ File parentDir = junitFile.getParentFile();
+ if (parentDir != null && !parentDir.exists()) {
+ parentDir.mkdirs();
+ }
+ BufferedWriter outputJunitXml = new BufferedWriter(new FileWriter(junitFilePath));
+ testSuite.writeToFile(outputJunitXml);
+ logger.info("Output junit results to: " + new File(junitFilePath).getAbsolutePath());
+ } catch (IOException e) {
+ throw new MojoExecutionException("Failed due to IO error executing transformation", e);
+ }
+ }
+
+
+ private void validateOutputPath(String outputPath, String expectedMimeType) {
+ try {
+ Path path = Paths.get(outputPath);
+
+ if (Files.isSymbolicLink(path)) {
+ throw new SecurityException(
+ "The output " + outputPath + " file must not be a symlink for security reasons");
+ }
+
+ if (Files.exists(path, LinkOption.NOFOLLOW_LINKS)) {
+ BasicFileAttributes attr = Files.readAttributes(
+ path, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
+
+ if (!attr.isRegularFile()) {
+ throw new SecurityException(
+ "The specified output file " + outputPath
+ + " exists, but is not a regular file. Cannot be overwritten.");
+ } else if (!tika.detect(path).equals(expectedMimeType)) {
+ throw new SecurityException(
+ "The specified output file " + outputPath + " exists, but is not a "
+ + expectedMimeType
+ + " file as expected, detected type was " + tika.detect(path)
+ + ". Cannot be overwritten");
}
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Could not validate output file path " + outputPath, e);
}
+ }
+
+ private void validateInputZipFile(String inputZipFilePath) {
+ try {
+ Path path = Paths.get(inputZipFilePath);
+
+ if (Files.isSymbolicLink(path)) {
+ throw new SecurityException(
+ "The input ZIP file must not be a symlink for security reasons");
+ } else if (!Files.exists(path, LinkOption.NOFOLLOW_LINKS)) {
+ throw new IllegalArgumentException(
+ "Specified input ZIP file " + inputZipFilePath + " does not exist");
+ }
+
+ BasicFileAttributes attr = Files.readAttributes(
+ path, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
+
+ if (!attr.isRegularFile()) {
+ throw new SecurityException("The input ZIP file must be a regular file");
+ } else if (attr.size() > MAX_ZIP_FILE_SIZE) {
+ throw new SecurityException("For security, input ZIP files over 1.5GB are not accepted. "
+ + "This should permit RF2 ALL or SNAPSHOT bundles requiring the required files - file size was "
+ + attr.size());
+ } else if (!tika.detect(path).equals("application/zip") && !tika.detect(path)
+ .equals("application/java-archive")) {
+ throw new SecurityException(
+ "The input ZIP file " + inputZipFilePath
+ + " is not a zip file as expected, detected type was "
+ + tika.detect(path));
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException(
+ "Could not validate input ZIP file path " + inputZipFilePath, e);
+ }
+ }
- private void writeFlatFile(Path path) throws IOException {
- if (path.getParent() != null && !Files.exists(path.getParent())) {
- Files.createDirectory(path.getParent());
+ private void writeFlatFile(Path path, FileFormat format) throws IOException {
+ if (path.getParent() != null && !Files.exists(path.getParent())) {
+ Files.createDirectory(path.getParent());
+ }
+ try (
+ BufferedWriter writer = Files.newBufferedWriter(path, StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
+
+ writer.write(
+ String.join(format.getDelimiter(), "CTPP SCTID", "CTPP PT", "ARTG_ID",
+ "TPP SCTID", "TPP PT", "TPUU SCTID", "TPUU PT",
+ "TPP TP SCTID", "TPP TP PT", "TPUU TP SCTID", "TPUU TP PT", "MPP SCTID", "MPP PT",
+ "MPUU SCTID",
+ "MPUU PT", "MP SCTID", "MP PT"));
+ writer.write("\r\n");
+
+ int count = 1;
+ int ctppCount = conceptCache.getCtppsFromRefset().values().size();
+ long startTime = System.currentTimeMillis();
+ for (Concept ctpp : conceptCache.getCtppsFromRefset().values()) {
+ // TPPs
+ Concept tpp = getParent(AmtRefset.TPP, ctpp);
+ if (tpp == null) {
+ tpp = ctpp;
}
- try (
- BufferedWriter writer = Files.newBufferedWriter(path, StandardOpenOption.CREATE,
- StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
-
- writer.write(
- String.join(",", "CTPP SCTID", "CTPP PT", "ARTG_ID", "TPP SCTID", "TPP PT", "TPUU SCTID", "TPUU PT",
- "TPP TP SCTID", "TPP TP PT", "TPUU TP SCTID", "TPUU TP PT", "MPP SCTID", "MPP PT", "MPUU SCTID",
- "MPUU PT", "MP SCTID", "MP PT"));
- writer.newLine();
-
- for (Concept ctpp : conceptCache.getCtpps().values()) {
- Concept tpp = getParent(AmtConcept.TPP, AmtConcept.CTPP, ctpp);
- Concept tppTp = null;
- if (tpp.getTps().size() == 1) {
- tppTp = tpp.getTps().iterator().next();
- } else {
- String message = "TPUU " + tpp + " has too many TPs " + tpp.getTps();
- testSuite.addTestCase("TPUU error", message, "TPUU has too many TPs (" + tpp + ")", "ERROR");
- if (exitOnError) {
- throw new RuntimeException(message);
- }
- continue;
- }
-
- Concept mpp = getParent(AmtConcept.MPP, AmtConcept.TPP, tpp);
- Set tpuus = tpp.getUnits();
-
- Set addedMpuus = new HashSet<>();
- for (Concept tpuu : tpuus) {
- Concept tpuuTp = getParent(AmtConcept.TP, AmtConcept.TPUU, tpuu);
- Concept mpuu = getParent(AmtConcept.MPUU, AmtConcept.TPUU, tpuu);
- addedMpuus.add(mpuu);
-
- Set mps = getParents(AmtConcept.MP, AmtConcept.MPUU, mpuu);
-
- Set artgids = ctpp.getArtgIds();
- if (artgids == null || artgids.size() == 0) {
- artgids = Collections.singleton("");
- }
-
- artgids = artgids.stream().map(String::trim).collect(Collectors.toSet());;
-
- if(tpuuTp == null || mpuu == null) {
- continue;
- }
-
- for (Concept mp : mps) {
- for (String artgid : artgids) {
- writer.write(
- String.join(",",
- ctpp.getId() + "", "\"" + ctpp.getPreferredTerm() + "\"",
- artgid,
- tpp.getId() + "", "\"" + tpp.getPreferredTerm() + "\"",
- tpuu.getId() + "", "\"" + tpuu.getPreferredTerm() + "\"",
- tppTp.getId() + "", "\"" + tppTp.getPreferredTerm() + "\"",
- tpuuTp.getId() + "", "\"" + tpuuTp.getPreferredTerm() + "\"",
- mpp.getId() + "", "\"" + mpp.getPreferredTerm() + "\"",
- mpuu.getId() + "", "\"" + mpuu.getPreferredTerm() + "\"",
- mp.getId() + "", "\"" + mp.getPreferredTerm() + "\""));
- writer.newLine();
- }
- }
- }
-
- if (!mpp.getUnits().containsAll(addedMpuus) || !addedMpuus.containsAll(mpp.getUnits())) {
-
- String message = "Mismatch between MPUUs from MPP "
- + mpp.getUnits().stream().map(c -> c.getId()).collect(Collectors.toList())
- + " and MPUUs added from TPUUs "
- + addedMpuus.stream().map(c -> c.getId()).collect(Collectors.toList())
- + " for MPP " + mpp;
- logger.warning(message);
-
- testSuite.addTestCase("Mismatch", message, "MPP mismatch (" + mpp.getId() + ")", "ERROR");
- }
- }
+ Concept tppTp = null;
+ if (tpp.getTps().size() == 1) {
+ tppTp = tpp.getTps().iterator().next();
+ } else {
+ String message = "TPP " + tpp + " has too many TPs " + tpp.getTps();
+ testSuite.addTestCase("TPP error", message, this.getClass().getName(),
+ "TPP has too many TPs (" + tpp + ")", "ERROR");
+ if (exitOnError) {
+ throw new RuntimeException(message);
+ }
+ logger.severe(message);
+ continue;
}
- }
-
- private void writeReplacementsFile(Path path) throws IOException {
- if (path.getParent() != null && !Files.exists(path.getParent())) {
- Files.createDirectory(path.getParent());
+ Concept mpp = getParent(AmtRefset.MPP, tpp);
+ if (mpp == null) {
+ logger.severe("No MPP found for TPP " + tpp);
}
- try (
- BufferedWriter writer = Files.newBufferedWriter(path, StandardOpenOption.CREATE,
- StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
-
- writer.write(
- String.join(",", "INACTIVE SCTID", "INACTIVE PT", "REPLACEMENT TYPE SCTID", "REPLACEMENT TYPE PT", "REPLACEMENT SCTID",
- "REPLACEMENT PT"));
- writer.newLine();
- for (Triple entry : conceptCache.getReplacementConcepts()) {
- writer.write(
- String.join(",",
- entry.getLeft().getId() + "", "\"" + entry.getLeft().getPreferredTerm() + "\"",
- entry.getMiddle().getId() + "", "\"" + entry.getMiddle().getPreferredTerm() + "\"",
- entry.getRight().getId() + "", "\"" + entry.getRight().getPreferredTerm() + "\""));
- writer.newLine();
- }
- }
- }
-
- private Concept getParent(AmtConcept parentType, AmtConcept current, Concept concept) {
- Set parents = getParents(parentType, current, concept).stream()
- .collect(Collectors.toSet());
-
- if (parents.size() != 1) {
- String message = "Expected 1 parent of type " + parentType + " for concept " + concept + " but got " + parents;
- testSuite.addTestCase("multiple parents", message, "Multiple parents (" + concept.getId() + ")", "ERROR");
-
+ // TPUUS
+ Set tpuus = tpp.getUnits();
+
+ Set addedMpuus = new HashSet<>();
+ for (Concept tpuu : tpuus) {
+ Concept tpuuTp = conceptCache.isAmtV3() ? getParent(AmtRefset.TP, tpuu) : null;
+ if (tpuuTp == null) {
+ if (tpuu.getTps().size() > 1) {
+ throw new RuntimeException("TPUU " + tpuu + " has too many TPs " + tpuu.getTps());
+ } else if (tpuu.getTps().size() == 1) {
+ tpuuTp = tpuu.getTps().iterator().next();
+ } else {
+ logger.severe("TPUU " + tpuu + " has no TPs");
+ testSuite.addTestCase("TPUU error", "TPUU has no TPs " + tpuu,
+ this.getClass().getName(), "TPUU has no TPs (" + tpuu.getId() + ")", "ERROR");
+ }
+ }
+ Concept mpuu = getParent(AmtRefset.MPUU, tpuu);
+ if (mpuu == null) {
+ throw new RuntimeException("TPUU " + tpuu + " has no MPUU");
+ }
+ addedMpuus.add(mpuu);
+
+ Set mps = getParents(AmtRefset.MP, mpuu);
+
+ if (mps.size() > 1) {
+ Set filteredMps = mps.stream().max(
+ (mp1, mp2) -> mp1.getIngredients().size() > mp2.getIngredients().size() ? 1 : -1)
+ .map(Collections::singleton).orElse(mps);
+ if (filteredMps.size() == 1) {
+ mps = filteredMps;
+ } else {
+ String message = "Expected 1 MP parent for MPUU " + mpuu.getId()
+ + " but got " + mps.size() + ". Filtering by ingredients yielded "
+ + filteredMps.size() + " not 1 as required. Full set was "
+ + mps.stream().map(c -> c.getId() + " | " + c.getPreferredTerm() + " | ")
+ .collect(Collectors.joining(", "));
+ logger.warning(message);
+ testSuite.addTestCase("multiple parents", message, this.getClass().getName(),
+ "Multiple parents (" + mpuu.getId() + ")", "ERROR");
+ if (exitOnError) {
+ throw new RuntimeException(message);
+ }
+ }
+ } else if (mps.isEmpty()) {
+ String message = "No MP parent for MPUU " + mpuu.getId();
+ logger.warning(message);
+ testSuite.addTestCase("no parents", message, this.getClass().getName(),
+ "No parents (" + mpuu.getId() + ")", "ERROR");
if (exitOnError) {
- throw new RuntimeException(message);
+ throw new RuntimeException(message);
}
- return null;
- }
- return parents.iterator().next();
- }
-
- private Set getParents(AmtConcept parentType, AmtConcept current, Concept concept) {
- return getParents(parentType, current, Collections.singleton(concept));
- }
-
- private Set getParents(AmtConcept parentType, AmtConcept current, Set concepts) {
- Set leafParents = new HashSet<>();
+ }
+
+ Set artgids = ctpp.getArtgIds();
+ if (artgids == null || artgids.isEmpty()) {
+ artgids = Collections.singleton("");
+ }
+
+ artgids = artgids.stream().map(String::trim).collect(Collectors.toSet());
+
+ if (tpuuTp == null || mpuu == null) {
+ continue;
+ }
+
+ for (Concept mp : mps) {
+ for (String artgid : artgids) {
+ writer.write(
+ String.join(format.getDelimiter(), ctpp.getId() + "",
+ format.getFieldQuote() + ctpp.getPreferredTerm()
+ + format.getFieldQuote(),
+ artgid,
+ tpp.getId() + "",
+ format.getFieldQuote() + tpp.getPreferredTerm()
+ + format.getFieldQuote(),
+ tpuu.getId() + "",
+ format.getFieldQuote() + tpuu.getPreferredTerm()
+ + format.getFieldQuote(),
+ tppTp.getId() + "",
+ format.getFieldQuote() + tppTp.getPreferredTerm()
+ + format.getFieldQuote(),
+ tpuuTp.getId() + "",
+ format.getFieldQuote() + tpuuTp.getPreferredTerm()
+ + format.getFieldQuote(),
+ mpp.getId() + "",
+ format.getFieldQuote() + mpp.getPreferredTerm()
+ + format.getFieldQuote(),
+ mpuu.getId() + "",
+ format.getFieldQuote() + mpuu.getPreferredTerm()
+ + format.getFieldQuote(),
+ mp.getId() + "",
+ format.getFieldQuote() + mp.getPreferredTerm()
+ + format.getFieldQuote()));
+ writer.write("\r\n");
+ }
+ }
+ }
- leafParents.addAll(concepts.stream()
- .flatMap(c -> c.getAncestors(parentType).stream())
- .filter(p -> !AmtConcept.isEnumValue(p.getId()))
- .filter(p -> p.hasAtLeastOneMatchingAncestor(parentType))
- .filter(p -> !p.hasAtLeastOneMatchingAncestor(current))
- .collect(Collectors.toSet()));
+ if (!mpp.getUnits().containsAll(addedMpuus) || !addedMpuus.containsAll(mpp.getUnits())) {
- Set redunantAncestors =
- leafParents.stream().flatMap(p -> p.getAncestors(parentType).stream()).collect(Collectors.toSet());
+ String message = "Mismatch between MPUUs from MPP "
+ + mpp.getUnits().stream().map(c -> c.getId()).collect(Collectors.toList())
+ + " and MPUUs added from TPUUs "
+ + addedMpuus.stream().map(c -> c.getId()).collect(Collectors.toList())
+ + " for MPP " + mpp;
+ logger.warning(message);
- leafParents.removeAll(redunantAncestors);
+ testSuite.addTestCase("Mismatch", message, this.getClass().getName(),
+ "MPP mismatch (" + mpp.getId() + ")", "ERROR");
+ }
- return leafParents;
- }
+ if (count % 1000 == 0) {
+ long endTime = System.currentTimeMillis();
+ logger.info("Processed CTPP [" + count + " of " + ctppCount
+ + "] Time spent processing 1000 CTPPs is " + (endTime - startTime) + "ms");
+ startTime = System.currentTimeMillis();
+ }
+ count++;
+ }
+ }
+ }
+
+ private Set memberOf(Set concepts, AmtRefset refset) {
+ Set refsetMembers = new HashSet<>();
+ Map refsetConcepts = conceptCache.getAmtRefsets().get(refset.getIdString());
+ for (Concept concept : concepts) {
+ if (refsetConcepts.containsKey(concept.getId())) {
+ refsetMembers.add(concept);
+ }
+ }
+ return refsetMembers;
+ }
- public String getInputZipFilePath() {
- return inputZipFilePath;
- }
+ private void writeReplacementsFile(Path path, FileFormat format) throws IOException {
+ if (path.getParent() != null && !Files.exists(path.getParent())) {
+ Files.createDirectory(path.getParent());
+ }
+ try (
+ BufferedWriter writer = Files.newBufferedWriter(path, StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
+
+ writer.write(
+ String.join(format.getDelimiter(), "INACTIVE SCTID", "INACTIVE PT",
+ "REPLACEMENT TYPE SCTID", "REPLACEMENT TYPE PT", "REPLACEMENT SCTID",
+ "REPLACEMENT PT", "DATE"));
+ writer.write("\r\n");
+ for (Replacement entry : conceptCache.getReplacementConcepts()) {
+ if (entry.getInactiveConcept() == null || entry.getReplacementType() == null
+ || entry.getActiveConcept() == null) {
+ throw new RuntimeException("Null replacement concept: " + entry);
+ }
+ writer.write(
+ String.join(format.getDelimiter(), entry.getInactiveConcept().getId() + "",
+ format.getFieldQuote()
+ + entry.getInactiveConcept().getPreferredTerm()
+ + format.getFieldQuote(),
+ entry.getReplacementType().getId() + "",
+ format.getFieldQuote()
+ + entry.getReplacementType().getPreferredTerm()
+ + format.getFieldQuote(),
+ entry.getActiveConcept().getId() + "",
+ format.getFieldQuote() + entry.getActiveConcept().getPreferredTerm()
+ + format.getFieldQuote(),
+ entry.getVersion()));
+ writer.write("\r\n");
+ }
+ }
+ }
+
+ private Concept getParent(AmtRefset parentType, Concept concept) {
+ Set parents = getParents(parentType, concept);
+
+ if (parents.size() != 1) {
+ String message =
+ "Expected 1 parent of type " + parentType + " for concept " + concept + " but got "
+ + parents;
+ testSuite.addTestCase("multiple parents", message, this.getClass().getName(),
+ "Multiple parents (" + concept.getId() + ")", "ERROR");
+
+ if (exitOnError) {
+ throw new RuntimeException(message);
+ }
+ return null;
+ }
+ return parents.iterator().next();
+ }
- public void setInputZipFilePath(String inputZipFilePath) {
- this.inputZipFilePath = inputZipFilePath;
- }
+ private Set getParents(AmtRefset parentType, Concept concept) {
+ Set leafParents = new HashSet<>();
- public String getOutputFilePath() {
- return outputFilePath;
- }
+ leafParents.addAll(concept.getAncestors().stream().collect(Collectors.toSet()));
+ leafParents = memberOf(leafParents, parentType);
- public void setOutputFilePath(String outputFilePath) {
- this.outputFilePath = outputFilePath;
- }
+ Set redunantAncestors =
+ leafParents.stream().flatMap(p -> p.getAncestors().stream()).collect(Collectors.toSet());
- public void setExitOnError(boolean exitOnError) {
- this.exitOnError = exitOnError;
- }
+ leafParents.removeAll(redunantAncestors);
- public void setJunitFilePath(String path) {
- this.junitFilePath = path;
- }
+ return leafParents;
+ }
- public void setReplacementsFilePath(String path) {
- this.replacementsOutputFilePath = path;
- }
+ public void setReplacementsFilePath(String path) {
+ this.replacementsOutputFilePath = path;
+ }
}
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtCache.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtCache.java
index 9903add..d45fe67 100644
--- a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtCache.java
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtCache.java
@@ -4,22 +4,19 @@
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
-import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.commons.lang3.tuple.Triple;
import org.jgrapht.alg.TransitiveClosure;
import org.jgrapht.graph.SimpleDirectedGraph;
-import org.openmbee.junit.model.JUnitFailure;
+import au.gov.digitalhealth.terminology.amtflatfile.Junit.JUnitFailure;
+import au.gov.digitalhealth.terminology.amtflatfile.Junit.JUnitTestCase;
+import au.gov.digitalhealth.terminology.amtflatfile.Junit.JUnitTestSuite;
public class AmtCache {
@@ -34,23 +31,81 @@ public class AmtCache {
private static final String AMT_MODULE_ID = "900062011000036108";
private static final Logger logger = Logger.getLogger(AmtCache.class.getCanonicalName());
+ private static final String INTERNATIONAL_MODULE = "900000000000207008";
+ private static final String AU_MODULE = "32506021000036107";
+
+ private static final List HISTORICAL_ASSOCAITION_IDS = List.of("900000000000523009",
+ "900000000000530003",
+ "900000000000525002",
+ "900000000000524003",
+ "1186924009",
+ "900000000000523009",
+ "1186921001",
+ "900000000000531004",
+ "900000000000526001",
+ "900000000000527005",
+ "900000000000529008",
+ "900000000000528000");
private SimpleDirectedGraph graph = new SimpleDirectedGraph<>(Edge.class);
private Map conceptCache = new HashMap<>();
- private Set preferredDescriptionIdCache = new HashSet<>();
+ public Map getConceptCache() {
+ return conceptCache;
+ }
- private Map ctpps = new HashMap<>();
+ private Set preferredDescriptionIdCache = new HashSet<>();
- private Set> replacements = new HashSet<>();
+ private Set replacements = new HashSet<>();
private boolean exitOnError;
- private JUnitTestSuite_EXT testSuite;
- private JUnitTestCase_EXT graphCase;
+ private JUnitTestSuite testSuite;
+ private JUnitTestCase graphCase;
+
+ private Map ctppsFromRefset = new HashMap<>();
+ private Map tppsFromRefset = new HashMap<>();
+ private Map tpuusFromRefset = new HashMap<>();
+ private Map tpsFromRefset = new HashMap<>();
+ private Map mppsFromRefset = new HashMap<>();
+ private Map mpuusFromRefset = new HashMap<>();
+ private Map mpsFromRefset = new HashMap<>();
+ private Map> amtRefsets = Map.ofEntries(
+ Map.entry(AmtRefset.CTPP.getIdString(), ctppsFromRefset),
+ Map.entry(AmtRefset.TPP.getIdString(), tppsFromRefset),
+ Map.entry(AmtRefset.TPUU.getIdString(), tpuusFromRefset),
+ Map.entry(AmtRefset.TP.getIdString(), tpsFromRefset),
+ Map.entry(AmtRefset.MPP.getIdString(), mppsFromRefset),
+ Map.entry(AmtRefset.MPUU.getIdString(), mpuusFromRefset),
+ Map.entry(AmtRefset.MP.getIdString(), mpsFromRefset)
+ );
+
+ private static final Map amtRefsetToV3Concept = Map.ofEntries(
+ Map.entry(AmtRefset.CTPP.getIdString(), AmtConcept.CTPP),
+ Map.entry(AmtRefset.TPP.getIdString(), AmtConcept.TPP),
+ Map.entry(AmtRefset.TPUU.getIdString(), AmtConcept.TPUU),
+ Map.entry(AmtRefset.TP.getIdString(), AmtConcept.TP),
+ Map.entry(AmtRefset.MPP.getIdString(), AmtConcept.MPP),
+ Map.entry(AmtRefset.MPUU.getIdString(), AmtConcept.MPUU),
+ Map.entry(AmtRefset.MP.getIdString(), AmtConcept.MP)
+ );
+
+ private Map> conceptMaps = new HashMap<>();
+
+ private boolean amtV3 = false;
+
+ public boolean isAmtV3() {
+ return amtV3;
+ }
+
+
+ public Map> getAmtRefsets() {
+ return amtRefsets;
+ }
+
- public AmtCache(FileSystem amtZip, JUnitTestSuite_EXT testSuite, boolean exitOnError) throws IOException {
+ public AmtCache(FileSystem amtZip, boolean exitOnError, JUnitTestSuite testSuite) throws IOException {
this.testSuite = testSuite;
this.exitOnError = exitOnError;
processAmtFiles(amtZip);
@@ -58,7 +113,7 @@ public AmtCache(FileSystem amtZip, JUnitTestSuite_EXT testSuite, boolean exitOnE
private void processAmtFiles(FileSystem amtZip) throws IOException {
- graphCase = new JUnitTestCase_EXT().setName("Graph errors");
+ graphCase = new JUnitTestCase().setName("Graph errors");
TerminologyFileVisitor visitor = new TerminologyFileVisitor();
@@ -71,10 +126,24 @@ private void processAmtFiles(FileSystem amtZip) throws IOException {
readFile(visitor.getLanguageRefsetFile(), s -> handleLanguageRefsetRow(s), true, "\t");
readFile(visitor.getDescriptionFile(), s -> handleDescriptionRow(s), true, "\t");
readFile(visitor.getArtgIdRefsetFile(), s -> handleArtgIdRefsetRow(s), true, "\t");
+ int refsetFileCount = 0;
+ for (Path amtRefsetFile : visitor.getAMTRefsetFiles()) {
+ readFile(amtRefsetFile, s -> handleAMTRefsetRow(s), true, "\t");
+ refsetFileCount++;
+ }
+ if (refsetFileCount > 1) {
+ this.amtV3 = true;
+ }
for (Path historicalFile : visitor.getHistoricalAssociationRefsetFiles()) {
readFile(historicalFile, s -> handleHistoricalAssociationRefsetRow(s), true, "\t");
}
+ if (this.isAmtV3()) {
+ logger.info("######### Working with AMT V3 Release #########");
+ } else {
+ logger.info("######### Working with AMT V4 Release #########");
+ }
+
try {
calculateTransitiveClosure();
} catch (Exception e) {
@@ -87,19 +156,13 @@ private void processAmtFiles(FileSystem amtZip) throws IOException {
}
}
- graph.incomingEdgesOf(AmtConcept.CTPP.getId())
- .stream()
- .map(e -> e.getSource())
- .filter(id -> !AmtConcept.isEnumValue(Long.toString(id)))
- .forEach(id -> ctpps.put(id, conceptCache.get(id)));
-
- Iterator> it = ctpps.entrySet().iterator();
+ Iterator> it = ctppsFromRefset.entrySet().iterator();
while (it.hasNext()) {
Entry entry = it.next();
if (!entry.getValue().isActive()) {
String message = "Found inactive CTPP! " + entry.getValue();
logger.warning(message);
- testSuite.addTestCase("Inactive CTPP found", entry.getValue().toString(), "Inactive_CTPP", "ERROR");
+ testSuite.addTestCase("Inactive CTPP found", entry.getValue().toString(), this.getClass().getName(), "Inactive_CTPP", "ERROR");
if (exitOnError) {
throw new RuntimeException(message);
}
@@ -117,7 +180,7 @@ private void processAmtFiles(FileSystem amtZip) throws IOException {
validateConceptCache();
- logger.info("Loaded " + ctpps.size() + " CTPPs " + conceptCache.size() + " concepts ");
+ logger.info("Loaded " + ctppsFromRefset.size() + " CTPPs " + conceptCache.size() + " concepts ");
validateUnits();
@@ -167,7 +230,7 @@ private void assertConceptCache(Predicate predicate, String message, St
if (!errors.isEmpty()) {
logger.warning(message + " " + errors);
- testSuite.addTestCase(message, errors.toString(), testCaseName, "ERROR");
+ testSuite.addTestCase(message, errors.toString(), this.getClass().getName(), testCaseName, "ERROR");
if (exitOnError || fix == null) {
throw new RuntimeException(message + " " + errors);
@@ -231,7 +294,7 @@ private void validateUnits() {
.collect(Collectors.toSet());
testSuite.addTestCase("Detected pack concepts with no units and/or MPPs with TPUU units and/or TPP/CTPPs with MPUU units",
- detail, "heirarchy_error", "ERROR");
+ detail, this.getClass().getName(), "heirarchy_error", "ERROR");
if (exitOnError) {
throw new RuntimeException(detail);
@@ -239,19 +302,43 @@ private void validateUnits() {
}
}
- public Map getCtpps() {
- return ctpps;
+ public Map getCtppsFromRefset() {
+ return ctppsFromRefset;
+ }
+
+ public Map getTppsFromRefset() {
+ return tppsFromRefset;
+ }
+
+ public Map getTpuusFromRefset() {
+ return tpuusFromRefset;
+ }
+
+ public Map getTpsFromRefset() {
+ return tpsFromRefset;
+ }
+
+ public Map getMppsFromRefset() {
+ return mppsFromRefset;
+ }
+
+ public Map getMpuusFromRefset() {
+ return mpuusFromRefset;
+ }
+
+ public Map getMpsFromRefset() {
+ return mpsFromRefset;
}
private void handleConceptRow(String[] row) {
try {
- if (isAmtOrMetadataModule(row)) {
+ if (isAuModule(row) || isAmtOrMetadataModule(row) || isIntModule(row)) {
long conceptId = Long.parseLong(row[0]);
graph.addVertex(conceptId);
conceptCache.put(conceptId, new Concept(conceptId, isActive(row)));
}
} catch (Exception e) {
- throw new RuntimeException("Failed processing row: " + row + " of Concepts file", e);
+ throw new RuntimeException("Failed processing row: " + String.join(", ",row) + " of Concepts file", e);
}
}
@@ -262,7 +349,7 @@ private void handleRelationshipRow(String[] row) {
long destination = Long.parseLong(row[5]);
String type = row[7];
- if (isActive(row) && isAmtModule(row) && AttributeType.isEnumValue(type) && graph.containsVertex(source)
+ if (isActive(row) && (isAmtModule(row) || isAuModule(row) || isIntModule(row)) && AttributeType.isEnumValue(type) && graph.containsVertex(source)
&& graph.containsVertex(destination)) {
Concept sourceConcept = conceptCache.get(source);
@@ -272,12 +359,25 @@ private void handleRelationshipRow(String[] row) {
sourceConcept.addParent(conceptCache.get(destination));
break;
+ case HAS_INTENDED_ACTIVE_INGREDIENT:
+ case HAS_ACTIVE_INGREDIENT:
+ case HAS_PRECISE_ACTIVE_INGREDIENT:
+ sourceConcept.addIngredient(conceptCache.get(destination));
+ break;
+
case HAS_MPUU:
case HAS_TPUU:
+ case CONTAINS_CLINICAL_DRUG:
+ case CONTAINS_DEVICE:
sourceConcept.addUnit(conceptCache.get(destination));
break;
+ case CONTAINS_PACKAGED_CLINICAL_DRUG:
+ sourceConcept.addSubpack(conceptCache.get(destination));
+ break;
+
case HAS_TP:
+ case HAS_PRODUCT_NAME:
sourceConcept.addTp(conceptCache.get(destination));
default:
@@ -285,7 +385,7 @@ private void handleRelationshipRow(String[] row) {
}
}
} catch (Exception e) {
- throw new RuntimeException("Failed processing row: " + row + " of Relationships file", e);
+ throw new RuntimeException("Failed processing row: " + String.join(", ",row) + " of Relationships file", e);
}
}
@@ -296,7 +396,7 @@ private void handleDescriptionRow(String[] row) {
Long conceptId = Long.parseLong(row[4]);
- if (isActive(row) && isAmtOrMetadataModule(row) && conceptCache.containsKey(conceptId)) {
+ if (isActive(row) && (isAmtOrMetadataModule(row) || isIntModule(row) || isAuModule(row)) && conceptCache.containsKey(conceptId)) {
String descriptionId = row[0];
String term = row[7];
Concept concept = conceptCache.get(conceptId);
@@ -307,18 +407,18 @@ private void handleDescriptionRow(String[] row) {
}
}
} catch (Exception e) {
- throw new RuntimeException("Failed processing row: " + row + " of Descriptions file", e);
+ throw new RuntimeException("Failed processing row: " + String.join(", ",row) + " of Descriptions file", e);
}
}
private void handleLanguageRefsetRow(String[] row) {
try {
- if (isActive(row) && isAmtOrMetadataModule(row) && row[6].equals(PREFERRED)) {
+ if (isActive(row) && (isAmtOrMetadataModule(row) || isAuModule(row)) && row[6].equals(PREFERRED)) {
preferredDescriptionIdCache.add(Long.parseLong(row[5]));
}
} catch (Exception e) {
- throw new RuntimeException("Failed processing row: " + row + " of Language file", e);
+ throw new RuntimeException("Failed processing row: " + String.join(", ",row) + " of Language file", e);
}
}
@@ -326,25 +426,48 @@ private void handleLanguageRefsetRow(String[] row) {
private void handleArtgIdRefsetRow(String[] row) {
try {
long conceptId = Long.parseLong(row[5]);
- if (isActive(row) && isAmtModule(row)) {
+ if (isActive(row) && (isAmtModule(row) || isAuModule(row))) {
conceptCache.get(conceptId).addArtgIds(row[6]);
}
} catch (Exception e) {
- throw new RuntimeException("Failed processing row: " + row + " of ARTG file", e);
+ throw new RuntimeException("Failed processing row: " + String.join(", ",row) + " of ARTG file", e);
+ }
+ }
+
+ private void handleAMTRefsetRow(String[] row) {
+ try {
+ long conceptId = Long.parseLong(row[5]);
+ if (isActive(row) && (isAmtModule(row) || isAuModule(row))) {
+ String refsetId = row[4];
+ AmtConcept v3Concept = amtRefsetToV3Concept.get(refsetId);
+ if (v3Concept != null) {
+ Concept concept = conceptCache.get(conceptId);
+ concept.setType(v3Concept);
+ amtRefsets.get(refsetId).put(conceptId, concept);
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Failed processing row: " + String.join(", ",row) + " of ARTG file", e);
}
}
private void handleHistoricalAssociationRefsetRow(String[] row) {
try {
- if (isActive(row) && isAmtModule(row) && !isDescriptionId(row[5])) {
+ if (isActive(row) && (isAmtModule(row) || isAuModule(row)) && !isDescriptionId(row[5]) && HISTORICAL_ASSOCAITION_IDS.contains(row[4])) {
Concept replacementType = conceptCache.get(Long.parseLong(row[4]));
Concept inactiveConcept = conceptCache.get(Long.parseLong(row[5]));
Concept replacementConcept = conceptCache.get(Long.parseLong(row[6]));
- replacements.add(Triple.of(inactiveConcept, replacementType, replacementConcept));
+ if (replacementType == null || inactiveConcept == null || replacementConcept == null) {
+ throw new RuntimeException("Failed processing row: " + String.join("\t", row) + " of History file.\nOne of the concepts is null. "
+ + "\nreplacementType: " + replacementType + " replacement id is " + Long.parseLong(row[4]) + "\ninactiveConcept: " + inactiveConcept + "\nreplacementConcept: "
+ + replacementConcept);
+ }
+
+ replacements.add(new Replacement(inactiveConcept, replacementType, replacementConcept, row[1]));
}
} catch (Exception e) {
- throw new RuntimeException("Failed processing row: " + row + " of History file", e);
+ throw new RuntimeException("Failed processing row: " + String.join(", ",row) + " of History file", e);
}
}
@@ -369,8 +492,13 @@ private boolean isAmtModule(String[] row) {
private boolean isAmtOrMetadataModule(String[] row) {
return row[3].equals(AMT_MODULE_ID) || row[3].equals(INTERNATIONAL_METADATA_MODULE) || row[3].equals(AU_METADATA_MODULE);
}
-
- public Set> getReplacementConcepts() {
+ private boolean isIntModule(String[] row) {
+ return row[3].equals(INTERNATIONAL_MODULE);
+ }
+ private boolean isAuModule(String[] row) {
+ return row[3].equals(AU_MODULE);
+ }
+ public Set getReplacementConcepts() {
return replacements;
}
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtConcept.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtConcept.java
index 62120c9..75297f7 100644
--- a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtConcept.java
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtConcept.java
@@ -15,7 +15,10 @@ public enum AmtConcept {
SUBSTANCE(30344011000036106L, "Australian substance"),
REPLACED_BY(900000000000526001L, "REPLACED BY"),
POSSIBLY_EQUIVALENT_TO(900000000000523009L, "POSSIBLY EQUIVALENT TO"),
- SAME_AS(900000000000527005L, "SAME AS");
+ SAME_AS(900000000000527005L, "SAME AS"),
+ INT_MPP(781405001, "Medicinal product pack"),
+ INT_MP(763158003, "Medicinal product"),
+ INT_MPF(91000001109L, "Medicinal product form");
// @formatter:on
private static HashMap instanceMap = new HashMap<>();
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtRefset.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtRefset.java
new file mode 100644
index 0000000..fc14ad5
--- /dev/null
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AmtRefset.java
@@ -0,0 +1,29 @@
+package au.gov.digitalhealth.terminology.amtflatfile;
+
+public enum AmtRefset {
+ // @formatter:off
+ CTPP(929360051000036108L),
+ TPP(929360041000036105L),
+ TPUU(929360031000036100L),
+ TP(929360021000036102L),
+ MPP(929360081000036101L),
+ MPUU(929360071000036103L),
+ MP(929360061000036106L);
+ // @formatter:on
+
+ private long id;
+
+ private AmtRefset(long id) {
+ this.id = id;
+ }
+
+ public long getId() {
+ return id;
+ }
+
+ public String getIdString() {
+ return Long.toString(id);
+ }
+
+}
+
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AttributeType.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AttributeType.java
index a233319..326c1ba 100644
--- a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AttributeType.java
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/AttributeType.java
@@ -18,13 +18,26 @@ public enum AttributeType {
HAS_UNIT_OF_USE(30548011000036101L),
HAS_COMPONENT_PACK(700000061000036106L),
HAS_DENOMINATOR_UNITS(700000071000036103L),
- HAS_INTENDED_ACTIVE_INGREDIENT(700000081000036101L),
+ HAS_INTENDED_ACTIVE_INGREDIENT(700000081000036101L),
+ HAS_ACTIVE_INGREDIENT(127489000),
+ HAS_PRECISE_ACTIVE_INGREDIENT(762949000),
HAS_NUMERATOR_UNITS(700000091000036104L),
HAS_TP(700000101000036108L),
STRENGTH(700000111000036105L),
UNIT_OF_USE_SIZE(700000141000036106L),
UNIT_OF_USE_QUANTITY(700000131000036101L),
- SUBPACK_QUANTITY(700000121000036103L);
+ SUBPACK_QUANTITY(700000121000036103L),
+ HAS_CONCENTRATION_STRENGH_VALUE(999000021000168100L),
+ HAS_CONCENTRATION_STRENGH_UNIT(999000031000168102L),
+ HAS_TOTAL_QUANTITY_VALUE(999000041000168106L),
+ HAS_TOTAL_QUANTITY_UNIT(999000051000168108L),
+ CONTAINS_CLINICAL_DRUG(774160008L),
+ CONTAINS_DEVICE(999000081000168101L),
+ HAS_DEVICE_TYPE(999000061000168105L),
+ PACKAGE(999000071000168104L),
+ CONTAINS_PACKAGED_CLINICAL_DRUG(999000011000168107L),
+ HAS_PRODUCT_NAME(774158006),
+ HAS_OTHER_IDENTIFYING_INFORMATION(999000001000168109L);
// @formatter:on
private static HashMap instanceMap = new HashMap<>();
@@ -39,7 +52,7 @@ public static AttributeType fromId(long id) {
if (instanceMap.containsKey(id)) {
return instanceMap.get(id);
} else {
- throw new RuntimeException("Cannot find enum for id " + id);
+ throw new RuntimeException("Cannot find enum for id " + id + " " + instanceMap);
}
}
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Concept.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Concept.java
index 5f8b45d..a7470ce 100644
--- a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Concept.java
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Concept.java
@@ -7,6 +7,7 @@
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
public class Concept {
@@ -14,11 +15,14 @@ public class Concept {
private String fullSpecifiedName;
private String preferredTerm;
private Set units = new HashSet<>();
+ private Set subpack = new HashSet<>();
private Map parents = new HashMap<>();
private Map ancestors = new HashMap<>();
private Set tps = new HashSet<>();
+ private Set ingredients = new HashSet<>();
private Set artgIds = new HashSet<>();
private boolean active;
+ private AmtConcept type;
public Concept(long id, boolean active) {
this.id = id;
@@ -61,9 +65,23 @@ public void addUnit(Concept unit) {
}
public Set getUnits() {
+ if (units.isEmpty()) {
+ return getSubpack().stream().map(Concept::getUnits).flatMap(Set::stream).collect(Collectors.toSet());
+ }
return units;
}
+ public void addIngredient(Concept concept) {
+ if (ingredients == null) {
+ ingredients = new HashSet<>();
+ }
+ ingredients.add(concept);
+ }
+
+ public Set getIngredients() {
+ return ingredients;
+ }
+
public boolean hasOneMatchingParent(AmtConcept... amtConcept) {
for (AmtConcept parent : amtConcept) {
if (parents.containsKey(parent.getId())) {
@@ -112,6 +130,10 @@ public Collection getAncestors(AmtConcept concept) {
return result;
}
+ public Collection getAncestors() {
+ return ancestors.values();
+ }
+
@Override
public String toString() {
return "Concept [id=" + id + ", fullSpecifiedName=" + fullSpecifiedName + ", parents=" + parents + "]";
@@ -164,4 +186,20 @@ public Set getArtgIds() {
public boolean isActive() {
return active;
}
+
+ public void addSubpack(Concept concept) {
+ subpack.add(concept);
+ }
+
+ public Set getSubpack() {
+ return subpack;
+ }
+
+ public void setType(AmtConcept type) {
+ this.type = type;
+ }
+
+ public AmtConcept getType() {
+ return type;
+ }
}
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/FileFormat.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/FileFormat.java
new file mode 100644
index 0000000..65a41c0
--- /dev/null
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/FileFormat.java
@@ -0,0 +1,51 @@
+package au.gov.digitalhealth.terminology.amtflatfile;
+
+public enum FileFormat {
+ TSV("tsv"), CSV("csv");
+
+ private static final String DOUBLEQUOTE = "\"";
+ private final String value;
+
+ FileFormat(String value) {
+ this.value = value;
+ }
+
+ public String getFilePath(String outputFilePath) {
+ return outputFilePath + "." + this.getExtension();
+ }
+
+ public String getExtension() {
+ return this.value;
+ }
+
+ public String getMimeType() {
+ return "text/" + this.value;
+ }
+
+ public String getDelimiter() {
+ if (this.value.equals("csv")) {
+ return ",";
+ } else if (this.value.equals("tsv")) {
+ return "\t";
+ }
+ return "";
+ }
+
+ public String getFieldQuote() {
+ if (this.equals(TSV)) {
+ return "";
+ } else {
+ return DOUBLEQUOTE;
+ }
+ }
+
+
+ public static FileFormat fromValue(String value) {
+ for (FileFormat kind : values()) {
+ if (kind.value.equals(value)) {
+ return kind;
+ }
+ }
+ throw new IllegalArgumentException("Unknown File Format: " + value);
+ }
+}
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/JUnitTestCase_EXT.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/JUnitTestCase_EXT.java
deleted file mode 100644
index d3ef87b..0000000
--- a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/JUnitTestCase_EXT.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package au.gov.digitalhealth.terminology.amtflatfile;
-
-import java.util.ArrayList;
-
-import org.openmbee.junit.model.JUnitFailure;
-import org.openmbee.junit.model.JUnitTestCase;
-
-public class JUnitTestCase_EXT extends JUnitTestCase{
-
- public JUnitTestCase_EXT addFailure(JUnitFailure failure) {
- if(this.getFailures() == null)
- this.setFailures(new ArrayList());
- this.getFailures().add(failure);
- return this;
- }
-
- //Return proper class after setname
- @Override
- public JUnitTestCase_EXT setName(String name) {
- super.setName(name);
- return this;
- }
-}
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Junit/JUnitFailure.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Junit/JUnitFailure.java
new file mode 100644
index 0000000..a12db7c
--- /dev/null
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Junit/JUnitFailure.java
@@ -0,0 +1,29 @@
+package au.gov.digitalhealth.terminology.amtflatfile.Junit;
+
+import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
+import jakarta.xml.bind.annotation.XmlAttribute;
+import jakarta.xml.bind.annotation.XmlValue;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+@Getter
+@Setter
+@Accessors(chain = true)
+public class JUnitFailure {
+
+ @XmlAttribute
+ private String message;
+
+ @XmlAttribute
+ private String type;
+
+ @XmlValue
+ private String value;
+
+ @Override
+ public String toString() {
+ return ReflectionToStringBuilder.toString(this);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Junit/JUnitTestCase.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Junit/JUnitTestCase.java
new file mode 100644
index 0000000..4452be3
--- /dev/null
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Junit/JUnitTestCase.java
@@ -0,0 +1,41 @@
+package au.gov.digitalhealth.terminology.amtflatfile.Junit;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
+
+import jakarta.xml.bind.annotation.XmlAttribute;
+import jakarta.xml.bind.annotation.XmlElement;
+import java.util.ArrayList;
+import java.util.List;
+
+@Getter
+@Setter
+@Accessors(chain = true)
+public class JUnitTestCase {
+
+ @XmlAttribute(name = "classname", required = true)
+ private String className;
+
+ @XmlAttribute(required = true)
+ private String name;
+
+ @XmlAttribute
+ private Double time;
+
+ @XmlElement(name = "failure")
+ private List failures;
+
+ public JUnitTestCase addFailure(JUnitFailure failure) {
+ if(this.getFailures() == null)
+ this.setFailures(new ArrayList());
+ this.getFailures().add(failure);
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return ReflectionToStringBuilder.toStringExclude(this);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/JUnitTestSuite_EXT.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Junit/JUnitTestSuite.java
similarity index 58%
rename from src/main/java/au/gov/digitalhealth/terminology/amtflatfile/JUnitTestSuite_EXT.java
rename to src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Junit/JUnitTestSuite.java
index 3ee3285..a49062c 100644
--- a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/JUnitTestSuite_EXT.java
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Junit/JUnitTestSuite.java
@@ -1,9 +1,12 @@
-package au.gov.digitalhealth.terminology.amtflatfile;
+package au.gov.digitalhealth.terminology.amtflatfile.Junit;
+import static org.junit.Assert.fail;
import java.io.BufferedWriter;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.stream.Collectors;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -15,24 +18,37 @@
import javax.xml.transform.TransformerFactoryConfigurationError;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
-
-import org.openmbee.junit.model.JUnitFailure;
-import org.openmbee.junit.model.JUnitTestCase;
-import org.openmbee.junit.model.JUnitTestSuite;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
-/**
- * The opembee JUnit parser was only ever meant to read JUnit files. Not create them.
- * This extension to the JUnitTestSuite class adds writing functionality, as well as
- * methods that make constructing models easier
- *
- * @author patrick
- *
- */
-public class JUnitTestSuite_EXT extends JUnitTestSuite {
-
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import jakarta.xml.bind.annotation.XmlAttribute;
+import jakarta.xml.bind.annotation.XmlElement;
+
+@Getter
+@Setter
+@Accessors(chain = true)
+ public class JUnitTestSuite {
+
+ @XmlAttribute
+ private int errors;
+
+ @XmlAttribute
+ private int failures;
+
+ @XmlAttribute(required = true)
+ private int tests;
+
+ @XmlAttribute(required = true)
+ private String name;
+
+ @XmlElement(name = "testcase")
+ private List testCases;
+
public void writeToFile(BufferedWriter stream) throws IOException {
-
+
Document doc = null;
try {
doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
@@ -46,7 +62,7 @@ public void writeToFile(BufferedWriter stream) throws IOException {
testSuiteEl.setAttribute("tests", "0");
testSuiteEl.setAttribute("errors", "0");
testSuiteEl.setAttribute("name", "validation.errors");
-
+
if(this.getTestCases() != null) {
this.setFailures((int) this.getTestCases().stream().flatMap(x -> x.getFailures().stream()).count());
this.setTests(this.getTestCases() != null ? this.getTestCases().size() : 0);
@@ -58,7 +74,7 @@ public void writeToFile(BufferedWriter stream) throws IOException {
for (JUnitTestCase testCase : this.getTestCases()) {
Element testCaseEl = doc.createElement("testcase");
testCaseEl.setAttribute("name", testCase.getName());
- testCaseEl.setAttribute("classname", "flatfile." + testCase.getName());
+ testCaseEl.setAttribute("classname", "flatfile." + testCase.getClassName());
for (JUnitFailure failure : testCase.getFailures()) {
Element failEl = doc.createElement("failure");
@@ -70,10 +86,10 @@ public void writeToFile(BufferedWriter stream) throws IOException {
testSuiteEl.appendChild(testCaseEl);
}
}
-
+
testSuitesEl.appendChild(testSuiteEl);
doc.appendChild(testSuiteEl);
-
+
Transformer transformer;
try {
transformer = TransformerFactory.newInstance().newTransformer();
@@ -85,38 +101,65 @@ public void writeToFile(BufferedWriter stream) throws IOException {
throw new RuntimeException("Error writing out xml: " + e.getMessage());
}
}
-
-
+
+
/*
* Adds a new test case. If a test case with the same name already exists within the
* testSuite, the add all the test cases failures to the existing test case
*/
- public void addTestCase(JUnitTestCase_EXT testCase){
+ public void addTestCase(JUnitTestCase testCase){
if(this.getTestCases() == null)
this.setTestCases(new ArrayList());
-
- List existingTestCase = this.getTestCases()
+
+ List existingTestCases = this.getTestCases()
.stream()
.filter(aCase -> aCase.getName().equals(testCase.getName()))
.collect(Collectors.toList());
-
- if(existingTestCase.size() == 1) {
- for(JUnitFailure fail : testCase.getFailures())
- ((JUnitTestCase_EXT) existingTestCase.get(0)).addFailure(fail);
- }else if(existingTestCase.size() == 0){
+
+ JUnitTestCase existingTestCase;
+ if (existingTestCases.size() == 1) {
+ existingTestCase = existingTestCases.get(0);
+ for (JUnitFailure fail : existingTestCase.getFailures()) {
+ Optional existingFailure = Optional.ofNullable(testCase.getFailures())
+ .orElse(Collections.emptyList()).stream()
+ .filter(f -> f.getMessage().equals(fail.getMessage())
+ && f.getValue().equals(fail.getValue())
+ && f.getType().equals(fail.getType()))
+ .findAny();
+
+ if (!existingFailure.isPresent()) {
+ existingTestCase.addFailure(fail);
+ }
+ }
+ } else if (existingTestCases.isEmpty()) {
this.getTestCases().add(testCase);
- }
-
+ } else {
+ throw new IllegalStateException(
+ "More than one test case with the same name already exists!"
+ + testCase.getName());
+ }
+
}
- public void addTestCase(String message, String detail, String testCaseName, String failType) {
+ public void addTestCase(String message, String detail, String className, String testCaseName, String failType) {
JUnitFailure fail = new JUnitFailure();
fail.setMessage(message);
fail.setValue(detail);
fail.setType(failType);
- JUnitTestCase_EXT testCase = new JUnitTestCase_EXT().setName(testCaseName);
- testCase.addFailure(fail);
+ JUnitTestCase testCase = new JUnitTestCase().setName(testCaseName).setClassName(className);
+ Optional existingFailure =
+ Optional.ofNullable(testCase.getFailures()).orElse(Collections.emptyList()).stream()
+ .filter(f -> f.getMessage().equals(fail.getMessage())
+ && f.getValue().equals(fail.getValue())
+ && f.getType().equals(fail.getType()))
+ .findAny();
+
+ if (!existingFailure.isPresent()) {
+ testCase.addFailure(fail);
+ }
+
this.addTestCase(testCase);
}
-
+
}
+
\ No newline at end of file
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Replacement.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Replacement.java
new file mode 100644
index 0000000..056ff51
--- /dev/null
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/Replacement.java
@@ -0,0 +1,31 @@
+package au.gov.digitalhealth.terminology.amtflatfile;
+
+public class Replacement {
+ Concept inactiveConcept;
+ Concept activeConcept;
+ Concept replacementType;
+ String version;
+
+ public Replacement(Concept inactiveConcept, Concept replacementType, Concept activeConcept, String version) {
+ this.inactiveConcept = inactiveConcept;
+ this.activeConcept = activeConcept;
+ this.replacementType = replacementType;
+ this.version = version;
+ }
+
+ public Concept getInactiveConcept() {
+ return inactiveConcept;
+ }
+
+ public Concept getActiveConcept() {
+ return activeConcept;
+ }
+
+ public Concept getReplacementType() {
+ return replacementType;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+}
diff --git a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/TerminologyFileVisitor.java b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/TerminologyFileVisitor.java
index ec415b9..b2dac72 100644
--- a/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/TerminologyFileVisitor.java
+++ b/src/main/java/au/gov/digitalhealth/terminology/amtflatfile/TerminologyFileVisitor.java
@@ -20,6 +20,7 @@ class TerminologyFileVisitor extends SimpleFileVisitor {
private Path conceptFile, relationshipFile, descriptionFile, languageRefsetFile, artgIdRefsetFile;
private List historicalAssociationRefsetFiles = new ArrayList<>();
+ private List AMTRefsetFiles = new ArrayList<>();
private Tika tika = new Tika();
@@ -27,58 +28,94 @@ class TerminologyFileVisitor extends SimpleFileVisitor {
public FileVisitResult visitFile(Path file, BasicFileAttributes attr) throws IOException {
if (attr.isRegularFile()) {
String fileName = file.getFileName().toString();
- if (fileName.matches("sct2_Concept_Snapshot_AU1000036_\\d{8}\\.txt")) {
+ if (fileName.matches("[x]?sct2_Concept_Snapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
conceptFile = file;
}
- } else if (fileName.matches("sct2_Relationship_Snapshot_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?sct2_Relationship_Snapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
relationshipFile = file;
}
- } else if (fileName.matches("sct2_Description_Snapshot-en-AU_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?sct2_Description_Snapshot-en-[Aa][Uu]_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
descriptionFile = file;
}
- } else if (fileName.matches("der2_cRefset_LanguageSnapshot-en-AU_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?der2_cRefset_LanguageSnapshot-en-[Aa][Uu]_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
languageRefsetFile = file;
}
- } else if (fileName.matches("der2_iRefset_ARTGIdSnapshot_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?der2_iRefset_ARTGIdSnapshot_AU1000036_\\d{8}\\.txt") || fileName.matches("[x]?der2_iRefset_SimpleMapSnapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
artgIdRefsetFile = file;
}
- } else if (fileName.matches("der2_cRefset_AssociationReferenceSnapshot_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?der2_cRefset_AssociationReferenceSnapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
historicalAssociationRefsetFiles.add(file);
}
- } else if (fileName.matches("der2_cRefset_AlternativeAssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?der2_cRefset_AlternativeAssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
historicalAssociationRefsetFiles.add(file);
}
- } else if (fileName.matches("der2_cRefset_MovedFromAssociationReferenceSnapshot_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?der2_cRefset_MovedFromAssociationReferenceSnapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
historicalAssociationRefsetFiles.add(file);
}
- } else if (fileName.matches("der2_cRefset_MovedToAssociationReferenceSnapshot_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?der2_cRefset_MovedToAssociationReferenceSnapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
historicalAssociationRefsetFiles.add(file);
}
- } else if (fileName.matches("der2_cRefset_PossiblyEquivalentToAssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?der2_cRefset_PossiblyEquivalentToAssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
historicalAssociationRefsetFiles.add(file);
}
- } else if (fileName.matches("der2_cRefset_ReplacedByAssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?der2_cRefset_ReplacedByAssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
historicalAssociationRefsetFiles.add(file);
}
- } else if (fileName.matches("der2_cRefset_SameAsAssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?der2_cRefset_SameAsAssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
historicalAssociationRefsetFiles.add(file);
}
- } else if (fileName.matches("der2_cRefset_WasAAssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
+ } else if (fileName.matches("[x]?der2_cRefset_WasAAssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
if (verifyFile(file)) {
historicalAssociationRefsetFiles.add(file);
}
+ } else if (fileName.matches("[x]?der2_cRefset_AssociationSnapshot_AU1000036_\\d{8}\\.txt")) {
+ if (verifyFile(file)) {
+ historicalAssociationRefsetFiles.add(file);
+ }
+ } else if (fileName.matches("[x]?der2_Refset_SimpleSnapshot_AU1000036_\\d{8}\\.txt")) {
+ if (verifyFile(file)) {
+ AMTRefsetFiles.add(file);
+ }
+ } else if (fileName.matches("[x]?der2_Refset_MedicinalProductPackSnapshot_AU1000036_\\d{8}\\.txt")) {
+ if (verifyFile(file)) {
+ AMTRefsetFiles.add(file);
+ }
+ } else if (fileName.matches("[x]?der2_Refset_MedicinalProductSnapshot_AU1000036_\\d{8}\\.txt")) {
+ if (verifyFile(file)) {
+ AMTRefsetFiles.add(file);
+ }
+ } else if (fileName.matches("[x]?der2_Refset_MedicinalProductUnitOfUseSnapshot_AU1000036_\\d{8}\\.txt")) {
+ if (verifyFile(file)) {
+ AMTRefsetFiles.add(file);
+ }
+ } else if (fileName.matches("[x]?der2_Refset_ContaineredTradeProductPackSnapshot_AU1000036_\\d{8}\\.txt")) {
+ if (verifyFile(file)) {
+ AMTRefsetFiles.add(file);
+ }
+ } else if (fileName.matches("[x]?der2_Refset_TradeProductPackSnapshot_AU1000036_\\d{8}\\.txt")) {
+ if (verifyFile(file)) {
+ AMTRefsetFiles.add(file);
+ }
+ } else if (fileName.matches("[x]?der2_Refset_TradeProductSnapshot_AU1000036_\\d{8}\\.txt")) {
+ if (verifyFile(file)) {
+ AMTRefsetFiles.add(file);
+ }
+ } else if (fileName.matches("[x]?der2_Refset_TradeProductUnitOfUseSnapshot_AU1000036_\\d{8}\\.txt")) {
+ if (verifyFile(file)) {
+ AMTRefsetFiles.add(file);
+ }
}
}
return FileVisitResult.CONTINUE;
@@ -118,6 +155,11 @@ public Path getArtgIdRefsetFile() {
return artgIdRefsetFile;
}
+ public List getAMTRefsetFiles() {
+ return AMTRefsetFiles;
+ }
+
+
public List getHistoricalAssociationRefsetFiles() {
return historicalAssociationRefsetFiles;
}
diff --git a/src/test/java/au/gov/digitalhealth/JUnitFileParser.java b/src/test/java/au/gov/digitalhealth/JUnitFileParser.java
new file mode 100644
index 0000000..9bbc12a
--- /dev/null
+++ b/src/test/java/au/gov/digitalhealth/JUnitFileParser.java
@@ -0,0 +1,62 @@
+package au.gov.digitalhealth;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.DocumentBuilder;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+import au.gov.digitalhealth.terminology.amtflatfile.Junit.JUnitFailure;
+import au.gov.digitalhealth.terminology.amtflatfile.Junit.JUnitTestCase;
+import org.w3c.dom.Node;
+import org.w3c.dom.Element;
+
+public class JUnitFileParser {
+
+ public static List parse(File junitXmlFile) throws IOException {
+
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder;
+ Document doc = null;
+ try {
+ dBuilder = dbFactory.newDocumentBuilder();
+ doc = dBuilder.parse(junitXmlFile);
+ doc.getDocumentElement().normalize();
+ } catch (ParserConfigurationException | SAXException e) {
+ throw new IOException("Failed to parse JUnit XML", e);
+ }
+ //Get the root element
+ NodeList nList = doc.getElementsByTagName("testcase");
+
+ List testCases = new ArrayList<>();
+
+ for(int i=0; i failures = new ArrayList<>();
+ NodeList failureNodes = element.getElementsByTagName("failure");
+ for (int j=0; j testCases = JUnitFileParser.parse(
+ new File("target/JUnitContainsCorrectErrors.xml"));
+ List fails = new ArrayList<>();
+ testCases.forEach(testCase -> {
+ List failures = testCase.getFailures();
+ fails.addAll(
+ failures.stream()
+ .map(fail -> fail.getValue())
+ .collect(Collectors.toList()));
+ });
+ Assert.assertTrue(fails.stream().anyMatch(fail -> fail.contains("1212261000168108")));
+ Assert.assertTrue(fails.stream().anyMatch(fail -> fail.contains("1209811000168100")));
+ Assert.assertEquals(fails.size(), 10);
+ }
+
+
+ @Test(groups = "files", priority = 1, description = "An exception is thrown when the provided input zip file doesn't exist", expectedExceptions = IllegalArgumentException.class)
+ public void fileDoesNotExistThrowException()
+ throws MojoExecutionException, MojoFailureException, IOException {
+
+ Amt2FlatFile amt2FlatFile = new Amt2FlatFile();
+ amt2FlatFile.setInputZipFilePath(testResDirectory + "nonExistent.zip");
+ amt2FlatFile.setOutputFilePath(outFilePath);
+ amt2FlatFile.execute();
+
+ }
+
+ @Test(groups = "files", priority = 1, description = "An exception is thrown when the provided input zip file is missing critical files", expectedExceptions = RuntimeException.class)
+ public void fileMissingFromReleaseExceptionThrown()
+ throws MojoExecutionException, MojoFailureException, IOException {
+
+ Amt2FlatFile amt2FlatFile = new Amt2FlatFile();
+ amt2FlatFile.setInputZipFilePath(testResDirectory + "incomplete.zip");
+ amt2FlatFile.setOutputFilePath(outFilePath);
+ amt2FlatFile.execute();
+ }
+
+ @Test(groups = "parse", priority = 2, description = "The output file should match the expected result. Test line by line, regardless fo order")
+ public void outputMatchesExpectedImproved()
+ throws MojoExecutionException, MojoFailureException, IOException {
+
+ Amt2FlatFile amt2FlatFile = new Amt2FlatFile();
+ String inFile =
+ testResDirectory + "NCTS_SCT_RF2_DISTRIBUTION_32506021000036107-20180430-SNAPSHOT.zip";
+ amt2FlatFile.setInputZipFilePath(inFile);
+ amt2FlatFile.setOutputFilePath(outFilePath);
+ String replacementFilePath = "target/test-out/replacement";
+ amt2FlatFile.setReplacementsFilePath(replacementFilePath);
+ amt2FlatFile.execute();
+
+ checkFile(outFilePath, "expected");
+ checkFile(replacementFilePath, "expectedReplacement");
+ }
+
+ private void checkFile(String outFilePath, String expected) throws IOException {
+ assertDosLineTermination(FileFormat.CSV.getFilePath(outFilePath));
+ assertDosLineTermination(FileFormat.TSV.getFilePath(outFilePath));
+
+ List outFileLinesCSV =
+ FileUtils.readLines(new File(FileFormat.CSV.getFilePath(outFilePath)), "UTF-8");
+ List outFileLinesTSV =
+ FileUtils.readLines(new File(FileFormat.TSV.getFilePath(outFilePath)), "UTF-8");
+
+ String expectedFilePath = testResDirectory + expected;
+ List expectedLinesCSV = FileUtils
+ .readLines(new File(FileFormat.CSV.getFilePath(expectedFilePath)), "UTF-8");
+ List expectedLinesTSV = FileUtils
+ .readLines(new File(FileFormat.TSV.getFilePath(expectedFilePath)), "UTF-8");
+ Collections.sort(outFileLinesCSV);
+ Collections.sort(outFileLinesTSV);
+ Collections.sort(expectedLinesCSV);
+ Collections.sort(expectedLinesTSV);
+
+ checkFiles(outFileLinesCSV, expectedLinesCSV, FileFormat.CSV);
+ checkFiles(outFileLinesTSV, expectedLinesTSV, FileFormat.TSV);
+ }
+
+ private void checkFiles(List outFileLines, List expectedLines,
+ FileFormat format) {
+ // Assert and explain TSV differences
+ if (!outFileLines.equals(expectedLines)) {
+ for (int i = 0; i < Math.min(outFileLines.size(), expectedLines.size()); i++) {
+ if (!outFileLines.get(i).equals(expectedLines.get(i))) {
+ fail("Difference found in " + format.toString() + " file at line " + (i + 1)
+ + ": expected '" + expectedLines.get(i) + "' but was '"
+ + outFileLines.get(i) + "'");
}
- }
-
- @Test(groups="files", priority = 1, description = "Tests that the JUnit file gets generated")
- public void JUnitGenerated() throws MojoExecutionException, MojoFailureException, IOException {
- Amt2FlatFile amt2FlatFile = new Amt2FlatFile();
- amt2FlatFile.setInputZipFilePath("target/test-classes/rf2-fails-flat-file-generation-1.0.zip");
- amt2FlatFile.setOutputFilePath(outFile);
- amt2FlatFile.execute();
- File validXml = new File("target/ValidationErrors.xml");
- Assert.assertTrue(validXml.exists());
- }
-
- @Test(groups="files", priority = 1, description = "Tests that the the correct errors are reported by the JUnit xml")
- public void JUnitContainsCorrectErrors() throws MojoExecutionException, MojoFailureException, IOException, JAXBException, XMLStreamException {
- Amt2FlatFile amt2FlatFile = new Amt2FlatFile();
- amt2FlatFile.setInputZipFilePath("target/test-classes/rf2-fails-flat-file-generation-1.0.zip");
- amt2FlatFile.setOutputFilePath(outFile);
- amt2FlatFile.setJunitFilePath("target/JUnitContainsCorrectErrors.xml");
- amt2FlatFile.execute();
- File validXml = new File("target/JUnitContainsCorrectErrors.xml");
- JUnitTestSuite info = JUnitMarshalling.unmarshalTestSuite(new FileInputStream(validXml));
- List