Merge branch 'main' into azure-load-tests

app/build.gradle

@@ -23,12 +23,12 @@ dependencies {
     implementation project(':etor')
     testImplementation testFixtures(project(':shared'))
 
-    implementation 'io.javalin:javalin:6.3.0'
+    implementation 'io.javalin:javalin:6.4.0'
 
     testImplementation 'org.apache.groovy:groovy:4.0.24'
     testImplementation 'org.spockframework:spock-core:2.3-groovy-4.0'
     testImplementation 'com.openpojo:openpojo:0.9.1'
-    testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.17.5'
+    testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.18'
 }
 
 jacocoTestCoverageVerification {

app/src/main/java/gov/hhs/cdc/trustedintermediary/external/javalin/App.java

@@ -48,7 +48,12 @@ public static void main(String[] args) {
 
         // apply this security header to all responses, but allow it to be overwritten by a specific
         // endpoint by using `before` if needed
-        app.before(ctx -> ctx.header("X-Content-Type-Options", "nosniff"));
+        app.before(
+                ctx -> {
+                    ctx.header("X-Content-Type-Options", "nosniff");
+                    // Fix for https://www.zaproxy.org/docs/alerts/90004
+                    ctx.header("Cross-Origin-Resource-Policy", "cross-origin");
+                });
 
         try {
             app.get(HEALTH_API_ENDPOINT, ctx -> ctx.result("Operational"));

e2e/build.gradle

@@ -19,10 +19,10 @@ dependencies {
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2'
 
     //fhir
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-base:7.6.0'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:7.6.0'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-caching-caffeine:7.6.0'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:7.6.0'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-base:7.6.1'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:7.6.1'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-caching-caffeine:7.6.1'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:7.6.1'
     implementation 'org.fhir:ucum:1.0.9'
 
     testImplementation 'org.apache.groovy:groovy:4.0.24'

etor/build.gradle

@@ -19,7 +19,7 @@ dependencies {
     testImplementation 'org.apache.groovy:groovy:4.0.24'
     testImplementation 'org.spockframework:spock-core:2.3-groovy-4.0'
     testImplementation 'com.openpojo:openpojo:0.9.1'
-    testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.17.5'
+    testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.18'
 }
 
 jacocoTestCoverageVerification {

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/CopyOrcOrderProviderToObrOrderProvider.java

@@ -28,16 +28,8 @@ public void transform(HealthData<?> resource, Map<String, Object> args) {
             return;
         }
 
-        // Extract or create the OBR-16 extension from the ServiceRequest
         Extension obrExtension =
                 HapiHelper.ensureExtensionExists(serviceRequest, HapiHelper.EXTENSION_OBR_URL);
-
-        // Extract or create the OBR-16 data type extension
-        Extension obr16Extension =
-                HapiHelper.ensureSubExtensionExists(
-                        obrExtension, HapiHelper.EXTENSION_OBR16_DATA_TYPE.toString());
-
-        // Set the ORC-12 Practitioner in the OBR-16 extension
-        HapiHelper.setOBR16WithPractitioner(obr16Extension, practitionerRole);
+        HapiHelper.setOBR16WithPractitioner(obrExtension, practitionerRole);
     }
 }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/RemovePatientIdentifiers.java

@@ -15,7 +15,8 @@ public class RemovePatientIdentifiers implements CustomFhirTransformation {
     @Override
     public void transform(HealthData<?> resource, Map<String, Object> args) {
         Bundle bundle = (Bundle) resource.getUnderlyingData();
-        HapiHelper.setPID3_4Value(bundle, ""); // remove PID.3-4
-        HapiHelper.setPID3_5Value(bundle, ""); // remove PID.3-5
+
+        HapiHelper.setPID3_4Value(bundle, null);
+        HapiHelper.removePID3_5Value(bundle);
     }
 }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/RemovePatientNameTypeCode.java

@@ -12,8 +12,6 @@ public class RemovePatientNameTypeCode implements CustomFhirTransformation {
     @Override
     public void transform(final HealthData<?> resource, final Map<String, Object> args) {
         Bundle bundle = (Bundle) resource.getUnderlyingData();
-        // Need to set the value for extension to empty instead of removing the extension,
-        // otherwise RS will set its own value in its place
-        HapiHelper.setPID5_7ExtensionValue(bundle, null);
+        HapiHelper.removePID5_7Value(bundle);
     }
 }

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/CopyOrcOrderProviderToObrOrderProviderTest.groovy

@@ -7,8 +7,6 @@ import gov.hhs.cdc.trustedintermediary.external.hapi.HapiHelper
 import gov.hhs.cdc.trustedintermediary.wrappers.MetricMetadata
 import org.hl7.fhir.r4.model.Bundle
 import org.hl7.fhir.r4.model.DiagnosticReport
-import org.hl7.fhir.r4.model.Practitioner
-import org.hl7.fhir.r4.model.Reference
 import org.hl7.fhir.r4.model.ServiceRequest
 import spock.lang.Specification
 
@@ -119,7 +117,7 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
         evaluateOrc12Values(bundle, EXPECTED_NPI, EXPECTED_FIRST_NAME, EXPECTED_LAST_NAME, EXPECTED_NAME_TYPE_CODE, EXPECTED_IDENTIFIER_TYPE_CODE)
 
         // OBR16 should not exist initially
-        def obr16Practitioner = getObr16ExtensionPractitioner(serviceRequest)
+        def obr16Practitioner = HapiHelper.getObr16ExtensionPractitioner(serviceRequest)
         obr16Practitioner == null
 
         when:
@@ -191,7 +189,7 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
     }
 
     void evaluateOrc12IsNull(Bundle bundle) {
-        assert getOrc12ExtensionPractitioner(bundle) == null
+        assert HapiHelper.getOrc12ExtensionPractitioner(bundle) == null
     }
 
     void evaluateOrc12Values(
@@ -201,7 +199,7 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
             String expectedLastName,
             String expectedNameTypeCode,
             String expectedIdentifierTypeCode) {
-        def practitioner = getOrc12ExtensionPractitioner(bundle)
+        def practitioner = HapiHelper.getOrc12ExtensionPractitioner(bundle)
         def xcnExtension = practitioner.getExtensionByUrl(PRACTITIONER_EXTENSION_URL)
 
         assert practitioner.identifier[0]?.value == expectedNpi
@@ -214,7 +212,8 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
     }
 
     void evaluateObr16IsNull(ServiceRequest serviceRequest) {
-        assert getObr16ExtensionPractitioner(serviceRequest) == null
+        assert HapiHelper.getObr16Extension(serviceRequest) == null
+        assert HapiHelper.getObr16ExtensionPractitioner(serviceRequest) == null
     }
 
     void evaluateObr16Values(
@@ -224,7 +223,7 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
             String expectedLastName,
             String expectedNameTypeCode,
             String expectedIdentifierTypeCode) {
-        def practitioner = getObr16ExtensionPractitioner(serviceRequest)
+        def practitioner = HapiHelper.getObr16ExtensionPractitioner(serviceRequest)
         def xcnExtension = practitioner.getExtensionByUrl(PRACTITIONER_EXTENSION_URL)
 
         assert practitioner.identifier[0]?.value == expectedNpi
@@ -234,40 +233,4 @@ class CopyOrcOrderProviderToObrOrderProviderTest extends Specification{
         def codingSystem = practitioner.identifier[0]?.type?.coding
         assert codingSystem == null || codingSystem[0]?.code == expectedIdentifierTypeCode
     }
-
-    Practitioner getObr16ExtensionPractitioner (serviceRequest) {
-        def resource
-        try {
-            def extensionByUrl1 =  serviceRequest.getExtensionByUrl(HapiHelper.EXTENSION_OBR_URL)
-            def extensionByUrl2 = extensionByUrl1.getExtensionByUrl(HapiHelper.EXTENSION_OBR16_DATA_TYPE.toString())
-            def value = extensionByUrl2.value
-            resource = value.getResource()
-            return resource
-        } catch(Exception ignored) {
-            resource = null
-            return resource
-        }
-    }
-
-    Practitioner getOrc12ExtensionPractitioner(Bundle bundle) {
-        def diagnosticReport = HapiHelper.getDiagnosticReport(bundle)
-        def serviceRequest = HapiHelper.getServiceRequest(diagnosticReport)
-
-        def orcExtension = serviceRequest.getExtensionByUrl(HapiHelper.EXTENSION_ORC_URL)
-        def orc12Extension = orcExtension.getExtensionByUrl(HapiHelper.EXTENSION_ORC12_URL)
-
-        if (orc12Extension == null) {
-            return null
-        }
-
-        def practitionerReference = (Reference) orc12Extension.getValue()
-        def practitionerUrl = practitionerReference.getReference()
-
-        for (Bundle.BundleEntryComponent entry : bundle.getEntry()) {
-            if (Objects.equals(entry.getFullUrl(), practitionerUrl) && entry.getResource() instanceof Practitioner)
-                return (Practitioner) entry.getResource()
-        }
-
-        return null
-    }
 }

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/RemovePatientIdentifierTest.groovy

@@ -33,10 +33,10 @@ class RemovePatientIdentifierTest extends Specification {
 
         when:
         transformClass.transform(fhirResource, null)
-        def actualPid3_4 = HapiFhirHelper.getPID3_4Value(bundle)
-        def actualPid3_5 = HapiFhirHelper.getPID3_5Value(bundle)
 
         then:
+        def actualPid3_4 = HapiFhirHelper.getPID3_4Value(bundle)
+        def actualPid3_5 = HapiFhirHelper.getPID3_5Value(bundle)
         actualPid3_4 == null || actualPid3_4.isEmpty()
         actualPid3_5 == null || actualPid3_5.isEmpty()
     }

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/ruleengine/transformation/custom/RemovePatientNameTypeCodeTest.groovy

@@ -22,18 +22,22 @@ class RemovePatientNameTypeCodeTest extends Specification {
 
     def "remove PID.5-7 from Bundle"() {
         given:
-        def fhirResource = ExamplesHelper.getExampleFhirResource("../CA/002_CA_ORU_R01_initial_translation.fhir")
+        def fhirResource = ExamplesHelper.getExampleFhirResource("../CA/007_CA_ORU_R01_CDPH_produced_UCSD2024-07-11-16-02-17-749_1_hl7_translation.fhir")
         def bundle = fhirResource.getUnderlyingData() as Bundle
-        def pid5_7 = HapiFhirHelper.getPID5_7Value(bundle)
+        def pid5Extension = HapiHelper.getPID5Extension(bundle)
+        def patientName = HapiHelper.getPIDPatient(bundle).getNameFirstRep()
+        def patientNameUse = patientName.getUse()
 
         expect:
-        pid5_7 != null
+        pid5Extension.getExtensionByUrl(HapiHelper.EXTENSION_XPN7_URL) != null
+        patientNameUse.toString() == "OFFICIAL"
 
         when:
         transformClass.transform(fhirResource, null)
 
         then:
-        HapiFhirHelper.getPID5_7Value(bundle) == null
+        pid5Extension.getExtensionByUrl(HapiHelper.EXTENSION_XPN7_URL) == null
+        !patientName.hasUse()
     }
 
     def "don't throw exception if patient resource not present"() {

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -86,8 +86,7 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
-' "$PWD" ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum

operations/template/logs.tf

@@ -47,14 +47,24 @@ resource "azurerm_log_analytics_query_pack" "application_logs_pack" {
   }
 }
 
-resource "azurerm_log_analytics_query_pack_query" "application_logs" {
-  display_name = "TI's Raw Application Logs"
-  description  = "View all TI's application logs in a structured format"
+resource "azurerm_log_analytics_query_pack_query" "live_application_logs" {
+  display_name = "TI's Live Slot Raw Application Logs"
+  description  = "View all TI's live slot application logs in a structured format"
 
   query_pack_id = azurerm_log_analytics_query_pack.application_logs_pack.id
   categories    = ["applications"]
 
-  body = "AppServiceConsoleLogs | project JsonResult = parse_json(ResultDescription) | evaluate bag_unpack(JsonResult) | project-reorder ['@timestamp'], level, message"
+  body = "AppServiceConsoleLogs | where _ResourceId !contains 'pre-live' | project JsonResult = parse_json(ResultDescription) | evaluate bag_unpack(JsonResult) | project-reorder ['@timestamp'], level, message"
+}
+
+resource "azurerm_log_analytics_query_pack_query" "prelive_application_logs" {
+  display_name = "TI's Pre-Live Slot Raw Application Logs"
+  description  = "View all TI's pre-live slot application logs in a structured format"
+
+  query_pack_id = azurerm_log_analytics_query_pack.application_logs_pack.id
+  categories    = ["applications"]
+
+  body = "AppServiceConsoleLogs | where _ResourceId contains 'pre-live' | project JsonResult = parse_json(ResultDescription) | evaluate bag_unpack(JsonResult) | project-reorder ['@timestamp'], level, message"
 }
 
 resource "azurerm_log_analytics_query_pack_query" "application_error_logs" {

rs-e2e/README.md

@@ -10,6 +10,7 @@ Information on how to set up the sample files evaluated by the tests can be foun
 
 - The output files generated by the framework are stored in an Azure blob storage container. Every time the tests are run, a cleanup task moves the files to a folder with the year/month/day format for better organization. The files are retained in the container for 90 days before being deleted
 - The code that organizes the files is using EST time zone. This means that if you are in PST, you may run into an issue if you submit a run before 9 PM PST and then run the tests after 9pm. You'd need to make sure to run both tasks before or after 9pm so the files are where they are expected to be
+- The HL7 parser and expression evaluator returns an empty string when the value is not found. It will only throw an exception if the path is not a valid HL7 notation
 
 ## Running the tests
 

rs-e2e/build.gradle

@@ -20,10 +20,6 @@ dependencies {
     // azure
     implementation 'com.azure:azure-storage-blob:12.29.0'
 
-    // hapi hl7
-    implementation 'ca.uhn.hapi:hapi-base:2.5.1'
-    implementation 'ca.uhn.hapi:hapi-structures-v251:2.5.1'
-
     testImplementation 'org.apache.groovy:groovy:4.0.24'
     testImplementation 'org.spockframework:spock-core:2.3-groovy-4.0'
 }

rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/FileFetcher.java

@@ -1,5 +1,6 @@
 package gov.hhs.cdc.trustedintermediary.rse2e;
 
+import gov.hhs.cdc.trustedintermediary.rse2e.hl7.HL7FileStream;
 import java.util.List;
 
 /**

rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobFileFetcher.java → rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobFileFetcher.java

@@ -1,10 +1,12 @@
-package gov.hhs.cdc.trustedintermediary.rse2e;
+package gov.hhs.cdc.trustedintermediary.rse2e.external.azure;
 
 import com.azure.storage.blob.BlobClient;
 import com.azure.storage.blob.BlobContainerClient;
 import com.azure.storage.blob.BlobContainerClientBuilder;
 import com.azure.storage.blob.models.BlobItem;
 import com.azure.storage.blob.models.ListBlobsOptions;
+import gov.hhs.cdc.trustedintermediary.rse2e.FileFetcher;
+import gov.hhs.cdc.trustedintermediary.rse2e.hl7.HL7FileStream;
 import java.time.LocalDate;
 import java.time.ZoneId;
 import java.time.ZoneOffset;

rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobHelper.java → rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobHelper.java

@@ -1,4 +1,4 @@
-package gov.hhs.cdc.trustedintermediary.rse2e;
+package gov.hhs.cdc.trustedintermediary.rse2e.external.azure;
 
 import java.time.LocalDate;
 

rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobOrganizer.java → rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/external/azure/AzureBlobOrganizer.java

@@ -1,4 +1,4 @@
-package gov.hhs.cdc.trustedintermediary.rse2e;
+package gov.hhs.cdc.trustedintermediary.rse2e.external.azure;
 
 import com.azure.storage.blob.BlobClient;
 import com.azure.storage.blob.BlobContainerClient;