Merge branch 'main' into feature/engagement/apham/16852

.github/ISSUE_TEMPLATE/platform-user-story.md

@@ -21,34 +21,30 @@ so that _[Outcome - what is the value add to the user]_."
 
 ### Description/Use Case
 <!--
-_Use this section to describe the 'Why', and/or provide an example scenario in which this feature/functionality would be valueable._
+Use this section to describe the 'Why', and/or provide an example scenario in which this feature/functionality would be valueable.
 -->
 
 ### Risks/Impacts/Considerations
 <!--
-_Use this section to briefly list out any risks/impacts that may come about, as a result of the proposed solution._
-- _System performance may be slowed_
-- _Only a single search parameter can be used_
+Use this section to briefly list out any risks/impacts that may come about, as a result of the proposed solution.
+- _[System performance may be slowed]_
+- _[Only a single search parameter can be used]_
 -->
 
 ### Dev Notes
 <!--
-_Use this section to describe any useful technical information to duplicate an issue or explain requirements related to this ticket without providing implementation details._
-- _Provided is the data that was used to replicate the issue..._
-- _To test, use SimpleReport upload CSV page to ..._
+Use this section to describe any useful technical information to duplicate an issue or explain requirements related to this ticket without providing implementation details.
+- _[Provided is the data that was used to replicate the issue...]_
+- _[To test, use SimpleReport upload CSV page to ...]_
 -->
 
 ### Acceptance Criteria
 <!--
 What is Acceptance Criteria?
 A set of conditions or business rules, as defined by the Product Owner, which the functionality or feature should satisfy, in order to be accepted by the Product Owner.
 
-Use the following template when creating new Acceptance Criteria:
+Write the acceptance criteria as a list that can be checked off as work progresses. For example:
 
-"Given _[describe the precondition]_, when I _[describe the action performed]_, then I expect _[describe the expected outcome]_."
-
-_OR... it may be written as a bulleted list._
-
-- _Time must be displayed as HH:MM:SS_
-- _Delivery rate must be shown as a percentage_
+- [ ] _[Time must be displayed as HH:MM:SS]_
+- [ ] _[Delivery rate must be shown as a percentage]_
 -->

.github/workflows/alert_MBUsers_Inactive.yml

@@ -40,7 +40,6 @@ jobs:
           echo -e "$messageInactiveUsers">> $GITHUB_OUTPUT
           echo "$EOF" >> $GITHUB_OUTPUT
 
-     
       - name: Slack Notification
         uses: ./.github/actions/notifications
         with:
@@ -49,8 +48,7 @@ jobs:
           message: |
             ${{ steps.users_out.outputs.MESSAGE_RESPONSE }}
           icon-emoji: ':hourglass_flowing_sand:'
-          channel: pagerduty-alert-dump
-          webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
+          channel: temp-cdc-rs-notifications  # Updated channel
+          webhook-url: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK_URL }}  # Updated webhook secret
           color: warning
           slackify-markdown: true
-

.github/workflows/alert_cert_expire.yml

@@ -61,8 +61,8 @@ jobs:
           message: |
             ${{ steps.format_out.outputs.LIST }}
           icon-emoji: ':bell:'
-          channel: pagerduty-alert-dump
-          webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
+          channel: temp-cdc-rs-notifications  # Updated channel
+          webhook-url: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK_URL }}  # Updated webhook secret
           color: warning
 
       - name: Remove Runner IP from the KeyVault Firewalls

.github/workflows/alert_resource_costs.yml

@@ -65,7 +65,7 @@ jobs:
             * **Provisioned by: \`${{ steps.last-pusher.outputs.username }}\`**
             * **Last Change Date: \`${{ steps.last-pusher.outputs.last_change_date }}\`**
           icon-emoji: ':money_with_wings:'
-          channel: prime-reportstream-engineering
-          webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
+          channel: temp-cdc-rs-notifications  # Updated channel
+          webhook-url: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK_URL }}  # Updated webhook secret
           color: failure
           slackify-markdown: true

.github/workflows/alert_terraform_changes.yml

@@ -32,7 +32,6 @@ jobs:
           tf-auth: true
 
       - name: Collect Terraform stats
-
         uses: josiahsiegel/terraform-stats@68b8cbe42c494333fbf6f8d90ac86da1fb69dcc2
         ## DevSecOps - Aquia (Replace) - uses: ./.github/actions/terraform-stats
 
@@ -66,6 +65,6 @@ jobs:
             "resource-drifts": "${{ env.resource-drifts }}"
 
           icon-emoji: ':bell:'
-          channel: pagerduty-alert-dump
-          webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
+          channel: temp-cdc-rs-notifications  # Updated channel
+          webhook-url: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK_URL }}  # Updated webhook secret
           color: warning

prime-router/settings/STLTs/WA/wa-phd.yml

@@ -90,6 +90,7 @@
       reverseTheQualityFilter: false
       conditionFilter:
         #Accept COVID only, "matches(abnormal_flag, A)"
+        #Verified that Covid only, "matches(abnormal_flag, A)" Abi Philip Jan 6 2025
         - "%resource.interpretation.coding.code = 'A' and (%resource.code.coding.extension('https://reportstream.cdc.gov/fhir/StructureDefinition/condition-code').value.where(code  in ('840539006')).exists())"
       mappedConditionFilter: []
       deidentify: false

prime-router/src/main/kotlin/Receiver.kt

@@ -3,6 +3,7 @@ package gov.cdc.prime.router
 import com.fasterxml.jackson.annotation.JsonIgnore
 import gov.cdc.prime.router.azure.BlobAccess
 import gov.cdc.prime.router.common.DateUtilities
+import gov.cdc.prime.router.common.Environment
 import gov.cdc.prime.router.fhirengine.translation.hl7.FhirToHl7Converter
 import gov.cdc.prime.router.fhirengine.translation.hl7.SchemaException
 import java.time.LocalTime
@@ -249,6 +250,7 @@ open class Receiver(
      * Validate the object and return null or an error message
      */
     fun consistencyErrorMessage(metadata: Metadata): String? {
+        // TODO: The logic in this method is slated to be removed as part of #17020
         if (conditionFilter.isNotEmpty() || mappedConditionFilter.isNotEmpty()) {
             if (!topic.isUniversalPipeline) {
                 return "Condition filter(s) not allowed for receivers with topic '${topic.jsonVal}'"
@@ -258,8 +260,13 @@ open class Receiver(
         if (translation is CustomConfiguration) {
             if (this.topic.isUniversalPipeline) {
                 try {
-                    // This is already scheduled for deletion in https://github.com/CDCgov/prime-reportstream/pull/13313
-                    FhirToHl7Converter(translation.schemaName, BlobAccess.defaultBlobMetadata)
+                    FhirToHl7Converter(
+                        translation.schemaName,
+                        BlobAccess.BlobContainerMetadata.build(
+                            "metadata",
+                            Environment.get().storageEnvVar
+                        )
+                    )
                 } catch (e: SchemaException) {
                     return e.message
                 }

prime-router/src/main/kotlin/SettingsProvider.kt

@@ -5,7 +5,6 @@ import com.fasterxml.jackson.annotation.JsonValue
 import gov.cdc.prime.router.CustomerStatus.ACTIVE
 import gov.cdc.prime.router.CustomerStatus.INACTIVE
 import gov.cdc.prime.router.CustomerStatus.TESTING
-import gov.cdc.prime.router.fhirengine.utils.HL7Reader
 import gov.cdc.prime.router.validation.IItemValidator
 import gov.cdc.prime.router.validation.MarsOtcElrOnboardingValidator
 import gov.cdc.prime.router.validation.MarsOtcElrValidator
@@ -55,7 +54,6 @@ enum class Topic(
     val isUniversalPipeline: Boolean = true,
     val isSendOriginal: Boolean = false,
     val validator: IItemValidator = NoopItemValidator(),
-    val hl7ParseConfiguration: HL7Reader.Companion.HL7MessageParseAndConvertConfiguration? = null,
 ) {
     FULL_ELR("full-elr", true, false),
     ETOR_TI("etor-ti", true, false),

prime-router/src/main/kotlin/SubmissionReceiver.kt

@@ -9,8 +9,8 @@ import gov.cdc.prime.router.azure.ReportWriter
 import gov.cdc.prime.router.azure.WorkflowEngine
 import gov.cdc.prime.router.azure.db.enums.TaskAction
 import gov.cdc.prime.router.fhirengine.engine.FhirConvertQueueMessage
-import gov.cdc.prime.router.fhirengine.engine.MessageType
 import gov.cdc.prime.router.fhirengine.utils.FhirTranscoder
+import gov.cdc.prime.router.fhirengine.utils.HL7MessageHelpers
 import gov.cdc.prime.router.fhirengine.utils.HL7Reader
 
 /**
@@ -268,14 +268,14 @@ class UniversalPipelineReceiver : SubmissionReceiver {
 
         when (sender.format) {
             MimeFormat.HL7 -> {
-                val messages = HL7Reader(actionLogs).getMessages(content)
-                val isBatch = HL7Reader(actionLogs).isBatch(content, messages.size)
-                // create a Report for this incoming HL7 message to use for tracking in the database
+                val messageCount = HL7MessageHelpers.messageCount(content)
+                val isBatch = HL7Reader.isBatch(content, messageCount)
 
+                // create a Report for this incoming HL7 message to use for tracking in the database
                 report = Report(
                     if (isBatch) MimeFormat.HL7_BATCH else MimeFormat.HL7,
                     sources,
-                    messages.size,
+                    messageCount,
                     metadata = metadata,
                     nextAction = TaskAction.convert,
                     topic = sender.topic,
@@ -290,11 +290,8 @@ class UniversalPipelineReceiver : SubmissionReceiver {
 //                        actionLogs
 //                    )
 //                }
-
-                // check for valid message type
-                messages.forEachIndexed {
-                    idx, element ->
-                    MessageType.validateMessageType(element, actionLogs, idx + 1)
+                if (messageCount == 0 && !actionLogs.hasErrors()) {
+                    actionLogs.error(InvalidReportMessage("Unable to find HL7 messages in provided data."))
                 }
             }
 

prime-router/src/main/kotlin/azure/service/SubmissionResponseBuilder.kt

@@ -5,12 +5,12 @@ import com.google.common.net.HttpHeaders
 import com.microsoft.azure.functions.HttpRequestMessage
 import com.microsoft.azure.functions.HttpResponseMessage
 import com.microsoft.azure.functions.HttpStatus
-import gov.cdc.prime.router.ActionLogger
 import gov.cdc.prime.router.Sender
 import gov.cdc.prime.router.azure.HttpUtilities
 import gov.cdc.prime.router.azure.HttpUtilities.Companion.isSuccessful
 import gov.cdc.prime.router.common.JacksonMapperUtilities
 import gov.cdc.prime.router.fhirengine.translation.hl7.utils.HL7ACKUtils
+import gov.cdc.prime.router.fhirengine.utils.HL7MessageHelpers
 import gov.cdc.prime.router.fhirengine.utils.HL7Reader
 import gov.cdc.prime.router.history.DetailedSubmissionHistory
 import org.apache.logging.log4j.kotlin.Logging
@@ -106,12 +106,11 @@ class SubmissionResponseBuilder(
             contentType == HttpUtilities.hl7V2MediaType &&
             requestBody != null
         ) {
-            val hl7Reader = HL7Reader(ActionLogger())
-            val messages = hl7Reader.getMessages(requestBody)
-            val isBatch = hl7Reader.isBatch(requestBody, messages.size)
+            val messageCount = HL7MessageHelpers.messageCount(requestBody)
+            val isBatch = HL7Reader.isBatch(requestBody, messageCount)
 
-            if (!isBatch && messages.size == 1) {
-                val message = messages.first()
+            if (!isBatch && messageCount == 1) {
+                val message = HL7Reader.parseHL7Message(requestBody)
                 val acceptAcknowledgementType = HL7Reader.getAcceptAcknowledgmentType(message)
                 val ackResponseRequired = acceptAcknowledgmentTypeRespondValues.contains(acceptAcknowledgementType)
                 if (ackResponseRequired) {

prime-router/src/main/kotlin/cli/ProcessFhirCommands.kt

@@ -223,8 +223,8 @@ class ProcessFhirCommands : CliktCommand(
                 (isCli && outputFormat == MimeFormat.HL7.toString()) ||
                     (
                         receiver != null &&
-                        (receiver.format == MimeFormat.HL7 || receiver.format == MimeFormat.HL7_BATCH)
-                    )
+                            (receiver.format == MimeFormat.HL7 || receiver.format == MimeFormat.HL7_BATCH)
+                        )
                 ) -> {
                 val (bundle2, inputMessage) = convertHl7ToFhir(contents, receiver)
 
@@ -297,7 +297,7 @@ class ProcessFhirCommands : CliktCommand(
         }
     }
 
-        private fun evaluateReceiverFilters(receiver: Receiver?, messageOrBundle: MessageOrBundle, isCli: Boolean) {
+    private fun evaluateReceiverFilters(receiver: Receiver?, messageOrBundle: MessageOrBundle, isCli: Boolean) {
         if (receiver != null && messageOrBundle.bundle != null) {
             val reportStreamFilters = mutableListOf<Pair<String, ReportStreamFilter>>()
             reportStreamFilters.add(Pair("Jurisdictional Filter", receiver.jurisdictionalFilter))
@@ -522,10 +522,8 @@ class ProcessFhirCommands : CliktCommand(
         // However, the library used to encode the HL7 message throws an error it there are more than 4 encoding
         // characters, so this work around exists for that scenario
         val stringToEncode = hl7String.replace("MSH|^~\\&#|", "MSH|^~\\&|")
-        val hl7message = HL7Reader.parseHL7Message(
-            stringToEncode,
-            null
-        )
+        val hl7message = HL7Reader.parseHL7Message(stringToEncode)
+
         // if a hl7 parsing failure happens, throw error and show the message
         if (hl7message.toString().lowercase().contains("failed")) {
             throw CliktError("HL7 parser failure. $hl7message")
@@ -534,12 +532,8 @@ class ProcessFhirCommands : CliktCommand(
             val msh = hl7message.get("MSH") as Segment
             Terser.set(msh, 2, 0, 1, 1, "^~\\&#")
         }
-        val hl7profile = HL7Reader.getMessageProfile(hl7message.toString())
         // search hl7 profile map and create translator with config path if found
-        var fhirMessage = when (val configPath = HL7Reader.profileDirectoryMap[hl7profile]) {
-            null -> HL7toFhirTranslator(inputSchema).translate(hl7message)
-            else -> HL7toFhirTranslator(configPath).translate(hl7message)
-        }
+        var fhirMessage = HL7toFhirTranslator(inputSchema).translate(hl7message)
 
         val stamper = ConditionStamper(LookupTableConditionMapper(Metadata.getInstance()))
         fhirMessage.getObservations().forEach { observation ->

prime-router/src/main/kotlin/cli/ProcessHl7Commands.kt

@@ -1,11 +1,11 @@
 package gov.cdc.prime.router.cli
 
+import ca.uhn.hl7v2.util.Hl7InputStreamMessageStringIterator
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.core.CliktError
 import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.options.required
 import com.github.ajalt.clikt.parameters.types.file
-import gov.cdc.prime.router.ActionLogger
 import gov.cdc.prime.router.cli.helpers.HL7DiffHelper
 import gov.cdc.prime.router.fhirengine.utils.HL7Reader
 
@@ -45,9 +45,14 @@ class ProcessHl7Commands : CliktCommand(
         val comparisonFile = comparisonFile.inputStream().readBytes().toString(Charsets.UTF_8)
         if (comparisonFile.isBlank()) throw CliktError("File ${this.comparisonFile.absolutePath} is empty.")
 
-        val actionLogger = ActionLogger()
-        val starterMessages = HL7Reader(actionLogger).getMessages(starterFile)
-        val comparisonMessages = HL7Reader(actionLogger).getMessages(comparisonFile)
+        val starterMessages = Hl7InputStreamMessageStringIterator(starterFile.byteInputStream()).asSequence()
+            .map { rawItem ->
+                HL7Reader.parseHL7Message(rawItem)
+            }.toList()
+        val comparisonMessages = Hl7InputStreamMessageStringIterator(comparisonFile.byteInputStream()).asSequence()
+            .map { rawItem ->
+                HL7Reader.parseHL7Message(rawItem)
+            }.toList()
 
         starterMessages.forEachIndexed { counter, message ->
             val differences = hl7DiffHelper.diffHl7(message, comparisonMessages[counter])