Add syphilis history field to bulk uploader (#7809)

* Add syphilis history field to bulk uploader

* Use string literals

* Refactor using list of fields

backend/src/main/java/gov/cdc/usds/simplereport/api/model/filerow/TestResultRow.java

@@ -15,8 +15,8 @@
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateEthnicity;
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateGendersOfSexualPartners;
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validatePhoneNumber;
-import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validatePositiveHIVRequiredAOEFields;
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateRace;
+import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateRequiredFieldsForPositiveResult;
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateResidence;
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateSpecimenType;
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateState;
@@ -30,6 +30,7 @@
 import gov.cdc.usds.simplereport.config.FeatureFlagsConfig;
 import gov.cdc.usds.simplereport.db.model.auxiliary.ResultUploadErrorSource;
 import gov.cdc.usds.simplereport.db.model.auxiliary.ResultUploadErrorType;
+import gov.cdc.usds.simplereport.service.DiseaseService;
 import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService;
 import gov.cdc.usds.simplereport.service.ResultsUploaderDeviceService;
 import gov.cdc.usds.simplereport.service.model.reportstream.FeedbackMessage;
@@ -105,6 +106,7 @@ public class TestResultRow implements FileRow {
   final ValueOrError testResultStatus;
   final ValueOrError testOrderedCode;
   final ValueOrError gendersOfSexualPartners;
+  final ValueOrError syphilisHistory;
 
   static final String PATIENT_LAST_NAME = "patient_last_name";
   static final String PATIENT_FIRST_NAME = "patient_first_name";
@@ -149,6 +151,7 @@ public class TestResultRow implements FileRow {
   public static final String ORDERING_FACILITY_ZIP_CODE = "ordering_facility_zip_code";
   public static final String ORDERING_FACILITY_PHONE_NUMBER = "ordering_facility_phone_number";
   public static final String GENDERS_OF_SEXUAL_PARTNERS = "genders_of_sexual_partners";
+  public static final String SYPHILIS_HISTORY = "syphilis_history";
 
   public static final ImmutableMap<String, String> diseaseSpecificLoincMap =
       new ImmutableMap.Builder<String, String>()
@@ -452,6 +455,7 @@ public TestResultRow(Map<String, String> rawRow) {
     testOrderedCode = getValue(rawRow, "test_ordered_code", isRequired("test_ordered_code"));
     gendersOfSexualPartners =
         getValue(rawRow, GENDERS_OF_SEXUAL_PARTNERS, isRequired(GENDERS_OF_SEXUAL_PARTNERS));
+    syphilisHistory = getValue(rawRow, SYPHILIS_HISTORY, isRequired(SYPHILIS_HISTORY));
   }
 
   private List<FeedbackMessage> validateDeviceModelAndTestPerformedCode(
@@ -487,6 +491,17 @@ private boolean isHivResult() {
                 equipmentModelName.getValue(), testPerformedCode.getValue()));
   }
 
+  private boolean isSyphilisResult() {
+    if (equipmentModelName.getValue() == null || testPerformedCode.getValue() == null) {
+      return false;
+    }
+    return resultsUploaderCachingService
+        .getSyphilisEquipmentModelAndTestPerformedCodeSet()
+        .contains(
+            ResultsUploaderCachingService.getKey(
+                equipmentModelName.getValue(), testPerformedCode.getValue()));
+  }
+
   private List<FeedbackMessage> generateInvalidDataErrorMessages() {
     String errorMessage =
         "Invalid " + EQUIPMENT_MODEL_NAME + " and " + TEST_PERFORMED_CODE + " combination";
@@ -599,7 +614,16 @@ public List<FeedbackMessage> validateIndividualValues() {
 
     if (isHivResult()) {
       errors.addAll(
-          validatePositiveHIVRequiredAOEFields(testResult, gendersOfSexualPartners, pregnant));
+          validateRequiredFieldsForPositiveResult(
+              testResult, DiseaseService.HIV_NAME, List.of(gendersOfSexualPartners, pregnant)));
+    }
+
+    if (isSyphilisResult()) {
+      errors.addAll(
+          validateRequiredFieldsForPositiveResult(
+              testResult,
+              DiseaseService.SYPHILIS_NAME,
+              List.of(gendersOfSexualPartners, pregnant, syphilisHistory, symptomaticForDisease)));
     }
 
     return errors;

backend/src/main/java/gov/cdc/usds/simplereport/config/CachingConfig.java

@@ -14,6 +14,8 @@ public class CachingConfig {
       "covidEquipmentModelAndTestPerformedCodeSet";
   public static final String HIV_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET =
       "hivEquipmentModelAndTestPerformedCodeSet";
+  public static final String SYPHILIS_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET =
+      "syphilisEquipmentModelAndTestPerformedCodeSet";
   public static final String DEVICE_MODEL_AND_TEST_PERFORMED_CODE_MAP =
       "deviceModelAndTestPerformedCodeMap";
   public static final String SPECIMEN_NAME_TO_SNOMED_MAP = "specimenTypeNameSNOMEDMap";
@@ -26,6 +28,7 @@ public CacheManager cacheManager() {
     return new ConcurrentMapCacheManager(
         COVID_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET,
         HIV_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET,
+        SYPHILIS_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET,
         DEVICE_MODEL_AND_TEST_PERFORMED_CODE_MAP,
         SPECIMEN_NAME_TO_SNOMED_MAP,
         SNOMED_TO_SPECIMEN_NAME_MAP,

backend/src/main/java/gov/cdc/usds/simplereport/service/DiseaseService.java

@@ -23,6 +23,7 @@ public class DiseaseService {
   public static final String FLU_RNA_NAME = "Flu RNA";
   public static final String RSV_NAME = "RSV";
   public static final String HIV_NAME = "HIV";
+  public static final String SYPHILIS_NAME = "Syphilis";
 
   private final DiseaseCacheService diseaseCacheService;
 

backend/src/main/java/gov/cdc/usds/simplereport/service/ResultsUploaderCachingService.java

@@ -6,6 +6,7 @@
 import static gov.cdc.usds.simplereport.config.CachingConfig.HIV_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET;
 import static gov.cdc.usds.simplereport.config.CachingConfig.SNOMED_TO_SPECIMEN_NAME_MAP;
 import static gov.cdc.usds.simplereport.config.CachingConfig.SPECIMEN_NAME_TO_SNOMED_MAP;
+import static gov.cdc.usds.simplereport.config.CachingConfig.SYPHILIS_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET;
 
 import gov.cdc.usds.simplereport.db.model.DeviceType;
 import gov.cdc.usds.simplereport.db.model.SpecimenType;
@@ -150,6 +151,12 @@ public Set<String> getHivEquipmentModelAndTestPerformedCodeSet() {
     return getDiseaseSpecificEquipmentModelAndTestPerformedCodeSet(DiseaseService.HIV_NAME);
   }
 
+  @Cacheable(SYPHILIS_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET)
+  public Set<String> getSyphilisEquipmentModelAndTestPerformedCodeSet() {
+    log.info("generating syphilisEquipmentModelAndTestPerformedCodeSet cache");
+    return getDiseaseSpecificEquipmentModelAndTestPerformedCodeSet(DiseaseService.SYPHILIS_NAME);
+  }
+
   @Cacheable(COVID_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET)
   public Set<String> getCovidEquipmentModelAndTestPerformedCodeSet() {
     log.info("generating covidEquipmentModelAndTestPerformedCodeSet cache");

backend/src/main/java/gov/cdc/usds/simplereport/service/TestResultUploadService.java

@@ -9,6 +9,7 @@
 import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.ORDERING_FACILITY_STREET;
 import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.ORDERING_FACILITY_STREET2;
 import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.ORDERING_FACILITY_ZIP_CODE;
+import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.SYPHILIS_HISTORY;
 import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.TESTING_LAB_CITY;
 import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.TESTING_LAB_NAME;
 import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.TESTING_LAB_PHONE_NUMBER;
@@ -305,6 +306,7 @@ private Map<String, String> transformCsvRow(Map<String, String> row) {
     row.put(DATE_RESULT_RELEASED_COLUMN_NAME, dateResultReleased.toOffsetDateTime().toString());
 
     row.remove(GENDERS_OF_SEXUAL_PARTNERS);
+    row.remove(SYPHILIS_HISTORY);
 
     return row;
   }

backend/src/main/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtils.java

@@ -72,6 +72,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
 
 public class CsvValidatorUtils {
 
@@ -138,6 +139,12 @@ public class CsvValidatorUtils {
   private static final String NOT_HISPANIC_LITERAL = "not hispanic or latino";
   private static final String NOT_HISPANIC_CODE = "2186-5";
   private static final String NOT_HISPANIC_DB_VALUE = "not_hispanic";
+  private static final String POSITIVE_LITERAL = "positive";
+  private static final String POSITIVE_CODE = "10828004";
+  private static final String DETECTED_LITERAL = "detected";
+  private static final String DETECTED_CODE = "260373001";
+  private static final Set<String> POSITIVE_TEST_RESULT_VALUES =
+      Set.of(POSITIVE_LITERAL, DETECTED_LITERAL, POSITIVE_CODE, DETECTED_CODE);
   private static final Set<String> GENDER_VALUES =
       Set.of(
           "m", MALE_LITERAL,
@@ -176,7 +183,7 @@ public class CsvValidatorUtils {
           "n", "no",
           "u", UNKNOWN_CODE);
   private static final Set<String> TEST_RESULT_VALUES =
-      Set.of("positive", "negative", "not detected", "detected", "invalid result");
+      Set.of(POSITIVE_LITERAL, "negative", "not detected", DETECTED_LITERAL, "invalid result");
 
   private static final Set<String> RESIDENCE_VALUES =
       Set.of(
@@ -254,10 +261,13 @@ private static String getRequiredValueErrorMessage(String columnName) {
     return "File is missing data in the " + columnName + " column.";
   }
 
-  private static String getRequiredHivAoeValuesErrorMessage(String columnName) {
+  private static String getPositiveResultRequiredValueErrorMessage(
+      String columnName, String diseaseName) {
     return "File is missing data in the "
         + columnName
-        + " column. This is required because the row contains a positive HIV test result.";
+        + " column. This is required because the row contains a positive "
+        + diseaseName
+        + " test result.";
   }
 
   public static List<FeedbackMessage> validateTestResult(ValueOrError input) {
@@ -637,39 +647,35 @@ public static List<FeedbackMessage> validateGendersOfSexualPartners(ValueOrError
     return errors;
   }
 
-  public static List<FeedbackMessage> validatePositiveHIVRequiredAOEFields(
-      ValueOrError testResult, ValueOrError gendersOfSexualPartners, ValueOrError pregnant) {
+  public static List<FeedbackMessage> validateRequiredFieldsForPositiveResult(
+      ValueOrError testResult, String diseaseName, List<ValueOrError> fields) {
     List<FeedbackMessage> errors = new ArrayList<>();
-    // includes SNOMED values for positive and detected
-    Set<String> positiveTestResultValues = Set.of("positive", "detected", "260373001", "10828004");
-    if (!positiveTestResultValues.contains(testResult.getValue().toLowerCase())) {
+
+    if (testResult.getValue() == null) {
+      // if test result is null, then it should already give an error when validating required
+      // fields
       return errors;
     }
 
-    if (gendersOfSexualPartners.getValue() == null
-        || gendersOfSexualPartners.getValue().isBlank()) {
-      errors.add(
-          FeedbackMessage.builder()
-              .scope(ITEM_SCOPE)
-              .fieldHeader(gendersOfSexualPartners.getHeader())
-              .source(ResultUploadErrorSource.SIMPLE_REPORT)
-              .message(getRequiredHivAoeValuesErrorMessage(gendersOfSexualPartners.getHeader()))
-              .errorType(ResultUploadErrorType.MISSING_DATA)
-              .fieldRequired(gendersOfSexualPartners.isRequired())
-              .build());
+    if (!POSITIVE_TEST_RESULT_VALUES.contains(testResult.getValue().toLowerCase())) {
+      return errors;
     }
 
-    if (pregnant.getValue() == null || pregnant.getValue().isBlank()) {
-      errors.add(
-          FeedbackMessage.builder()
-              .scope(ITEM_SCOPE)
-              .fieldHeader(pregnant.getHeader())
-              .source(ResultUploadErrorSource.SIMPLE_REPORT)
-              .message(getRequiredHivAoeValuesErrorMessage(pregnant.getHeader()))
-              .errorType(ResultUploadErrorType.MISSING_DATA)
-              .fieldRequired(pregnant.isRequired())
-              .build());
-    }
+    fields.forEach(
+        field -> {
+          if (StringUtils.isBlank(field.getValue())) {
+            errors.add(
+                FeedbackMessage.builder()
+                    .scope(ITEM_SCOPE)
+                    .fieldHeader(field.getHeader())
+                    .source(ResultUploadErrorSource.SIMPLE_REPORT)
+                    .message(
+                        getPositiveResultRequiredValueErrorMessage(field.getHeader(), diseaseName))
+                    .errorType(ResultUploadErrorType.MISSING_DATA)
+                    .fieldRequired(field.isRequired())
+                    .build());
+          }
+        });
     return errors;
   }
 

backend/src/test/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtilsTest.java

@@ -11,7 +11,7 @@
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateGendersOfSexualPartners;
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validatePartialUnkAddress;
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validatePhoneNumber;
-import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validatePositiveHIVRequiredAOEFields;
+import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateRequiredFieldsForPositiveResult;
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateSpecimenType;
 import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateZipCode;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -24,6 +24,7 @@
 import com.fasterxml.jackson.databind.MappingIterator;
 import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
 import gov.cdc.usds.simplereport.api.model.errors.CsvProcessingException;
+import gov.cdc.usds.simplereport.service.DiseaseService;
 import gov.cdc.usds.simplereport.service.model.reportstream.FeedbackMessage;
 import gov.cdc.usds.simplereport.utils.UnknownAddressUtils;
 import java.io.IOException;
@@ -293,22 +294,31 @@ void validNegativeHIVNoRequiredAOEFields() {
     ValueOrError testResult = new ValueOrError("negative", "test_result");
     ValueOrError genders = new ValueOrError("", "genders_of_sexual_partners");
     ValueOrError pregnant = new ValueOrError("", "pregnant");
-    assertThat(validatePositiveHIVRequiredAOEFields(testResult, genders, pregnant)).isEmpty();
+    assertThat(
+            validateRequiredFieldsForPositiveResult(
+                testResult, DiseaseService.HIV_NAME, List.of(genders, pregnant)))
+        .isEmpty();
   }
 
   @Test
   void validPositiveHIVRequiredAOEFields() {
     ValueOrError testResult = new ValueOrError("positive", "test_result");
     ValueOrError genders = new ValueOrError("m, f, tm, tw", "genders_of_sexual_partners");
     ValueOrError pregnant = new ValueOrError("n", "pregnant");
-    assertThat(validatePositiveHIVRequiredAOEFields(testResult, genders, pregnant)).isEmpty();
+    assertThat(
+            validateRequiredFieldsForPositiveResult(
+                testResult, DiseaseService.HIV_NAME, List.of(genders, pregnant)))
+        .isEmpty();
   }
 
   @Test
   void invalidPositiveHIVRequiredAOEFields() {
     ValueOrError testResult = new ValueOrError("positive", "test_result");
     ValueOrError genders = new ValueOrError("", "genders_of_sexual_partners");
     ValueOrError pregnant = new ValueOrError("", "pregnant");
-    assertThat(validatePositiveHIVRequiredAOEFields(testResult, genders, pregnant)).hasSize(2);
+    assertThat(
+            validateRequiredFieldsForPositiveResult(
+                testResult, DiseaseService.HIV_NAME, List.of(genders, pregnant)))
+        .hasSize(2);
   }
 }

backend/src/test/java/gov/cdc/usds/simplereport/validators/TestResultRowTest.java

@@ -461,6 +461,40 @@ void validatePositiveHivRequiredAoeFields() {
                 .contains("This is required because the row contains a positive HIV test result."));
   }
 
+  @Test
+  void validatePositiveSyphilisRequiredAoeFields() {
+    var missingSyphilisRequiredAoeFields = validRowMap;
+    missingSyphilisRequiredAoeFields.put("equipment_model_name", "Syphilis model");
+    missingSyphilisRequiredAoeFields.put("test_performed_code", "80387-4");
+    missingSyphilisRequiredAoeFields.put("specimen_type", "123456789");
+    missingSyphilisRequiredAoeFields.put("test_result", "Detected");
+    missingSyphilisRequiredAoeFields.put("pregnant", "");
+    missingSyphilisRequiredAoeFields.put("genders_of_sexual_partners", "");
+    missingSyphilisRequiredAoeFields.put("previous_syphilis_diagnosis", "");
+    missingSyphilisRequiredAoeFields.put("symptomatic_for_disease", "");
+
+    var resultsUploaderCachingService = mock(ResultsUploaderCachingService.class);
+    when(resultsUploaderCachingService.getModelAndTestPerformedCodeToDeviceMap())
+        .thenReturn(Map.of("syphilis model|80387-4", TestDataBuilder.createDeviceType()));
+    when(resultsUploaderCachingService.getSyphilisEquipmentModelAndTestPerformedCodeSet())
+        .thenReturn(Set.of("syphilis model|80387-4"));
+
+    var testResultRow =
+        new TestResultRow(
+            missingSyphilisRequiredAoeFields,
+            resultsUploaderCachingService,
+            mock(FeatureFlagsConfig.class));
+
+    var actual = testResultRow.validateIndividualValues();
+
+    assertThat(actual).hasSize(4);
+    actual.forEach(
+        message ->
+            assertThat(message.getMessage())
+                .contains(
+                    "This is required because the row contains a positive Syphilis test result."));
+  }
+
   private ResultsUploaderCachingService mockResultsUploaderCachingService() {
     var resultsUploaderCachingService = mock(ResultsUploaderCachingService.class);
     when(resultsUploaderCachingService.getModelAndTestPerformedCodeToDeviceMap())