diff --git a/care/emr/migrations/0033_migrate_daily_rounds_to_obseravations.py b/care/emr/migrations/0033_migrate_daily_rounds_to_obseravations.py new file mode 100644 index 0000000000..19929174e0 --- /dev/null +++ b/care/emr/migrations/0033_migrate_daily_rounds_to_obseravations.py @@ -0,0 +1,143 @@ +# Generated by Django 5.1.1 on 2024-12-11 08:29 + +from django.db import migrations +from django.core.paginator import Paginator +from django.db.models import F +from pathlib import Path +import json + +migration_id = 31 + +quantity_value_types = {"decimal", "integer", "quantity"} + + +def get_nested_value(obj, key): + # jq + keys = key.split(".") + value = getattr(obj, keys[0], None) + for key in keys[1:]: + if isinstance(value, dict): + value = value.get(key, None) + else: + return None + return value + +def get_observations(field_mapping, daily_round): + observations = [] + + if not daily_round.created_by_id: + # bad data in staging + print(f"Daily round {daily_round.id} has missing values {daily_round.created_by_id=}") + return observations + + default_observation = { + "status": "final", + "subject_type": "patient", + "subject_id": daily_round.patient_external_id, + "patient_id": daily_round.patient_id, + "encounter_id": daily_round.consultation_id, + "effective_datetime": daily_round.taken_at or daily_round.created_date, + "created_date": daily_round.created_date, + "data_entered_by_id": daily_round.created_by_id, + "performer": {}, + "meta": { + "is_migration": True, + "migration_id": migration_id, + "orignal_model": "DailyRound", + "orignal_model_id": daily_round.id, + "created_by_telemedicine": daily_round.created_by_telemedicine, + "round_type": daily_round.rounds_type, + "in_prone_position": daily_round.in_prone_position, + } + } + + # set nested json values as attributes on the object so that we can access them easily + for array_field in ("output", "nursing", "pressure_sore"): + for item in getattr(daily_round, array_field, []): + for key, value in item.items(): + # TODO: check if we need deduplication + setattr(daily_round, f"{array_field}__{key.lower()}", value) + + for field in field_mapping: + try: + daily_round_value = None + daily_round_note = None + + #TODO: instead of always using the helper we can ask it explicitly from the field mapping data + if field["value_key"]: + daily_round_value = get_nested_value(daily_round, field["value_key"]) + if field["notes_key"]: + daily_round_note = get_nested_value(daily_round, field["value_key"]) + + if daily_round_value or daily_round_note: + value = {} + if daily_round_value: + value["value"] = str(daily_round_value) + + if field["value_map"]: + value["value_code"] = field["value_map"].get(daily_round_value, {}) + + if field["value_type"] in quantity_value_types: + value["value_quantity"] = { + "value": float(daily_round_value), + } + if field["unit_code"]: + value["value_quantity"]["unit"] = field["unit_code"] + # if field["quantity_code"]: + # TODO: maybe this needs to be derived from the value? + # value["value_quantity"]["code"] = field["quantity_code"] + + observations.append(dict( + **default_observation, + category = field["category_code"], + main_code = field["main_code"], + alternate_coding = field["alternate_coding"], + value_type = field["value_type"], + value = value, + body_site = field["body_site"], + method = field["method"], + )) + except Exception as e: + print(f"Error while processing {field['value_key']} for {daily_round.id=}: {e}") + return observations + + +def migrate_daily_rounds_to_observation(apps, schema_editor): + DailyRound = apps.get_model("facility", "DailyRound") + Observation = apps.get_model("emr", "Observation") + + #load filed mapping from json + field_mapping_file = Path(__file__).parent / "daily_round_to_observations_field_mapping.json" + with field_mapping_file.open() as f: + field_mapping = json.load(f) + + queryset = DailyRound.objects.all().order_by("id").annotate( + patient_id=F("consultation__patient_id"), + patient_external_id=F("consultation__patient__external_id"), + ) + paginator = Paginator(queryset, 3000) # TODO: come up with a better batch size + for page_num in paginator.page_range: + bulk_observations = [] + daily_rounds = paginator.get_page(page_num) + for daily_round in daily_rounds: + bulk_observations.extend(get_observations(field_mapping, daily_round)) + Observation.objects.bulk_create( + [Observation(**observation) for observation in bulk_observations], + ) + + +def reverse_migration(apps, schema_editor): + # drop all observations created by this migration + schema_editor.execute("DELETE FROM emr_observation WHERE meta->>'migration_id' = %s", [str(migration_id)]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('emr', '0032_alter_medicationstatement_information_source'), + ] + + operations = [ + migrations.RunPython(migrate_daily_rounds_to_observation, + reverse_code=reverse_migration), + ] diff --git a/care/emr/migrations/daily_round_to_observations_field_mapping.json b/care/emr/migrations/daily_round_to_observations_field_mapping.json new file mode 100644 index 0000000000..bf09e9dc78 --- /dev/null +++ b/care/emr/migrations/daily_round_to_observations_field_mapping.json @@ -0,0 +1,711 @@ +[ + { + "value_key": "bp.systolic", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display_name": "Vital Signs" + }, + "main_code": { + "system": "http://loinc.org/", + "code": "8480-6", + "display_name": "BP Systolic" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "271649006", + "display_name": "BP Systolic" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "bp.diastolic", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display_name": "Vital Signs" + }, + "main_code": { + "system": "http://loinc.org/", + "code": "8462-4", + "display_name": "BP Diastolic" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "271650006", + "display_name": "BP Diastolic" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "ventilator_spo2", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display_name": "Vital Signs" + }, + "main_code": {}, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "250554003", + "display_name": "SpO2 (Measurement)" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "temperature", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display_name": "Vital Signs" + }, + "main_code": { + "system": "http://loinc.org/", + "code": "8329-5", + "display_name": "Temperature (observable entity)" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "703421000", + "display_name": "Temperature (observable entity)" + }, + "value_type": "decimal", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "resp", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display_name": "Vital Signs" + }, + "main_code": { + "system": "http://loinc.org/", + "code": "9279-1", + "display_name": "Respiratory Rate" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "86290005", + "display_name": "Respiratory Rate" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "pain", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "38212-7", + "display_name": "Pain" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "406127006", + "display_name": "Pain" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "pulse", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display_name": "Vital Signs" + }, + "main_code": {}, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "364075005", + "display_name": "Pulse (observable entity)" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "rhythm", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "vital-signs", + "display_name": "Vital Signs" + }, + "main_code": { + "system": "http://loinc.org/", + "code": "67519-9", + "display_name": "Rhythm (Pulse rhythm observable entity)" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "364095004", + "display_name": "Rhythm (Pulse rhythm observable entity)" + }, + "value_type": "string", + "unit_code": {}, + "quantity_code": {}, + "value_map": {}, + "body_site": {}, + "method": {} + }, + { + "value_key": "consciousness_level", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "80288-4", + "display_name": "Level of Consciousness (observable entity)" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "6942003", + "display_name": "Level of Consciousness (observable entity)" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "left_pupil_size", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "exam", + "display_name": "Exam" + }, + "main_code": { + "system": "http://loinc.org/", + "code": "8640-5", + "display_name": "Left Pupil Diameter" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "363953003", + "display_name": "Left Pupil Diameter" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "exam", + "display_name": "Exam" + }, + "main_code": { + "system": "http://loinc.org/", + "code": "75944-9", + "display_name": "Left Pupil Reaction" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "271733001", + "display_name": "Left Pupil Reaction" + }, + "value_type": "string", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "right_pupil_size", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "exam", + "display_name": "Exam" + }, + "main_code": { + "system": "http://loinc.org/", + "code": "8642-1", + "display_name": "Right Pupil Diameter" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "363953003", + "display_name": "Right Pupil Diameter" + }, + "value_type": "string", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "exam", + "display_name": "Exam" + }, + "main_code": { + "system": "http://loinc.org/", + "code": "75945-6", + "display_name": "Right Pupil Reaction" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "271733001", + "display_name": "Right Pupil Reaction" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "glasgow_eye_open", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "9267-6", + "display_name": "- Eye opening response" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "281395000", + "display_name": "- Eye opening response" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "glasgow_verbal_response", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "9270-0", + "display_name": "- verbal" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "281397008", + "display_name": "- verbal" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "glasgow_motor_response", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "9268-4", + "display_name": "- motor" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "281396004", + "display_name": "- motor" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "etco2", + "notes_key": "", + "category_code": {}, + "main_code": {}, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "250790007", + "display_name": "etco2" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "bilateral_air_entry", + "notes_key": "", + "category_code": {}, + "main_code": {}, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "58840004", + "display_name": "Bilateral air entry" + }, + "value_type": "boolean", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "po2", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "703-7", + "display_name": "po2" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "25579001", + "display_name": "po2" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "pco2", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "2019-8", + "display_name": "pco2" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "1304152003", + "display_name": "pco2" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "ph", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "2744-1", + "display_name": "ph" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "27051004", + "display_name": "ph" + }, + "value_type": "decimal", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "hco3", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "1960-4", + "display_name": "HCO3" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "443685006", + "display_name": "HCO3" + }, + "value_type": "decimal", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "base_excess", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "1925-7", + "display_name": "Base Excess" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "129907001", + "display_name": "Base Excess" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "lactate", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "30242-2", + "display_name": "Lactate" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "394960005", + "display_name": "Lactate" + }, + "value_type": "decimal", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "sodium", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "32717-1", + "display_name": "Sodium" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "312469006", + "display_name": "Sodium" + }, + "value_type": "decimal", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "potassium", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "32713-0", + "display_name": "Potassium" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "312468003", + "display_name": "Potassium" + }, + "value_type": "decimal", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "blood_sugar_level", + "notes_key": "", + "category_code": { + "system": "http://terminology.hl7.org/CodeSystem/observation-category", + "code": "laboratory", + "display_name": "Laboratory" + }, + "main_code": { + "system": "http://loinc.org/", + "code": "2339-0", + "display_name": "Glucose" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "365812005", + "display_name": "Glucose" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "insulin_intake_dose", + "notes_key": "", + "category_code": {}, + "main_code": {}, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "789480007", + "display_name": "Insulin Intake" + }, + "value_type": "decimal", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "infusions", + "notes_key": "", + "category_code": {}, + "main_code": {}, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "36576007", + "display_name": "Infusions" + }, + "value_type": "string", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "iv_fluids", + "notes_key": "", + "category_code": {}, + "main_code": {}, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "118431008", + "display_name": "IV Fluids" + }, + "value_type": "string", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "feeds", + "notes_key": "", + "category_code": {}, + "main_code": {}, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "229912004", + "display_name": "Feed" + }, + "value_type": "string", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "dialysis_fluid_balance", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "104103-7", + "display_name": "Fluid Balance" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "251856003", + "display_name": "Fluid Balance" + }, + "value_type": "integer", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + }, + { + "value_key": "pressure_sore", + "notes_key": "", + "category_code": {}, + "main_code": { + "system": "http://loinc.org/", + "code": "46535-1", + "display_name": "Pressure sore (Pressure injury)" + }, + "alternate_coding": { + "system": "http://www.snomed.info/sct", + "code": "225393005", + "display_name": "Pressure sore (Pressure injury)" + }, + "value_type": "string", + "unit_code": {}, + "quantity_code": {}, + "value_map": "", + "body_site": {}, + "method": {} + } +] diff --git a/gen_observations_field_mappings.py b/gen_observations_field_mappings.py new file mode 100644 index 0000000000..8e68c1f0a6 --- /dev/null +++ b/gen_observations_field_mappings.py @@ -0,0 +1,189 @@ +import csv +import json +import logging +from pathlib import Path + +import requests + +logger = logging.getLogger(__name__) + + +""" +This is a helper script to generate the field mapping for the daily_round to observations migration. + + +Fields in the CSV: + +Any one of value_key or notes_key should be present + +- value_key: The key to access the value in the daily_round object +- notes_key: The key to access the notes in the daily_round object + +- value_type: The type of value for the observation: + group | boolean | decimal | integer | string | text | display | date | dateTime | + time | choice | open-choice | attachment | reference | quantity | structured + +- value_map: A json mapping of values to be used for the value field, useful for mapping values to codes +eg: for rhythm +{ + 0: {system: "http://loinc.org/", code: "8867-4", display_name: "Normal sinus rhythm"}, + 1: {system: "http://loinc.org/", code: "271594007", display_name: "Atrial fibrillation"} + ... +} + +Below are the fields that are used to generate Coding objects: + +- category_code: The code for the category of the observation, uses the HL7 observation system by default +- main_code: The main code for the observation, uses the LOINC system by default +- alternate_coding: The alternate coding for the observation, uses the SNOMED system by default +- body_site: The body site for the observation +- method: The method used for the observation +- unit_code: The code for the unit of the observation +- quantity_code: The code for the quantity of the observation + +unit_code and quantity_code are only used when the value_type is quantifiable + + +All code fields have a corresponding display_name field which is used to +generate the display field in the Coding object, the system value is +inferred as LOINC or SNOMED if the code is a URL + +""" + + +def make_codeable_concept(code, display_name, system=None): + code = str(code) + if not code: + return {} + + # try to infer the system from the code if its a url + if "loinc" in code: + code = [x for x in code.split("/") if x][-1] + system = "http://loinc.org/" + elif "snomed" in code: + code = [x for x in code.split("/") if x][-1] + system = "http://www.snomed.info/sct" + + if not system: + system = "http://example.com" + + return { + "system": system, + "code": code, + "display_name": display_name, + # TODO: add fields like version, display, userSelected, etc. + } + + +def clean_csv_and_save_as_json(sheet_id): + csv_url = ( + f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?gid=0&format=csv" + ) + logger.info("Downloading CSV from %s", csv_url) + + # Download the CSV content + response = requests.get(csv_url, timeout=10) + response.raise_for_status() + csv_data = response.text.splitlines() + + reader = csv.DictReader( + csv_data, + fieldnames=[ + "value_key", + "notes_key", + "category_code", + "category_code_display_name", + "main_code", + "main_code_display_name", + "alternate_coding", + "alternate_coding_display_name", + "value_type", + "unit_code", + "unit_code_display_name", + "quantity_code", + "quantity_code_display_name", + "value_map", + "body_site", + "body_site_display_name", + "method", + "method_display_name", + ], + ) + + cleaned_rows = [] + + logger.info("Cleaning CSV") + for row in reader: + # Ensure we skip the header line if it's repeated (in case of specifying fieldnames) + if row["value_key"] == "value_key": + continue + + row["notes_key"] = "" + + row["value_type"] = row["value_type"].lower() or "string" + + # Convert codes to coding objects + row["category_code"] = make_codeable_concept( + row["category_code"], + row["category_code_display_name"], + "http://terminology.hl7.org/CodeSystem/observation-category", + ) + + row["main_code"] = make_codeable_concept( + row["main_code"], + row["main_code_display_name"], + "http://loinc.org/", + ) + + row["alternate_coding"] = make_codeable_concept( + row["alternate_coding"], + row["alternate_coding_display_name"], + "http://www.snomed.info/sct", + ) + + row["unit_code"] = make_codeable_concept( + row["unit_code"], row["unit_code_display_name"] + ) + + row["quantity_code"] = make_codeable_concept( + row["quantity_code"], row["quantity_code_display_name"] + ) + + row["body_site"] = make_codeable_concept( + row["body_site"], row["body_site_display_name"] + ) + + row["method"] = make_codeable_concept(row["method"], row["method_display_name"]) + + if row["value_map"]: + row["value_map"] = json.loads(row["value_map"]) + + # Remove display_name fields + for col in [ + "category_code_display_name", + "main_code_display_name", + "alternate_coding_display_name", + "unit_code_display_name", + "quantity_code_display_name", + "body_site_display_name", + "method_display_name", + ]: + row.pop(col, None) + + cleaned_rows.append(row) + + # Convert to JSON lines + json_file_path = ( + Path.cwd() + / "care/emr/migrations" + / "daily_round_to_observations_field_mapping.json" + ) + with json_file_path.open("w", encoding="utf-8") as f: + json.dump(cleaned_rows, f, ensure_ascii=False, indent=2) + + logger.info("Cleaned data saved to %s", json_file_path) + + +# https://docs.google.com/spreadsheets/d/1xgR5G1NFAKo0mODKUULOZapCtuaP-wd5kEacky0RNJA/edit +sheet_id = "1xgR5G1NFAKo0mODKUULOZapCtuaP-wd5kEacky0RNJA" +clean_csv_and_save_as_json(sheet_id)