Merge pull request #9 from I-TECH-UW/use-poetry

poetry.toml

@@ -0,0 +1,2 @@
+[virtualenvs]
+in-project = true

pyproject.toml

@@ -0,0 +1,31 @@
+[tool.poetry]
+name = "who-l3-smart-tools"
+version = "0.2.0"
+description = "Tools for automating extraction of L3 technical resources from L2 packages"
+authors = ["I-TECH DIGI"]
+license = "MPL-2.0"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.12"
+pandas = ">=1.0"
+xlrd = ">=1.2"
+openpyxl = ">=3.0"
+fhirpy = ">=1.4"
+"fhir.resources" = ">=7.0"
+Faker = ">=25.0"
+
+[tool.poetry.group.dev.dependencies]
+pytest = ">=6"
+flake8 = ">=3.8"
+Sphinx = ">=3.2"
+stringcase = ">=1.2"
+pyright = "^1.1.367"
+
+[tool.pyright]
+venvPath = "."
+venv = ".venv"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

tests/data/example_fsh/HIV27_library.fsh

@@ -0,0 +1,19 @@
+Instance: HIVIND27Logic
+InstanceOf: Library
+Title: "HIV.IND.27 Logic"
+Description: "Number and % of people on ART among all people living with HIV at the end of the reporting period"
+Usage: #definition
+* meta.profile[+] = "http://hl7.org/fhir/uv/crmi/StructureDefinition/crmi-shareablelibrary"
+* meta.profile[+] = "http://hl7.org/fhir/uv/crmi/StructureDefinition/crmi-publishablelibrary"
+* meta.profile[+] = "http://hl7.org/fhir/uv/cql/StructureDefinition/cql-library"
+* meta.profile[+] = "http://hl7.org/fhir/uv/cql/StructureDefinition/cql-module"
+* url = "http://smart.who.int/immunizations-measles/Library/HIVIND27Logic"
+* extension[+]
+  * url = "http://hl7.org/fhir/StructureDefinition/cqf-knowledgeCapability"
+  * valueCode = #computable
+* name = "HIVIND27Logic"
+* status = #draft
+* experimental = true
+* publisher = "World Health Organization (WHO)"
+* type = $library-type#logic-library
+* content.id = "ig-loader-HIVIND27Logic.cql"

tests/data/example_fsh/HIV27_measure.fsh

@@ -0,0 +1,46 @@
+Instance: HIVIND27
+InstanceOf: http://hl7.org/fhir/us/cqfmeasures/StructureDefinition/proportion-measure-cqfm
+Title: "HIV.IND.27 People living with HIV on ART"
+* meta.profile[+] = "http://hl7.org/fhir/uv/crmi/StructureDefinition/crmi-shareablemeasure"
+* meta.profile[+] = "http://hl7.org/fhir/uv/crmi/StructureDefinition/crmi-publishablemeasure"
+* extension[http://hl7.org/fhir/us/cqfmeasures/StructureDefinition/cqfm-populationBasis].valueCode = #boolean
+* description = "Number and % of people on ART among all people living with HIV at the end of the reporting period"
+* url = "http://smart.who.int/immunizations-measles/Measure/HIVIND27"
+* status = #draft
+* experimental = true
+* date = "2024-06-14"
+* name = "HIVIND27"
+* title = "HIV.IND.27 People living with HIV on ART"
+* publisher = "World Health Organization (WHO)"
+* library = "http://smart.who.int/immunizations-measles/Library/HIVIND27Logic"
+* scoring = $measure-scoring#proportion "Proportion"
+* group[+]
+  * population[denominator]
+    * id = "HIV.IND.27.DEN"
+    * description = "1. To determine treatment coverage: estimated number of people living with HIV (from models, such as Spectrum AIM)
+2. To gauge progress toward the second 95 target: number of people living with HIV who know their HIV status (from surveys or models)"
+    * code = $measure-population#denominator "Denominator"
+    * criteria.language = #text/cql-identifier
+    * criteria.expression = "Denominator"
+  * population[numerator]
+    * id = "HIV.IND.27.NUM"
+    * description = "Number of people on ART at the end of the reporting period (HIV patient monitoring data from, for example, ART registers, patient records or EMRs). For key populations survey data may be required."
+    * code = $measure-population#numerator "Numerator"
+    * criteria.language = #text/cql-identifier
+    * criteria.expression = "Numerator"
+  * stratifier[+]
+    * id = "HIV.IND.27.S.AG"
+    * criteria.language = #text/cql-identifier
+    * criteria.expression = "Administrative Gender Stratifier"
+  * stratifier[+]
+    * id = "HIV.IND.27.S.A"
+    * criteria.language = #text/cql-identifier
+    * criteria.expression = "Age Stratifier"
+  * stratifier[+]
+    * id = "HIV.IND.27.S.GR"
+    * criteria.language = #text/cql-identifier
+    * criteria.expression = "Geographic Region Stratifier"
+  * stratifier[+]
+    * id = "HIV.IND.27.S.P"
+    * criteria.language = #text/cql-identifier
+    * criteria.expression = "patientGroups Stratifier"

tests/test_cql_tools.py

@@ -1,4 +1,5 @@
 # Generated by CodiumAI
+import datetime
 import os
 import re
 from who_l3_smart_tools.core.indicator_generation.cql_tools import (
@@ -24,7 +25,7 @@ def test_generate_cql_file_headers(self):
 
         generator.print_to_files(output_dir)
 
-        self.assertTrue(os.path.exists(output_dir + "HIV.IND.2.cql"))
+        assert os.path.exists(os.path.join(output_dir, "HIVIND2Logic.cql"))
 
     def test_generate_cql_template(self):
         input_file = "tests/data/indicator_dak_input_MINI.xlsx"
@@ -38,30 +39,34 @@ def test_generate_cql_template(self):
         # Test the first row
         cql_template = generator.generate_cql_template(indicator_artifact.iloc[0])
 
-        self.assertIsNotNone(cql_template)
+        assert cql_template is not None
 
 
 class TestCqlResourceGenerator(unittest.TestCase):
     def setUp(self):
+        # since we're comparing text, it's useful to have large diffs
+        self.maxDiff=5000
+
         # Load example CQL from data directory
         cql_file_path = "tests/data/example_cql_HIV27.cql"
         indicator_file_path = "tests/data/indicator_dak_input_MINI.xlsx"
 
         # Load content and close file
-        cql_file = open(cql_file_path, "r")
-        self.cql_content = cql_file.read()
-        cql_file.close()
+        with open(cql_file_path, "r") as cql_file:
+            self.cql_content = cql_file.read()
 
         indicator_file = pd.read_excel(
             indicator_file_path, sheet_name="Indicator definitions"
         )
 
-        self.indicator_row = indicator_file.iloc[2]
+        self.indicator_row = indicator_file[indicator_file['DAK ID'] == 'HIV.IND.27'].head(1).squeeze()
 
-        self.generator = CQLResourceGenerator(self.indicator_row, self.cql_content)
+        self.generator = CQLResourceGenerator(self.cql_content, {
+            self.indicator_row['DAK ID']: self.indicator_row
+        })
 
     def test_parse_cql_with_valid_content(self):
-        parsed_cql = self.generator.parse_cql()
+        parsed_cql = self.generator.parsed_cql
 
         self.assertIsNotNone(parsed_cql)
         self.assertEqual(parsed_cql["library_name"], "HIV.IND.27")
@@ -70,13 +75,12 @@ def test_parse_cql_with_valid_content(self):
         self.assertIn("numerator", parsed_cql.keys())
         self.assertIn("populations", parsed_cql.keys())
         self.assertGreater(len(parsed_cql["stratifiers"]), 0)
-        self.assertGreater(len(parsed_cql["populations"]), 0)
+        # self.assertGreater(len(parsed_cql["populations"]), 0)
         self.assertIsNotNone(parsed_cql["denominator"])
         self.assertIsNotNone(parsed_cql["numerator"])
 
     def test_generate_library_fsh(self):
-        p = self.generator.parse_cql()
-        library_fsh = self.generator.generate_library_fsh()
+        library_fsh = self.generator.generate_library_fsh().strip()
 
         output_file = "tests/output/fsh/HIV27_library.fsh"
 
@@ -85,35 +89,40 @@ def test_generate_library_fsh(self):
         with open(output_file, "w") as f:
             f.write(library_fsh)
 
-        expected_lib_file = f"tests/data/example_fsh/{stringcase.alphanumcase(p["library_name"])}_library.fsh"
-        expected_lib_file = open(expected_lib_file, "r")
-
-        expected_library_fsh = expected_lib_file.read()
+        expected_lib_file = f"tests/data/example_fsh/HIV27_library.fsh"
+        with open(expected_lib_file, "r") as expected_lib_fsh_file:
+            expected_library_fsh = expected_lib_fsh_file.read().rstrip()
 
         self.assertIsNotNone(library_fsh)
-
-        self.assertEqual(library_fsh.strip(), expected_library_fsh.strip())
+        self.assertEqual(expected_library_fsh, library_fsh)
 
     def test_generate_measure_fsh(self):
-        p = self.generator.parse_cql()
+        p = self.generator.parsed_cql
         measure_fsh = self.generator.generate_measure_fsh()
 
+        assert measure_fsh is not None
+
         output_file = f"tests/output/fsh/{stringcase.alphanumcase(p["library_name"])}_measure.fsh"
 
         if os.path.exists(output_file):
             os.remove(output_file)
         with open(output_file, "w") as f:
-            f.write(measure_fsh)
+            f.write(measure_fsh.strip())
 
         expected_measure_file = "tests/data/example_fsh/HIV27_measure.fsh"
-        expected_measure_file = open(expected_measure_file, "r")
-
-        expected_measure_fsh = expected_measure_file.read()
+        with open(expected_measure_file, "r") as expected_measure_file:
+            expected_measure_fsh = expected_measure_file.read().rstrip()
+        # The date is always the date the measure was generated, so we need to update it
+        expected_measure_fsh = expected_measure_fsh.replace(
+            '* date = "2024-06-14"',
+            f'* date = "{datetime.datetime.now(datetime.timezone.utc).date():%Y-%m-%d}"'
+        )
 
         self.assertIsNotNone(measure_fsh)
-        self.assertEqual(measure_fsh.strip(), expected_measure_fsh.strip())
+        self.assertEqual(expected_measure_fsh, measure_fsh)
+
 class TestCqlGeneratorOnAllFiles(unittest.TestCase):
-    
+
     def test_resource_gen_for_all(self):
         input_directory = "tests/data/cql/"
         indicator_file_path = "tests/data/l2/test_indicators.xlsx"
@@ -128,9 +137,12 @@ def test_resource_gen_for_all(self):
             indicator_dict[row["DAK ID"]] = row
 
         # Create output dir if not exists
-        output_directory = "tests/output/fsh/"
+        output_directory = os.path.join("tests", "output", "fsh")
         if not os.path.exists(output_directory):
             os.makedirs(output_directory)
+        for subfolder in ["measures", "libraries"]:
+            if not os.path.exists(os.path.join(output_directory, subfolder)):
+                os.makedirs(os.path.join(output_directory, subfolder))
 
 
         # For each cql file, generate library resources. Only generate measures for
@@ -147,9 +159,9 @@ def test_resource_gen_for_all(self):
 
             # Create Library file and save to file
             library_fsh = generator.generate_library_fsh()
-            if(library_fsh):
+            if library_fsh:
                 file_name = f"{stringcase.alphanumcase(generator.get_library_name())}"
-                if(generator.is_indicator()):
+                if generator.is_indicator():
                     file_name += "Logic"
                 file_name += ".fsh"
                 output_file = os.path.join(output_directory, "libraries", file_name)
@@ -158,10 +170,10 @@ def test_resource_gen_for_all(self):
 
             # Create Measure file and save to file
             measure_fsh = generator.generate_measure_fsh()
-            if(measure_fsh):
+            if measure_fsh:
                 output_file = os.path.join(output_directory, "measures", f"{stringcase.alphanumcase(generator.get_library_name())}.fsh")
                 with open(output_file, "w") as f:
-                    f.write(measure_fsh)    
+                    f.write(measure_fsh)
 
 if __name__ == "__main__":
     unittest.main()

who_l3_smart_tools/cli/indicator_testing.py

@@ -3,17 +3,16 @@
 import argparse
 from pathlib import Path
 import pandas as pd
-from fhirclient import send_to_fhir_server
-from bundle_generator import BundleGenerator
-from data_generator import DataGenerator
-from scaffolding_generator import ScaffoldingGenerator
+from who_l3_smart_tools.core.indicator_testing.bundle_generator import BundleGenerator
+from who_l3_smart_tools.core.indicator_testing.data_generator import DataGenerator
+from who_l3_smart_tools.core.indicator_testing.scaffolding_generator import ScaffoldingGenerator
 
 
 def generate_test_scaffold(input_file):
     scaffolding_generator = ScaffoldingGenerator(
         input_file,
         "Indicator_Scaffold_"
-        + datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
+        + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
         + ".xlsx",
     )
     scaffolding_generator.generate_test_scaffolding()
@@ -23,7 +22,7 @@ def generate_test_values(input_file):
     data_generator = DataGenerator(input_file)
     data_generator.generate_data_file(
         "Indicator_Test_Data_"
-        + datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
+        + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
         + ".xlsx",
         1000,
     )

who_l3_smart_tools/cli/logical_model_gen.py

@@ -1,5 +1,5 @@
 import argparse
-from who_l3_smart_tools.core.logical_models.logical_model_generator import generate_fsh_from_excel
+from who_l3_smart_tools.core.logical_models.logical_model_generator import LogicalModelGenerator
 
 
 def main():
@@ -21,7 +21,7 @@ def main():
 
     args = parser.parse_args()
 
-    generate_fsh_from_excel(args.input, args.output)
+    LogicalModelGenerator(args.input, args.output).generate_fsh_from_excel()
 
 
 if __name__ == "__main__":