ORDO Mappings

- Add: New release output: ordo-icd11.sssom.tsv - Completed work in mappings.py - Add: utils.py for this: This file now has several functions that are exactly the same in several repositories. Needs to be put into a package. - Update: config/icd11.sssom-metadata.yml: removed some unnecessary entries. Added necessary ones. - Add: requirements*.txt files: for installation of Python packages; and updated readme about that. - Delete: Temporary anlalysis files in src/compare_*/ - Add: icd11mms-exact-matches.sparql for querying ordo.owl for mappings. But this may not be necessary and may get removed. - Add: src/temp_compare_matches_owl_and_nomenclature.py: Temporary file for analysis for comparing Ordo->ICD11MMS exact matches from nomenclature file and from OWL release. - Bug fix: run.sh: Updated to an older version that works.
monarch-initiative · Mar 30, 2024 · 12dc3d1 · 12dc3d1
1 parent 9b03e83
commit 12dc3d1
Show file tree

Hide file tree

Showing 16 changed files with 286 additions and 38,891 deletions.
diff --git a/README.md b/README.md
@@ -1,11 +1,14 @@
-# ICD11 Foundation Ingest
-
-## Pre-reqs
-- Docker
-- Docker images  
-  One or both of the following, depending on if you want to run the stable build `latest` or `dev`:
-  - a. `docker pull obolibrary/odkfull:latest`
-  - b. `docker pull obolibrary/odkfull:dev`
-
-## Running
-`sh run.sh make all`
+# ICD11 Foundation Ingest
+
+## Pre-reqs
+- Docker
+- Docker images  
+  One or both of the following, depending on if you want to run the stable build `latest` or `dev`:
+  - a. `docker pull obolibrary/odkfull:latest`
+  - b. `docker pull obolibrary/odkfull:dev`
+
+## Local development setup
+`pip install -r requirements.txt`
+
+## Running
+`sh run.sh make all`
diff --git a/config/icd11.sssom-metadata.yml b/config/icd11.sssom-metadata.yml
@@ -1,14 +1,7 @@
 creator_id: 0000-0002-2906-7319
 curie_map:
-  GTR: http://purl.obolibrary.org/obo/mondo/mappings/unknown_prefix/GTR/
-  HP: http://purl.obolibrary.org/obo/HP_
-  MESH: http://identifiers.org/mesh/
-  MONDO: http://purl.obolibrary.org/obo/MONDO_
-  NCIT: http://purl.obolibrary.org/obo/NCIT_
-  OMIM: https://omim.org/entry/
   Orphanet: http://www.orpha.net/ORDO/Orphanet_
-  SCTID: http://identifiers.org/snomedct/
-  UMLS: http://purl.obolibrary.org/obo/UMLS_
+  icd11.foundation: http://id.who.int/icd/entity/
   oboInOwl: http://www.geneontology.org/formats/oboInOwl#
   owl: http://www.w3.org/2002/07/owl#
   rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns#

diff --git a/makefile b/makefile
@@ -6,7 +6,7 @@ SOURCE_URL=https://icd11files.blob.core.windows.net/tmp/whofic-2023-04-08.owl.gz
 
 
 # MAIN COMMANDS / GOALS ------------------------------------------------------------------------------------------------
-all: tmp/output/release/icd11foundation.owl
+all: tmp/output/release/icd11foundation.owl tmp/output/release/ordo-icd11.sssom.tsv
 
 clean:
 	rm -rf tmp/
@@ -37,14 +37,22 @@ release: | tmp/output/release/
 	gh release create $(VERSION) --notes "New release." --title "$(VERSION)" tmp/output/release/*
 
 # Mappings
-tmp/input/mappings.json: | tmp/input/
-	wget -qO tmp/input/en_product1.json.tar.gz https://www.orphadata.com/data/json/en_product1.json.tar.gz
-	tar -xvf tmp/input/en_product1.json.tar.gz -C tmp/input/
-	mv tmp/input/en_product1.json $@
-	rm tmp/input/en_product1.json.tar.gz
-
-tmp/output/release/ordo-icd11.sssom.tsv: tmp/input/mappings.json | tmp/output/release/
-	python3 src/make_sssom.py $< > $@
+# todo: Stable URI/filename issue: https://github.com/monarch-initiative/icd11/pull/12#discussion_r1542187711
+tmp/input/Orphanet_Nomenclature_Pack_EN/ORPHA_ICD11_mapping_en_newversion_2023.xml: | tmp/input/
+	wget https://www.orphadata.com/data/nomenclature/packs/Orphanet_Nomenclature_Pack_EN.zip -O tmp/input/Orphanet_Nomenclature_Pack_EN.zip
+	unzip tmp/input/Orphanet_Nomenclature_Pack_EN.zip -d tmp/input/Orphanet_Nomenclature_Pack_EN
+
+tmp/input/ordo.owl: | tmp/input/
+	wget http://www.orphadata.org/data/ORDO/ordo_orphanet.owl -O $@
+
+tmp/output/icd11mms-exact-matches.tsv: tmp/input/ordo.owl
+	robot query -i $< --query src/icd11mms-exact-matches.sparql $@
+
+tmp/output/release/ordo-icd11.sssom.tsv: tmp/input/Orphanet_Nomenclature_Pack_EN/ORPHA_ICD11_mapping_en_newversion_2023.xml tmp/output/icd11mms-exact-matches.tsv | tmp/output/release/
+	python3 src/mappings.py \
+	--input-nomenclature-xml tmp/input/Orphanet_Nomenclature_Pack_EN/ORPHA_ICD11_mapping_en_newversion_2023.xml \
+	--input-sssom-config config/icd11.sssom-metadata.yml \
+	--outpath $@
 
 # HELP -----------------------------------------------------------------------------------------------------------------
 help:

diff --git a/requirements-unlocked.txt b/requirements-unlocked.txt
@@ -0,0 +1,2 @@
+pandas
+pyyaml
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,14 @@
+distlib==0.3.6
+filelock==3.9.0
+numpy==1.25.1
+pandas==2.0.3
+pbr==5.11.1
+platformdirs==3.1.0
+python-dateutil==2.8.2
+pytz==2023.3
+PyYAML==6.0.1
+six==1.16.0
+stevedore==5.0.0
+tzdata==2023.3
+virtualenv==20.20.0
+virtualenv-clone==0.5.7
diff --git a/run.sh b/run.sh
@@ -14,8 +14,6 @@
 #
 # See README-editors.md for more details.
 
-set -e
-
 if [ -f run.sh.conf ]; then
     . ./run.sh.conf
 fi
@@ -31,22 +29,6 @@ elif [ -f "$HOME/Library/Application Support/ontology-development-kit/github/tok
     GH_TOKEN=$(cat "$HOME/Library/Application Support/ontology-development-kit/github/token")
 fi
 
-# SSH agent socket
-# On macOS, we cannot use $SSH_AUTH_SOCK directly,
-# we need to use a "magic" socket instead.
-case "$(uname)" in
-Darwin)
-    ODK_SSH_AUTH_SOCKET=/run/host-services/ssh-auth.sock
-    ;;
-*)
-    ODK_SSH_AUTH_SOCKET=$SSH_AUTH_SOCK
-    ;;
-esac
-ODK_SSH_BIND=
-if [ -n "$ODK_SSH_AUTH_SOCKET" ]; then
-    ODK_SSH_BIND=",$ODK_SSH_AUTH_SOCKET:/run/host-services/ssh-auth.sock"
-fi
-
 ODK_IMAGE=${ODK_IMAGE:-odkfull}
 TAG_IN_IMAGE=$(echo $ODK_IMAGE | awk -F':' '{ print $2 }')
 if [ -n "$TAG_IN_IMAGE" ]; then
@@ -58,9 +40,6 @@ ODK_TAG=${ODK_TAG:-latest}
 ODK_JAVA_OPTS=${ODK_JAVA_OPTS:--Xmx20G}
 ODK_DEBUG=${ODK_DEBUG:-no}
 
-ODK_USER_ID=${ODK_USER_ID:-$(id -u)}
-ODK_GROUP_ID=${ODK_GROUP_ID:-$(id -g)}
-
 # Convert OWLAPI_* environment variables to the OWLAPI as Java options
 # See http://owlcs.github.io/owlapi/apidocs_4/org/semanticweb/owlapi/model/parameters/ConfigurationOptions.html
 # for a list of allowed options
@@ -77,26 +56,25 @@ if [ x$ODK_DEBUG = xyes ]; then
     echo "Running ${IMAGE} with ${ODK_JAVA_OPTS} of memory for ROBOT and Java-based pipeline steps."
     TIMECMD="/usr/bin/time -f ### DEBUG STATS ###\nElapsed time: %E\nPeak memory: %M kb"
 fi
-rm -f tmp/debug.log
 
-VOLUME_BIND=$PWD/../../:/work$ODK_SSH_BIND
-WORK_DIR=/work/src/ontology
+VOLUME_BIND=$PWD:/work
+WORK_DIR=/work
 
 if [ -n "$ODK_BINDS" ]; then
     VOLUME_BIND="$VOLUME_BIND,$ODK_BINDS"
 fi
 
 if [ -n "$USE_SINGULARITY" ]; then
-    
+
     singularity exec --cleanenv $ODK_SINGULARITY_OPTIONS \
-        --env "ROBOT_JAVA_ARGS=$ODK_JAVA_OPTS,JAVA_OPTS=$ODK_JAVA_OPTS,SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock,ODK_USER_ID=$ODK_USER_ID,ODK_GROUP_ID=$ODK_GROUP_ID,ODK_DEBUG=$ODK_DEBUG" \
+        --env "ROBOT_JAVA_ARGS=$ODK_JAVA_OPTS,JAVA_OPTS=$ODK_JAVA_OPTS" \
         --bind $VOLUME_BIND \
         -W $WORK_DIR \
         docker://obolibrary/$ODK_IMAGE:$ODK_TAG $TIMECMD "$@"
 else
     BIND_OPTIONS="-v $(echo $VOLUME_BIND | sed 's/,/ -v /')"
     docker run $ODK_DOCKER_OPTIONS $BIND_OPTIONS -w $WORK_DIR \
-        -e ROBOT_JAVA_ARGS="$ODK_JAVA_OPTS" -e JAVA_OPTS="$ODK_JAVA_OPTS" -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock -e ODK_USER_ID=$ODK_USER_ID -e ODK_GROUP_ID=$ODK_GROUP_ID -e ODK_DEBUG=$ODK_DEBUG \
+        -e ROBOT_JAVA_ARGS="$ODK_JAVA_OPTS" -e JAVA_OPTS="$ODK_JAVA_OPTS" \
         --rm -ti obolibrary/$ODK_IMAGE:$ODK_TAG $TIMECMD "$@"
 fi