Catalogue download

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "smpc-global"]
 	path = smpc-global
-	url = https://github.com/Athena-MHMD/smpc-global.git
+	url = https://github.com/athenarc/smpc-global.git

scripts/Pipfile

@@ -8,6 +8,8 @@ verify_ssl = true
 [packages]
 numpy = "*"
 pandas = "*"
+requests = "*"
+tqdm = "*"
 
 [requires]
 python_version = "3.7"

scripts/catalogue.py

@@ -1,29 +1,24 @@
 #!/usr/bin/python3
 
-import json
 import argparse
+import requests
+import re
+from tqdm import tqdm
+import xml.etree.ElementTree as ET
 
-
-def read_file(file):
-    with open(file) as f:
-        return json.load(f)
-
-
-def write_file(file, out):
-    with open(file, 'w') as f:
-        json.dump(out, f, indent=4)
+from utils import read_json, write_json
 
 
 def process(args, callback):
     catalogue = None
     out = {}
 
-    catalogue = read_file(args.catalogue)
+    catalogue = read_json(args.catalogue)
 
     for entry in catalogue:
         callback(entry, out)
 
-    write_file(args.output, out)
+    write_json(args.output, out)
 
 
 def normal(args):
@@ -38,12 +33,51 @@ def code(args):
     process(args, lambda entry, out: out.update({entry['code']: entry}))
 
 
+def download(args):
+    mesh_codes = read_json('../smpc-global/meshTermsByCode.json')
+    xml = ET.parse(args.catalogue)
+    root = xml.getroot()
+
+    string = '{(.*?)}'
+    prefix = re.search(string, root.tag)[0]
+    attributes = set()
+    values = set()
+    mesh_terms = []
+
+    for tname in root.findall('.//' + prefix + 'ClinicalVariables'):
+        attribute = tname.find(prefix + 'TypeName').text
+        value = tname.find(prefix + 'Value').text
+        attributes.add(attribute)
+        values.add(value)
+
+    for attr in tqdm(attributes):
+        data = {
+            'term': attr,
+            'terminology': 'text',
+            'result_format': 'json'
+        }
+
+        response = requests.post(args.url[0].strip('\t\n\r'), data=data)
+        terms = response.json()
+
+        tmp = {'attribute': attr, 'terms': []}
+        for t in terms:
+            if t['mesh_code'] in mesh_codes:
+                tmp['terms'].append(mesh_codes[t['mesh_code']])
+            else:
+                tmp['terms'].append(t)
+
+        mesh_terms.append(tmp)
+
+    write_json(args.output, mesh_terms)
+
+
 def group(args):
     catalogue = None
     normalized_catalogue = {}
     out = []
 
-    catalogue = read_file(args.catalogue)
+    catalogue = read_json(args.catalogue)
 
     for entry in catalogue:
         normalized_catalogue[entry['id']] = entry
@@ -55,14 +89,28 @@ def group(args):
             normalized_catalogue[parent_id]['children'].append({key: normalized_catalogue[id][key] for key in normalized_catalogue[id] if key != 'children'})
         out.append(normalized_catalogue[id])
 
-    write_file(args.output, out)
+    write_json(args.output, out)
+
+
+def keywords(args):
+    mesh_terms = read_json('../smpc-global/meshTermsInversed.json')
+    headers = {'accept': 'application/json'}
+    response = requests.get(args.keywords[0].strip('\t\n\r'), headers=headers)
+    keywords = response.json()
+    out = []
+
+    for k in keywords['data']:
+        if k['label'] in mesh_terms:
+            out.append(mesh_terms[k['label']])
+
+    write_json(args.keywords[1], out)
 
 
 def main():
     parser = argparse.ArgumentParser(
         description='SMPC catalogue processor')
-    parser.add_argument('catalogue', help='Catalogue file')
-    parser.add_argument('output', help='Output file')
+    parser.add_argument('catalogue', nargs='?', help='Catalogue file')
+    parser.add_argument('output', nargs='?', help='Output file')
     parser.add_argument(
         '-i',
         '--inverse',
@@ -83,6 +131,17 @@ def main():
         '--group',
         action='store_true',
         help='Create a file where each mesh term has its childer.')
+    parser.add_argument(
+        '-d',
+        '--download',
+        nargs=1,
+        dest='url',
+        help='Create a file where each attribute and value of the dataset is mapped to a mesh term by quering the catalogue.')
+    parser.add_argument(
+        '-k',
+        '--keywords',
+        nargs=2,
+        help='Create a file where each keyword from the given url is mapped to a mesh term.')
     parser.add_argument('--version', action='version', version='%(prog)s 0.1')
     args = parser.parse_args()
 
@@ -95,6 +154,10 @@ def main():
         code(args)
     elif(args.group):
         group(args)
+    elif(args.url):
+        download(args)
+    elif(args.keywords):
+        keywords(args)
     else:
         normal(args)
 

scripts/generate_attributes.py

@@ -12,13 +12,13 @@ def generate(dataset, output):
 
     string = '{(.*?)}'
     prefix = re.search(string, root.tag)[0]
-    attrributes = set()
+    attributes = set()
 
     for tname in root.findall('.//' + prefix + 'TypeName'):
-        attrributes.add(tname.text)
+        attributes.add(tname.text)
 
     with open(output, 'w') as f:
-        json.dump(list(attrributes), f, indent=4)
+        json.dump(list(attributes), f, indent=4)
 
 
 def main():

scripts/requirements.txt

@@ -1,2 +1,4 @@
 numpy==1.16.0
 pandas==0.23.4
+requests==2.22.0
+tqdm==4.32.1