caching locations into sql for performance improvement

intrahealth · Nov 14, 2023 · 87b4b47 · 87b4b47
1 parent e51e513
commit 87b4b47
Show file tree

Hide file tree

Showing 27 changed files with 1,278 additions and 1,026 deletions.
diff --git a/dhis2App/gofr.zip b/dhis2App/gofr.zip
diff --git a/dhis2App/index.html b/dhis2App/index.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="favicon.ico"><title>gofr2</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@mdi/font@latest/css/materialdesignicons.min.css"><link href="css/chunk-32294727.a5c908f7.css" rel="prefetch"><link href="css/chunk-72162224.7cb64c1b.css" rel="prefetch"><link href="css/chunk-b33551c6.76a811a3.css" rel="prefetch"><link href="css/install-app.76a811a3.css" rel="prefetch"><link href="js/apps.94ead83a.js" rel="prefetch"><link href="js/chunk-0779b602.f3796e8a.js" rel="prefetch"><link href="js/chunk-08a9b5b7.e7f7b41e.js" rel="prefetch"><link href="js/chunk-0a6b076a.622831ac.js" rel="prefetch"><link href="js/chunk-0a6f3fea.3d2cafe2.js" rel="prefetch"><link href="js/chunk-19a7ffa0.5d9dbaf2.js" rel="prefetch"><link href="js/chunk-1b3d2af8.c6cebede.js" rel="prefetch"><link href="js/chunk-1f83e9dc.161b2de4.js" rel="prefetch"><link href="js/chunk-2d0a4639.a4c7229f.js" rel="prefetch"><link href="js/chunk-2d0a50ef.b4b405ab.js" rel="prefetch"><link href="js/chunk-2d0aa9e6.faedc767.js" rel="prefetch"><link href="js/chunk-2d0ac384.6d11f961.js" rel="prefetch"><link href="js/chunk-2d0af298.db89198d.js" rel="prefetch"><link href="js/chunk-2d0afa68.155cf1de.js" rel="prefetch"><link href="js/chunk-2d0b2c3f.ff712f12.js" rel="prefetch"><link href="js/chunk-2d0b2ea8.fe000e1f.js" rel="prefetch"><link href="js/chunk-2d0b8b5d.2dddd8ca.js" rel="prefetch"><link href="js/chunk-2d0b9022.d1b71ea0.js" rel="prefetch"><link href="js/chunk-2d0b9df2.3c312ef0.js" rel="prefetch"><link href="js/chunk-2d0c17b7.924c29ec.js" rel="prefetch"><link href="js/chunk-2d0c2116.4a9b5ff0.js" rel="prefetch"><link href="js/chunk-2d0cef8e.192ffe91.js" rel="prefetch"><link href="js/chunk-2d0d34f6.c3659bf9.js" rel="prefetch"><link href="js/chunk-2d0da301.748b6f0c.js" rel="prefetch"><link href="js/chunk-2d0da3b2.1d31624a.js" rel="prefetch"><link href="js/chunk-2d0e2c53.59bfc68b.js" rel="prefetch"><link href="js/chunk-2d0e6654.754901bd.js" rel="prefetch"><link href="js/chunk-2d0efc9d.03b458f9.js" rel="prefetch"><link href="js/chunk-2d207782.c2f833ce.js" rel="prefetch"><link href="js/chunk-2d2106b9.1dc098fb.js" rel="prefetch"><link href="js/chunk-2d210a08.d7c738ac.js" rel="prefetch"><link href="js/chunk-2d210f9a.808b6982.js" rel="prefetch"><link href="js/chunk-2d216051.5371850f.js" rel="prefetch"><link href="js/chunk-2d2174d7.08f464e8.js" rel="prefetch"><link href="js/chunk-2d21a3c6.cbfbf879.js" rel="prefetch"><link href="js/chunk-2d21a437.49798103.js" rel="prefetch"><link href="js/chunk-2d225dc6.7d49a2a7.js" rel="prefetch"><link href="js/chunk-2d226188.3c69d482.js" rel="prefetch"><link href="js/chunk-2f5db3ea.fde25947.js" rel="prefetch"><link href="js/chunk-32294727.bb5c0f89.js" rel="prefetch"><link href="js/chunk-5369b150.b1793ec7.js" rel="prefetch"><link href="js/chunk-53797504.653668e7.js" rel="prefetch"><link href="js/chunk-58ec08a2.0d114804.js" rel="prefetch"><link href="js/chunk-5cafeba1.2be0cb37.js" rel="prefetch"><link href="js/chunk-5e700835.e0bcd751.js" rel="prefetch"><link href="js/chunk-5e71a7f0.8fc0698e.js" rel="prefetch"><link href="js/chunk-63fba70c.96bd4e5d.js" rel="prefetch"><link href="js/chunk-69088730.bc3e8613.js" rel="prefetch"><link href="js/chunk-69e0cc28.c404b9d7.js" rel="prefetch"><link href="js/chunk-72162224.46f1e738.js" rel="prefetch"><link href="js/chunk-77a0b274.5281d4fb.js" rel="prefetch"><link href="js/chunk-78a0fc3a.c36989e6.js" rel="prefetch"><link href="js/chunk-b33551c6.ed68db25.js" rel="prefetch"><link href="js/chunk-d3e34a68.63cad3e9.js" rel="prefetch"><link href="js/custom.d6a13f9f.js" rel="prefetch"><link href="js/install-app.509b4435.js" rel="prefetch"><link href="js/uninstall-app.5616a164.js" rel="prefetch"><link href="css/app.5cab2a52.css" rel="preload" as="style"><link href="css/chunk-vendors.7de2f704.css" rel="preload" as="style"><link href="js/app.7dc03f11.js" rel="preload" as="script"><link href="js/chunk-vendors.b63c1243.js" rel="preload" as="script"><link href="css/chunk-vendors.7de2f704.css" rel="stylesheet"><link href="css/app.5cab2a52.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but gofr2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="js/chunk-vendors.b63c1243.js"></script><script src="js/app.7dc03f11.js"></script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="favicon.ico"><title>gofr2</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@mdi/font@latest/css/materialdesignicons.min.css"><link href="css/chunk-32294727.a5c908f7.css" rel="prefetch"><link href="css/chunk-72162224.7cb64c1b.css" rel="prefetch"><link href="css/chunk-b33551c6.76a811a3.css" rel="prefetch"><link href="css/install-app.76a811a3.css" rel="prefetch"><link href="js/apps.94ead83a.js" rel="prefetch"><link href="js/chunk-0779b602.f3796e8a.js" rel="prefetch"><link href="js/chunk-08a9b5b7.e7f7b41e.js" rel="prefetch"><link href="js/chunk-0a6b076a.622831ac.js" rel="prefetch"><link href="js/chunk-0a6f3fea.3d2cafe2.js" rel="prefetch"><link href="js/chunk-19a7ffa0.5d9dbaf2.js" rel="prefetch"><link href="js/chunk-1b3d2af8.c6cebede.js" rel="prefetch"><link href="js/chunk-1f83e9dc.161b2de4.js" rel="prefetch"><link href="js/chunk-2d0a4639.a4c7229f.js" rel="prefetch"><link href="js/chunk-2d0a50ef.b4b405ab.js" rel="prefetch"><link href="js/chunk-2d0aa9e6.faedc767.js" rel="prefetch"><link href="js/chunk-2d0ac384.6d11f961.js" rel="prefetch"><link href="js/chunk-2d0af298.db89198d.js" rel="prefetch"><link href="js/chunk-2d0afa68.155cf1de.js" rel="prefetch"><link href="js/chunk-2d0b2c3f.ff712f12.js" rel="prefetch"><link href="js/chunk-2d0b2ea8.fe000e1f.js" rel="prefetch"><link href="js/chunk-2d0b8b5d.2dddd8ca.js" rel="prefetch"><link href="js/chunk-2d0b9022.d1b71ea0.js" rel="prefetch"><link href="js/chunk-2d0b9df2.3c312ef0.js" rel="prefetch"><link href="js/chunk-2d0c17b7.924c29ec.js" rel="prefetch"><link href="js/chunk-2d0c2116.4a9b5ff0.js" rel="prefetch"><link href="js/chunk-2d0cef8e.192ffe91.js" rel="prefetch"><link href="js/chunk-2d0d34f6.c3659bf9.js" rel="prefetch"><link href="js/chunk-2d0da301.748b6f0c.js" rel="prefetch"><link href="js/chunk-2d0da3b2.1d31624a.js" rel="prefetch"><link href="js/chunk-2d0e2c53.59bfc68b.js" rel="prefetch"><link href="js/chunk-2d0e6654.754901bd.js" rel="prefetch"><link href="js/chunk-2d0efc9d.03b458f9.js" rel="prefetch"><link href="js/chunk-2d207782.c2f833ce.js" rel="prefetch"><link href="js/chunk-2d2106b9.1dc098fb.js" rel="prefetch"><link href="js/chunk-2d210a08.d7c738ac.js" rel="prefetch"><link href="js/chunk-2d210f9a.808b6982.js" rel="prefetch"><link href="js/chunk-2d216051.5371850f.js" rel="prefetch"><link href="js/chunk-2d2174d7.08f464e8.js" rel="prefetch"><link href="js/chunk-2d21a3c6.cbfbf879.js" rel="prefetch"><link href="js/chunk-2d21a437.49798103.js" rel="prefetch"><link href="js/chunk-2d225dc6.7d49a2a7.js" rel="prefetch"><link href="js/chunk-2d226188.3c69d482.js" rel="prefetch"><link href="js/chunk-2f5db3ea.fde25947.js" rel="prefetch"><link href="js/chunk-32294727.bb5c0f89.js" rel="prefetch"><link href="js/chunk-5369b150.b1793ec7.js" rel="prefetch"><link href="js/chunk-53797504.653668e7.js" rel="prefetch"><link href="js/chunk-58ec08a2.0d114804.js" rel="prefetch"><link href="js/chunk-5cafeba1.2be0cb37.js" rel="prefetch"><link href="js/chunk-5e700835.e0bcd751.js" rel="prefetch"><link href="js/chunk-5e71a7f0.8fc0698e.js" rel="prefetch"><link href="js/chunk-63fba70c.96bd4e5d.js" rel="prefetch"><link href="js/chunk-69088730.bc3e8613.js" rel="prefetch"><link href="js/chunk-69e0cc28.c404b9d7.js" rel="prefetch"><link href="js/chunk-72162224.46f1e738.js" rel="prefetch"><link href="js/chunk-77a0b274.5281d4fb.js" rel="prefetch"><link href="js/chunk-78a0fc3a.c36989e6.js" rel="prefetch"><link href="js/chunk-b33551c6.ed68db25.js" rel="prefetch"><link href="js/chunk-d3e34a68.63cad3e9.js" rel="prefetch"><link href="js/custom.d6a13f9f.js" rel="prefetch"><link href="js/install-app.509b4435.js" rel="prefetch"><link href="js/uninstall-app.5616a164.js" rel="prefetch"><link href="css/app.5cab2a52.css" rel="preload" as="style"><link href="css/chunk-vendors.7de2f704.css" rel="preload" as="style"><link href="js/app.8b20f208.js" rel="preload" as="script"><link href="js/chunk-vendors.b63c1243.js" rel="preload" as="script"><link href="css/chunk-vendors.7de2f704.css" rel="stylesheet"><link href="css/app.5cab2a52.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but gofr2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="js/chunk-vendors.b63c1243.js"></script><script src="js/app.8b20f208.js"></script></body></html>
diff --git a/dhis2App/js/app.7dc03f11.js.map b/dhis2App/js/app.7dc03f11.js.map
diff --git a/dhis2App/js/app.7dc03f11.js → dhis2App/js/app.8b20f208.js b/dhis2App/js/app.7dc03f11.js → dhis2App/js/app.8b20f208.js
diff --git a/dhis2App/js/app.8b20f208.js.map b/dhis2App/js/app.8b20f208.js.map
diff --git a/fshcustom/input/fsh/GofrUser.fsh b/fshcustom/input/fsh/GofrUser.fsh
@@ -61,7 +61,7 @@ Description:    "GOFR password extension for local users."
 * extension[salt].valueString ^label = "Salt"
 * extension[salt].valueString 1..1 MS
 
-Instance:       e9b41c35-7c85-46df-aeea-a4e8dbf0364e
+Instance:       pJryMYLm5MFB6Skk5EsE1m
 InstanceOf:     GofrPersonUser
 Title:          "GOFR Admin User"
 Usage:          #example

diff --git a/fshmcsd/fsh-generated/resources/Parameters-gofr-config.json b/fshmcsd/fsh-generated/resources/Parameters-gofr-config.json
@@ -12,13 +12,13 @@
       "valueSignature": {
         "type": [
           {
-            "system": "urn:iso-astm:E1762-95:2013",
-            "code": "1.2.840.10065.1.12.1.14"
+            "code": "1.2.840.10065.1.12.1.14",
+            "system": "urn:iso-astm:E1762-95:2013"
           }
         ],
-        "when": "2023-09-25T08:05:15.964Z",
+        "when": "2022-09-20T03:44:20.575Z",
         "who": {
-          "reference": "http://ihris.org/fhir/Organization/ihris"
+          "reference": "http://gofr.org/fhir/Organization/gofr"
         },
         "data": "Td3sLJzy//AlrMy/w+FMYkv1V2zd7vro0+sQiQK32CwXun1B9MYMTLLA214BWsOmnUhi82Dojo3Jn3U2GpPf0BCu47wlln1weYsLA169HDLkG50ch89p3YZ87TyNiNidctCaAHAQ4gz8W+X20szMZeqTkOy/EoEGW0+GuPNGbpw="
       }
@@ -961,4 +961,4 @@
       ]
     }
   ]
-}
+}
diff --git a/...26e19ebd-65e5-4181-84b6-589bcf3bc44b.json → ...ources/Person-kLsxhfrrrB3rMGwBkDS4kY.json b/...26e19ebd-65e5-4181-84b6-589bcf3bc44b.json → ...ources/Person-kLsxhfrrrB3rMGwBkDS4kY.json
@@ -1,6 +1,6 @@
 {
   "resourceType": "Person",
-  "id": "26e19ebd-65e5-4181-84b6-589bcf3bc44b",
+  "id": "kLsxhfrrrB3rMGwBkDS4kY",
   "meta": {
     "profile": [
       "http://gofr.org/fhir/StructureDefinition/gofr-person-user"

diff --git a/...e9b41c35-7c85-46df-aeea-a4e8dbf0364e.json → ...ources/Person-pJryMYLm5MFB6Skk5EsE1m.json b/...e9b41c35-7c85-46df-aeea-a4e8dbf0364e.json → ...ources/Person-pJryMYLm5MFB6Skk5EsE1m.json
@@ -1,6 +1,6 @@
 {
   "resourceType": "Person",
-  "id": "e9b41c35-7c85-46df-aeea-a4e8dbf0364e",
+  "id": "pJryMYLm5MFB6Skk5EsE1m",
   "meta": {
     "profile": [
       "http://gofr.org/fhir/StructureDefinition/gofr-person-user"

diff --git a/fshmcsd/input/fsh/GofrUser.fsh b/fshmcsd/input/fsh/GofrUser.fsh
@@ -74,7 +74,7 @@ Description:    "DHIS2 Organization Unit ID"
 * valueString 1..1 MS
 * valueString ^label = "DHIS2 Organization Unit ID"
 
-Instance:       e9b41c35-7c85-46df-aeea-a4e8dbf0364e
+Instance:       pJryMYLm5MFB6Skk5EsE1m
 InstanceOf:     GofrPersonUser
 Title:          "GOFR Admin User"
 Usage:          #example
@@ -89,7 +89,7 @@ Usage:          #example
 * extension[password].extension[hash].valueString = "727c00bcb3d604db9b807155240b97347951e5e89e4c69b823279287694501fcaa683d883f5854a05c2c50c5b31413c6bb4a5949876a42b5c5bd74247e5777fc"
 * extension[role][0].valueReference = Reference(Basic/gofr-role-admin)
 
-Instance:       26e19ebd-65e5-4181-84b6-589bcf3bc44b
+Instance:       kLsxhfrrrB3rMGwBkDS4kY
 InstanceOf:     GofrPersonUser
 Title:          "GOFR Public User"
 Usage:          #example

diff --git a/gofr-backend/lib/defaultSetup.js b/gofr-backend/lib/defaultSetup.js
@@ -103,7 +103,7 @@ const addDataPartition = () => new Promise((resolve, reject) => {
             extension: [{
               url: 'userID',
               valueReference: {
-                reference: 'Person/e9b41c35-7c85-46df-aeea-a4e8dbf0364e',
+                reference: 'Person/pJryMYLm5MFB6Skk5EsE1m',
               },
             }],
           }, {

diff --git a/gofr-backend/lib/dhis.js b/gofr-backend/lib/dhis.js
@@ -1,6 +1,6 @@
 /* eslint-disable func-names */
 const request = require('request');
-const async = require('async');
+const uuid4 = require('uuid/v4');
 const URI = require('urijs');
 const http = require('http');
 const https = require('https');
@@ -10,6 +10,7 @@ const mixin = require('./mixin');
 const logger = require('./winston');
 const fhirAxios = require('./modules/fhirAxios');
 const config = require('./config');
+const uploadToSql = require("./modules/uploadToSql")
 
 const redisClient = redis.createClient({
   host: process.env.REDIS_HOST || '127.0.0.1',
@@ -152,6 +153,9 @@ const dhis = {
     });
   },
   sync: (host, username, password, name, clientId, topOrgName, reset, full, dousers, doservices) => {
+    if(!host.endsWith("/")) {
+      host += "/"
+    }
     const dhis2URL = new URI(host);
     const auth = `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
     credentials.dhis2URL = dhis2URL;
@@ -237,7 +241,7 @@ const dhis = {
   },
 };
 
-function processOrgUnit(metadata, hasKey) {
+async function processOrgUnit(metadata, hasKey) {
   logger.info('Now writting org units into the database');
   const {
     name,
@@ -283,10 +287,40 @@ function processOrgUnit(metadata, hasKey) {
     }
   });
 
+  let queries = []
+  let fakeOrgId = {
+    name: config.get('mCSD:fakeOrgName'),
+    parent: null,
+    uuid: credentials.topOrgId,
+    parentUUID: null,
+  };
+  await uploadToSql.createTable(database).catch((err) => {
+    console.log(err);
+  })
+  uploadToSql.buildSQL(fakeOrgId, queries, database)
+  let bundle = {
+    id: uuid4(),
+    resourceType: 'Bundle',
+    type: 'transaction',
+    entry: [],
+  }
   let i = 0;
   metadata.organisationUnits.sort((a, b) => a.level - b.level);
-  async.eachSeries(metadata.organisationUnits, (org, nxtOrg) => {
+  for(org of metadata.organisationUnits) {
     logger.info(`Processing (${++i}/${max}) ${org.id}`);
+    let location = {
+      uuid: org.id,
+      id: [org.id],
+      code: [],
+      name: org.name,
+      lat: "",
+      long: "",
+      parent: "",
+      parentUUID: "",
+    }
+    if(org.code) {
+      location.code.push(org.code)
+    }
     const fhir = {
       meta: {
         profile: [],
@@ -361,6 +395,8 @@ function processOrgUnit(metadata, hasKey) {
     if (org.featureType == 'POINT' && org.coordinates) {
       try {
         const coords = JSON.parse(org.coordinates);
+        location.lat = coords[1]
+        location.long = coords[0]
         fhir.position = {
           longitude: coords[0],
           latitude: coords[1],
@@ -373,11 +409,13 @@ function processOrgUnit(metadata, hasKey) {
       fhir.partOf = {
         reference: `Location/${org.parent.id}`,
       };
+      location.parentUUID = org.parent.id
     } else {
       fhir.partOf = {
         reference: `Location/${credentials.topOrgId}`,
         display: credentials.topOrgName,
       };
+      location.parentUUID = credentials.topOrgId
     }
     if (org.attributeValues) {
       for (const attr of org.attributeValues) {
@@ -386,52 +424,67 @@ function processOrgUnit(metadata, hasKey) {
             system: 'http://dhis2.org/id',
             value: attr.value,
           });
+          location.id.push(attr.value)
         }
         if (attr.attribute.id == 'Ed6SCy0OXfx') {
           fhir.identifier.push({
             system: 'http://dhis2.org/code',
             value: attr.value,
           });
+          location.code.push(attr.value)
         }
       }
     }
-
-    hostURL = URI(fhirAxios.__genUrl(database))
-      .segment('Location')
-      .segment(fhir.id)
-      .toString();
-    options = {
-      url: hostURL.toString(),
-      headers: {
-        'Content-Type': 'application/fhir+json',
-      },
-      json: fhir,
-    };
-    request.put(options, (err, res, body) => {
-      counter += 1;
-      const percent = parseFloat((counter * 100 / max).toFixed(2));
-      let status = '2/2 - Saving DHIS2 locations into FHIR server';
-      if (counter === max) {
-        status = 'Done';
+    bundle.entry.push({
+      resource: fhir,
+      request: {
+        method: 'PUT',
+        url: `Location/${fhir.id}`,
       }
+    })
+    uploadToSql.buildSQL(location, queries, database)
+    counter++
+    if(bundle.entry.length > 250) {
+      await fhirAxios.create(bundle, database).then(() => {
+        bundle.entry = []
+        const percent = parseFloat((counter * 100 / max).toFixed(2));
+        let status = '2/2 - Saving DHIS2 locations into FHIR server';
+        if (counter === max) {
+          status = 'Done';
+        }
+        const dhisSyncRequestId = `dhisSyncRequest${clientId}`;
+        const dhisSyncRequest = JSON.stringify({
+          status,
+          error: null,
+          percent,
+        });
+        redisClient.set(dhisSyncRequestId, dhisSyncRequest);
+      }).catch((err) => {
+        logger.error(err);
+      })
+      await uploadToSql.saveSQL(queries)
+      queries = []
+    }
+  }
+  if(queries.length > 0) {
+    uploadToSql.saveSQL(queries)
+  }
+  if(bundle.entry.length > 0) {
+    await fhirAxios.create(bundle, database).then(() => {
+      bundle.entry = []
       const dhisSyncRequestId = `dhisSyncRequest${clientId}`;
       const dhisSyncRequest = JSON.stringify({
-        status,
+        status: "Done",
         error: null,
-        percent,
+        percent: 100,
       });
       redisClient.set(dhisSyncRequestId, dhisSyncRequest);
-      if (err) {
-        logger.error(err);
-      } else {
-        logger.info(body);
-      }
-      return nxtOrg();
-    });
-  }, () => {
-    const thisRunTime = new Date().toISOString();
-    setLastUpdate(hasKey, thisRunTime);
-  });
+    }).catch((err) => {
+      logger.error(err);
+    })
+  }
+  const thisRunTime = new Date().toISOString();
+  setLastUpdate(hasKey, thisRunTime);
 }
 
 function checkLoaderDataStore() {