Changed Webserver.conf + Db.conf to Environment variable + Removed se…

…d / jq use Details of files changed 1. Start scripts .docker/docker_start_script.sh emission/integrationTests/start_integration_tests.sh setup/tests/start_script.sh - Changed seq / jq usage to directly set Environment variable to desired value; no need of saving sample file as actual conf json file. 2. Config.py files emission/core/config.py emission/net/api/config.py emission/net/ext_service/push/config.py - Based this file on emission/analysis/config.py - Added these to read from conf files if present or environment variables instead of sample files. - Default values set are taken from sample files. - check_unset_env_vars() can be used to check whether ALL environment variables are unset. 3. DB, Webapp, Push application usage files emission/core/get_database.py emission/net/api/cfc_webapp.py emission/net/ext_service/push/notify_interface.py - Changed logic to read using the config.py files that read the non-sample actual config files if present or from the Environment variables instead of sample files. 4. Test Files emission/integrationTests/storageTests/TestMongodbAuth.py emission/tests/netTests/TestWebserver.py emission/tests/netTests/TestPush.py - Test files that exercise the functionality of the logic in the files in (3). - Earlier, config files were being replaced with test values and copied over for testing purposes. - Now, switching to using environment variables - call sent to config files in (2) indirectly via application usage files in (3) - Following flow is followed in reading from and restoring original environment variables values setup() - Sets ENV vars by storing original vars if set, then uses test ENV vars as new values TestFunc() - importing modules named in (3) causes values to be read in, which now reads in newer test values since they set the ENV vars in setup() - only those ENV vars in test values are set; unchanged ones left untouched or default values read using os.getenv(var name, default) teardown() - Unset test ENV vars by using del os.environ[var_name] - Restore original values from original dict
e-mission · Apr 12, 2024 · 2531672 · 2531672
1 parent 4aeb4a6
commit 2531672
Show file tree

Hide file tree

Showing 12 changed files with 173 additions and 62 deletions.
diff --git a/.docker/docker_start_script.sh b/.docker/docker_start_script.sh
@@ -7,9 +7,8 @@
 echo ${DB_HOST}
 if [ -z ${DB_HOST} ] ; then
     local_host=`hostname -i`
-    jq --arg db_host "$local_host" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf
-else
-    jq --arg db_host "$DB_HOST" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf
+    export DB_HOST=$local_host
+    echo "Setting db host environment variable to localhost"
 fi
 cat conf/storage/db.conf
 

diff --git a/emission/core/config.py b/emission/core/config.py
@@ -0,0 +1,39 @@
+import json
+import logging
+import os
+
+def get_config_data_from_env():
+    config_data_env = {
+        "url": os.getenv('DB_HOST', "localhost"),
+        "result_limit": os.getenv('DB_TS_RESULT_LIMIT', 250000)
+    }
+    return config_data_env
+
+def check_unset_env_vars():
+    config_data_env = {
+        "url": os.getenv('DB_HOST'),
+        "result_limit": os.getenv('DB_TS_RESULT_LIMIT')
+    }
+    return not any(config_data_env.values())
+
+def get_config_data():
+    try:
+        config_file = open('conf/storage/db.conf')
+        ret_val = json.load(config_file)
+        config_file.close()
+    except:
+        # Check if all DB environment variables are not set
+        # if check_unset_env_vars():
+            # print("All DB environment variables are set to None")
+        logging.debug("storage not configured, falling back to sample, default configuration")
+        ret_val = get_config_data_from_env()
+    return ret_val
+
+config_data = get_config_data()
+
+def get_config():
+    return config_data
+
+def reload_config():
+    global config_data
+    config_data = get_config_data()
diff --git a/emission/core/get_database.py b/emission/core/get_database.py
@@ -10,16 +10,10 @@
 import os
 import json
 
-try:
-    config_file = open('conf/storage/db.conf')
-except:
-    print("storage not configured, falling back to sample, default configuration")
-    config_file = open('conf/storage/db.conf.sample')
+import emission.core.config as ecc
 
-config_data = json.load(config_file)
-url = config_data["timeseries"]["url"]
-result_limit = config_data["timeseries"]["result_limit"]
-config_file.close()
+url = ecc.get_config()["url"]
+result_limit = ecc.get_config()["result_limit"]
 
 try:
     parsed=pymongo.uri_parser.parse_uri(url)

diff --git a/emission/integrationTests/start_integration_tests.sh b/emission/integrationTests/start_integration_tests.sh
@@ -6,9 +6,8 @@ cd /src/e-mission-server
 echo ${DB_HOST}
 if [ -z ${DB_HOST} ] ; then
     local_host=`hostname -i`
-    sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf
-else
-    sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf
+    export DB_HOST=$local_host
+    echo "Setting db host environment variable to localhost"
 fi
 cat conf/storage/db.conf
 

diff --git a/emission/integrationTests/storageTests/TestMongodbAuth.py b/emission/integrationTests/storageTests/TestMongodbAuth.py
@@ -47,10 +47,19 @@ def setUp(self):
         self.uuid = uuid.uuid4()
         self.testUserId = self.uuid
         self.db_conf_file = "conf/storage/db.conf"
+        self.originalDBEnvVars = {}
         self.createAdmin()
 
     def tearDown(self):
         self.admin_auth.command({"dropAllUsersFromDatabase": 1})
+        logging.debug("Deleting test db environment variables")
+        for env_var_name, env_var_value in self.testModifiedEnvVars.items():
+            del os.environ[env_var_name]
+		# Restoring original db environment variables
+        for env_var_name, env_var_value in self.originalDBEnvVars.items():
+            os.environ[env_var_name] = env_var_value
+        logging.debug("Finished restoring original db environment variables")
+        logging.debug("Restored original values are = %s" % self.originalDBEnvVars)
         try:
             os.remove(self.db_conf_file)
         except FileNotFoundError as e:
@@ -67,14 +76,20 @@ def createAdmin(self):
         self.admin_auth = pymongo.MongoClient(self.getURL(self.test_username, self.test_password)).admin
 
     def configureDB(self, url):
-        config = {
-            "timeseries": {
-                "url": url,
-                "result_limit": 250000
-            }
+        self.testModifiedEnvVars = {
+            'DB_HOST' : url
         }
-        with open(self.db_conf_file, "w") as fp:
-            json.dump(config, fp, indent=4)
+
+        for env_var_name, env_var_value in self.testModifiedEnvVars.items():
+            if os.getenv(env_var_name) is not None:
+                # Storing original db environment variables before modification
+                self.originalDBEnvVars[env_var_name] = os.getenv(env_var_name)
+            # Setting db environment variables with test values
+            os.environ[env_var_name] = env_var_value
+
+        logging.debug("Finished setting up test db environment variables")
+        logging.debug("Current original values are = %s" % self.originalDBEnvVars)
+        logging.debug("Current modified values are = %s" % self.testModifiedEnvVars)
 
     def getURL(self, username, password, dbname="admin"):
         return "mongodb://%s:%s@localhost/%s?authSource=admin&authMechanism=SCRAM-SHA-1" % (username, password, dbname)

diff --git a/emission/net/api/cfc_webapp.py b/emission/net/api/cfc_webapp.py
@@ -51,22 +51,17 @@
 import emission.storage.timeseries.cache_series as esdc
 import emission.core.timer as ect
 import emission.core.get_database as edb
-
-try:
-    config_file = open('conf/net/api/webserver.conf')
-except:
-    logging.debug("webserver not configured, falling back to sample, default configuration")
-    config_file = open('conf/net/api/webserver.conf.sample')
+import emission.net.api.config as enac
 
 STUDY_CONFIG = os.getenv('STUDY_CONFIG', "stage-program")
-config_file.close()
-static_path = os.getenv('WEB_SERVER_STATIC_PATH', "webapp/www/")
-server_host = os.getenv('WEB_SERVER_HOST', "0.0.0.0")
-server_port = os.getenv('WEB_SERVER_PORT', "8080")
-socket_timeout = os.getenv('WEB_SERVER_TIMEOUT', "3600")
-auth_method = os.getenv('WEB_SERVER_AUTH', "skip")
-aggregate_call_auth = os.getenv('WEB_SERVER_AGGREGATE_CALL_AUTH', "no_auth")
-not_found_redirect = os.getenv('WEB_SERVER_OPENPATH_URL', "https://www.nrel.gov/transportation/openpath.html")   
+enac.reload_config()
+static_path = enac.get_config()["static_path"]
+server_host = enac.get_config()["server_host"]
+server_port = enac.get_config()["server_port"]
+socket_timeout = enac.get_config()["socket_timeout"]
+auth_method = enac.get_config()["auth_method"]
+aggregate_call_auth = enac.get_config()["aggregate_call_auth"]
+not_found_redirect = enac.get_config()["not_found_redirect"]
 
 BaseRequest.MEMFILE_MAX = 1024 * 1024 * 1024 # Allow the request size to be 1G
 # to accomodate large section sizes

diff --git a/emission/net/api/config.py b/emission/net/api/config.py
@@ -0,0 +1,48 @@
+import json
+import logging
+import os
+
+def get_config_data_from_env():
+    config_data_env = {
+        "static_path": os.getenv('WEB_SERVER_STATIC_PATH', "webapp/www/"),
+        "server_host": os.getenv('WEB_SERVER_HOST', "0.0.0.0"),
+        "server_port": os.getenv('WEB_SERVER_PORT', "8080"),
+        "socket_timeout": os.getenv('WEB_SERVER_TIMEOUT', "3600"),
+        "auth_method": os.getenv('WEB_SERVER_AUTH', "skip"),
+        "aggregate_call_auth": os.getenv('WEB_SERVER_AGGREGATE_CALL_AUTH', "no_auth"),
+        "not_found_redirect": os.getenv('WEB_SERVER_REDIRECT_URL', "https://www.nrel.gov/transportation/openpath.html")
+    }
+    return config_data_env
+
+def check_unset_env_vars():
+    config_data_env = {
+        "static_path": os.getenv('WEB_SERVER_STATIC_PATH'),
+        "server_host": os.getenv('WEB_SERVER_HOST'),
+        "server_port": os.getenv('WEB_SERVER_PORT'),
+        "socket_timeout": os.getenv('WEB_SERVER_TIMEOUT'),
+        "auth_method": os.getenv('WEB_SERVER_AUTH'),
+        "aggregate_call_auth": os.getenv('WEB_SERVER_AGGREGATE_CALL_AUTH'),
+        "not_found_redirect": os.getenv('WEB_SERVER_REDIRECT_URL')
+    }
+    return not any(config_data_env.values())
+
+def get_config_data():
+    try:
+        config_file = open('conf/storage/db.conf')
+        ret_val = json.load(config_file)
+        config_file.close()
+    except:
+        # Check if all Webserver environment variables are not set
+        # if check_unset_env_vars():
+        logging.debug("webserver not configured, falling back to sample, default configuration")
+        ret_val = get_config_data_from_env()
+    return ret_val
+
+config_data = get_config_data()
+
+def get_config():
+    return config_data
+
+def reload_config():
+    global config_data
+    config_data = get_config_data()
diff --git a/emission/net/ext_service/push/config.py b/emission/net/ext_service/push/config.py
@@ -17,7 +17,7 @@ def get_config_data():
         ret_val = json.load(config_file)
         config_file.close()
     except:
-        logging.debug("net.ext_service.push.json not configured, checking environment variables...")
+        logging.warning("net.ext_service.push.json not configured, checking environment variables...")
         ret_val = get_config_data_from_env()
         # Check if all PUSH environment variables are not set
         if (not any(ret_val.values())):
@@ -27,7 +27,7 @@ def get_config_data():
 try:
     config_data = get_config_data()
 except:
-    logging.debug("All push environment variables are set to None")
+    logging.warning("All push environment variables are set to None")
 
 def get_config():
     return config_data

diff --git a/emission/net/ext_service/push/notify_interface.py b/emission/net/ext_service/push/notify_interface.py
@@ -19,7 +19,7 @@
 # We can revisit this if push providers eventually decide to standardize...
 
 try:
-    push_config = pc.get_config_data()
+    push_config = pc.get_config()
 except:
     logging.warning("push service not configured, push notifications not supported")
 

diff --git a/emission/tests/netTests/TestPush.py b/emission/tests/netTests/TestPush.py
@@ -40,22 +40,33 @@ def generate_fake_result(successful_tokens, failed_tokens):
 
 class TestPush(unittest.TestCase):
     def setUp(self):
-        import shutil
-        self.push_conf_path = "conf/net/ext_service/push.json"
-        shutil.copyfile("%s.sample" % self.push_conf_path,
-                        self.push_conf_path)
-        with open(self.push_conf_path, "w") as fd:
-            fd.write(json.dumps({
-                "provider": "firebase",
-                "server_auth_token": "firebase_api_key",
-                "ios_token_format": "apns"
-            }))
-        logging.debug("Finished setting up %s" % self.push_conf_path)
-        with open(self.push_conf_path) as fd:
-            logging.debug("Current values are %s" % json.load(fd))
+        self.originalPushEnvVars = {}
+        self.testModifiedEnvVars = {
+            'PUSH_PROVIDER' : "firebase",
+            'PUSH_SERVER_AUTH_TOKEN' : "firebase_api_key",
+            'PUSH_IOS_TOKEN_FORMAT' : "apns"
+        }
+
+        for env_var_name, env_var_value in self.testModifiedEnvVars.items():
+            if os.getenv(env_var_name) is not None:
+                # Storing original push environment variables before modification
+                self.originalPushEnvVars[env_var_name] = os.getenv(env_var_name)
+            # Setting push environment variables with test values
+            os.environ[env_var_name] = env_var_value
+
+        logging.debug("Finished setting up test push environment variables")
+        logging.debug("Current original values are = %s" % self.originalPushEnvVars)
+        logging.debug("Current modified values are = %s" % self.testModifiedEnvVars)
 
     def tearDown(self):
-        os.remove(self.push_conf_path)
+        logging.debug("Deleting test push environment variables")
+        for env_var_name, env_var_value in self.testModifiedEnvVars.items():
+            del os.environ[env_var_name]
+        # Restoring original push environment variables
+        for env_var_name, env_var_value in self.originalPushEnvVars.items():
+            os.environ[env_var_name] = env_var_value
+        logging.debug("Finished restoring original push environment variables")
+        logging.debug("Restored original values are = %s" % self.originalPushEnvVars)
 
     def testGetInterface(self):
         import emission.net.ext_service.push.notify_interface as pni
@@ -177,6 +188,15 @@ def testFcmNoMapping(self):
 
         # and there will be no entries in the token mapping database
         self.assertEqual(edb.get_push_token_mapping_db().count_documents({}), 0)
+
+    def testNoEnvVarSetUp(self):
+        self.tearDown()
+        import emission.net.ext_service.push.notify_interface as pni
+        # import emission.net.ext_service.push.config as pc
+        # print("Fetching push config from ENV variables by deleting existing non-sample JSON file")
+        # self.tearDown()
+        # self.assertRaises(TypeError, pc.get_config_data())
+        self.setUp()
 
 if __name__ == '__main__':
     import emission.tests.common as etc

diff --git a/emission/tests/netTests/TestWebserver.py b/emission/tests/netTests/TestWebserver.py
@@ -25,7 +25,7 @@ class TestWebserver(unittest.TestCase):
     def setUp(self):
         self.originalWebserverEnvVars = {}
         self.testModifiedEnvVars = {
-            'WEB_SERVER_OPENPATH_URL' : "http://somewhere.else"
+            'WEB_SERVER_REDIRECT_URL' : "http://somewhere.else"
         }
 
         for env_var_name, env_var_value in self.testModifiedEnvVars.items():
@@ -35,16 +35,19 @@ def setUp(self):
             # Setting webserver environment variables with test values
             os.environ[env_var_name] = env_var_value
 
-        logging.debug("Finished setting up test webserver environment variables")
-        logging.debug("Current original values are = %s" % self.originalWebserverEnvVars)
-        logging.debug("Current modified values are = %s" % self.testModifiedEnvVars)
+        print("Finished setting up test webserver environment variables")
+        print("Current original values are = %s" % self.originalWebserverEnvVars)
+        print("Current modified values are = %s" % self.testModifiedEnvVars)
 
     def tearDown(self):
+        print("Deleting test webserver environment variables")
+        for env_var_name, env_var_value in self.testModifiedEnvVars.items():
+            del os.environ[env_var_name]
         # Restoring original webserver environment variables
         for env_var_name, env_var_value in self.originalWebserverEnvVars.items():
             os.environ[env_var_name] = env_var_value
-        logging.debug("Finished restoring original webserver environment variables")
-        logging.debug("Restored original values are = %s" % self.originalWebserverEnvVars)
+        print("Finished restoring original webserver environment variables")
+        print("Restored original values are = %s" % self.originalWebserverEnvVars)
 
     def test404Redirect(self):
         from emission.net.api.bottle import response

diff --git a/setup/tests/start_script.sh b/setup/tests/start_script.sh
@@ -6,9 +6,8 @@ cd /src/e-mission-server
 echo ${DB_HOST}
 if [ -z ${DB_HOST} ] ; then
     local_host=`hostname -i`
-    sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf
-else
-    sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf
+    export DB_HOST=$local_host
+    echo "Setting db host environment variable to localhost"
 fi
 cat conf/storage/db.conf