M117 - Made 'root_dir' to a configurable option.

coverage_model/config.py

@@ -1,6 +1,7 @@
 __author__ = 'casey'
 
 from ooi.logging import log
+from pyon.util.config import Config
 import datetime
 
 
@@ -25,8 +26,10 @@ class CoverageConfig(object):
     _default_vertical_db_key = 'vertical_range'
     _default_span_id_db_key = 'span_address'
     _default_span_coverage_id_db_key = 'coverage_id'
+    _default_storage_location = None
 
     def __init__(self):
+        print 'setting defaults'
         self.ordered_time_key_preferences = self._default_ordered_time_key_preferences
         self.ordered_lat_key_preferences = self._default_ordered_lat_key_preferences
         self.ordered_lon_key_preferences = self._default_ordered_lon_key_preferences
@@ -36,16 +39,34 @@ def __init__(self):
         self.vertical_db_key = self._default_vertical_db_key
         self.span_id_db_key = self._default_span_id_db_key
         self.span_coverage_id_db_key = self._default_span_coverage_id_db_key
+        self.top_level_storage_location = self._default_storage_location
         self.using_default_config = True
         self.config_time = 0
         self.read_and_set_config()
 
     def read_and_set_config(self):
-        is_success = False
-        if is_success is True:
+        one_from_config = False
+        try:
+            data = Config(["res/config/coverage.yml"]).data['CoverageConfig']
+            for k, v in data.iteritems():
+                self.__setattr__(k, v)
+                one_from_config = True
             self.using_default_config = False
             self.config_time = datetime.datetime.utcnow().time()
-        return is_success
+        except Exception as ex:
+            if one_from_config:
+                log.info("Load from config failed with '%s'.  Using hybrid default/config file configuration" % ex.message)
+                return False
+            else:
+                log.info("load from config failed with '%s'.  Using default config" % ex.message)
+        return True
+
+    def __setattr__(self, key, value):
+        if key.endswith('_'):
+            for k, v in value.iteritems():
+                self.__setattr__(k, v)
+        else:
+            super(CoverageConfig, self).__setattr__(key,value)
 
     def get_preferred_key(self, options, preferences):
         for key in preferences:

coverage_model/coverage.py

@@ -45,6 +45,7 @@
 from pyon.util.async import spawn
 from coverage_model.basic_types import AbstractIdentifiable, AxisTypeEnum, MutabilityEnum, VariabilityEnum, Dictable, \
     Span
+from coverage_model.config import CoverageConfig
 from coverage_model.parameter import Parameter, ParameterDictionary, ParameterContext
 from coverage_model.parameter_values import get_value_class, AbstractParameterValue
 from coverage_model.persistence import InMemoryPersistenceLayer, is_persisted
@@ -148,6 +149,9 @@ def load(cls, root_dir, persistence_guid=None, mode=None):
         mode             - I/O mode
         returns an instantiated coverage model instance
         '''
+        if root_dir != None:
+            log.info("\'root_dir\' specification is OBE.  Use coverage configuration file to specify root_dir")
+
         if not isinstance(root_dir, basestring):
             raise ValueError('\'root_dir\' must be a string')
 
@@ -993,7 +997,7 @@ def __init__(self, root_dir, persistence_guid, name=None, reference_coverage_loc
             def _doload(self):
                 # Make sure the coverage directory exists
                 if not is_persisted(root_dir, persistence_guid):
-                    raise SystemError('Cannot find specified coverage: {0}'.format(pth))
+                    raise SystemError('Cannot find specified coverage: {0}'.format(persistence_guid))
 
                 self._persistence_layer = PostgresPersistenceLayer(root_dir, persistence_guid, mode=self.mode)
                 if self._persistence_layer.version != self.version:
@@ -1024,7 +1028,7 @@ def _doload(self):
 
                 # If the coverage directory exists, load it instead!!
                 if is_persisted(root_dir, persistence_guid):
-                    log.warn('The specified coverage already exists - performing load of \'{0}\''.format(pth))
+                    log.warn('The specified coverage already exists - performing load of \'{0}\''.format(persistence_guid))
                     _doload(self)
                     return
 
@@ -1233,7 +1237,7 @@ def _existing_coverage(self, root_dir, persistence_guid):
         # If the path does exist, initialize the simple persistence layer
         pth = os.path.join(root_dir, persistence_guid)
         if not is_persisted(root_dir, persistence_guid):
-            raise SystemError('Cannot find specified coverage: {0}'.format(pth))
+            raise SystemError('Cannot find specified coverage: {0}'.format(persistence_guid))
         self._persistence_layer = PostgresPersistenceLayer(root_dir, persistence_guid, mode=self.mode)
         if self._persistence_layer.version != self.version:
             raise IOError('Coverage Model Version Mismatch: %s != %s' %(self.version, self._persistence_layer.version))
@@ -2132,20 +2136,17 @@ def __init__(self, root_dir, persistence_guid, name=None, parameter_dictionary=N
         """
         AbstractCoverage.__init__(self, mode=mode)
         try:
-            # Make sure root_dir and persistence_guid are both not None and are strings
-            if root_dir is not None and not isinstance(root_dir, basestring):
-                raise TypeError('root_dir must be instance of basestring')
             if not isinstance(persistence_guid, basestring):
                 raise TypeError('persistence_guid must be instance of basestring')
 
-            root_dir = root_dir if not root_dir.endswith(persistence_guid) else os.path.split(root_dir)[0]
-
-            pth = os.path.join(root_dir, persistence_guid)
+            if root_dir != None:
+                log.info("\'root_dir\' specification is OBE.  Use coverage configuration file to specify root_dir")
+            root_dir = CoverageConfig().top_level_storage_location
 
             def _doload(self):
                 # Make sure the coverage directory exists
                 if not is_persisted(root_dir, persistence_guid):
-                    raise SystemError('Cannot find specified coverage: {0}'.format(pth))
+                    raise SystemError('Cannot find specified coverage: {0}'.format(persistence_guid))
 
                 # All appears well - load it up!
                 self._persistence_layer = PostgresPersistenceLayer(root_dir, persistence_guid, mode=self.mode)
@@ -2193,13 +2194,13 @@ def _doload(self):
                 if name is None or parameter_dictionary is None:
                     raise SystemError('\'name\' and \'parameter_dictionary\' cannot be None')
 
-                # Make sure the specified root_dir exists
-                if not in_memory_storage and not os.path.exists(root_dir):
-                    raise SystemError('Cannot find specified \'root_dir\': {0}'.format(root_dir))
+                # # Make sure the specified root_dir exists
+                # if not in_memory_storage and not os.path.exists(root_dir):
+                #     raise SystemError('Cannot find specified \'root_dir\': {0}'.format(root_dir))
 
                 # If the coverage directory exists, load it instead!!
                 if is_persisted(root_dir, persistence_guid):
-                    log.warn('The specified coverage already exists - performing load of \'{0}\''.format(pth))
+                    log.warn('The specified coverage already exists - performing load of \'{0}\''.format(persistence_guid))
                     _doload(self)
                     return
 

coverage_model/coverages/aggregate_coverage.py

@@ -20,14 +20,15 @@ def __init__(self, root_dir, persistence_guid, name=None, reference_coverage_loc
 
         # Initializes base class with proper mode.
         super(AggregateCoverage, self).__init__(mode)
+        if root_dir != None:
+            log.info("\'root_dir\' specification is OBE.  Use coverage configuration file to specify root_dir")
 
         try:
-            # Make sure root_dir and persistence_guid are both not None and are strings
-            if not isinstance(root_dir, basestring) or not isinstance(persistence_guid, basestring):
-                raise TypeError('\'root_dir\' and \'persistence_guid\' must be instances of basestring')
-
-            root_dir = root_dir if not root_dir.endswith(persistence_guid) else os.path.split(root_dir)[0]
+            # Make sure persistence_guid is string
+            if not isinstance(persistence_guid, basestring):
+                raise TypeError('\'persistence_guid\' must be instance of basestring')
 
+            root_dir = CoverageConfig().top_level_storage_location
             if is_persisted(root_dir, persistence_guid):
                 self._existing_coverage(root_dir, persistence_guid)
             else:

coverage_model/db_backed_metadata.py

@@ -28,7 +28,10 @@ def dirExists(directory):
     @staticmethod
     def isPersisted(directory, guid):
         if DbBackedMetadataManager.is_persisted_in_db(guid) is False:
-            return MasterManager.isPersisted(directory, guid)
+            if directory is not None:
+                return MasterManager.isPersisted(directory, guid)
+            else:
+                return False
         return True
 
     @staticmethod

coverage_model/parameter_functions.py

@@ -288,11 +288,8 @@ def __init__(self, name, external_guid, external_name):
 
     def load_coverage(self, pdir):
         from coverage_model.coverage import AbstractCoverage
-        root_path, guid = os.path.split(pdir)
         external_guid = self.param_map[self.external_name]
-        path = os.path.join(root_path, external_guid)
-        # cov = AbstractCoverage.resurrect(external_guid, mode='r')
-        cov = AbstractCoverage.load(path, mode='r')
+        cov = AbstractCoverage.resurrect(external_guid, mode='r')
         return cov
 
     def evaluate(self, pval_callback, pdir, time_segment, fill_value=-9999):

coverage_model/test/coverage_test_base.py

@@ -282,6 +282,7 @@ def test_create_multi_bricks(self):
         except:
             raise
 
+    @unittest.skip('root_dir moved to configuration file.')
     def test_create_dir_not_exists(self):
         # Tests creation of SimplexCoverage fails using an incorrect path
         pdict = get_parameter_dict()

coverage_model/test/test_complex_coverage.py

@@ -455,7 +455,7 @@ def test_temporal_aggregation(self):
         np.testing.assert_array_equal(comp_cov.get_parameter_values('data_c').get_data()['data_c'], third)
 
         # Check that the head_coverage_path is correct
-        self.assertEqual(comp_cov.head_coverage_path, covc_pth)
+        self.assertEqual(os.path.relpath(comp_cov.head_coverage_path), os.path.relpath(covc_pth))
 
         # Add some data to the last coverage (covc) and make sure it comes in
         cov_c = AbstractCoverage.load(covc_pth, mode='a')
@@ -478,7 +478,7 @@ def test_temporal_aggregation(self):
         np.testing.assert_array_equal(comp_cov.get_parameter_values('data_c').get_data()['data_c'], third)
 
         # Check that the head_coverage_path is still correct
-        self.assertEqual(comp_cov.head_coverage_path, covc_pth)
+        self.assertEqual(os.path.abspath(comp_cov.head_coverage_path), os.path.abspath(covc_pth))
 
     def _setup_allparams(self, size=10, num_covs=2, sequential_covs=True):
         # Setup types
@@ -709,14 +709,14 @@ def test_head_coverage_path(self):
                                          complex_type=ComplexCoverageType.TEMPORAL_BROADCAST)
 
             # Ensure the correct path is returned from NewComplexCoverage.head_coverage_path in CC --> [SC & SC] scenario
-            self.assertEqual(comp_cov.head_coverage_path, covb_pth)
+            self.assertEqual(os.path.abspath(comp_cov.head_coverage_path), os.path.abspath(covb_pth))
 
             # Ensure the correct path is returned from NewComplexCoverage.head_coverage_path in CC --> [SC & VC] scenario
-            self.assertEqual(comp_cov2.head_coverage_path, cova_pth)
-            self.assertEqual(comp_cov3.head_coverage_path, covb_pth)
+            self.assertEqual(os.path.abspath(comp_cov2.head_coverage_path), os.path.abspath(cova_pth))
+            self.assertEqual(os.path.abspath(comp_cov3.head_coverage_path), os.path.abspath(covb_pth))
 
             # Ensure the correct path is returned from NewComplexCoverage.head_coverage_path in CC --> [SC & CC --> [VC & SC]] scenario
-            self.assertEqual(comp_cov3.head_coverage_path, covb_pth)
+            self.assertEqual(os.path.abspath(comp_cov3.head_coverage_path), os.path.abspath(covb_pth))
 
     def make_timeseries_cov(self):
         cova_pth = _make_cov(self.working_dir, ['value_set'], data_dict={'time': np.arange(10,20),'value_set':np.ones(10)})
@@ -1076,7 +1076,7 @@ def test_external_refs(self):
         offset.param_map = {'x':'value_set'}
         ctx = ParameterContext('offset', param_type=ParameterFunctionType(offset, value_encoding='<f4'))
 
-        cova_pth = _make_cov(self.working_dir, ['value_set', ctx], data_dict={'time':np.arange(10), 'value_set':np.arange(20,30)})
+        cova_pth = _make_cov(self.working_dir, ['value_set', ctx], data_dict={'time':np.arange(0,10,0.7), 'value_set':np.arange(20,30, 0.7)})
         cova = SimplexCoverage.load(cova_pth, mode='r')
 
 
@@ -1086,7 +1086,9 @@ def test_external_refs(self):
         covb_pth = _make_cov(self.working_dir, [ctx], data_dict={'time':np.arange(0.5, 10.5, 1)})
         cov = SimplexCoverage.load(covb_pth, mode='r')
         # Assert that the values are correctly interpolated
-        np.testing.assert_array_equal(cov.get_parameter_values('example').get_data()['example'], np.arange(20.5, 30.5, 1))
+        data = cov.get_parameter_values().get_data()
+        np.testing.assert_allclose(data['example'], np.arange(20.5, 30.5, 1))
+        np.testing.assert_array_equal(data['time'], np.arange(0.5, 10.5, 1))
 
 
 def create_all_params():

coverage_model/test/test_simplex_coverage.py

@@ -510,3 +510,34 @@ def test_ptypescov_get_values(self):
     @unittest.skip('Does not apply to empty coverage.')
     def test_get_all_data_metrics(self):
         pass
+
+    def test_load_without_dir(self):
+        ptypes_cov, cov_name = self.get_cov(nt=2000)
+        self.assertIsInstance(ptypes_cov, AbstractCoverage)
+
+        cov_id = ptypes_cov.persistence_guid
+        ptypes_cov.close()
+
+        cov = AbstractCoverage.resurrect(cov_id, mode='r')
+        # QuantityType
+        function_params = {}
+        res = cov.get_parameter_values(['time', 'const_int', 'const_rng_int'], function_params=function_params)
+        np.testing.assert_array_equal( res.get_data()['time'], np.arange(2000))
+        # ConstantType
+        self.assertTrue('const_int' in function_params)
+        self.assertTrue('const_rng_int' in function_params)
+        self.assertEqual(len(function_params['const_int']), 1)
+        self.assertEqual(len(function_params['const_rng_int']), 1)
+        for val in function_params['const_int'].values():
+            self.assertEqual(ConstantOverTime('const_int', 45.32), val)
+        for val in function_params['const_rng_int'].values():
+            self.assertEqual(ConstantOverTime('const_rng_int', (-10,10)), val)
+
+        expected_const_int_arr = np.empty(2000)
+        expected_const_int_arr.fill(45)
+        np.testing.assert_array_equal(res.get_data()['const_int'], expected_const_int_arr)
+        # ConstantRangeType
+        expected_const_rng_int_arr = np.empty(2000, dtype=np.object)
+        expected_const_rng_int_arr.fill((-10,10))
+        np.testing.assert_array_equal(res.get_data()['const_rng_int'], expected_const_rng_int_arr)
+        ptypes_cov.close()