From c8296bad8a58758c29977fe991af262c70ec2d0d Mon Sep 17 00:00:00 2001 From: Casey Bryant Date: Sun, 6 Jul 2014 17:42:56 -0700 Subject: [PATCH] M117 - Made 'root_dir' to a configurable option. --- coverage_model/config.py | 27 ++++++++++++++-- coverage_model/coverage.py | 29 ++++++++--------- .../coverages/aggregate_coverage.py | 11 ++++--- coverage_model/db_backed_metadata.py | 5 ++- coverage_model/parameter_functions.py | 5 +-- coverage_model/test/coverage_test_base.py | 1 + coverage_model/test/test_complex_coverage.py | 18 ++++++----- coverage_model/test/test_simplex_coverage.py | 31 +++++++++++++++++++ 8 files changed, 92 insertions(+), 35 deletions(-) diff --git a/coverage_model/config.py b/coverage_model/config.py index 6de47bd..2c146db 100644 --- a/coverage_model/config.py +++ b/coverage_model/config.py @@ -1,6 +1,7 @@ __author__ = 'casey' from ooi.logging import log +from pyon.util.config import Config import datetime @@ -25,8 +26,10 @@ class CoverageConfig(object): _default_vertical_db_key = 'vertical_range' _default_span_id_db_key = 'span_address' _default_span_coverage_id_db_key = 'coverage_id' + _default_storage_location = None def __init__(self): + print 'setting defaults' self.ordered_time_key_preferences = self._default_ordered_time_key_preferences self.ordered_lat_key_preferences = self._default_ordered_lat_key_preferences self.ordered_lon_key_preferences = self._default_ordered_lon_key_preferences @@ -36,16 +39,34 @@ def __init__(self): self.vertical_db_key = self._default_vertical_db_key self.span_id_db_key = self._default_span_id_db_key self.span_coverage_id_db_key = self._default_span_coverage_id_db_key + self.top_level_storage_location = self._default_storage_location self.using_default_config = True self.config_time = 0 self.read_and_set_config() def read_and_set_config(self): - is_success = False - if is_success is True: + one_from_config = False + try: + data = Config(["res/config/coverage.yml"]).data['CoverageConfig'] + for k, v in data.iteritems(): + self.__setattr__(k, v) + one_from_config = True self.using_default_config = False self.config_time = datetime.datetime.utcnow().time() - return is_success + except Exception as ex: + if one_from_config: + log.info("Load from config failed with '%s'. Using hybrid default/config file configuration" % ex.message) + return False + else: + log.info("load from config failed with '%s'. Using default config" % ex.message) + return True + + def __setattr__(self, key, value): + if key.endswith('_'): + for k, v in value.iteritems(): + self.__setattr__(k, v) + else: + super(CoverageConfig, self).__setattr__(key,value) def get_preferred_key(self, options, preferences): for key in preferences: diff --git a/coverage_model/coverage.py b/coverage_model/coverage.py index c795f4e..1789c4e 100644 --- a/coverage_model/coverage.py +++ b/coverage_model/coverage.py @@ -45,6 +45,7 @@ from pyon.util.async import spawn from coverage_model.basic_types import AbstractIdentifiable, AxisTypeEnum, MutabilityEnum, VariabilityEnum, Dictable, \ Span +from coverage_model.config import CoverageConfig from coverage_model.parameter import Parameter, ParameterDictionary, ParameterContext from coverage_model.parameter_values import get_value_class, AbstractParameterValue from coverage_model.persistence import InMemoryPersistenceLayer, is_persisted @@ -148,6 +149,9 @@ def load(cls, root_dir, persistence_guid=None, mode=None): mode - I/O mode returns an instantiated coverage model instance ''' + if root_dir != None: + log.info("\'root_dir\' specification is OBE. Use coverage configuration file to specify root_dir") + if not isinstance(root_dir, basestring): raise ValueError('\'root_dir\' must be a string') @@ -993,7 +997,7 @@ def __init__(self, root_dir, persistence_guid, name=None, reference_coverage_loc def _doload(self): # Make sure the coverage directory exists if not is_persisted(root_dir, persistence_guid): - raise SystemError('Cannot find specified coverage: {0}'.format(pth)) + raise SystemError('Cannot find specified coverage: {0}'.format(persistence_guid)) self._persistence_layer = PostgresPersistenceLayer(root_dir, persistence_guid, mode=self.mode) if self._persistence_layer.version != self.version: @@ -1024,7 +1028,7 @@ def _doload(self): # If the coverage directory exists, load it instead!! if is_persisted(root_dir, persistence_guid): - log.warn('The specified coverage already exists - performing load of \'{0}\''.format(pth)) + log.warn('The specified coverage already exists - performing load of \'{0}\''.format(persistence_guid)) _doload(self) return @@ -1233,7 +1237,7 @@ def _existing_coverage(self, root_dir, persistence_guid): # If the path does exist, initialize the simple persistence layer pth = os.path.join(root_dir, persistence_guid) if not is_persisted(root_dir, persistence_guid): - raise SystemError('Cannot find specified coverage: {0}'.format(pth)) + raise SystemError('Cannot find specified coverage: {0}'.format(persistence_guid)) self._persistence_layer = PostgresPersistenceLayer(root_dir, persistence_guid, mode=self.mode) if self._persistence_layer.version != self.version: raise IOError('Coverage Model Version Mismatch: %s != %s' %(self.version, self._persistence_layer.version)) @@ -2132,20 +2136,17 @@ def __init__(self, root_dir, persistence_guid, name=None, parameter_dictionary=N """ AbstractCoverage.__init__(self, mode=mode) try: - # Make sure root_dir and persistence_guid are both not None and are strings - if root_dir is not None and not isinstance(root_dir, basestring): - raise TypeError('root_dir must be instance of basestring') if not isinstance(persistence_guid, basestring): raise TypeError('persistence_guid must be instance of basestring') - root_dir = root_dir if not root_dir.endswith(persistence_guid) else os.path.split(root_dir)[0] - - pth = os.path.join(root_dir, persistence_guid) + if root_dir != None: + log.info("\'root_dir\' specification is OBE. Use coverage configuration file to specify root_dir") + root_dir = CoverageConfig().top_level_storage_location def _doload(self): # Make sure the coverage directory exists if not is_persisted(root_dir, persistence_guid): - raise SystemError('Cannot find specified coverage: {0}'.format(pth)) + raise SystemError('Cannot find specified coverage: {0}'.format(persistence_guid)) # All appears well - load it up! self._persistence_layer = PostgresPersistenceLayer(root_dir, persistence_guid, mode=self.mode) @@ -2193,13 +2194,13 @@ def _doload(self): if name is None or parameter_dictionary is None: raise SystemError('\'name\' and \'parameter_dictionary\' cannot be None') - # Make sure the specified root_dir exists - if not in_memory_storage and not os.path.exists(root_dir): - raise SystemError('Cannot find specified \'root_dir\': {0}'.format(root_dir)) + # # Make sure the specified root_dir exists + # if not in_memory_storage and not os.path.exists(root_dir): + # raise SystemError('Cannot find specified \'root_dir\': {0}'.format(root_dir)) # If the coverage directory exists, load it instead!! if is_persisted(root_dir, persistence_guid): - log.warn('The specified coverage already exists - performing load of \'{0}\''.format(pth)) + log.warn('The specified coverage already exists - performing load of \'{0}\''.format(persistence_guid)) _doload(self) return diff --git a/coverage_model/coverages/aggregate_coverage.py b/coverage_model/coverages/aggregate_coverage.py index 5f89103..5a54750 100644 --- a/coverage_model/coverages/aggregate_coverage.py +++ b/coverage_model/coverages/aggregate_coverage.py @@ -20,14 +20,15 @@ def __init__(self, root_dir, persistence_guid, name=None, reference_coverage_loc # Initializes base class with proper mode. super(AggregateCoverage, self).__init__(mode) + if root_dir != None: + log.info("\'root_dir\' specification is OBE. Use coverage configuration file to specify root_dir") try: - # Make sure root_dir and persistence_guid are both not None and are strings - if not isinstance(root_dir, basestring) or not isinstance(persistence_guid, basestring): - raise TypeError('\'root_dir\' and \'persistence_guid\' must be instances of basestring') - - root_dir = root_dir if not root_dir.endswith(persistence_guid) else os.path.split(root_dir)[0] + # Make sure persistence_guid is string + if not isinstance(persistence_guid, basestring): + raise TypeError('\'persistence_guid\' must be instance of basestring') + root_dir = CoverageConfig().top_level_storage_location if is_persisted(root_dir, persistence_guid): self._existing_coverage(root_dir, persistence_guid) else: diff --git a/coverage_model/db_backed_metadata.py b/coverage_model/db_backed_metadata.py index e407b0a..d2b9543 100644 --- a/coverage_model/db_backed_metadata.py +++ b/coverage_model/db_backed_metadata.py @@ -28,7 +28,10 @@ def dirExists(directory): @staticmethod def isPersisted(directory, guid): if DbBackedMetadataManager.is_persisted_in_db(guid) is False: - return MasterManager.isPersisted(directory, guid) + if directory is not None: + return MasterManager.isPersisted(directory, guid) + else: + return False return True @staticmethod diff --git a/coverage_model/parameter_functions.py b/coverage_model/parameter_functions.py index 4a9d6ed..09dedef 100644 --- a/coverage_model/parameter_functions.py +++ b/coverage_model/parameter_functions.py @@ -288,11 +288,8 @@ def __init__(self, name, external_guid, external_name): def load_coverage(self, pdir): from coverage_model.coverage import AbstractCoverage - root_path, guid = os.path.split(pdir) external_guid = self.param_map[self.external_name] - path = os.path.join(root_path, external_guid) - # cov = AbstractCoverage.resurrect(external_guid, mode='r') - cov = AbstractCoverage.load(path, mode='r') + cov = AbstractCoverage.resurrect(external_guid, mode='r') return cov def evaluate(self, pval_callback, pdir, time_segment, fill_value=-9999): diff --git a/coverage_model/test/coverage_test_base.py b/coverage_model/test/coverage_test_base.py index e7b1f90..86ab579 100644 --- a/coverage_model/test/coverage_test_base.py +++ b/coverage_model/test/coverage_test_base.py @@ -282,6 +282,7 @@ def test_create_multi_bricks(self): except: raise + @unittest.skip('root_dir moved to configuration file.') def test_create_dir_not_exists(self): # Tests creation of SimplexCoverage fails using an incorrect path pdict = get_parameter_dict() diff --git a/coverage_model/test/test_complex_coverage.py b/coverage_model/test/test_complex_coverage.py index 49c390e..637a550 100644 --- a/coverage_model/test/test_complex_coverage.py +++ b/coverage_model/test/test_complex_coverage.py @@ -455,7 +455,7 @@ def test_temporal_aggregation(self): np.testing.assert_array_equal(comp_cov.get_parameter_values('data_c').get_data()['data_c'], third) # Check that the head_coverage_path is correct - self.assertEqual(comp_cov.head_coverage_path, covc_pth) + self.assertEqual(os.path.relpath(comp_cov.head_coverage_path), os.path.relpath(covc_pth)) # Add some data to the last coverage (covc) and make sure it comes in cov_c = AbstractCoverage.load(covc_pth, mode='a') @@ -478,7 +478,7 @@ def test_temporal_aggregation(self): np.testing.assert_array_equal(comp_cov.get_parameter_values('data_c').get_data()['data_c'], third) # Check that the head_coverage_path is still correct - self.assertEqual(comp_cov.head_coverage_path, covc_pth) + self.assertEqual(os.path.abspath(comp_cov.head_coverage_path), os.path.abspath(covc_pth)) def _setup_allparams(self, size=10, num_covs=2, sequential_covs=True): # Setup types @@ -709,14 +709,14 @@ def test_head_coverage_path(self): complex_type=ComplexCoverageType.TEMPORAL_BROADCAST) # Ensure the correct path is returned from NewComplexCoverage.head_coverage_path in CC --> [SC & SC] scenario - self.assertEqual(comp_cov.head_coverage_path, covb_pth) + self.assertEqual(os.path.abspath(comp_cov.head_coverage_path), os.path.abspath(covb_pth)) # Ensure the correct path is returned from NewComplexCoverage.head_coverage_path in CC --> [SC & VC] scenario - self.assertEqual(comp_cov2.head_coverage_path, cova_pth) - self.assertEqual(comp_cov3.head_coverage_path, covb_pth) + self.assertEqual(os.path.abspath(comp_cov2.head_coverage_path), os.path.abspath(cova_pth)) + self.assertEqual(os.path.abspath(comp_cov3.head_coverage_path), os.path.abspath(covb_pth)) # Ensure the correct path is returned from NewComplexCoverage.head_coverage_path in CC --> [SC & CC --> [VC & SC]] scenario - self.assertEqual(comp_cov3.head_coverage_path, covb_pth) + self.assertEqual(os.path.abspath(comp_cov3.head_coverage_path), os.path.abspath(covb_pth)) def make_timeseries_cov(self): cova_pth = _make_cov(self.working_dir, ['value_set'], data_dict={'time': np.arange(10,20),'value_set':np.ones(10)}) @@ -1076,7 +1076,7 @@ def test_external_refs(self): offset.param_map = {'x':'value_set'} ctx = ParameterContext('offset', param_type=ParameterFunctionType(offset, value_encoding='