From b6d9fea264c9c1f1723553b6d8aa83079d57bc68 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 4 Jan 2025 13:16:11 -0800 Subject: [PATCH 1/9] remove anti_alias_filter accessors from FCDecimation and AuroraDecimationLevel - anti_alias_filter is now accessed by self.decimation.anti_alias_filter for AuroraDecimationLevel - anti_alias_filter is now accessed by self.time_series_decimation.anti_alias_filter for FCDecimation --- .../processing/aurora/decimation_level.py | 39 +++++++------------ .../fourier_coefficients/decimation.py | 19 --------- .../processing/time_series_decimation.py | 30 ++------------ 3 files changed, 18 insertions(+), 70 deletions(-) diff --git a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py index 5d30a47e..a62ed8a4 100644 --- a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py +++ b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py @@ -96,9 +96,6 @@ def __init__(self, **kwargs): super().__init__(attr_dict=attr_dict, **kwargs) - # if self.decimation.level == 0: - # self.anti_alias_filter = None - @property def window(self) -> Window: """ @@ -111,14 +108,6 @@ def window(self) -> Window: """ return self.stft.window - @property - def anti_alias_filter(self) -> str: - """ - get anti_alais_filter from TimeSeriesDecimation. - - """ - return self.decimation.anti_alias_filter - @property def bands(self) -> list: """ @@ -344,15 +333,15 @@ def is_consistent_with_archived_fc_parameters( Iterates over FCDecimation attributes: "channels_estimated": to ensure all expected channels are in the group - "anti_alias_filter": check that the expected AAF was applied - "sample_rate, - "method", - "prewhitening_type", - "recoloring", - "pre_fft_detrend_type", - "min_num_stft_windows", - "window", - "harmonic_indices", + "decimation.anti_alias_filter": check that the expected AAF was applied + "decimation.sample_rate, + "decimation.method", + "stft.prewhitening_type", + "stft.recoloring", + "stft.pre_fft_detrend_type", + "stft.min_num_stft_windows", + "stft.window", + "stft.harmonic_indices", Returns ------- @@ -373,18 +362,18 @@ def is_consistent_with_archived_fc_parameters( self.logger.info(msg) return False - # anti_alias_filter: Check that the data were + # anti_alias_filter: Check that the data were filtered the same way try: - assert fc_decimation.decimation_anti_alias_filter == self.decimation.anti_alias_filter + assert fc_decimation.time_series_decimation.anti_alias_filter == self.decimation.anti_alias_filter except AssertionError: - cond1 = self.anti_alias_filter == "default" - cond2 = fc_decimation.decimation_anti_alias_filter is None + cond1 = self.time_series_decimation.anti_alias_filter == "default" + cond2 = fc_decimation.time_series_decimation.anti_alias_filter is None if cond1 & cond2: pass else: msg = ( "Antialias Filters Not Compatible -- need to add handling for " - f"{msg} FCdec {fc_decimation.decimation_anti_alias_filter} and " + f"{msg} FCdec {fc_decimation.time_series_decimation.anti_alias_filter} and " f"{msg} processing config:{self.decimation.anti_alias_filter}" ) raise NotImplementedError(msg) diff --git a/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py b/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py index 484bac0b..1460a165 100644 --- a/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py +++ b/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py @@ -70,9 +70,6 @@ def __init__(self, **kwargs): logger.debug(msg) self.short_time_fourier_transform.per_window_detrend_type = "" - # if self.time_series_decimation.level == 0: - # self.time_series_decimation.anti_alias_filter = None - @property def window(self): return self.stft.window @@ -158,22 +155,6 @@ def decimation_method(self, value: str) -> None: """ self.time_series_decimation.method = value - @property - def decimation_anti_alias_filter(self) -> str: - """ - Access the decimation anti_alias_filter description from the TSDecimation - :return: Description of how anti_alias_filtering is performed - :rtype: str - """ - return self.time_series_decimation.anti_alias_filter - - @decimation_method.setter - def decimation_anti_alias_filter(self, value: str) -> None: - """ - Set the decimation_anti_alias_filter in the TSDecimation - """ - self.time_series_decimation.anti_alias_filter = value - @property def decimation_sample_rate(self) -> float: """ diff --git a/mt_metadata/transfer_functions/processing/time_series_decimation.py b/mt_metadata/transfer_functions/processing/time_series_decimation.py index dc7a2575..79a99390 100644 --- a/mt_metadata/transfer_functions/processing/time_series_decimation.py +++ b/mt_metadata/transfer_functions/processing/time_series_decimation.py @@ -10,7 +10,7 @@ ["level", "factor", "method", "sample_rate", "anti_alias_filter"], TODO: Consider adding a parent_sample_rate attribute to this class - + Created on Thu Dec 26 12:00:00 2024 @author: kkappler @@ -45,28 +45,6 @@ def __init__(self, **kwargs): super().__init__(attr_dict=attr_dict, **kwargs) - # Temporary workarounds while replacing legacy Decimation class - # @property - # def level(self): - # return self.decimation_level - # - # @property - # def factor(self): - # return self.decimation_factor - # - # @property - # def method(self): - # return self.decimation_method - # - # @property - # def sample_rate(self): - # return self.sample_rate_decimation - - -# -# def main(): -# pass -# -# -# if __name__ == "__main__": -# main() + # TODO: add this logic to __init__ and a test + # if self.level == 0: + # self.anti_alias_filter = None From a0a46d10f7ad22ceaaabb37362c4cc8f01f3178e Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 4 Jan 2025 13:31:31 -0800 Subject: [PATCH 2/9] add logging --- .../transfer_functions/processing/aurora/decimation_level.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py index a62ed8a4..ee44bf0f 100644 --- a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py +++ b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py @@ -467,6 +467,8 @@ def is_consistent_with_archived_fc_parameters( # if harmonic_indices is -1, it means the archive kept all so we can skip this check. pass else: + msg = "WIP: harmonic indices in AuroraDecimationlevel are derived from processing bands -- Not robustly tested to compare with FCDecimation" + self.logger.debug(msg) harmonic_indices_requested = self.harmonic_indices fcdec_group_set = set(fc_decimation.harmonic_indices) processing_set = set(harmonic_indices_requested) From e737ac5020150e45d7cae5f4732ac8c1a9d7f978 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 4 Jan 2025 13:37:28 -0800 Subject: [PATCH 3/9] window attr of FCDecimation now accessed via stft --- .../processing/fourier_coefficients/decimation.py | 12 ++++-------- tests/tf/processing/fcs/test_decimation.py | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py b/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py index 1460a165..d532b084 100644 --- a/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py +++ b/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py @@ -70,10 +70,6 @@ def __init__(self, **kwargs): logger.debug(msg) self.short_time_fourier_transform.per_window_detrend_type = "" - @property - def window(self): - return self.stft.window - def __len__(self) -> int: return len(self.channels) @@ -427,14 +423,14 @@ def is_valid_for_time_series_length(self, n_samples_ts: int) -> bool: """ required_num_samples = ( - self.window.num_samples - + (self.stft.min_num_stft_windows - 1) * self.window.num_samples_advance + self.stft.window.num_samples + + (self.stft.min_num_stft_windows - 1) * self.stft.window.num_samples_advance ) if n_samples_ts < required_num_samples: msg = ( f"{n_samples_ts} not enough samples for minimum of " f"{self.stft.min_num_stft_windows} stft windows of length " - f"{self.window.num_samples} and overlap {self.window.overlap}" + f"{self.stft.window.num_samples} and overlap {self.stft.window.overlap}" ) self.logger.warning(msg) return False @@ -444,7 +440,7 @@ def is_valid_for_time_series_length(self, n_samples_ts: int) -> bool: @property def fft_frequencies(self) -> np.ndarray: """ Returns the one-sided fft frequencies (without Nyquist)""" - return self.window.fft_harmonics(self.sample_rate) + return self.stft.window.fft_harmonics(self.sample_rate) def fc_decimations_creator( diff --git a/tests/tf/processing/fcs/test_decimation.py b/tests/tf/processing/fcs/test_decimation.py index b6ec63c7..979be84d 100644 --- a/tests/tf/processing/fcs/test_decimation.py +++ b/tests/tf/processing/fcs/test_decimation.py @@ -116,7 +116,7 @@ def test_update(self): def test_fft_frequencies(self): dl1 = self.dl.copy() freqs = dl1.fft_frequencies - assert (len(freqs) == dl1.window.num_samples/2) + assert (len(freqs) == dl1.stft.window.num_samples/2) def test_update_with_match(self): From 53241f22f6036effd06020647c70a2b8e3d37f9c Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 4 Jan 2025 13:39:25 -0800 Subject: [PATCH 4/9] window attr of AuroraDecimationLevel now accessed via stft --- .../processing/aurora/decimation_level.py | 14 +------------- tests/tf/processing/fcs/test_decimation.py | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py index ee44bf0f..baf1a105 100644 --- a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py +++ b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py @@ -96,18 +96,6 @@ def __init__(self, **kwargs): super().__init__(attr_dict=attr_dict, **kwargs) - @property - def window(self) -> Window: - """ - Convenience access to STFT window metadata. - - This was placed here to allow access to legacy Decimation's window attribute. - - Note: This maybe deprecated in future to use only direct access via self.stft.window. - - """ - return self.stft.window - @property def bands(self) -> list: """ @@ -213,7 +201,7 @@ def frequency_sample_interval(self) -> float: Returns the delta_f in frequency domain df = 1 / (N * dt) Here dt is the sample interval after decimation """ - return self.sample_rate_decimation / self.window.num_samples + return self.sample_rate_decimation / self.stft.window.num_samples @property def band_edges(self) -> np.ndarray: diff --git a/tests/tf/processing/fcs/test_decimation.py b/tests/tf/processing/fcs/test_decimation.py index 979be84d..f868f528 100644 --- a/tests/tf/processing/fcs/test_decimation.py +++ b/tests/tf/processing/fcs/test_decimation.py @@ -253,7 +253,7 @@ def test_pre_fft_detrend_type_false(self): ) def test_window_false(self): - self.adl.window.type = "dpss" + self.adl.stft.window.type = "dpss" self.assertEqual( False, self.adl.is_consistent_with_archived_fc_parameters(self.dl, None) ) From 8e0d4d2196c4ec446dc16a6d900f9781f2e26ad5 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 4 Jan 2025 14:30:40 -0800 Subject: [PATCH 5/9] add a couple missed stft.window accessors --- .../transfer_functions/processing/aurora/decimation_level.py | 4 ++-- .../transfer_functions/processing/aurora/processing.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py index baf1a105..009ad372 100644 --- a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py +++ b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py @@ -216,7 +216,7 @@ def band_edges(self) -> np.ndarray: ).T return band_edges - def frequency_bands_obj(self) -> FrequencyBands: # TODO: FIXME circular import when correctly dtyped -> FrequencyBands: + def frequency_bands_obj(self) -> FrequencyBands: """ Gets a FrequencyBands object that is used as input to processing. @@ -268,7 +268,7 @@ def fft_frequencies(self) -> np.ndarray: :return freqs: The frequencies at which the stft will be available. :rtype freqs: np.ndarray """ - freqs = self.window.fft_harmonics(self.decimation.sample_rate) + freqs = self.stft.window.fft_harmonics(self.decimation.sample_rate) return freqs @property diff --git a/mt_metadata/transfer_functions/processing/aurora/processing.py b/mt_metadata/transfer_functions/processing/aurora/processing.py index 145bbe15..bf2ba30e 100644 --- a/mt_metadata/transfer_functions/processing/aurora/processing.py +++ b/mt_metadata/transfer_functions/processing/aurora/processing.py @@ -216,7 +216,7 @@ def assign_bands( ) # self.decimations_dict[key] decimation_obj.decimation.factor = d decimation_obj.decimation.sample_rate = sr - decimation_obj.window.num_samples = num_samples_window[i_level] + decimation_obj.stft.window.num_samples = num_samples_window[i_level] frequencies = decimation_obj.fft_frequencies for low, high in band_edges: From f56dced457dcfe021156a8f7c58e9950d263b5cb Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 4 Jan 2025 14:30:55 -0800 Subject: [PATCH 6/9] remove commented block - added temporary test for this TODO in aurora --- .../processing/aurora/decimation_level.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py index 009ad372..221b7422 100644 --- a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py +++ b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py @@ -233,33 +233,6 @@ def frequency_bands_obj(self) -> FrequencyBands: frequency_bands = FrequencyBands(band_edges=self.band_edges) return frequency_bands - # # TODO: FIXME WIP - # def to_frequency_bands_obj(self): - # """ - # Define band_edges array from decimation_level object, - # - # Development Notes. - # This function was originally in FrequencyBands class, it was called: - # from_decimation_object. Circular imports were encountered when it was correctly dtyped. - # There is no reason to have FrequencyBands.from_decimation_object(decimation_level) - # _and_ decimation_level.to_frequency_bands_obj() - # The function above already does the task of generating a frequency bands. - # Keeping this commented until documentation improves. - # Below looks like an alternative, and more readable way to get band_edges, - # without passing through the dataframe. At a minimum a test should be created - # that makes band edges both ways and asserts equal. - # (a few command line tests showed that they are, Dec 2024). - # - # """ - # df = self.frequency_sample_interval - # half_df = df / 2.0 - # - # lower_edges = (self.lower_bounds * df) - half_df - # upper_edges = (self.upper_bounds * df) + half_df - # band_edges = np.vstack((lower_edges, upper_edges)).T - # return FrequencyBands(band_edges=band_edges) - - @property def fft_frequencies(self) -> np.ndarray: """ From da533ed5d1186a37b3119b20a860a5d0d0a150cd Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 4 Jan 2025 14:36:31 -0800 Subject: [PATCH 7/9] replace sample_rate_decimation properties with direct access - use self.decimation.sample_rate instead - this will trigger updates in mth5, and aurora --- .../processing/aurora/decimation_level.py | 13 ++--------- .../fourier_coefficients/decimation.py | 23 ------------------- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py index 221b7422..0d449f43 100644 --- a/mt_metadata/transfer_functions/processing/aurora/decimation_level.py +++ b/mt_metadata/transfer_functions/processing/aurora/decimation_level.py @@ -201,7 +201,7 @@ def frequency_sample_interval(self) -> float: Returns the delta_f in frequency domain df = 1 / (N * dt) Here dt is the sample interval after decimation """ - return self.sample_rate_decimation / self.stft.window.num_samples + return self.decimation.sample_rate / self.stft.window.num_samples @property def band_edges(self) -> np.ndarray: @@ -244,15 +244,6 @@ def fft_frequencies(self) -> np.ndarray: freqs = self.stft.window.fft_harmonics(self.decimation.sample_rate) return freqs - @property - def sample_rate_decimation(self) -> float: - """ - Returns the sample rate of the data after decimation. - TODO: Delete this method and replace calls to self.sample_rate_decimation with self.decimation.sample_rate - - """ - return self.decimation.sample_rate - @property def harmonic_indices(self) -> List[int]: """ @@ -494,7 +485,7 @@ def to_fc_decimation( fc_dec_obj.stft.pre_fft_detrend_type = self.stft.pre_fft_detrend_type fc_dec_obj.stft.prewhitening_type = self.stft.prewhitening_type fc_dec_obj.stft.recoloring = self.stft.recoloring - fc_dec_obj.time_series_decimation.sample_rate = self.sample_rate_decimation + fc_dec_obj.time_series_decimation.sample_rate = self.decimation.sample_rate fc_dec_obj.stft.window = self.stft.window return fc_dec_obj diff --git a/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py b/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py index d532b084..ba6995ce 100644 --- a/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py +++ b/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py @@ -160,29 +160,6 @@ def decimation_sample_rate(self) -> float: """ return self.time_series_decimation.sample_rate - @property - def sample_rate_decimation(self) -> float: - """ - TODO: Delete this function in 2025. - - Access the decimation sample rate from the TSDecimation - :return:Time series sample rate after decimation (from the TSDecimation) - :rtype: str - """ - msg = "sample_rate_decimation is deprecated -- use self.time_series_decimation.sample_rate" - logger.warning(msg) - return self.time_series_decimation.sample_rate - - @sample_rate_decimation.setter - def sample_rate_decimation(self, value: float) -> None: - """ - TODO: Delete this function in 2025. - - Set the decimation sample_rate_decimation in the TSDecimation - """ - msg = "sample_rate_decimation is deprecated -- use self.time_series_decimation.sample_rate" - logger.warning(msg) - self.time_series_decimation.sample_rate = value #----- End (Possibly Temporary) methods for integrating TimeSeriesDecimation Class -----# From b1a01137368079fad94d859e6b257730586a8781 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 4 Jan 2025 14:43:26 -0800 Subject: [PATCH 8/9] fix test --- tests/tf/processing/fcs/test_decimation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tf/processing/fcs/test_decimation.py b/tests/tf/processing/fcs/test_decimation.py index f868f528..1320503c 100644 --- a/tests/tf/processing/fcs/test_decimation.py +++ b/tests/tf/processing/fcs/test_decimation.py @@ -203,7 +203,7 @@ def setUp(self): self.dl.decimation_factor = 4 self.dl.decimation_level = 1 self.dl.id = 1 - self.dl.sample_rate_decimation = 16 + self.dl.time_series_decimation.sample_rate = 16 for ch in ["ex", "ey", "hx", "hy", "hz"]: self.dl.add_channel(Channel(component=ch)) From f9c136f2318f789283e105b54579fc31e0f68f5c Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 4 Jan 2025 14:48:17 -0800 Subject: [PATCH 9/9] add decimation attr/shortcut for time_series_decimation --- .../processing/fourier_coefficients/decimation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py b/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py index ba6995ce..5c764598 100644 --- a/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py +++ b/mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py @@ -89,6 +89,14 @@ def __add__(self, other): raise TypeError(msg) #----- Begin (Possibly Temporary) methods for integrating TimeSeriesDecimation Class -----# + + @property + def decimation(self) -> TimeSeriesDecimation: + """ + Passthrough method to access self.time_series_decimation + """ + return self.time_series_decimation + @property def factor(self): """